#!/bin/ksh
###############################################################################
#
# This script is used to collect performance mustgather data for
# Websphere or Liberty on AIX
#
# ./aixperf.sh [PID(s)_of_the_problematic_JVM(s)_separated_by_spaces]
#
# Optional options
# -j javacore interval
# -s script span
# -t top dash interval
# -v vmstat interval
# -q quiet end message
# -z allow stats
# --disable-collecting-ifconfig to disable collecting ifconfig
# --disable-collecting-hostname to disable collecting hostname
#
# Example with some options against a Java PID:
# A javacore/thread dump every 5 seconds, running for 30 seconds:print
# ./aixperf.sh -j 5 -s 30 PID_of_the_problematic_JVM
#
# Example without PID for collecting stats, using -z flag:
# ./aixperf.sh -z
# 
#####################################################################################################
# WSADMIN START: If there are any code changes between below this and WSADMIN END, 
# be sure to update the aixperf_shell_script_content in the perf.py script and test the changes.
#####################################################################################################
#
# Variables
#
#####################################################################################################
SCRIPT_SPAN=240         # How long the whole script should take. Default=240
JAVACORE_INTERVAL=30    # How often javacores should be taken. Default=30  
TPROF_SPAN=60           # How long tprof should collect data. Default=60
VMSTAT_INTERVAL=5       # How often vmstat data should be taken. Default=5
KEEP_QUIET=0            # To disable the end print message, change to 1. Default=0
ALLOW_STATS=0			      # To collect OS data without provided PIDs, change to 1. Default=0
ROOT_ACCESS_REQUIRED=1        # Default=1 to require root for running the script.
DISABLE_COLLECTING_IFCONFIG=0 # Default=0 to collect ifconfig info.
DISABLE_COLLECTING_HOSTNAME=0 # Default=0 to collect hostname info.
#####################################################################################################
# * All the 'INTERVAL' values should divide into the 'SCRIPT_SPAN' by a whole
#   integer to obtain expected results.
# * Setting any 'INTERVAL' too low (especially JAVACORE) can result in data
#   that may not be useful towards resolving the issue.  This becomes a problem
#   when the process of collecting data obscures the real issue.
###############################################################################
SCRIPT_VERSION=2026.06.25
START_DAY="$(date +%Y%m%d)"
START_TIME="$(date +%H%M%S)"

provided_pids=""


####################################################################################################################################
# Currently not used but for future enchancements
CPU_THRESHOLD=          # Triggers data collection when the CPU usage is reached. Default empty.
                        # WARNING: ps -p $PID -o pcpu prints the process's lifetime CPU usage. Not supported but being explored.
CHECK_CPU_INTERVAL=30   # How often the script should check a process's CPU usage. Default=30.
                        # It only works with CPU_THRESHOLD=<value>, -c <value> or --cpu-threshold=<value>. 
MONITOR_ONLY=0          # Monitoring trigering events only with no data created. 
                        # It only works with -c <value> or --cpu-threshold=<value>. Default=0
####################################################################################################################################
while [[ $# -gt 0 ]]; do
	case "$1" in
		-j ) JAVACORE_INTERVAL="$2"; shift 2;;
		-q ) KEEP_QUIET=1; shift 1;;
		-s ) SCRIPT_SPAN="$2"; shift 2;;
		-t ) TPROF_SPAN="$2"; shift 2;;
		-v ) VMSTAT_INTERVAL="$2"; shift 2;;
		-z ) ALLOW_STATS=1; shift 1;;
		# Triggering event
		# -c ) CPU_THRESHOLD="$2"; shift 2;;
		
		# Disable issuing a kill -3
		--disable-kill ) disable_kill=1; shift 1;;
		# Disable writing to screen.out file. Only used by wsadmin script.
		--disable-screen-output ) DISABLE_SCREEN_OUTPUT=1; shift 1;;
		
    # Rename aixperf_results. Only used by a wsadmin script.
    --dir-name=* ) DIR_NAME="${1#*=}"; shift 1;;
    # Redirect aixperf_results data to a specified directory.
    --output-dir=* ) OUTPUT_DIR="${1#*=}"; shift 1;;

		# Monitoring a triggering event only without data created.
    --monitor-only ) MONITOR_ONLY=1; shift 1;;

		# Triggering event. Update: Not supported but being explored.
    # --cpu-threshold=* ) CPU_THRESHOLD="${1#*=}"; shift 1;;
    --check-cpu-interval=* ) CHECK_CPU_INTERVAL="${1#*=}"; shift 1;;

		# Only used by a wsadmin script.
    --hide-stamp ) HIDE_STAMP=1; shift 1;;

    # Disable collecting hostname and ifconfig
    --disable-collecting-ifconfig ) DISABLE_COLLECTING_IFCONFIG=1; shift 1;;
    --disable-collecting-hostname ) DISABLE_COLLECTING_HOSTNAME=1; shift 1;;

    # Disable root requirement
    --ignore-root ) ROOT_ACCESS_REQUIRED=0; shift 1;;

    # Clean up javacores after tarring. 
    --clean-up-javacores ) CLEAN_UP_JAVACORES=1; shift 1;;
		
		[0-9]* ) provided_pids="$provided_pids $1"; shift 1;;

		* ) print "Unknown option: $1"; exit 1;;
		
	esac
done

############################################
# Root access is required for complete data.
############################################
if [ "$(id -u)" -ne 0 ] && [ $ROOT_ACCESS_REQUIRED -eq 1 ]; then
  print "Root access is required. Use --ignore-root to bypass."
  exit 1
elif [ $ROOT_ACCESS_REQUIRED -eq 0 ]; then
  print $(date '+%Y-%m-%d %H:%M:%S') "\tWarning: Root access is disabled. Data may be incomplete."
fi

#############################################
# If PIDs are not provided, the script exits.
#############################################
if [ -z "$provided_pids" ] && [ $ALLOW_STATS -eq 0 ]; then
	print "Unable to find required PID argument. Please rerun the script as follows:"
	print "./aixperf.sh [PID(s)_of_the_problematic_JVM(s)_separated_by_spaces]"
	exit 1
fi

######################
# Verify option inputs
######################
if [ $MONITOR_ONLY -eq 1 ]; then
	if [ -z "${CPU_THRESHOLD}" ]; then
		echo "To run with --monitor-only, include --cpu-threshold=<value>. Exiting the script."
		exit 1
	fi
fi
# Adjust TPROF_SPAN if it exceeds SCRIPT_SPAN (set to SCRIPT_SPAN/4 or minimum of 5)
if [ $TPROF_SPAN -gt $SCRIPT_SPAN ]; then
  if [ $(( $SCRIPT_SPAN/4 )) -gt 5 ]; then
    TPROF_SPAN=$(( $SCRIPT_SPAN/4 ))
  else
   TPROF_SPAN=5
  fi
fi

################################################
# Verify running PIDs
# Remove duplicate PID and non-exist PID.
# The script exits if PIDs do not exist.
################################################
pids=""
if [ -n "$provided_pids" ]; then
	for pid in `echo $provided_pids | xargs -n1 | sort -u | xargs`
	do
		if test -d /proc/"$pid"/; then
			pids="$pid $pids"
		else
			print "PID $pid does not exist. Exiting the script."
			exit 1
		fi
	done
fi

################################
# Assign OUTPUT_DIR and DIR_NAME 
################################
if [ -n "$OUTPUT_DIR" ] && [ -n "$DIR_NAME" ]; then
	OUTPUT_DIR="$OUTPUT_DIR/$DIR_NAME"
elif [ -n "$OUTPUT_DIR" ] && [ -z "$DIR_NAME" ]; then
	readonly DIR_NAME="aixperf_results.$START_DAY.$START_TIME"
	OUTPUT_DIR="$OUTPUT_DIR/$DIR_NAME"
elif [ -z "$OUTPUT_DIR" ] && [ -n "$DIR_NAME" ]; then
	OUTPUT_DIR="$DIR_NAME"
else
	readonly DIR_NAME="aixperf_results.$START_DAY.$START_TIME"
	OUTPUT_DIR="$DIR_NAME"
fi
case "$OUTPUT_DIR" in
	*" "* ) print "\"$OUTPUT_DIR\" contains a space. Exiting the script."
	exit 1
esac

#####################################
# Create a temporary output directory
#####################################
if [ $MONITOR_ONLY -ne 1 ]; then
	mkdir -p $OUTPUT_DIR
	if [ $? -ne 0 ]; then
		print "Failed to create $OUTPUT_DIR."
		exit 1
	fi
	# Go into the created output directory and
	# get the full path to let the user know the current directory.
	cd $OUTPUT_DIR
	if [ $? -ne 0 ]; then
		print "Failed to go into $OUTPUT_DIR."
		exit 1
	fi
	readonly OUTPUT_DIR="$(pwd)"
fi

###############################
# Write a message to screen.out 
# Empty $1 does not write out   
###############################
log()
{
	# $1 - message
	if [ -z "${DISABLE_SCREEN_OUTPUT}" ] && [ -n "$1" ] && [ $MONITOR_ONLY -ne 1 ] && [ -z "${HIDE_STAMP}" ]; then
		print $(date '+%Y-%m-%d %H:%M:%S')  "\t$1" | tee -a screen.out
	elif [ -n "${HIDE_STAMP}" ] && [ -n "$1" ]; then
		print "$1"
	elif [ -n "$1" ]; then
		print $(date '+%Y-%m-%d %H:%M:%S')  "\t$1"
	fi
}
#######################################################################
# Remove PID either calling from cpu_trigger() or when kill -3 PID fail
# and return remaining $pids.
#######################################################################
no_longer_running_pids=""
remove_pid()
{
  local pid_to_remove=$1
  local pids=$2
  local new_pids=""
  # Remove PID from PIDs, replaces multiple spaces with single, remove trailing space
  new_pids=$(echo " $pids " | sed "s/ $pid_to_remove / /g" | sed 's/^ //;s/ $//')
  echo "$new_pids"
}

###########################################
# Loop until CPU usage threshold is reached
# TODO: ps -p $pid -o pcpu does not print the current CPU usage. Workaround is being explored.
###########################################
cpu_trigger()
{
	if [ -n "${CPU_THRESHOLD}" ] && [ -n "${pids}" ]; then
		log "CHECK_CPU_INTERVAL = $CHECK_CPU_INTERVAL"
		log "Monitoring PIDs: $pids"
		log "Waiting for a process to exceed $CPU_THRESHOLD% CPU usage."
		count_pids=$(echo $pids | awk '{print NF}')
		non_exist_pids=""
		while :
		do
			for pid in $pids
			do
				pid_CPU=$(ps -p $pid -o pcpu | tail -1 | sed 's/^[[:blank:]]*//;s/[[:blank:]]*$//')
				if [ "$pid_CPU" = "%CPU" ]; then
					log "PID $pid inaccessible."
					# Remove PID from PIDs and add it to non_exist_pids
					temp_pids=$(echo $pids| sed s/"$pid"//)
					pids=$temp_pids
					temp_non_exist_pid="$non_exist_pids $pid"
					non_exist_pids="$temp_non_exist_pid"

					count_non_pids=$(echo $non_exist_pids | awk '{print NF}')
					if [ $count_non_pids -eq $count_pids ]; then
						log "PIDs $non_exist_pids inaccessible. Exiting the script."
						exit 1
					else
						log "Monitoring remaining PIDs: $pids"
					fi

					# As PID is inaccessible, assign 0 to pid_CPU rather having it as '%CPU'
					# to avoid error for the next if statement below.
					pid_CPU=0
				fi
				
				if [ $pid_CPU -ge $CPU_THRESHOLD ] ; then
					log "PID $pid exceeded the CPU usage threshold, currently at $pid_CPU%"
					break 2
				fi
				pid_CPU=0
			done
			sleep $CHECK_CPU_INTERVAL
		done
	fi
	if [ $MONITOR_ONLY -eq 1 ]; then
		exit 0
	fi
}

#############################################################################
# If MONITOR_ONLY is true, no output data is created after this if statement.   
#############################################################################
if [ $MONITOR_ONLY -eq 1 ]; then
	log "aixperf version:  $SCRIPT_VERSION."
	# cpu_trigger # Not supported.
  # Future plan:
  # FileDescriptorTrigger?
  # SwapTrigger?
	exit 0
fi

#############################################
# Let user know path to a temporary directory
# and the inputs and beings are displayed.
#############################################
if [ $DISABLE_COLLECTING_HOSTNAME -eq 0 ]; then
  current_hostname="on $(hostname)"
fi
log "Output to $OUTPUT_DIR $current_hostname."
log "aixperf version:  $SCRIPT_VERSION."
if [ $ROOT_ACCESS_REQUIRED -eq 0 ]; then
  log "ROOT_ACCESS_REQUIRED = 0."
fi
for pid in $pids
do
	log "Provided PID:  $pid"
done
log "SCRIPT_SPAN = $SCRIPT_SPAN"
if [ -z "$disable_kill" ]; then
	log "JAVACORE_INTERVAL = $JAVACORE_INTERVAL"
else
	log "Disable kill -3 : true"
fi
log "TPROF_SPAN = $TPROF_SPAN"
log "VMSTAT_INTERVAL = $VMSTAT_INTERVAL"
log "Timezone: $(date +%Z)"
# Call cpu_trigger() to check if CPU_THRESHOLD is set.
# If set, it will check CPU usage and collect relevant data.
cpu_trigger

# Collect the user currently executing the script
date > whoami.out
whoami >> whoami.out 2>&1
log "Collection of user authority data complete."

################################################################################
# Start collection of:
#  * netstat x2
#  * ps
#  * uptime
#  * vmstat
#  * tprof
#  * javacores
#  * ifconfig -a if DISABLE_COLLECTING_IFCONFIG is 0. 
######################### 
wparname=`uname -W`
wparspec=""    
if ((wparname != 0))
	then wparspec="-@ $wparname"
fi

# Collect the first netstat: date at the top, data, and then a blank line
date >> netstat.out
netstat -an >> netstat.out 2>&1
print >> netstat.out
log "First netstat snapshot complete."

log "Collecting ps and memory-sorted snapshots."
date >> ps.out
ps $wparspec avwwwg >> ps.out 2>&1
print >> ps.out
collect_ps_mem_sorted(){
  local fn="ps_mem_sorted.out"
  date >> $fn
  ps auxww | (head -1; ps auxww | sort -rn +3 | head -n 20) >> $fn
  echo >> $fn
}
collect_ps_mem_sorted


# Collect the uptime
log "Collecting uptime."
uptime >> uptime.out

# Start the collection of tprof data.
# It runs in the background so that other tasks can be completed while this runs.
log "Starting collection of tprof data..."
date >> tprof.out
# Calculate TPROF_ARGS
# First, check if environment is SMT capable.  If so, proceed to next check; if not, set args without -R option
# Next, check if OS is 32 bit.  If so, set args without -R option (-R is not supported on 32 bit); if not, set args with -R option.
if  /usr/sbin/smtctl >/dev/null 2>&1; then
	KERTYPE=`getconf KERNEL_BITMODE`
	if [ "$KERTYPE" = 32 ]; then
		log "TPROF_ARGS set to -skex; SMT capable but 32 bit environment."
		TPROF_ARGS="-skex"
	else
		log "TPROF_ARGS set to -Rskex; SMT capable and NOT a 32 bit environment."
		TPROF_ARGS="-Rskex"
	fi
else
	log "TPROF_ARGS set to -skex; Not SMT capable."
	TPROF_ARGS="-skex"
fi
LDR_CNTRL=MAXDATA=0x80000000 tprof $wparspec $TPROF_ARGS sleep $TPROF_SPAN >> tprof.out 2>&1 &
log "Collection of tprof data started."

# Start the colletion of vmstat data.
# It runs in the background so that other tasks can be completed while this runs.
date >> vmstat.out
vmstat -t -w $wparspec $VMSTAT_INTERVAL `expr $SCRIPT_SPAN / $VMSTAT_INTERVAL + 1` >> vmstat.out 2>&1 &
log "Collection of vmstat data started."

if [ $DISABLE_COLLECTING_IFCONFIG -eq 0 ]; then
  log "Collecting ifconfig -a"
  ifconfig -a >> ifconfig.out
  ifconfig_file="ifconfig.out"
fi

###############################################################################
# Start collection of javacores.
# Loop the appropriate number of times, pausing for the given amount of time, 
# and iterate through each supplied PID.
# A kill -3 command will not be executed if disable_kill is true.
###############################################################################
n=1
m=`expr $SCRIPT_SPAN / $JAVACORE_INTERVAL`

if [ -n "$pids" ] && [ -z "$disable_kill" ]; then
	log "Issuing a kill -3 command for the provided PID(s) to create a javacore or a thread dump."
fi
while [ $n -le $m ]
do
	# Produce a javacore/thread dump
	if [ -z "$disable_kill" ];then
		for pid in $pids
		do
			kill_output=$(kill -3 $pid 2>&1)

      if [ $? -ne 0 ]; then
        log "PID $pid inaccessible. $kill_output"
        pids=$(remove_pid $pid "$pids")
        no_longer_running_pids="$no_longer_running_pids $pid"
        if [ -z "$pids" ]; then
          log "No PIDs remaining to process, finishing the script."
          # Kill -3 $PID is no longer needed due to above.
          # Support team guides for locating javacores/thread dumps if they were produced before PIDs are inaccessible/no longer.
          disable_kill=1
          break 3
        fi
      else
        log "Issued a kill -3 for PID $pid."
      fi
		done
	fi
  # Pause for JAVACORE_INTERVAL seconds.
	log "Continuing to collect data for $JAVACORE_INTERVAL seconds..."
	sleep $JAVACORE_INTERVAL
  # Increment counter
	n=`expr $n + 1`
done

log "Collecting final javacore, and ps memory-sorted snapshot."
collect_ps_mem_sorted

# Produce the final javacore/thread dump.
if [ -z "$disable_kill" ];then
	for pid in $pids
	do
		log $(kill -3 $pid 2>&1)
		log "Issued the final kill -3 for PID $pid."
	done
fi

# Collect a final netstat
date >> netstat.out
netstat -an >> netstat.out 2>&1
log "Final netstat snapshot complete."

#########################
# Other data collection #
######################### 
log "Collecting other data. This may take a few moments..."
/usr/sbin/emgr -lv3 > emgr-lv3.out 2>&1
oslevel -s > oslevel-s.out 2>&1
lslpp -la > lslpp-la.out 2>&1
instfix -i > instfix-i.out 2>&1
prtconf > prtconf.out 2>&1
errpt -a > errpt.out 2>&1
lparstat -i > lparstat-i.out 2>&1
lsattr -El sys0 > lsattr.out 2>&1
df > df.out 2>&1

print >> ps.out
date >> ps.out
ps $wparspec avwwwg >> ps.out 2>&1

################################################################################################
# WSADMIN END: If there are any changes between WSADMIN START and this, 
# be sure to update the aixperf_shell_script_content in the perf.py script and test the changes.
################################################################################################

################################################################################
# Compress & Cleanup
######################### 
# Brief pause to make sure all data is collected.
log "Preparing for packaging and cleanup..."
sleep 5

CleanUpJavacores()
{
  # $1 - directory path
  # $2 - javacore files
  if [ -n "$CLEAN_UP_JAVACORES" ]; then
    log "Cleaning up the javacores in $1"
    for file in $2; do
      local full_path="$1/$file"
      if [ -f "$full_path" ]; then
        rm "$full_path" && log "Removed: $full_path " || log "Failed to remove: $full_path"
      else
        log "File not found to remove: $full_path"
      fi
    done
  fi
}

# Tar javacores
tarred_javacore_string=""
pids_not_found_javacore="$no_longer_running_pids"
TarJavacores()
{
	# $1 - directory
	# $2 - javacore filenames
	# $3 - PID
	(cd ''"$1"'' && tar -cf ''"$OUTPUT_DIR/javacore.$3.tar"'' $2)
	temp_javacore_string="$tarred_javacore_string javacore.$3.tar "
	tarred_javacore_string="$temp_javacore_string"
  # Remove the javacores if --clean-up-javacores is used.
  CleanUpJavacores "$1" "$2"
}
if [ -z "$disable_kill" ];then
	for pid in $pids
	do
		# Check javacores in the default JVM directory
		# Check -Xdump:directory
		# Check environment entry IBM_JAVACOREDIR
		# If any of above is true, gather javacores. 
		# If javacores are not found, append PID to pids_not_found_javacore
		javacores="$(cd /proc/$pid/cwd/ && ls javacore* 2>/dev/null | awk -v pid="${pid}" -v startday="${START_DAY}" -v starttime="${START_TIME}" '/javacore\.[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\.txt/ { str=$0; gsub(/.*\//, "", str); split(str, pieces, /\./); if ( pieces[4] == pid && pieces[2] >= startday && pieces[3] >= starttime ) { printf("%s ", $0); } } END { printf("\n"); }' )"
		if [ -n "${javacores}" ]; then
			TarJavacores "/proc/$pid/cwd/" ''"$javacores"'' $pid
		elif $(ps -fp $pid | grep -q -e "-Xdump:directory"); then
			xdumpdir="$(ps -fp $pid | sed '1d' | sed 's/.*\Xdump:directory=//g' | sed 's/\ -.*//g')"
			javacores="$(cd ''"$xdumpdir"'' && ls javacore* 2>/dev/null | awk -v pid="${pid}" -v startday="${START_DAY}" -v starttime="${START_TIME}" '/javacore\.[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\.txt/ { str=$0; gsub(/.*\//, "", str); split(str, pieces, /\./); if ( pieces[4] == pid && pieces[2] >= startday && pieces[3] >= starttime ) { printf("%s ", $0); } } END { printf("\n"); }' )"
			TarJavacores ''"$xdumpdir"'' ''"$javacores"'' $pid
		elif $(ps ewww $pid | grep -q -e "IBM_JAVACOREDIR="); then
			IBM_javacoredir=$(ps ewww $pid | awk 'NR == 2 { for (i = 1; i < NF; i++) { if ($i ~ /.+=/) { state = 0; } if (state >= 1) { printf(" "); } if (state >= 1) { printf("%s", $i); state = 2; } if (state == 0 && $i ~ /^IBM_JAVACOREDIR=/) { state = 1; x = $i; gsub(/^IBM_JAVACOREDIR=/, "", x); printf("%s", x); } } printf("\n"); }')
			javacores="$(cd ''"$IBM_javacoredir"'' && ls javacore* 2>/dev/null | awk -v pid="${pid}" -v startday="${START_DAY}" -v starttime="${START_TIME}" '/javacore\.[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+\.txt/ { str=$0; gsub(/.*\//, "", str); split(str, pieces, /\./); if ( pieces[4] == pid && pieces[2] >= startday && pieces[3] >= starttime ) { printf("%s ", $0); } } END { printf("\n"); }' )"
			TarJavacores ''"$IBM_javacoredir"'' ''"$javacores"'' $pid
		else
			log "Cannot locate javacores for $pid."
			temp_pid_string="$pids_not_found_javacore $pid"
			pids_not_found_javacore="$temp_pid_string "
		fi
	done
fi

# Tar/GZip the output files together and then remove the output files.
log "Compressing the following files into $DIR_NAME.tar.gz."
files_string="netstat.out vmstat.out ps.out ps_mem_sorted.out sleep.prof tprof.out emgr-lv3.out oslevel-s.out lslpp-la.out instfix-i.out prtconf.out lparstat-i.out lsattr.out whoami.out df.out errpt.out screen.out uptime.out $tarred_javacore_string $ifconfig_file"
log "$files_string"

tar -cf ../$DIR_NAME.tar $files_string
if [ $? -ne 0 ]; then
	print $(date '+%Y-%m-%d %H:%M:%S') "\tDue to above tar issue, the mentioned files in $DIR_NAME are not removed."
	dir_end_message="$OUTPUT_DIR"
else
	gzip ../$DIR_NAME.tar
	rm $files_string
fi
# Check if the current directory is empty before removing it.
# Otherwise, print out the message and the output dir is not removed.
if [ -n "$(ls -A)" ]; then
	print $(date '+%Y-%m-%d %H:%M:%S') "\tFailed to remove $DIR_NAME."
else
	cd ..
	rm -r $DIR_NAME
	print $(date '+%Y-%m-%d %H:%M:%S') "\tTemporary directory $DIR_NAME removed."
	dir_end_message="$OUTPUT_DIR.tar.gz"
fi
print $(date '+%Y-%m-%d %H:%M:%S') "\taixperf.sh complete."

# Print the instruction message.
if  [ $KEEP_QUIET -eq 0 ]; then
	print "$(tput smso)\t\t\t\t\t\t\t\t\t\t$(tput rmso)"
	print "$(tput smso) $(tput rmso)"
	print "$(tput smso) $(tput rmso)  To share with IBM support, upload all the following files:"
	print "$(tput smso) $(tput rmso)"
	print "$(tput smso) $(tput rmso)  * $dir_end_message"
	print "$(tput smso) $(tput rmso)"
	print "$(tput smso) $(tput rmso)  For WebSphere Application Server:"
	print "$(tput smso) $(tput rmso)  * logs (systemout.log, native_stderr.log, etc)"
	if [ "${pids_not_found_javacore}" != "" ]; then
		print "$(tput smso) $(tput rmso)  * javacores from PID(s)$pids_not_found_javacore"
	fi
	print "$(tput smso) $(tput rmso)  * server.xml for the server(s) that you are providing data for"
	print "$(tput smso) $(tput rmso)"
	print "$(tput smso) $(tput rmso)  For Liberty:"
	print "$(tput smso) $(tput rmso)  * logs (messages.log, console.log, etc)"
	if [ "${pids_not_found_javacore}" != "" ]; then
		print "$(tput smso) $(tput rmso)  * javacores from PID(s)$pids_not_found_javacore (if running on an IBM JDK or IBM Semeru)"
	fi
	print "$(tput smso) $(tput rmso)  * server.env, server.xml, and jvm.options"
	print "$(tput smso) $(tput rmso)"
	print "$(tput smso)\t\t\t\t\t\t\t\t\t\t$(tput rmso)"
fi