#!/bin/sh
# /*******************************************************************************
#  * (c) Copyright IBM Corporation 2026.
#  *
#  * Licensed under the Apache License, Version 2.0 (the "License");
#  * you may not use this file except in compliance with the License.
#  * You may obtain a copy of the License at
#  *
#  *    https://www.apache.org/licenses/LICENSE-2.0
#  *
#  * Unless required by applicable law or agreed to in writing, software
#  * distributed under the License is distributed on an "AS IS" BASIS,
#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#  * See the License for the specific language governing permissions and
#  * limitations under the License.
#  *******************************************************************************/
#
# netprobe_collect.sh — NetProbe Standalone Network Statistics & Diagnostics Collector
#
# Collects /proc network snapshots compatible with netprobe.sh analysis,
# along with comprehensive Linux network diagnostics recommended by the
# IBM WebSphere Performance Cookbook:
# https://publib.boulder.ibm.com/httpserv/cookbook/Troubleshooting_Recipes-Troubleshooting_Linux_Recipes-Linux_network_issues.html
#
# POSIX-compliant (/bin/sh). Compatible with Linux and Unix systems.

set -u

SCRIPT_VERSION="1.1.20260923"
DEFAULT_DURATION=60
DEFAULT_INTERVAL=30
DEFAULT_LOG_LINES=5000
OUTPUT_DIR="."
TAR_OUTPUT=1
TARGET_HOST=""
VERBOSE=0

usage() {
    cat <<'EOF'
Usage:
  ./netprobe_collect.sh [OPTIONS]

Options:
  -d, --duration <sec>   Total collection duration in seconds (default: 60)
  -i, --interval <sec>   Interval between snapshots in seconds (default: 30)
  -l, --log-lines <num>  Number of log lines to capture (default: 5000)
  -t, --target <host/ip> Target IP/host to probe (ping, mtr, ip route get)
  -o, --output-dir <dir> Destination directory for archive (default: .)
  -p, --plain            Save uncompressed directory/file instead of tar.gz
  -V, --verbose          Enable verbose output (shell trace) for troubleshooting
  -v, --version          Show script version
  -h, --help             Show this help message

Diagnostics Collected (IBM Cookbook Recipes):
  - /proc snapshots: /proc/net/{netstat, snmp, snmp6, tcp, udp, dev, route,
                     softnet_stat, if_inet6}, /proc/{uptime, loadavg, cmdline,
                     cpuinfo, meminfo}, cgroup cpu.stat, sysctl TCP params
  - Command metrics: netstat (-s, -i, -antop), ss (-amponetOi), vmstat (1 4),
                     ip address, ip route show, ip -s link, ulimit -a
  - Kernel/system logs: journalctl (-k -n <lines> and -n <lines>), dmesg,
                        /var/log/{messages,syslog} (tail <lines>), sysctl -a
  - Optional target diagnostics: ping (-c 10), mtr (10 cycles), ip route get

Examples:
  ./netprobe_collect.sh                     # Default: 60s total, 30s interval, 5000 log lines
  ./netprobe_collect.sh -d 300 -i 60        # 5 min collection, 1 min snapshots
  ./netprobe_collect.sh -l 10000            # Capture 10,000 log lines
  ./netprobe_collect.sh -t 192.168.1.1      # Include ping/mtr/route tests to target
  ./netprobe_collect.sh -d 0                # Single point-in-time snapshot
  ./netprobe_collect.sh -o /tmp -d 120      # Save output in /tmp
  ./netprobe_collect.sh -V                  # Enable verbose/trace output for troubleshooting
EOF
    exit 0
}

# Parse command line options
DURATION="$DEFAULT_DURATION"
INTERVAL="$DEFAULT_INTERVAL"
LOG_LINES="$DEFAULT_LOG_LINES"

while [ $# -gt 0 ]; do
    case "$1" in
        -d|--duration)
            [ $# -ge 2 ] || { echo "Error: $1 requires an argument" >&2; exit 1; }
            DURATION="$2"
            shift 2
            ;;
        -i|--interval)
            [ $# -ge 2 ] || { echo "Error: $1 requires an argument" >&2; exit 1; }
            INTERVAL="$2"
            shift 2
            ;;
        -l|--log-lines)
            [ $# -ge 2 ] || { echo "Error: $1 requires an argument" >&2; exit 1; }
            LOG_LINES="$2"
            shift 2
            ;;
        -t|--target)
            [ $# -ge 2 ] || { echo "Error: $1 requires an argument" >&2; exit 1; }
            TARGET_HOST="$2"
            shift 2
            ;;
        -o|--output-dir)
            [ $# -ge 2 ] || { echo "Error: $1 requires an argument" >&2; exit 1; }
            OUTPUT_DIR="$2"
            shift 2
            ;;
        -p|--plain)
            TAR_OUTPUT=0
            shift
            ;;
        -V|--verbose)
            VERBOSE=1
            shift
            ;;
        -v|--version)
            echo "netprobe_collect.sh version $SCRIPT_VERSION"
            exit 0
            ;;
        -h|--help)
            usage
            ;;
        *)
            echo "Error: Unknown option: $1" >&2
            echo "Run '$0 --help' for usage." >&2
            exit 1
            ;;
    esac
done

# Enable shell trace if verbose mode requested
if [ "$VERBOSE" -eq 1 ]; then
    set -x
fi

# Validate numeric arguments
case "$DURATION" in
    ''|*[!0-9]*) echo "Error: Duration must be a non-negative integer" >&2; exit 1 ;;
esac

case "$INTERVAL" in
    ''|*[!0-9]*) echo "Error: Interval must be a positive integer" >&2; exit 1 ;;
esac

case "$LOG_LINES" in
    ''|*[!0-9]*) echo "Error: Log lines must be a positive integer" >&2; exit 1 ;;
esac

if [ "$DURATION" -gt 0 ] && [ "$INTERVAL" -le 0 ]; then
    echo "Error: Interval must be greater than 0" >&2
    exit 1
fi

if [ "$LOG_LINES" -le 0 ]; then
    echo "Error: Log lines must be greater than 0" >&2
    exit 1
fi

[ -d "$OUTPUT_DIR" ] || mkdir -p "$OUTPUT_DIR" || {
    echo "Error: Cannot create output directory $OUTPUT_DIR" >&2
    exit 1
}

# Network snapshot collection function (formatted for netprobe.sh parser)
collect_snapshot() {
    echo "=== NETWORK_SNAPSHOT_BEGIN ==="
    echo "=== TIMESTAMP: $(date) ==="

    echo "=== PROC_NET_NETSTAT_BEGIN ==="
    cat /proc/net/netstat 2>/dev/null || true
    echo "=== PROC_NET_NETSTAT_END ==="

    echo "=== PROC_NET_SNMP_BEGIN ==="
    cat /proc/net/snmp 2>/dev/null || true
    echo "=== PROC_NET_SNMP_END ==="

    echo "=== PROC_NET_SNMP6_BEGIN ==="
    cat /proc/net/snmp6 2>/dev/null || true
    echo "=== PROC_NET_SNMP6_END ==="

    echo "=== PROC_UPTIME_BEGIN ==="
    cat /proc/uptime 2>/dev/null || true
    echo "=== PROC_UPTIME_END ==="

    echo "=== PROC_NET_ROUTE_BEGIN ==="
    cat /proc/net/route 2>/dev/null || true
    echo "=== PROC_NET_ROUTE_END ==="

    echo "=== PROC_NET_TCP_BEGIN ==="
    cat /proc/net/tcp 2>/dev/null || true
    echo "=== PROC_NET_TCP_END ==="

    echo "=== PROC_NET_UDP_BEGIN ==="
    cat /proc/net/udp 2>/dev/null || true
    echo "=== PROC_NET_UDP_END ==="

    echo "=== PROC_NET_DEV_BEGIN ==="
    cat /proc/net/dev 2>/dev/null || true
    echo "=== PROC_NET_DEV_END ==="

    echo "=== PROC_NET_SOFTNET_STAT_BEGIN ==="
    cat /proc/net/softnet_stat 2>/dev/null || true
    echo "=== PROC_NET_SOFTNET_STAT_END ==="

    echo "=== TCP_CONGESTION_CONTROL_BEGIN ==="
    cat /proc/sys/net/ipv4/tcp_congestion_control 2>/dev/null || true
    echo "=== TCP_CONGESTION_CONTROL_END ==="

    echo "=== TCP_BUFFER_PARAMS_BEGIN ==="
    cat /proc/sys/net/ipv4/tcp_rmem 2>/dev/null || true
    cat /proc/sys/net/ipv4/tcp_wmem 2>/dev/null || true
    cat /proc/sys/net/core/somaxconn 2>/dev/null || true
    echo "=== TCP_BUFFER_PARAMS_END ==="

    echo "=== NETWORK_SNAPSHOT_END ==="
}

# Collect IBM Cookbook basic command diagnostics for a snapshot iteration
collect_cookbook_diagnostics() {
    _snap_dir="$1"
    _ts=$(date +%Y%m%d_%H%M%S)
    _host=$(hostname 2>/dev/null || echo "localhost")

    # Command outputs
    netstat -s > "$_snap_dir/diag_netstat_s_${_host}_${_ts}.txt" 2>&1 || true
    netstat -i > "$_snap_dir/diag_netstat_i_${_host}_${_ts}.txt" 2>&1 || true
    netstat -antop > "$_snap_dir/diag_netstat_antop_${_host}_${_ts}.txt" 2>&1 || true

    # Proc + cgroup consolidated dump from recipe
    cat /proc/net/dev /proc/net/snmp* /proc/net/netstat /proc/net/tcp* /proc/net/udp* \
        /proc/net/if_inet6 /proc/loadavg \
        /sys/fs/cgroup/cpu.stat /sys/fs/cgroup/cpu/*/*/cpu.stat \
        > "$_snap_dir/diag_proc_${_host}_${_ts}.txt" 2>&1 || true

    # System / Interface stats
    vmstat -twn 1 4 > "$_snap_dir/diag_vmstat_${_host}_${_ts}.txt" 2>&1 || \
        vmstat 1 4 > "$_snap_dir/diag_vmstat_${_host}_${_ts}.txt" 2>&1 || true

    ip -color=never address > "$_snap_dir/diag_ipaddress_${_host}_${_ts}.txt" 2>&1 || \
        ip address > "$_snap_dir/diag_ipaddress_${_host}_${_ts}.txt" 2>&1 || \
        ifconfig -a > "$_snap_dir/diag_ipaddress_${_host}_${_ts}.txt" 2>&1 || true

    ip -color=never route show > "$_snap_dir/diag_iprouteshow_${_host}_${_ts}.txt" 2>&1 || \
        ip route show > "$_snap_dir/diag_iprouteshow_${_host}_${_ts}.txt" 2>&1 || \
        netstat -rn > "$_snap_dir/diag_iprouteshow_${_host}_${_ts}.txt" 2>&1 || true

    ip -color=never -s link > "$_snap_dir/diag_iplink_${_host}_${_ts}.txt" 2>&1 || \
        ip -s link > "$_snap_dir/diag_iplink_${_host}_${_ts}.txt" 2>&1 || true

    ulimit -a > "$_snap_dir/diag_ulimit_${_host}_${_ts}.txt" 2>&1 || true
    ss -amponetOi > "$_snap_dir/diag_ss_${_host}_${_ts}.txt" 2>&1 || \
        ss -antop > "$_snap_dir/diag_ss_${_host}_${_ts}.txt" 2>&1 || true

    # Target path tests if specified
    if [ -n "$TARGET_HOST" ]; then
        ping -c 10 -n "$TARGET_HOST" > "$_snap_dir/diag_ping_${_host}_${_ts}.txt" 2>&1 || true
        mtr --report-wide --show-ips --aslookup --report-cycles 10 "$TARGET_HOST" > "$_snap_dir/diag_mtr_${_host}_${_ts}.txt" 2>&1 || \
            traceroute -n "$TARGET_HOST" > "$_snap_dir/diag_mtr_${_host}_${_ts}.txt" 2>&1 || true
        ip route get "$TARGET_HOST" > "$_snap_dir/diag_iprouteget_${_host}_${_ts}.txt" 2>&1 || true
    fi
}

# Collect kernel and OS configuration logs
collect_kernel_logs() {
    _out_dir="$1"
    _lines="$2"
    _host=$(hostname 2>/dev/null || echo "localhost")
    _klog="$_out_dir/diag_kernel_${_host}.txt"

    {
        echo "===== KERNEL MESSAGES (DMESG / JOURNALCTL -k) (LAST $_lines LINES) ====="
        if command -v journalctl >/dev/null 2>&1; then
            journalctl -k -n "$_lines" 2>&1 || true
        elif command -v dmesg >/dev/null 2>&1; then
            dmesg -T 2>&1 | tail -n "$_lines" || dmesg 2>&1 | tail -n "$_lines" || true
        fi

        echo "===== SYSTEM JOURNAL / SERVICE LOGS (LAST $_lines LINES) ====="
        if command -v journalctl >/dev/null 2>&1; then
            journalctl -n "$_lines" 2>&1 || true
        fi

        echo "===== SYSTEM INFO & BOOT CMDLINE ====="
        cat /proc/cmdline /proc/cpuinfo /proc/meminfo 2>&1 || true

        echo "===== SYSLOG / MESSAGES (LAST $_lines LINES) ====="
        for _f in /var/log/messages /var/log/syslog; do
            if [ -f "$_f" ]; then
                echo "--- $_f ---"
                tail -n "$_lines" "$_f" 2>&1 || true
            fi
        done

        echo "===== SYSCTL CONFIGURATION ====="
        sysctl -a 2>&1 || true
    } > "$_klog" 2>&1
}

# Create temp working directory inside output directory
TMP_DIR="${OUTPUT_DIR}/.netprobe_collect_$$"
mkdir -p "$TMP_DIR" || {
    echo "Error: Failed to create temp directory $TMP_DIR" >&2
    exit 1
}

DATA_FILE="$TMP_DIR/network_data.out"
: > "$DATA_FILE"

cleanup() {
    # Explicit file iteration cleanup without recursive rm
    if [ -n "${TMP_DIR:-}" ] && [ -d "$TMP_DIR" ]; then
        for _f in "$TMP_DIR"/network_data.out "$TMP_DIR"/diag_*.txt "$TMP_DIR"/netprobe_collect.*.tar.gz; do
            if [ -f "$_f" ]; then
                rm -f "$_f"
            fi
        done
        rmdir "$TMP_DIR" 2>/dev/null || true
    fi
}
trap cleanup EXIT
trap 'exit 130' INT
trap 'exit 143' TERM

# Collection run
echo "netprobe_collect.sh version $SCRIPT_VERSION"
TS_TAG=$(date +%Y%m%d.%H%M%S)

# Check for root / elevated permissions and notify user
if [ "$(id -u 2>/dev/null || echo 1)" -ne 0 ]; then
    echo "Note: Running as non-root user ($(whoami 2>/dev/null || echo "user"))."
    echo "      Core /proc metrics will be collected normally."
    echo "      Some extended diagnostics (e.g., restricted journalctl logs, /var/log/messages,"
    echo "      full socket PID/process mappings in netstat/ss) may be omitted or partial."
    echo "      Run as root or with sudo for full diagnostic coverage."
    echo ""
fi

if [ "$DURATION" -eq 0 ]; then
    echo "Collecting single point-in-time network snapshot and cookbook diagnostics..."
    collect_snapshot >> "$DATA_FILE"
    collect_cookbook_diagnostics "$TMP_DIR"
    collect_kernel_logs "$TMP_DIR" "$LOG_LINES"
    echo "Snapshot completed."
else
    ELAPSED=0
    SNAPSHOT_NUM=1
    echo "Starting network data collection (duration: ${DURATION}s, interval: ${INTERVAL}s, log lines: ${LOG_LINES})..."

    while [ "$ELAPSED" -le "$DURATION" ]; do
        echo "[$(date '+%Y-%m-%d %H:%M:%S')] Capturing snapshot & diagnostics #$SNAPSHOT_NUM (elapsed: ${ELAPSED}s / ${DURATION}s)..."
        collect_snapshot >> "$DATA_FILE"
        collect_cookbook_diagnostics "$TMP_DIR"
        SNAPSHOT_NUM=$((SNAPSHOT_NUM + 1))

        if [ "$ELAPSED" -ge "$DURATION" ]; then
            break
        fi

        REMAINING=$((DURATION - ELAPSED))
        SLEEP_TIME="$INTERVAL"
        if [ "$SLEEP_TIME" -gt "$REMAINING" ]; then
            SLEEP_TIME="$REMAINING"
        fi

        sleep "$SLEEP_TIME"
        ELAPSED=$((ELAPSED + SLEEP_TIME))
    done

    echo "Collecting kernel logs, sysctl configuration, and system info (last $LOG_LINES lines)..."
    collect_kernel_logs "$TMP_DIR" "$LOG_LINES"
    echo "Collection finished."
fi

# Package output
if [ "$TAR_OUTPUT" -eq 1 ]; then
    ARCHIVE_NAME="netprobe_collect.${TS_TAG}.tar.gz"
    ARCHIVE_PATH="${OUTPUT_DIR}/${ARCHIVE_NAME}"

    echo "Archiving to $ARCHIVE_PATH..."
    (cd "$TMP_DIR" && tar -czf "$ARCHIVE_NAME" network_data.out diag*) && \
        mv "$TMP_DIR/$ARCHIVE_NAME" "$ARCHIVE_PATH" || {
        echo "Error: Failed to create tar archive" >&2
        exit 1
    }

    echo "Success: Archive created at $ARCHIVE_PATH"
else
    OUT_DIR_PLAIN="${OUTPUT_DIR}/netprobe_data_${TS_TAG}"
    mkdir -p "$OUT_DIR_PLAIN"
    cp -r "$TMP_DIR"/* "$OUT_DIR_PLAIN"/ 2>/dev/null || true
    echo "Success: Network data written to $OUT_DIR_PLAIN/"
fi