#!/usr/bin/env bash
trap 'killm f' EXIT
#
# Watchdog Daemon Test Script
# Tests psmwatch and sdrwatch daemon auto-launch, ionrestart, and ionexit
#
# This test verifies:
# 1. ionadmin launches psmwatch and sdrwatch daemons via 'w' command
# 2. Data transfer works while daemons are running
# 3. ionrestart properly stops and restarts daemons
# 4. ionexit properly shuts down daemons
#

TOPDIR=$(pwd)
RETVAL=0

# Test configuration
BUNDLESIZE=10000
NUMBUNDLES=100
MAXSECONDS=60

# Portable timeout function (works on macOS and Linux)
run_with_timeout() {
    local timeout_sec=$1
    shift

    # Try native timeout first (Linux)
    if command -v timeout >/dev/null 2>&1; then
        timeout "$timeout_sec" "$@"
        return $?
    fi

    # macOS/BSD fallback using background process
    "$@" &
    local cmd_pid=$!

    ( sleep "$timeout_sec"; kill -TERM $cmd_pid 2>/dev/null ) &
    local sleep_pid=$!

    wait $cmd_pid 2>/dev/null
    local exit_code=$?

    # Clean up the sleep/kill process if command finished first
    kill $sleep_pid 2>/dev/null
    wait $sleep_pid 2>/dev/null

    return $exit_code
}

echo "###############################################################################"
echo "# Watchdog Daemon Test"
echo "###############################################################################"
echo "Date: $(date)"
echo "Hostname: $(hostname)"
echo ""

# Function to check if watchdog daemons are running
check_daemons() {
    local node=$1
    local check_type=$2  # "running" or "stopped"
    local psmwatch_running=0
    local sdrwatch_running=0

    if pgrep -f "psmwatch" > /dev/null 2>&1; then
        psmwatch_running=1
    fi
    if pgrep -f "sdrwatch" > /dev/null 2>&1; then
        sdrwatch_running=1
    fi

    if [ "$check_type" = "running" ]; then
        if [ $psmwatch_running -eq 1 ] && [ $sdrwatch_running -eq 1 ]; then
            echo "[PASS] Node $node: Both psmwatch and sdrwatch daemons are running"
            return 0
        else
            echo "[FAIL] Node $node: Expected daemons running - psmwatch=$psmwatch_running, sdrwatch=$sdrwatch_running"
            return 1
        fi
    else
        if [ $psmwatch_running -eq 0 ] && [ $sdrwatch_running -eq 0 ]; then
            echo "[PASS] Node $node: Both psmwatch and sdrwatch daemons are stopped"
            return 0
        else
            echo "[FAIL] Node $node: Expected daemons stopped - psmwatch=$psmwatch_running, sdrwatch=$sdrwatch_running"
            return 1
        fi
    fi
}

# Function to show daemon process info
show_daemon_info() {
    echo "=== Current watchdog daemon processes ==="
    ps aux | grep -E "(psmwatch|sdrwatch)" | grep -v grep || echo "(no daemons running)"
    echo ""
}

# Cleanup
echo "###############################################################################"
echo "# Step 0: Cleanup"
echo "###############################################################################"
./cleanup
sleep 1

export ION_NODE_LIST_DIR=$PWD
rm -f ./ion_nodes

echo ""
echo "###############################################################################"
echo "# Step 1: Start ION nodes with watchdog daemons enabled"
echo "###############################################################################"
echo ""

# Start nodes
echo "Starting node 2..."
(cd 2.watchdog.ltp; ./ionstart)
sleep 2

echo "Starting node 3..."
(cd 3.watchdog.ltp; ./ionstart)
sleep 2

echo ""
echo "Checking if watchdog daemons were launched..."
show_daemon_info

if ! check_daemons "2/3" "running"; then
    echo ""
    echo "[ERROR] Watchdog daemons not started by ionadmin!"
    RETVAL=1
fi

echo ""
echo "###############################################################################"
echo "# Step 2: Transfer data while daemons are running"
echo "###############################################################################"
echo ""

# Check ion.log for daemon activity
echo "Checking ion.log for daemon launch messages..."
if grep -l "psmwatch" 2.watchdog.ltp/ion.log 3.watchdog.ltp/ion.log 2>/dev/null; then
    echo "[INFO] Found psmwatch references in ion.log"
fi
if grep -l "sdrwatch" 2.watchdog.ltp/ion.log 3.watchdog.ltp/ion.log 2>/dev/null; then
    echo "[INFO] Found sdrwatch references in ion.log"
fi

echo ""
echo "Sending ${NUMBUNDLES} bundles of ${BUNDLESIZE} bytes from node 2 to node 3..."

# Start receiver on node 3
cd 3.watchdog.ltp
bpcounter ipn:3.1 ${NUMBUNDLES} &
BPCOUNTER_PID=$!
cd "$TOPDIR"
sleep 1

# Check if bpcounter started
if ! ps -p $BPCOUNTER_PID > /dev/null 2>&1; then
    echo "[ERROR] bpcounter couldn't be started."
    RETVAL=1
else
    # Send bundles from node 2
    (cd 2.watchdog.ltp; bpdriver ${NUMBUNDLES} ipn:2.1 ipn:3.1 -${BUNDLESIZE} > /dev/null) &
    BPDRIVER_PID=$!

    # Wait for transmission
    echo "Waiting for transmission to complete..."
    TIMER=0
    while ps -p $BPCOUNTER_PID > /dev/null 2>&1; do
        sleep 1
        TIMER=$((TIMER + 1))
        if [ $TIMER -gt $MAXSECONDS ]; then
            echo "[ERROR] Transmission timeout after $MAXSECONDS seconds"
            kill -9 $BPCOUNTER_PID $BPDRIVER_PID 2>/dev/null
            RETVAL=1
            break
        fi
    done

    if [ $TIMER -le $MAXSECONDS ]; then
        echo "[PASS] Data transfer completed successfully"
    fi
fi

echo ""
show_daemon_info

echo ""
echo "###############################################################################"
echo "# Step 3: Run ionrestart on node 2 and verify daemons restart"
echo "###############################################################################"
echo ""

# Capture all daemon PIDs before ionrestart (sorted, space-separated).
# Using "psmwatch -d" to match only daemon-mode processes.
echo "Current daemon PIDs before ionrestart:"
PSMWATCH_PIDS_BEFORE=$(pgrep -f "psmwatch -d" | sort | tr '\n' ' ')
SDRWATCH_PIDS_BEFORE=$(pgrep -f "sdrwatch -d" | sort | tr '\n' ' ')
echo "  psmwatch PIDs: ${PSMWATCH_PIDS_BEFORE:-none}"
echo "  sdrwatch PIDs: ${SDRWATCH_PIDS_BEFORE:-none}"

echo ""
echo "Running ionrestart on node 2 (from 2.watchdog.ltp directory)..."
echo "(ionrestart requires running from the node's config directory in multi-node mode)"
cd 2.watchdog.ltp
ionrestart &
IONRESTART_PID=$!
cd "$TOPDIR"

# Wait for ionrestart to complete. Bound it at 120 s so a truly stuck
# ionrestart still fails the test, but slower runners (e.g. ARC rhel9
# under load) get time to finish. The previous 30 s poll-and-continue
# let Step 4 start with ionrestart still mid-flight, which left the
# system half-restarted and made ionstop hang.
TIMER=0
while ps -p $IONRESTART_PID > /dev/null 2>&1; do
    sleep 1
    TIMER=$((TIMER + 1))
    if [ $TIMER -gt 120 ]; then
        echo "[FAIL] ionrestart did not complete within 120s; killing"
        kill -9 $IONRESTART_PID 2>/dev/null
        RETVAL=1
        break
    fi
done

sleep 3

echo ""
echo "Daemon PIDs after ionrestart:"
PSMWATCH_PIDS_AFTER=$(pgrep -f "psmwatch -d" | sort | tr '\n' ' ')
SDRWATCH_PIDS_AFTER=$(pgrep -f "sdrwatch -d" | sort | tr '\n' ' ')
echo "  psmwatch PIDs: ${PSMWATCH_PIDS_AFTER:-none}"
echo "  sdrwatch PIDs: ${SDRWATCH_PIDS_AFTER:-none}"

echo ""
show_daemon_info

# Check if daemons are still running (they should have been restarted)
if ! check_daemons "after-restart" "running"; then
    echo "[FAIL] Daemons not running after ionrestart!"
    RETVAL=1
else
    # Check if the set of PIDs changed (indicating node 2 daemons were restarted)
    if [ "$PSMWATCH_PIDS_BEFORE" != "$PSMWATCH_PIDS_AFTER" ]; then
        echo "[INFO] psmwatch PID set changed after ionrestart (restart detected)"
    else
        echo "[WARN] psmwatch PIDs unchanged after ionrestart"
    fi
    if [ "$SDRWATCH_PIDS_BEFORE" != "$SDRWATCH_PIDS_AFTER" ]; then
        echo "[INFO] sdrwatch PID set changed after ionrestart (restart detected)"
    else
        echo "[WARN] sdrwatch PIDs unchanged after ionrestart"
    fi
fi

echo ""
echo "###############################################################################"
echo "# Step 4: Shutdown both nodes and verify daemons stop"
echo "###############################################################################"
echo ""

# stop_node: Stop a node with ionstop; if it times out, run ionadmin .
# directly as fallback to ensure rfx_stop() runs and daemons get SIGTERM.
stop_node() {
    local node_dir=$1
    local node_name=$2
    cd "$TOPDIR/$node_dir"
    echo "Stopping $node_name using ionstop..."
    if ! run_with_timeout 45 ./ionstop; then
        echo "[WARN] ionstop for $node_name timed out, running ionadmin . as fallback"
        run_with_timeout 15 ionadmin .
    fi
}

# Stop both nodes in parallel to avoid LTP inter-node hangs caused by
# stopping one node while the other's LTP stack is still communicating.
echo "Stopping both nodes in parallel..."
stop_node "2.watchdog.ltp" "node 2" &
STOP2_PID=$!
stop_node "3.watchdog.ltp" "node 3" &
STOP3_PID=$!

wait $STOP2_PID 2>/dev/null
wait $STOP3_PID 2>/dev/null
cd "$TOPDIR"

# Poll for daemons to stop (up to 15 seconds).
# Daemons use a 10-second snooze() poll interval (DAEMON_POLL_INTERVAL_SEC),
# so they may need time to wake up and process the SIGTERM from rfx_stop().
echo ""
echo "Waiting for daemons to stop..."
WAIT=0
while [ $WAIT -lt 15 ]; do
    if ! pgrep -f "psmwatch" > /dev/null 2>&1 && \
       ! pgrep -f "sdrwatch" > /dev/null 2>&1; then
        echo "All daemons stopped after ${WAIT}s"
        break
    fi
    sleep 1
    WAIT=$((WAIT + 1))
done

echo ""
show_daemon_info

if ! check_daemons "after-ionexit" "stopped"; then
    echo "[FAIL] Daemons still running after ionexit!"
    # Force cleanup
    pkill -f psmwatch 2>/dev/null
    pkill -f sdrwatch 2>/dev/null
    RETVAL=1
else
    echo "[PASS] ionexit properly stopped all watchdog daemons"
fi

echo ""
echo "###############################################################################"
echo "# Test Summary"
echo "###############################################################################"
echo ""

if [ $RETVAL -eq 0 ]; then
    echo "=========================================="
    echo "  ALL TESTS PASSED"
    echo "=========================================="
else
    echo "=========================================="
    echo "  SOME TESTS FAILED"
    echo "=========================================="
fi

echo ""
echo "Cleaning up..."
pkill -f psmwatch 2>/dev/null
pkill -f sdrwatch 2>/dev/null

echo "Test completed."
exit $RETVAL
