#!/bin/bash
#
# Freeze Diagnostic Script
#
# Run this DURING a slowdown/lockup, BEFORE restarting Apache or PM2.
# It captures the state needed to identify root cause.
#
# Usage: ./scripts/diagnose-freeze.sh [port]
#   port: Next.js backend port (default: 3001)
#

PORT="${1:-3001}"
TIMESTAMP=$(date '+%Y-%m-%d_%H-%M-%S')
OUTFILE="/tmp/freeze-diagnostics-${TIMESTAMP}.txt"

echo "=== Freeze Diagnostics: $(date) ===" | tee "$OUTFILE"
echo "Next.js backend port: $PORT" | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 1. Connection states from Apache to Next.js backend
echo "--- 1. Connection states to backend (:$PORT) ---" | tee -a "$OUTFILE"
for state in established close-wait fin-wait-1 fin-wait-2 time-wait syn-sent; do
  count=$(ss -tn state "$state" "( dport = :$PORT )" 2>/dev/null | tail -n +2 | wc -l)
  echo "  $state: $count" | tee -a "$OUTFILE"
done
echo "" | tee -a "$OUTFILE"

# 2. Apache process count and status
echo "--- 2. Apache status ---" | tee -a "$OUTFILE"
echo "  httpd processes: $(pgrep -c httpd 2>/dev/null || echo 'N/A')" | tee -a "$OUTFILE"
echo "  Apache service:" | tee -a "$OUTFILE"
systemctl is-active httpd 2>/dev/null | sed 's/^/    /' | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 3. Can Next.js respond directly? (bypass Apache)
echo "--- 3. Direct Next.js response (bypass Apache) ---" | tee -a "$OUTFILE"
{ time curl -sI "http://127.0.0.1:$PORT/" 2>&1 | head -3 ; } 2>&1 | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 4. Node.js / Next.js process info
echo "--- 4. Node.js process info ---" | tee -a "$OUTFILE"
ps aux | grep -E '(next|node)' | grep -v grep | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 5. PM2 process status
echo "--- 5. PM2 status ---" | tee -a "$OUTFILE"
pm2 list 2>/dev/null | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 6. PostgreSQL connection state
echo "--- 6. PostgreSQL connections ---" | tee -a "$OUTFILE"
# Try to connect using the production database user
# Falls back gracefully if credentials aren't available
PGPASSWORD="${DB_PASSWORD:-}" psql -h 127.0.0.1 -U amsoil_dlp_production -d amsoil_dlp_production -c "
SELECT count(*) AS total,
       count(*) FILTER (WHERE state = 'active') AS active,
       count(*) FILTER (WHERE state = 'idle') AS idle,
       count(*) FILTER (WHERE state = 'idle in transaction') AS idle_in_tx,
       count(*) FILTER (WHERE wait_event_type IS NOT NULL) AS waiting
FROM pg_stat_activity WHERE datname = current_database();" 2>&1 | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 7. PostgreSQL blocking queries
echo "--- 7. Blocking queries ---" | tee -a "$OUTFILE"
PGPASSWORD="${DB_PASSWORD:-}" psql -h 127.0.0.1 -U amsoil_dlp_production -d amsoil_dlp_production -c "
SELECT blocked.pid AS blocked_pid,
       age(now(), blocked.query_start) AS blocked_for,
       LEFT(blocked.query, 80) AS blocked_query,
       blocker.pid AS blocker_pid,
       LEFT(blocker.query, 80) AS blocker_query
FROM pg_stat_activity blocked
JOIN pg_stat_activity blocker ON blocker.pid = ANY(pg_blocking_pids(blocked.pid))
WHERE blocked.wait_event_type = 'Lock' LIMIT 10;" 2>&1 | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 8. Recent Apache proxy errors
echo "--- 8. Recent Apache proxy errors ---" | tee -a "$OUTFILE"
tail -200 /etc/apache2/logs/error_log 2>/dev/null | grep -iE "proxy|timeout|AH0|connection" | tail -30 | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 9. System resource overview
echo "--- 9. System resources ---" | tee -a "$OUTFILE"
echo "  File descriptors (allocated / free / max):" | tee -a "$OUTFILE"
cat /proc/sys/fs/file-nr 2>/dev/null | sed 's/^/    /' | tee -a "$OUTFILE"
echo "  Memory:" | tee -a "$OUTFILE"
free -h 2>/dev/null | sed 's/^/    /' | tee -a "$OUTFILE"
echo "  Load average:" | tee -a "$OUTFILE"
uptime 2>/dev/null | sed 's/^/    /' | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

# 10. Full connection dump to backend port
echo "--- 10. All connections to :$PORT ---" | tee -a "$OUTFILE"
ss -tnp '( dport = :'"$PORT"' )' 2>/dev/null | tee -a "$OUTFILE"
echo "" | tee -a "$OUTFILE"

echo "=== Diagnostics complete ===" | tee -a "$OUTFILE"
echo "Full output saved to: $OUTFILE"