#!/bin/bash
# Post-crash / periodic host health bundle for Blair pve. Run as root on pve.
# Saves under /tank/blair-share/fleet/diagnostics/host-health-<timestamp>.txt
set -euo pipefail

SHARE="${BLAIR_SHARE:-/tank/blair-share}"
OUT_DIR="$SHARE/fleet/diagnostics"
STAMP=$(date -Is | tr ':' '-')
OUT="$OUT_DIR/host-health-$STAMP.txt"

mkdir -p "$OUT_DIR"
exec > >(tee "$OUT") 2>&1

echo "=== Blair host health collection $STAMP ==="
echo "hostname: $(hostname)"
uname -a
pveversion -v 2>/dev/null | head -3 || true

echo ""
echo "=== Previous boot errors (PCIe / MCE / AER) ==="
journalctl -b -1 -p err..alert --no-pager 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error|iommu|vfio' | tail -60 || echo "(no journal -1 or no matches)"
echo "--- current boot tail ---"
dmesg -T 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error' | tail -30 || true

echo ""
echo "=== Dell panel bus decode (decimal -> hex) ==="
for dec in 58 93 215; do
  hex=$(printf '%x' "$dec")
  echo "Panel bus $dec = 0x$hex"
  lspci -s "${hex}:" 2>/dev/null | head -5 || true
  next=$(printf '%x' $((dec + 1)))
  echo "  GPU neighbor bus 0x$next:"
  lspci -nn -s "${next}:00" 2>/dev/null || true
done

echo ""
echo "=== All passthrough GPUs (link status) ==="
for bdf in 18:00.0 3b:00.0 5e:00.0 5f:00.0 86:00.0 87:00.0 d8:00.0; do
  echo "--- $bdf ---"
  lspci -nn -s "$bdf" 2>/dev/null || echo "  (missing)"
  lspci -vvv -s "$bdf" 2>/dev/null | grep -iE 'LnkCap|LnkSta|Width|Speed|UESta|UEMsk|CESta|DevSta|AER' || true
done

echo ""
echo "=== PCIe topology (GPU + NVMe + bridges) ==="
lspci -t 2>/dev/null | head -80 || true

echo ""
echo "=== Physical slots (dmidecode) ==="
dmidecode -t slot 2>/dev/null | grep -E 'Designation|Type|Current Usage|Bus Address|Length' || echo "(dmidecode unavailable)"

echo ""
echo "=== Power supplies (dmidecode type 39) ==="
dmidecode -t 39 2>/dev/null || echo "(no type 39)"

echo ""
echo "=== IPMI / BMC sensors (if ipmitool present) ==="
if command -v ipmitool >/dev/null 2>&1; then
  ipmitool sensor list 2>/dev/null | grep -iE 'power|volt|temp|fan|ps|current|watt' || ipmitool sdr 2>/dev/null | head -40
else
  echo "ipmitool not installed — use iDRAC web UI: System > Health > Power / SEL"
fi

echo ""
echo "=== hwmon (onboard sensors) ==="
for h in /sys/class/hwmon/hwmon*/; do
  [[ -d "$h" ]] || continue
  name=$(cat "$h/name" 2>/dev/null || echo '?')
  echo "hwmon $name"
  for f in "$h"/temp*_input "$h"/power*_input "$h"/in*_input; do
    [[ -f "$f" ]] || continue
    base=$(basename "$f" _input)
    label_file="${f/_input/_label}"
    label=$(cat "$label_file" 2>/dev/null || echo "$base")
    val=$(cat "$f" 2>/dev/null)
    if [[ "$f" == *power* ]]; then
      echo "  $label: ${val} uW"
    elif [[ "$f" == *in* ]]; then
      echo "  $label: ${val} mV"
    else
      echo "  $label: $(( val / 1000 )) C"
    fi
  done
done 2>/dev/null | head -40

echo ""
echo "=== Load / memory ==="
uptime
free -h

echo ""
echo "=== GPU VM status ==="
for v in 230 231 232 233 234 235; do
  st=$(qm status "$v" 2>/dev/null | awk '{print $2}' || echo '?')
  onboot=$(grep -E '^onboot:' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null || true)
  hp=$(grep -E '^hostpci' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null | head -1 || true)
  echo "VM $v: $st  $onboot  $hp"
done

echo ""
echo "=== Rough power budget reminder ==="
echo "Dual Xeon 5218R ~250W + 6 GPU passthrough (guest-owned power) + storage/NVMe."
echo "Simultaneous VM start = simultaneous GPU reset/inrush on 12V — correlate with SEL + panel bus."

echo ""
echo "Saved: $OUT"