alexveley
107 lines
3.5 KiB
Bash
107 lines
3.5 KiB
Bash
#!/bin/bash
|
|
# Post-crash / periodic host health bundle for Blair pve. Run as root on pve.
|
|
# Saves under /tank/blair-share/fleet/diagnostics/host-health-<timestamp>.txt
|
|
set -euo pipefail
|
|
|
|
SHARE="${BLAIR_SHARE:-/tank/blair-share}"
|
|
OUT_DIR="$SHARE/fleet/diagnostics"
|
|
STAMP=$(date -Is | tr ':' '-')
|
|
OUT="$OUT_DIR/host-health-$STAMP.txt"
|
|
|
|
mkdir -p "$OUT_DIR"
|
|
exec > >(tee "$OUT") 2>&1
|
|
|
|
echo "=== Blair host health collection $STAMP ==="
|
|
echo "hostname: $(hostname)"
|
|
uname -a
|
|
pveversion -v 2>/dev/null | head -3 || true
|
|
|
|
echo ""
|
|
echo "=== Previous boot errors (PCIe / MCE / AER) ==="
|
|
journalctl -b -1 -p err..alert --no-pager 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error|iommu|vfio' | tail -60 || echo "(no journal -1 or no matches)"
|
|
echo "--- current boot tail ---"
|
|
dmesg -T 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error' | tail -30 || true
|
|
|
|
echo ""
|
|
echo "=== Dell panel bus decode (decimal -> hex) ==="
|
|
for dec in 58 93 215; do
|
|
hex=$(printf '%x' "$dec")
|
|
echo "Panel bus $dec = 0x$hex"
|
|
lspci -s "${hex}:" 2>/dev/null | head -5 || true
|
|
next=$(printf '%x' $((dec + 1)))
|
|
echo " GPU neighbor bus 0x$next:"
|
|
lspci -nn -s "${next}:00" 2>/dev/null || true
|
|
done
|
|
|
|
echo ""
|
|
echo "=== All passthrough GPUs (link status) ==="
|
|
for bdf in 18:00.0 3b:00.0 5e:00.0 5f:00.0 86:00.0 87:00.0 d8:00.0; do
|
|
echo "--- $bdf ---"
|
|
lspci -nn -s "$bdf" 2>/dev/null || echo " (missing)"
|
|
lspci -vvv -s "$bdf" 2>/dev/null | grep -iE 'LnkCap|LnkSta|Width|Speed|UESta|UEMsk|CESta|DevSta|AER' || true
|
|
done
|
|
|
|
echo ""
|
|
echo "=== PCIe topology (GPU + NVMe + bridges) ==="
|
|
lspci -t 2>/dev/null | head -80 || true
|
|
|
|
echo ""
|
|
echo "=== Physical slots (dmidecode) ==="
|
|
dmidecode -t slot 2>/dev/null | grep -E 'Designation|Type|Current Usage|Bus Address|Length' || echo "(dmidecode unavailable)"
|
|
|
|
echo ""
|
|
echo "=== Power supplies (dmidecode type 39) ==="
|
|
dmidecode -t 39 2>/dev/null || echo "(no type 39)"
|
|
|
|
echo ""
|
|
echo "=== IPMI / BMC sensors (if ipmitool present) ==="
|
|
if command -v ipmitool >/dev/null 2>&1; then
|
|
ipmitool sensor list 2>/dev/null | grep -iE 'power|volt|temp|fan|ps|current|watt' || ipmitool sdr 2>/dev/null | head -40
|
|
else
|
|
echo "ipmitool not installed — use iDRAC web UI: System > Health > Power / SEL"
|
|
fi
|
|
|
|
echo ""
|
|
echo "=== hwmon (onboard sensors) ==="
|
|
for h in /sys/class/hwmon/hwmon*/; do
|
|
[[ -d "$h" ]] || continue
|
|
name=$(cat "$h/name" 2>/dev/null || echo '?')
|
|
echo "hwmon $name"
|
|
for f in "$h"/temp*_input "$h"/power*_input "$h"/in*_input; do
|
|
[[ -f "$f" ]] || continue
|
|
base=$(basename "$f" _input)
|
|
label_file="${f/_input/_label}"
|
|
label=$(cat "$label_file" 2>/dev/null || echo "$base")
|
|
val=$(cat "$f" 2>/dev/null)
|
|
if [[ "$f" == *power* ]]; then
|
|
echo " $label: ${val} uW"
|
|
elif [[ "$f" == *in* ]]; then
|
|
echo " $label: ${val} mV"
|
|
else
|
|
echo " $label: $(( val / 1000 )) C"
|
|
fi
|
|
done
|
|
done 2>/dev/null | head -40
|
|
|
|
echo ""
|
|
echo "=== Load / memory ==="
|
|
uptime
|
|
free -h
|
|
|
|
echo ""
|
|
echo "=== GPU VM status ==="
|
|
for v in 230 231 232 233 234 235; do
|
|
st=$(qm status "$v" 2>/dev/null | awk '{print $2}' || echo '?')
|
|
onboot=$(grep -E '^onboot:' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null || true)
|
|
hp=$(grep -E '^hostpci' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null | head -1 || true)
|
|
echo "VM $v: $st $onboot $hp"
|
|
done
|
|
|
|
echo ""
|
|
echo "=== Rough power budget reminder ==="
|
|
echo "Dual Xeon 5218R ~250W + 6 GPU passthrough (guest-owned power) + storage/NVMe."
|
|
echo "Simultaneous VM start = simultaneous GPU reset/inrush on 12V — correlate with SEL + panel bus."
|
|
|
|
echo ""
|
|
echo "Saved: $OUT"
|