From 10233bd109d2430e4846edad558bb5dc483c6067 Mon Sep 17 00:00:00 2001
From: alexveley <alexv1129@gmail.com>
Date: Fri, 26 Jun 2026 01:00:02 -0400
Subject: [PATCH] Update pve RustDesk GA sync scripts

---
 collect-blair-host-health.sh   | 106 +++++++++++++++++++++++++++++++++
 install-collect-host-health.sh |  24 ++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 collect-blair-host-health.sh
 create mode 100644 install-collect-host-health.sh

diff --git a/collect-blair-host-health.sh b/collect-blair-host-health.sh
new file mode 100644
index 0000000..15724ae
--- /dev/null
+++ b/collect-blair-host-health.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+# Post-crash / periodic host health bundle for Blair pve. Run as root on pve.
+# Saves under /tank/blair-share/fleet/diagnostics/host-health-<timestamp>.txt
+set -euo pipefail
+
+SHARE="${BLAIR_SHARE:-/tank/blair-share}"
+OUT_DIR="$SHARE/fleet/diagnostics"
+STAMP=$(date -Is | tr ':' '-')
+OUT="$OUT_DIR/host-health-$STAMP.txt"
+
+mkdir -p "$OUT_DIR"
+exec > >(tee "$OUT") 2>&1
+
+echo "=== Blair host health collection $STAMP ==="
+echo "hostname: $(hostname)"
+uname -a
+pveversion -v 2>/dev/null | head -3 || true
+
+echo ""
+echo "=== Previous boot errors (PCIe / MCE / AER) ==="
+journalctl -b -1 -p err..alert --no-pager 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error|iommu|vfio' | tail -60 || echo "(no journal -1 or no matches)"
+echo "--- current boot tail ---"
+dmesg -T 2>/dev/null | grep -iE 'aer|pcie|fatal|mce|hardware error' | tail -30 || true
+
+echo ""
+echo "=== Dell panel bus decode (decimal -> hex) ==="
+for dec in 58 93 215; do
+  hex=$(printf '%x' "$dec")
+  echo "Panel bus $dec = 0x$hex"
+  lspci -s "${hex}:" 2>/dev/null | head -5 || true
+  next=$(printf '%x' $((dec + 1)))
+  echo "  GPU neighbor bus 0x$next:"
+  lspci -nn -s "${next}:00" 2>/dev/null || true
+done
+
+echo ""
+echo "=== All passthrough GPUs (link status) ==="
+for bdf in 18:00.0 3b:00.0 5e:00.0 5f:00.0 86:00.0 87:00.0 d8:00.0; do
+  echo "--- $bdf ---"
+  lspci -nn -s "$bdf" 2>/dev/null || echo "  (missing)"
+  lspci -vvv -s "$bdf" 2>/dev/null | grep -iE 'LnkCap|LnkSta|Width|Speed|UESta|UEMsk|CESta|DevSta|AER' || true
+done
+
+echo ""
+echo "=== PCIe topology (GPU + NVMe + bridges) ==="
+lspci -t 2>/dev/null | head -80 || true
+
+echo ""
+echo "=== Physical slots (dmidecode) ==="
+dmidecode -t slot 2>/dev/null | grep -E 'Designation|Type|Current Usage|Bus Address|Length' || echo "(dmidecode unavailable)"
+
+echo ""
+echo "=== Power supplies (dmidecode type 39) ==="
+dmidecode -t 39 2>/dev/null || echo "(no type 39)"
+
+echo ""
+echo "=== IPMI / BMC sensors (if ipmitool present) ==="
+if command -v ipmitool >/dev/null 2>&1; then
+  ipmitool sensor list 2>/dev/null | grep -iE 'power|volt|temp|fan|ps|current|watt' || ipmitool sdr 2>/dev/null | head -40
+else
+  echo "ipmitool not installed — use iDRAC web UI: System > Health > Power / SEL"
+fi
+
+echo ""
+echo "=== hwmon (onboard sensors) ==="
+for h in /sys/class/hwmon/hwmon*/; do
+  [[ -d "$h" ]] || continue
+  name=$(cat "$h/name" 2>/dev/null || echo '?')
+  echo "hwmon $name"
+  for f in "$h"/temp*_input "$h"/power*_input "$h"/in*_input; do
+    [[ -f "$f" ]] || continue
+    base=$(basename "$f" _input)
+    label_file="${f/_input/_label}"
+    label=$(cat "$label_file" 2>/dev/null || echo "$base")
+    val=$(cat "$f" 2>/dev/null)
+    if [[ "$f" == *power* ]]; then
+      echo "  $label: ${val} uW"
+    elif [[ "$f" == *in* ]]; then
+      echo "  $label: ${val} mV"
+    else
+      echo "  $label: $(( val / 1000 )) C"
+    fi
+  done
+done 2>/dev/null | head -40
+
+echo ""
+echo "=== Load / memory ==="
+uptime
+free -h
+
+echo ""
+echo "=== GPU VM status ==="
+for v in 230 231 232 233 234 235; do
+  st=$(qm status "$v" 2>/dev/null | awk '{print $2}' || echo '?')
+  onboot=$(grep -E '^onboot:' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null || true)
+  hp=$(grep -E '^hostpci' "/etc/pve/qemu-server/${v}.conf" 2>/dev/null | head -1 || true)
+  echo "VM $v: $st  $onboot  $hp"
+done
+
+echo ""
+echo "=== Rough power budget reminder ==="
+echo "Dual Xeon 5218R ~250W + 6 GPU passthrough (guest-owned power) + storage/NVMe."
+echo "Simultaneous VM start = simultaneous GPU reset/inrush on 12V — correlate with SEL + panel bus."
+
+echo ""
+echo "Saved: $OUT"
diff --git a/install-collect-host-health.sh b/install-collect-host-health.sh
new file mode 100644
index 0000000..8e0896b
--- /dev/null
+++ b/install-collect-host-health.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+# One-shot: download Blair host health collector and run it. Run as root on pve.
+set -euo pipefail
+
+BASE_URL="${BLAIR_HEALTH_BASE_URL:-https://git.easygoingaming.com/Davoguha/blair-pve-sync-temp/raw/main}"
+TARGET='/usr/local/sbin/collect-blair-host-health.sh'
+SHARE_COPY='/tank/blair-share/tools/golden/host/collect-blair-host-health.sh'
+
+echo "=== Download host health collector ==="
+curl -fsSL "$BASE_URL/collect-blair-host-health.sh" -o "$TARGET"
+chmod +x "$TARGET"
+sed -i 's/\r$//' "$TARGET" 2>/dev/null || true
+
+mkdir -p "$(dirname "$SHARE_COPY")" /tank/blair-share/fleet/diagnostics
+cp -f "$TARGET" "$SHARE_COPY"
+chmod +x "$SHARE_COPY"
+
+echo "=== Run collection (output also on stdout) ==="
+"$TARGET"
+
+echo ""
+echo "=== Report path (copy to Alex) ==="
+ls -1t /tank/blair-share/fleet/diagnostics/host-health-*.txt 2>/dev/null | head -1 || true
+echo "=== done ==="