diff --git a/install-fleet-monitor.sh b/install-fleet-monitor.sh new file mode 100644 index 0000000..c6ec36b --- /dev/null +++ b/install-fleet-monitor.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Install Blair fleet monitor on pve. Run as root. +set -euo pipefail + +BASE_URL="${BLAIR_MONITOR_BASE_URL:-https://git.easygoingaming.com/Davoguha/blair-pve-sync-temp/raw/main}" +TARGET='/usr/local/sbin/monitor-blair-fleet.sh' +SHARE_COPY='/tank/blair-share/tools/golden/host/monitor-blair-fleet.sh' +CRON='/etc/cron.d/blair-fleet-monitor' +LOG='/var/log/blair-fleet-monitor.jsonl' + +echo "=== Download monitor script ===" +curl -fsSL "$BASE_URL/monitor-blair-fleet.sh" -o "$TARGET" +chmod +x "$TARGET" +sed -i 's/\r$//' "$TARGET" 2>/dev/null || true + +mkdir -p "$(dirname "$SHARE_COPY")" /tank/blair-share/fleet/monitor +cp -f "$TARGET" "$SHARE_COPY" +chmod +x "$SHARE_COPY" + +echo "=== cron (every 5 min) ===" +printf '%s\n' "*/5 * * * * root $TARGET >> /var/log/blair-fleet-monitor-cron.log 2>&1" > "$CRON" +chmod 644 "$CRON" +cat "$CRON" + +echo "=== test run ===" +"$TARGET" +echo "=== latest snapshot ===" +head -c 2000 /tank/blair-share/fleet/monitor/latest.json; echo +echo "=== log tail ===" +tail -3 "$LOG" 2>/dev/null || true +echo "=== done ===" diff --git a/monitor-blair-fleet.sh b/monitor-blair-fleet.sh new file mode 100644 index 0000000..fb2a300 --- /dev/null +++ b/monitor-blair-fleet.sh @@ -0,0 +1,147 @@ +#!/bin/bash +# Lightweight GPU fleet monitor — run on pve cron (every 5 min). +# Logs JSONL to /var/log/blair-fleet-monitor.jsonl; updates blair-share latest snapshot. +# Alerts in log when IoPsiFull or flush latency exceed thresholds. +set -euo pipefail + +export BLAIR_SHARE="${BLAIR_SHARE:-/tank/blair-share}" +LOG="${BLAIR_FLEET_MONITOR_LOG:-/var/log/blair-fleet-monitor.jsonl}" +LATEST="${BLAIR_SHARE}/fleet/monitor/latest.json" +NODE="${BLAIR_PVE_NODE:-pve}" +GPU_VMIDS=(230 231 232 233 234 235) +IO_WARN=15 +IO_CRIT=30 +FLUSH_WARN_MS=20 + +if ! command -v python3 >/dev/null 2>&1; then + echo 'python3 required' >&2 + exit 1 +fi + +exec python3 - <<'PY' +import json +import os +import subprocess +from datetime import datetime, timezone + +LOG = os.environ.get('BLAIR_FLEET_MONITOR_LOG', '/var/log/blair-fleet-monitor.jsonl') +LATEST = os.path.join(os.environ.get('BLAIR_SHARE', '/tank/blair-share'), 'fleet/monitor/latest.json') +NODE = os.environ.get('BLAIR_PVE_NODE', 'pve') +GPU_VMIDS = [230, 231, 232, 233, 234, 235] +IO_WARN = int(os.environ.get('BLAIR_IO_WARN', '15')) +IO_CRIT = int(os.environ.get('BLAIR_IO_CRIT', '30')) +FLUSH_WARN_MS = float(os.environ.get('BLAIR_FLUSH_WARN_MS', '20')) + + +def pvesh_get(path: str) -> dict: + proc = subprocess.run( + ['pvesh', 'get', path, '--output-format', 'json'], + capture_output=True, text=True, check=False, + ) + if proc.returncode != 0: + return {} + return json.loads(proc.stdout) + + +def blockstat_latency(blockstat: dict) -> list[dict]: + rows = [] + if not blockstat: + return rows + for dev, d in blockstat.items(): + if not isinstance(d, dict): + continue + rd_ops = float(d.get('rd_operations') or 0) + wr_ops = float(d.get('wr_operations') or 0) + fl_ops = float(d.get('flush_operations') or 0) + rows.append({ + 'device': dev, + 'rd_avg_ms': round(float(d.get('rd_total_time_ns') or 0) / rd_ops / 1e6, 2) if rd_ops else 0, + 'wr_avg_ms': round(float(d.get('wr_total_time_ns') or 0) / wr_ops / 1e6, 2) if wr_ops else 0, + 'flush_avg_ms': round(float(d.get('flush_total_time_ns') or 0) / fl_ops / 1e6, 2) if fl_ops else 0, + 'rd_ops': int(rd_ops), + 'wr_ops': int(wr_ops), + 'flush_ops': int(fl_ops), + }) + return rows + + +def main() -> int: + now = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ') + alerts: list[dict] = [] + host = pvesh_get(f'/nodes/{NODE}/status') + stor = pvesh_get(f'/nodes/{NODE}/storage') + pool = next((s for s in stor if s.get('storage') == 'local-2tb'), {}) + pool_pct = round(100 * pool.get('used', 0) / pool['total'], 1) if pool.get('total') else None + + host_summary = { + 'cpu_pct': round(float(host.get('cpu', 0)) * 100, 1), + 'mem_used_gb': round(float(host.get('memory', {}).get('used', 0)) / (1024**3), 1), + 'mem_total_gb': round(float(host.get('memory', {}).get('total', 0)) / (1024**3), 1), + 'pool_used_pct': pool_pct, + } + + gpu_rows = [] + for vmid in GPU_VMIDS: + cfg = pvesh_get(f'/nodes/{NODE}/qemu/{vmid}/config') + cur = pvesh_get(f'/nodes/{NODE}/qemu/{vmid}/status/current') + if not cur or cur.get('status') != 'running': + gpu_rows.append({'vmid': vmid, 'name': cfg.get('name'), 'status': 'stopped'}) + continue + io_full = round(float(cur.get('pressureiofull') or 0), 1) + io_some = round(float(cur.get('pressureiosome') or 0), 1) + lat = blockstat_latency(cur.get('blockstat') or {}) + disk = cfg.get('sata0') or cfg.get('scsi0') or cfg.get('virtio0') + hostpci = next((v for k, v in cfg.items() if str(k).startswith('hostpci')), '') + row = { + 'vmid': vmid, + 'name': cur.get('name') or cfg.get('name'), + 'status': 'running', + 'cpu_pct': round(float(cur.get('cpu', 0)) * 100, 1), + 'mem_gb': round(float(cur.get('mem', 0)) / (1024**3), 2), + 'io_psi_full': io_full, + 'io_psi_some': io_some, + 'uptime_h': round(float(cur.get('uptime', 0)) / 3600, 1), + 'vga': cfg.get('vga'), + 'disk': disk, + 'gpu_mapping': hostpci, + 'latency': lat, + } + gpu_rows.append(row) + if io_full >= IO_CRIT: + alerts.append({'level': 'CRIT', 'vmid': vmid, 'msg': f'IO stall {io_full}%'}) + elif io_full >= IO_WARN: + alerts.append({'level': 'WARN', 'vmid': vmid, 'msg': f'IO stall {io_full}%'}) + for l in lat: + if l['flush_avg_ms'] >= FLUSH_WARN_MS and l['flush_ops'] > 10: + alerts.append({ + 'level': 'WARN', 'vmid': vmid, + 'msg': f"{l['device']} flush avg {l['flush_avg_ms']}ms", + }) + + record = { + 'timestamp': now, + 'host': host_summary, + 'gpu_fleet': gpu_rows, + 'alerts': alerts, + } + + os.makedirs(os.path.dirname(LOG), exist_ok=True) + with open(LOG, 'a', encoding='utf-8') as fh: + fh.write(json.dumps(record, separators=(',', ':')) + '\n') + + os.makedirs(os.path.dirname(LATEST), exist_ok=True) + tmp = LATEST + '.tmp' + with open(tmp, 'w', encoding='utf-8') as fh: + json.dump(record, fh, indent=2) + fh.write('\n') + os.replace(tmp, LATEST) + + if alerts: + for a in alerts: + print(f"[{a['level']}] VM {a.get('vmid', '-')} {a['msg']}") + return 0 + + +if __name__ == '__main__': + raise SystemExit(main()) +PY