adam
0f829e0380
Rearchitect so a single poller process is the sole owner of all SAS/SMART/ SES hardware I/O, serialized behind one gate and paced, writing results to Redis. The API/web and MQTT publisher become pure Redis readers — they no longer issue any subprocess and can restart freely without touching the bus. This addresses backplane/expander stress from concurrent + restart-triggered SMART/SES storms (the prior model re-ran a hardware sweep on every container start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/ Xyratex expanders). - poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS, host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker - services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1) - services/store.py: Redis as the only producer<->consumer interface + LED queue - services/health.py: shared drive-health classifier (fixes overview double-count) - gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path - routers + temps read-only from the store; main.py drops the in-process poller - compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
68 lines
2.3 KiB
Python
68 lines
2.3 KiB
Python
"""Per-enclosure temperature aggregation (read-only consumer).
|
|
|
|
Reads the poller's data out of Redis — never touches hardware. Each named
|
|
SES temperature sensor plus a derived per-enclosure hotspot (the hottest
|
|
reading across SES sensors *and* the drives housed in that enclosure —
|
|
drive temps are usually the real hotspot signal).
|
|
"""
|
|
import logging
|
|
import os
|
|
import socket
|
|
|
|
from services import store
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
def get_node_id() -> str:
|
|
"""Stable identifier for this monitor instance (defaults to hostname)."""
|
|
return os.environ.get("MQTT_NODE_ID") or socket.gethostname() or "jbod-monitor"
|
|
|
|
|
|
async def build_temp_snapshot() -> dict:
|
|
"""Collect per-enclosure temperature metrics from the store."""
|
|
inventory = await store.get_inventory()
|
|
out_enclosures: list[dict] = []
|
|
|
|
for enc in inventory.get("enclosures", []):
|
|
ses = await store.get_ses(enc["id"]) or {}
|
|
|
|
sensors: list[dict] = []
|
|
sensor_temps: list[float] = []
|
|
for t in ses.get("temps", []):
|
|
temp = t.get("temperature_c")
|
|
sensors.append({
|
|
"index": t["index"],
|
|
"name": t.get("name"),
|
|
"status": t.get("status", "Unknown"),
|
|
"temperature_c": temp,
|
|
})
|
|
if isinstance(temp, (int, float)) and temp > 0:
|
|
sensor_temps.append(float(temp))
|
|
|
|
# Drive temperatures for drives housed in this enclosure.
|
|
drive_temps: list[float] = []
|
|
for slot in enc.get("slots", []):
|
|
dev = slot.get("device")
|
|
if not dev:
|
|
continue
|
|
sm = await store.get_smart(dev)
|
|
if not sm:
|
|
continue
|
|
temp = sm.get("temperature_c")
|
|
if isinstance(temp, (int, float)) and temp > 0:
|
|
drive_temps.append(float(temp))
|
|
|
|
all_temps = sensor_temps + drive_temps
|
|
out_enclosures.append({
|
|
"id": enc["id"],
|
|
"vendor": enc.get("vendor", ""),
|
|
"model": enc.get("model", ""),
|
|
"hotspot_c": max(all_temps) if all_temps else None,
|
|
"drive_max_c": max(drive_temps) if drive_temps else None,
|
|
"drive_temp_count": len(drive_temps),
|
|
"sensors": sensors,
|
|
})
|
|
|
|
return {"node": get_node_id(), "enclosures": out_enclosures}
|