Split into dedicated hardware poller + read-only consumers
Rearchitect so a single poller process is the sole owner of all SAS/SMART/ SES hardware I/O, serialized behind one gate and paced, writing results to Redis. The API/web and MQTT publisher become pure Redis readers — they no longer issue any subprocess and can restart freely without touching the bus. This addresses backplane/expander stress from concurrent + restart-triggered SMART/SES storms (the prior model re-ran a hardware sweep on every container start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/ Xyratex expanders). - poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS, host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker - services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1) - services/store.py: Redis as the only producer<->consumer interface + LED queue - services/health.py: shared drive-health classifier (fixes overview double-count) - gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path - routers + temps read-only from the store; main.py drops the in-process poller - compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
This commit is contained in:
@@ -1,74 +1,57 @@
|
||||
"""Per-enclosure temperature aggregation.
|
||||
"""Per-enclosure temperature aggregation (read-only consumer).
|
||||
|
||||
Builds a compact snapshot of enclosure temperatures suitable for Home
|
||||
Assistant: each named SES temperature sensor plus a derived per-enclosure
|
||||
hotspot (the hottest reading across SES sensors *and* the drives housed in
|
||||
that enclosure — drive temps are usually the real hotspot signal).
|
||||
Reads the poller's data out of Redis — never touches hardware. Each named
|
||||
SES temperature sensor plus a derived per-enclosure hotspot (the hottest
|
||||
reading across SES sensors *and* the drives housed in that enclosure —
|
||||
drive temps are usually the real hotspot signal).
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
|
||||
from services.enclosure import discover_enclosures, get_enclosure_status, list_slots
|
||||
from services.smart import get_smart_data
|
||||
from services import store
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_node_id() -> str:
|
||||
"""Stable identifier for this monitor instance (defaults to hostname)."""
|
||||
import os
|
||||
|
||||
return os.environ.get("MQTT_NODE_ID") or socket.gethostname() or "jbod-monitor"
|
||||
|
||||
|
||||
async def build_temp_snapshot() -> dict:
|
||||
"""Collect per-enclosure temperature metrics for all enclosures."""
|
||||
enclosures = discover_enclosures()
|
||||
|
||||
async def _health(enc):
|
||||
if enc.get("sg_device"):
|
||||
return await get_enclosure_status(enc["sg_device"])
|
||||
return None
|
||||
|
||||
health_results = await asyncio.gather(
|
||||
*[_health(enc) for enc in enclosures], return_exceptions=True
|
||||
)
|
||||
|
||||
"""Collect per-enclosure temperature metrics from the store."""
|
||||
inventory = await store.get_inventory()
|
||||
out_enclosures: list[dict] = []
|
||||
for enc, health in zip(enclosures, health_results):
|
||||
if isinstance(health, Exception):
|
||||
logger.warning("SES health failed for %s: %s", enc["id"], health)
|
||||
health = None
|
||||
|
||||
for enc in inventory.get("enclosures", []):
|
||||
ses = await store.get_ses(enc["id"]) or {}
|
||||
|
||||
sensors: list[dict] = []
|
||||
sensor_temps: list[float] = []
|
||||
if isinstance(health, dict):
|
||||
for t in health.get("temps", []):
|
||||
temp = t.get("temperature_c")
|
||||
sensors.append({
|
||||
"index": t["index"],
|
||||
"name": t.get("name"),
|
||||
"status": t.get("status", "Unknown"),
|
||||
"temperature_c": temp,
|
||||
})
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
sensor_temps.append(float(temp))
|
||||
for t in ses.get("temps", []):
|
||||
temp = t.get("temperature_c")
|
||||
sensors.append({
|
||||
"index": t["index"],
|
||||
"name": t.get("name"),
|
||||
"status": t.get("status", "Unknown"),
|
||||
"temperature_c": temp,
|
||||
})
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
sensor_temps.append(float(temp))
|
||||
|
||||
# Drive temperatures for drives housed in this enclosure.
|
||||
drive_temps: list[float] = []
|
||||
devices = [s["device"] for s in list_slots(enc["id"]) if s.get("device")]
|
||||
if devices:
|
||||
smart_results = await asyncio.gather(
|
||||
*[get_smart_data(d) for d in devices], return_exceptions=True
|
||||
)
|
||||
for res in smart_results:
|
||||
if isinstance(res, Exception):
|
||||
continue
|
||||
data, _hit = res
|
||||
temp = data.get("temperature_c")
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
drive_temps.append(float(temp))
|
||||
for slot in enc.get("slots", []):
|
||||
dev = slot.get("device")
|
||||
if not dev:
|
||||
continue
|
||||
sm = await store.get_smart(dev)
|
||||
if not sm:
|
||||
continue
|
||||
temp = sm.get("temperature_c")
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
drive_temps.append(float(temp))
|
||||
|
||||
all_temps = sensor_temps + drive_temps
|
||||
out_enclosures.append({
|
||||
|
||||
Reference in New Issue
Block a user