Split into dedicated hardware poller + read-only consumers
Rearchitect so a single poller process is the sole owner of all SAS/SMART/ SES hardware I/O, serialized behind one gate and paced, writing results to Redis. The API/web and MQTT publisher become pure Redis readers — they no longer issue any subprocess and can restart freely without touching the bus. This addresses backplane/expander stress from concurrent + restart-triggered SMART/SES storms (the prior model re-ran a hardware sweep on every container start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/ Xyratex expanders). - poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS, host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker - services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1) - services/store.py: Redis as the only producer<->consumer interface + LED queue - services/health.py: shared drive-health classifier (fixes overview double-count) - gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path - routers + temps read-only from the store; main.py drops the in-process poller - compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
This commit is contained in:
@@ -4,6 +4,8 @@ import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from services.hwgate import gate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ENCLOSURE_BASE = Path("/sys/class/enclosure")
|
||||
@@ -153,14 +155,16 @@ def _parse_slot_number(entry: Path) -> int | None:
|
||||
|
||||
|
||||
async def _run_sg_ses(sg_device: str, page: str) -> str | None:
|
||||
"""Run sg_ses for a given page, returning decoded stdout or None."""
|
||||
"""Run sg_ses for a given page, returning decoded stdout or None.
|
||||
Hardware-gated."""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"sg_ses", f"--page={page}", sg_device,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
async with gate():
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"sg_ses", f"--page={page}", sg_device,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
logger.warning(
|
||||
"sg_ses page %s failed for %s: %s",
|
||||
@@ -177,22 +181,18 @@ async def _run_sg_ses(sg_device: str, page: str) -> str | None:
|
||||
|
||||
|
||||
async def get_enclosure_status(sg_device: str) -> dict | None:
|
||||
"""Run sg_ses status (0x02) + element descriptor (0x07) pages and parse.
|
||||
"""Run sg_ses status page (0x02) and parse enclosure health data.
|
||||
|
||||
The element descriptor page provides human-readable names for each
|
||||
element (e.g. "Temp Inlet", "Fan 1"); these are merged into the status
|
||||
elements so callers get labelled sensors. Descriptor lookup is
|
||||
best-effort — if page 0x07 is unsupported, elements fall back to bare
|
||||
indices.
|
||||
Deliberately only the status page: the element descriptor page (0x07)
|
||||
is omitted because it errored on the IOM6/Xyratex expanders here
|
||||
(``couldn't read config page, res=35``) and returned empty names
|
||||
anyway. Elements fall back to generic labels. If descriptor names are
|
||||
ever wanted, fetch 0x07 ONCE at startup and cache — never per poll.
|
||||
"""
|
||||
status_text, desc_text = await asyncio.gather(
|
||||
_run_sg_ses(sg_device, "0x02"),
|
||||
_run_sg_ses(sg_device, "0x07"),
|
||||
)
|
||||
status_text = await _run_sg_ses(sg_device, "0x02")
|
||||
if status_text is None:
|
||||
return None
|
||||
names = _parse_element_descriptors(desc_text) if desc_text else {}
|
||||
return _parse_ses_page02(status_text, names)
|
||||
return _parse_ses_page02(status_text)
|
||||
|
||||
|
||||
def _norm_type(element_type: str) -> str:
|
||||
|
||||
Reference in New Issue
Block a user