Split into dedicated hardware poller + read-only consumers

Rearchitect so a single poller process is the sole owner of all SAS/SMART/
SES hardware I/O, serialized behind one gate and paced, writing results to
Redis. The API/web and MQTT publisher become pure Redis readers — they no
longer issue any subprocess and can restart freely without touching the bus.

This addresses backplane/expander stress from concurrent + restart-triggered
SMART/SES storms (the prior model re-ran a hardware sweep on every container
start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/
Xyratex expanders).

- poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS,
  host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker
- services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1)
- services/store.py: Redis as the only producer<->consumer interface + LED queue
- services/health.py: shared drive-health classifier (fixes overview double-count)
- gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path
- routers + temps read-only from the store; main.py drops the in-process poller
- compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs

services/host.py

@@ -3,22 +3,28 @@ import json
 import logging
 
 from services.enclosure import discover_enclosures, list_slots
-from services.smart import get_smart_data, scan_megaraid_drives
-from services.zfs import get_zfs_pool_map
+from services.health import compute_health_status
+from services.hwgate import gate
+from services.smart import _run_smartctl, scan_megaraid_drives
 
 logger = logging.getLogger(__name__)
 
 
-async def get_host_drives() -> list[dict]:
-    """Discover non-enclosure block devices and return SMART data for each."""
+async def get_host_drives(pool_map: dict) -> list[dict]:
+    """Discover non-enclosure block devices and return SMART data for each.
+
+    Poller-side producer: serialized through the hardware gate. ``pool_map``
+    is passed in (computed once per sweep) to avoid a second zpool call.
+    """
     # Get all block devices via lsblk
     try:
-        proc = await asyncio.create_subprocess_exec(
-            "lsblk", "-d", "-o", "NAME,SIZE,TYPE,MODEL,ROTA,TRAN", "-J",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        stdout, _ = await proc.communicate()
+        async with gate():
+            proc = await asyncio.create_subprocess_exec(
+                "lsblk", "-d", "-o", "NAME,SIZE,TYPE,MODEL,ROTA,TRAN", "-J",
+                stdout=asyncio.subprocess.PIPE,
+                stderr=asyncio.subprocess.PIPE,
+            )
+            stdout, _ = await proc.communicate()
         lsblk_data = json.loads(stdout)
     except (FileNotFoundError, json.JSONDecodeError) as e:
         logger.warning("lsblk failed: %s", e)
@@ -59,10 +65,11 @@ async def get_host_drives() -> list[dict]:
 
         host_devices.append({"name": name, "drive_type": drive_type})
 
-    # Fetch SMART + ZFS data concurrently
-    pool_map = await get_zfs_pool_map()
-    smart_tasks = [get_smart_data(d["name"]) for d in host_devices]
-    smart_results = await asyncio.gather(*smart_tasks, return_exceptions=True)
+    # Fetch SMART for each host disk (serialized by the gate inside _run_smartctl)
+    smart_results = await asyncio.gather(
+        *[_run_smartctl(d["name"]) for d in host_devices],
+        return_exceptions=True,
+    )
 
     results: list[dict] = []
     for dev_info, smart_result in zip(host_devices, smart_results):
@@ -72,21 +79,9 @@ async def get_host_drives() -> list[dict]:
             logger.warning("SMART query failed for host drive %s: %s", name, smart_result)
             smart = {"device": name, "smart_supported": False}
         else:
-            smart, _ = smart_result
-
-        # Compute health_status (same logic as overview.py)
-        healthy = smart.get("smart_healthy")
-        realloc = smart.get("reallocated_sectors") or 0
-        pending = smart.get("pending_sectors") or 0
-        unc = smart.get("uncorrectable_errors") or 0
-
-        if healthy is False:
-            health_status = "error"
-        elif realloc > 0 or pending > 0 or unc > 0 or (healthy is None and smart.get("smart_supported", True)):
-            health_status = "warning"
-        else:
-            health_status = "healthy"
+            smart = smart_result
 
+        health_status = compute_health_status(smart)
         zfs_info = pool_map.get(name, {})
 
         results.append({
@@ -97,7 +92,7 @@ async def get_host_drives() -> list[dict]:
             "wwn": smart.get("wwn"),
             "firmware": smart.get("firmware"),
             "capacity_bytes": smart.get("capacity_bytes"),
-            "smart_healthy": healthy,
+            "smart_healthy": smart.get("smart_healthy"),
             "smart_supported": smart.get("smart_supported", True),
             "temperature_c": smart.get("temperature_c"),
             "power_on_hours": smart.get("power_on_hours"),
@@ -116,16 +111,7 @@ async def get_host_drives() -> list[dict]:
     if has_raid:
         megaraid_drives = await scan_megaraid_drives()
         for pd in megaraid_drives:
-            pd_healthy = pd.get("smart_healthy")
-            pd_realloc = pd.get("reallocated_sectors") or 0
-            pd_pending = pd.get("pending_sectors") or 0
-            pd_unc = pd.get("uncorrectable_errors") or 0
-            if pd_healthy is False:
-                pd["health_status"] = "error"
-            elif pd_realloc > 0 or pd_pending > 0 or pd_unc > 0:
-                pd["health_status"] = "warning"
-            else:
-                pd["health_status"] = "healthy"
+            pd["health_status"] = compute_health_status(pd)
             pd["drive_type"] = "physical"
             pd["physical_drives"] = []