Split into dedicated hardware poller + read-only consumers

Rearchitect so a single poller process is the sole owner of all SAS/SMART/
SES hardware I/O, serialized behind one gate and paced, writing results to
Redis. The API/web and MQTT publisher become pure Redis readers — they no
longer issue any subprocess and can restart freely without touching the bus.

This addresses backplane/expander stress from concurrent + restart-triggered
SMART/SES storms (the prior model re-ran a hardware sweep on every container
start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/
Xyratex expanders).

- poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS,
  host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker
- services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1)
- services/store.py: Redis as the only producer<->consumer interface + LED queue
- services/health.py: shared drive-health classifier (fixes overview double-count)
- gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path
- routers + temps read-only from the store; main.py drops the in-process poller
- compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs

main.py

@@ -1,5 +1,4 @@
 import asyncio
-import json
 import logging
 import os
 from contextlib import asynccontextmanager
@@ -11,11 +10,9 @@ from fastapi.staticfiles import StaticFiles
 
 from models.schemas import HealthCheck
 from routers import drives, enclosures, leds, overview, temps
-from services.cache import cache_set, close_cache, init_cache, redis_available
-from services.enclosure import discover_enclosures, list_slots
+from services import store
+from services.cache import close_cache, init_cache, redis_available
 from services.mqtt_publisher import MqttPublisher, _PAHO_AVAILABLE, mqtt_enabled
-from services.smart import SMART_CACHE_TTL, _run_smartctl, sg_ses_available, smartctl_available
-from services.zfs import get_zfs_pool_map
 
 logging.basicConfig(
     level=logging.INFO,
@@ -23,81 +20,20 @@ logging.basicConfig(
 )
 logger = logging.getLogger(__name__)
 
-SMART_POLL_INTERVAL = int(os.environ.get("SMART_POLL_INTERVAL", "90"))
-
 _tool_status: dict[str, bool] = {}
-_poll_task: asyncio.Task | None = None
 _mqtt_task: asyncio.Task | None = None
 _mqtt_publisher: MqttPublisher | None = None
 
 
-async def smart_poll_loop():
-    """Pre-warm Redis with SMART data for all drives."""
-    await asyncio.sleep(2)  # let app finish starting
-    while True:
-        try:
-            # Discover all enclosure devices
-            enclosures_raw = discover_enclosures()
-            devices: set[str] = set()
-            for enc in enclosures_raw:
-                for slot in list_slots(enc["id"]):
-                    if slot["device"]:
-                        devices.add(slot["device"])
-
-            # Discover host block devices via lsblk
-            try:
-                proc = await asyncio.create_subprocess_exec(
-                    "lsblk", "-d", "-o", "NAME,TYPE", "-J",
-                    stdout=asyncio.subprocess.PIPE,
-                    stderr=asyncio.subprocess.PIPE,
-                )
-                stdout, _ = await proc.communicate()
-                if stdout:
-                    for dev in json.loads(stdout).get("blockdevices", []):
-                        if dev.get("type") == "disk":
-                            name = dev.get("name", "")
-                            if name and name not in devices:
-                                devices.add(name)
-            except Exception as e:
-                logger.warning("lsblk discovery failed in poller: %s", e)
-
-            # Poll each drive and cache result
-            for device in sorted(devices):
-                try:
-                    result = await _run_smartctl(device)
-                    await cache_set(f"jbod:smart:{device}", result, SMART_CACHE_TTL)
-                except Exception as e:
-                    logger.warning("Poll failed for %s: %s", device, e)
-
-            # Pre-warm ZFS map (bypasses cache by calling directly)
-            await get_zfs_pool_map()
-
-            logger.info("SMART poll complete: %d devices", len(devices))
-        except Exception as e:
-            logger.error("SMART poll loop error: %s", e)
-        await asyncio.sleep(SMART_POLL_INTERVAL)
-
-
 @asynccontextmanager
 async def lifespan(app: FastAPI):
-    global _poll_task, _mqtt_task, _mqtt_publisher
-    # Startup
-    _tool_status["smartctl"] = smartctl_available()
-    _tool_status["sg_ses"] = sg_ses_available()
-
-    if not _tool_status["smartctl"]:
-        logger.warning("smartctl not found — install smartmontools for SMART data")
-    if not _tool_status["sg_ses"]:
-        logger.warning("sg_ses not found — install sg3-utils for enclosure SES data")
-    if os.geteuid() != 0:
-        logger.warning("Not running as root — smartctl may fail on some devices")
+    """API/web + MQTT consumer. Reads everything from Redis; the dedicated
+    poller process is the only thing that touches hardware."""
+    global _mqtt_task, _mqtt_publisher
 
     await init_cache()
     _tool_status["redis"] = redis_available()
 
-    if redis_available():
-        _poll_task = asyncio.create_task(smart_poll_loop())
-
     # Optional MQTT publisher for Home Assistant (opt-in via MQTT_HOST).
     _tool_status["mqtt"] = False
     if mqtt_enabled():
@@ -114,7 +50,6 @@ async def lifespan(app: FastAPI):
 
     yield
 
-    # Shutdown
     if _mqtt_task is not None:
         _mqtt_task.cancel()
         try:
@@ -123,19 +58,13 @@ async def lifespan(app: FastAPI):
             pass
     if _mqtt_publisher is not None:
         _mqtt_publisher.stop()
-    if _poll_task is not None:
-        _poll_task.cancel()
-        try:
-            await _poll_task
-        except asyncio.CancelledError:
-            pass
     await close_cache()
 
 
 app = FastAPI(
     title="JBOD Monitor",
     description="Drive health monitoring for JBOD enclosures",
-    version="0.1.0",
+    version="0.2.0",
     lifespan=lifespan,
 )
 
@@ -155,8 +84,18 @@ app.include_router(temps.router)
 
 @app.get("/api/health", response_model=HealthCheck, tags=["health"])
 async def health():
-    _tool_status["redis"] = redis_available()
-    return HealthCheck(status="ok", tools=_tool_status)
+    tools = dict(_tool_status)
+    tools["redis"] = redis_available()
+
+    # Poller freshness: is the producer keeping the store warm?
+    meta = await store.get_meta()
+    if meta and meta.get("last_sweep_ts"):
+        age = store.now() - meta["last_sweep_ts"]
+        tools["poller_fresh"] = age < meta.get("sweep_interval", 300) * 3
+    else:
+        tools["poller_fresh"] = False
+
+    return HealthCheck(status="ok", tools=tools)
 
 
 # Serve built frontend static files — mounted last so /api routes take priority.