adamksmith/jbod-monitor
1
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Split into dedicated hardware poller + read-only consumers

Browse Source 
Rearchitect so a single poller process is the sole owner of all SAS/SMART/
SES hardware I/O, serialized behind one gate and paced, writing results to
Redis. The API/web and MQTT publisher become pure Redis readers — they no
longer issue any subprocess and can restart freely without touching the bus.

This addresses backplane/expander stress from concurrent + restart-triggered
SMART/SES storms (the prior model re-ran a hardware sweep on every container
start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/
Xyratex expanders).

- poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS,
  host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker
- services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1)
- services/store.py: Redis as the only producer<->consumer interface + LED queue
- services/health.py: shared drive-health classifier (fixes overview double-count)
- gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path
- routers + temps read-only from the store; main.py drops the in-process poller
- compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
This commit is contained in:
adam
2026-06-16 15:12:34 +00:00
parent 7a50d1261f
commit 0f829e0380
19 changed files with 717 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
routers/drives.py
View File

@@ -1,30 +1,37 @@
import re
from fastapi import APIRouter, HTTPException, Response
from models.schemas import DriveDetail
from services.smart import get_smart_data
from services.zfs import get_zfs_pool_map
from services import store
router = APIRouter(prefix="/api/drives", tags=["drives"])
@router.get("/{device}", response_model=DriveDetail)
async def get_drive_detail(device: str, response: Response):
"""Get SMART detail for a specific block device."""
try:
data, cache_hit = await get_smart_data(device)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
"""Get SMART detail for a specific block device (from the store)."""
if not re.match(r"^[a-zA-Z0-9\-]+$", device):
raise HTTPException(status_code=400, detail=f"Invalid device name: {device}")
data = await store.get_smart(device)
if data is None:
raise HTTPException(
status_code=404,
detail=f"No SMART data for '{device}' yet (poller may not have swept it)",
)
if "error" in data:
raise HTTPException(status_code=502, detail=data["error"])
pool_map = await get_zfs_pool_map()
pool_map = await store.get_zfs_map()
zfs_info = pool_map.get(device)
if zfs_info:
data["zfs_pool"] = zfs_info["pool"]
data["zfs_vdev"] = zfs_info["vdev"]
data["zfs_state"] = zfs_info.get("state")
response.headers["X-Cache"] = "HIT" if cache_hit else "MISS"
meta = await store.get_meta()
if meta and meta.get("last_sweep_ts"):
response.headers["X-Poll-Age"] = str(int(store.now() - meta["last_sweep_ts"]))
return DriveDetail(**data)

21
routers/enclosures.py
View File

@@ -1,24 +1,23 @@
from fastapi import APIRouter, HTTPException
from models.schemas import Enclosure, SlotInfo
from services.enclosure import discover_enclosures, list_slots
from services import store
router = APIRouter(prefix="/api/enclosures", tags=["enclosures"])
@router.get("", response_model=list[Enclosure])
async def get_enclosures():
"""Discover all SES enclosures."""
return discover_enclosures()
"""List all discovered SES enclosures (from the poller inventory)."""
inventory = await store.get_inventory()
return [Enclosure(**e) for e in inventory.get("enclosures", [])]
@router.get("/{enclosure_id}/drives", response_model=list[SlotInfo])
async def get_enclosure_drives(enclosure_id: str):
"""List all drive slots for an enclosure."""
slots = list_slots(enclosure_id)
if not slots:
# Check if the enclosure exists at all
enclosures = discover_enclosures()
if not any(e["id"] == enclosure_id for e in enclosures):
raise HTTPException(status_code=404, detail=f"Enclosure '{enclosure_id}' not found")
return slots
"""List all drive slots for an enclosure (from the poller inventory)."""
inventory = await store.get_inventory()
for enc in inventory.get("enclosures", []):
if enc["id"] == enclosure_id:
return [SlotInfo(**s) for s in enc.get("slots", [])]
raise HTTPException(status_code=404, detail=f"Enclosure '{enclosure_id}' not found")

21
routers/leds.py
View File

@@ -1,7 +1,9 @@
import re
from fastapi import APIRouter, HTTPException
from pydantic import BaseModel, field_validator
from services.leds import set_led
from services import store
router = APIRouter(prefix="/api/drives", tags=["leds"])
@@ -19,11 +21,12 @@ class LedRequest(BaseModel):
@router.post("/{device}/led")
async def set_drive_led(device: str, body: LedRequest):
"""Set the locate LED for a drive."""
try:
result = await set_led(device, body.state)
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except RuntimeError as e:
raise HTTPException(status_code=502, detail=str(e))
return result
"""Queue a locate-LED change. The poller (sole hardware owner) executes it."""
if not re.fullmatch(r"[a-zA-Z0-9]+", device):
raise HTTPException(status_code=400, detail=f"Invalid device name: {device}")
queued = await store.enqueue_led(device, body.state)
if not queued:
raise HTTPException(status_code=503, detail="LED queue unavailable (Redis down)")
return {"device": device, "state": body.state, "queued": True}

165
routers/overview.py
View File

@@ -1,4 +1,3 @@
import asyncio
import logging
from fastapi import APIRouter, Response
@@ -11,10 +10,8 @@ from models.schemas import (
Overview,
SlotWithDrive,
)
from services.enclosure import discover_enclosures, get_enclosure_status, list_slots
from services.host import get_host_drives
from services.smart import get_smart_data
from services.zfs import get_zfs_pool_map
from services import store
from services.health import compute_health_status
logger = logging.getLogger(__name__)
@@ -23,142 +20,92 @@ router = APIRouter(prefix="/api/overview", tags=["overview"])
@router.get("", response_model=Overview)
async def get_overview(response: Response):
"""Aggregate view of all enclosures, slots, and drive health."""
enclosures_raw = discover_enclosures()
pool_map = await get_zfs_pool_map()
"""Aggregate view of all enclosures, slots, and drive health.
# Fetch SES health data for all enclosures concurrently
async def _get_health(enc):
if enc.get("sg_device"):
return await get_enclosure_status(enc["sg_device"])
return None
health_results = await asyncio.gather(
*[_get_health(enc) for enc in enclosures_raw],
return_exceptions=True,
)
Pure read from the store — the poller is the only thing that touched
hardware to produce this data.
"""
inventory = await store.get_inventory()
pool_map = await store.get_zfs_map()
enc_results: list[EnclosureWithDrives] = []
total_drives = 0
warnings = 0
errors = 0
all_healthy = True
all_cache_hits = True
any_lookups = False
for enc_idx, enc in enumerate(enclosures_raw):
slots_raw = list_slots(enc["id"])
# Gather SMART data for all populated slots concurrently
populated = [(s, s["device"]) for s in slots_raw if s["populated"] and s["device"]]
smart_tasks = [get_smart_data(dev) for _, dev in populated]
smart_results = await asyncio.gather(*smart_tasks, return_exceptions=True)
smart_map: dict[str, dict] = {}
for (slot_info, dev), result in zip(populated, smart_results):
if isinstance(result, Exception):
logger.warning("SMART query failed for %s: %s", dev, result)
smart_map[dev] = {"device": dev, "smart_supported": False}
all_cache_hits = False
else:
data, hit = result
smart_map[dev] = data
any_lookups = True
if not hit:
all_cache_hits = False
for enc in inventory.get("enclosures", []):
slots_out: list[SlotWithDrive] = []
for s in slots_raw:
for s in enc.get("slots", []):
dev = s.get("device")
drive_summary = None
if s["device"] and s["device"] in smart_map:
sd = smart_map[s["device"]]
if dev:
total_drives += 1
sd = await store.get_smart(dev)
if sd:
status = compute_health_status(sd)
if status == "error":
errors += 1
all_healthy = False
elif status == "warning":
warnings += 1
healthy = sd.get("smart_healthy")
if healthy is False:
errors += 1
all_healthy = False
elif healthy is None and sd.get("smart_supported", True):
warnings += 1
# Check for concerning SMART values
if sd.get("reallocated_sectors") and sd["reallocated_sectors"] > 0:
warnings += 1
if sd.get("pending_sectors") and sd["pending_sectors"] > 0:
warnings += 1
if sd.get("uncorrectable_errors") and sd["uncorrectable_errors"] > 0:
warnings += 1
# Compute health_status for frontend
realloc = sd.get("reallocated_sectors") or 0
pending = sd.get("pending_sectors") or 0
unc = sd.get("uncorrectable_errors") or 0
if healthy is False:
health_status = "error"
elif realloc > 0 or pending > 0 or unc > 0 or (healthy is None and sd.get("smart_supported", True)):
health_status = "warning"
else:
health_status = "healthy"
drive_summary = DriveHealthSummary(
device=sd["device"],
model=sd.get("model"),
serial=sd.get("serial"),
wwn=sd.get("wwn"),
firmware=sd.get("firmware"),
capacity_bytes=sd.get("capacity_bytes"),
smart_healthy=healthy,
smart_supported=sd.get("smart_supported", True),
temperature_c=sd.get("temperature_c"),
power_on_hours=sd.get("power_on_hours"),
reallocated_sectors=sd.get("reallocated_sectors"),
pending_sectors=sd.get("pending_sectors"),
uncorrectable_errors=sd.get("uncorrectable_errors"),
zfs_pool=pool_map.get(sd["device"], {}).get("pool"),
zfs_vdev=pool_map.get(sd["device"], {}).get("vdev"),
zfs_state=pool_map.get(sd["device"], {}).get("state"),
health_status=health_status,
)
elif s["populated"]:
zinfo = pool_map.get(dev, {})
drive_summary = DriveHealthSummary(
device=sd.get("device", dev),
model=sd.get("model"),
serial=sd.get("serial"),
wwn=sd.get("wwn"),
firmware=sd.get("firmware"),
capacity_bytes=sd.get("capacity_bytes"),
smart_healthy=sd.get("smart_healthy"),
smart_supported=sd.get("smart_supported", True),
temperature_c=sd.get("temperature_c"),
power_on_hours=sd.get("power_on_hours"),
reallocated_sectors=sd.get("reallocated_sectors"),
pending_sectors=sd.get("pending_sectors"),
uncorrectable_errors=sd.get("uncorrectable_errors"),
zfs_pool=zinfo.get("pool"),
zfs_vdev=zinfo.get("vdev"),
zfs_state=zinfo.get("state"),
health_status=status,
)
elif s.get("populated"):
total_drives += 1
slots_out.append(SlotWithDrive(
slot=s["slot"],
populated=s["populated"],
device=s["device"],
device=dev,
drive=drive_summary,
))
# Attach enclosure health from SES
health_data = health_results[enc_idx]
ses = await store.get_ses(enc["id"])
enc_health = None
if isinstance(health_data, dict):
enc_health = EnclosureHealth(**health_data)
# Count enclosure-level issues
if ses:
enc_health = EnclosureHealth(**ses)
if enc_health.overall_status == "CRITICAL":
errors += 1
all_healthy = False
elif enc_health.overall_status == "WARNING":
warnings += 1
elif isinstance(health_data, Exception):
logger.warning("SES health failed for %s: %s", enc["id"], health_data)
enc_results.append(EnclosureWithDrives(
id=enc["id"],
sg_device=enc.get("sg_device"),
vendor=enc["vendor"],
model=enc["model"],
revision=enc["revision"],
total_slots=enc["total_slots"],
populated_slots=enc["populated_slots"],
vendor=enc.get("vendor", ""),
model=enc.get("model", ""),
revision=enc.get("revision", ""),
total_slots=enc.get("total_slots", 0),
populated_slots=enc.get("populated_slots", 0),
slots=slots_out,
health=enc_health,
))
# Host drives (non-enclosure)
host_drives_raw = await get_host_drives()
# Host (non-enclosure) drives — fully precomputed by the poller.
host_drives_out: list[HostDrive] = []
for hd in host_drives_raw:
for hd in await store.get_host_drives():
total_drives += 1
hs = hd.get("health_status", "healthy")
if hs == "error":
@@ -166,7 +113,6 @@ async def get_overview(response: Response):
all_healthy = False
elif hs == "warning":
warnings += 1
# Count physical drives behind RAID controllers
for pd in hd.get("physical_drives", []):
total_drives += 1
pd_hs = pd.get("health_status", "healthy")
@@ -177,7 +123,10 @@ async def get_overview(response: Response):
warnings += 1
host_drives_out.append(HostDrive(**hd))
response.headers["X-Cache"] = "HIT" if (any_lookups and all_cache_hits) else "MISS"
# Surface poller freshness so stale data is visible.
meta = await store.get_meta()
if meta and meta.get("last_sweep_ts"):
response.headers["X-Poll-Age"] = str(int(store.now() - meta["last_sweep_ts"]))
return Overview(
healthy=all_healthy and errors == 0,