adam
0f829e0380
Rearchitect so a single poller process is the sole owner of all SAS/SMART/ SES hardware I/O, serialized behind one gate and paced, writing results to Redis. The API/web and MQTT publisher become pure Redis readers — they no longer issue any subprocess and can restart freely without touching the bus. This addresses backplane/expander stress from concurrent + restart-triggered SMART/SES storms (the prior model re-ran a hardware sweep on every container start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/ Xyratex expanders). - poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS, host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker - services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1) - services/store.py: Redis as the only producer<->consumer interface + LED queue - services/health.py: shared drive-health classifier (fixes overview double-count) - gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path - routers + temps read-only from the store; main.py drops the in-process poller - compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
139 lines
4.7 KiB
Python
139 lines
4.7 KiB
Python
import logging
|
|
|
|
from fastapi import APIRouter, Response
|
|
|
|
from models.schemas import (
|
|
DriveHealthSummary,
|
|
EnclosureHealth,
|
|
EnclosureWithDrives,
|
|
HostDrive,
|
|
Overview,
|
|
SlotWithDrive,
|
|
)
|
|
from services import store
|
|
from services.health import compute_health_status
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
router = APIRouter(prefix="/api/overview", tags=["overview"])
|
|
|
|
|
|
@router.get("", response_model=Overview)
|
|
async def get_overview(response: Response):
|
|
"""Aggregate view of all enclosures, slots, and drive health.
|
|
|
|
Pure read from the store — the poller is the only thing that touched
|
|
hardware to produce this data.
|
|
"""
|
|
inventory = await store.get_inventory()
|
|
pool_map = await store.get_zfs_map()
|
|
|
|
enc_results: list[EnclosureWithDrives] = []
|
|
total_drives = 0
|
|
warnings = 0
|
|
errors = 0
|
|
all_healthy = True
|
|
|
|
for enc in inventory.get("enclosures", []):
|
|
slots_out: list[SlotWithDrive] = []
|
|
for s in enc.get("slots", []):
|
|
dev = s.get("device")
|
|
drive_summary = None
|
|
|
|
if dev:
|
|
total_drives += 1
|
|
sd = await store.get_smart(dev)
|
|
if sd:
|
|
status = compute_health_status(sd)
|
|
if status == "error":
|
|
errors += 1
|
|
all_healthy = False
|
|
elif status == "warning":
|
|
warnings += 1
|
|
|
|
zinfo = pool_map.get(dev, {})
|
|
drive_summary = DriveHealthSummary(
|
|
device=sd.get("device", dev),
|
|
model=sd.get("model"),
|
|
serial=sd.get("serial"),
|
|
wwn=sd.get("wwn"),
|
|
firmware=sd.get("firmware"),
|
|
capacity_bytes=sd.get("capacity_bytes"),
|
|
smart_healthy=sd.get("smart_healthy"),
|
|
smart_supported=sd.get("smart_supported", True),
|
|
temperature_c=sd.get("temperature_c"),
|
|
power_on_hours=sd.get("power_on_hours"),
|
|
reallocated_sectors=sd.get("reallocated_sectors"),
|
|
pending_sectors=sd.get("pending_sectors"),
|
|
uncorrectable_errors=sd.get("uncorrectable_errors"),
|
|
zfs_pool=zinfo.get("pool"),
|
|
zfs_vdev=zinfo.get("vdev"),
|
|
zfs_state=zinfo.get("state"),
|
|
health_status=status,
|
|
)
|
|
elif s.get("populated"):
|
|
total_drives += 1
|
|
|
|
slots_out.append(SlotWithDrive(
|
|
slot=s["slot"],
|
|
populated=s["populated"],
|
|
device=dev,
|
|
drive=drive_summary,
|
|
))
|
|
|
|
ses = await store.get_ses(enc["id"])
|
|
enc_health = None
|
|
if ses:
|
|
enc_health = EnclosureHealth(**ses)
|
|
if enc_health.overall_status == "CRITICAL":
|
|
errors += 1
|
|
all_healthy = False
|
|
elif enc_health.overall_status == "WARNING":
|
|
warnings += 1
|
|
|
|
enc_results.append(EnclosureWithDrives(
|
|
id=enc["id"],
|
|
sg_device=enc.get("sg_device"),
|
|
vendor=enc.get("vendor", ""),
|
|
model=enc.get("model", ""),
|
|
revision=enc.get("revision", ""),
|
|
total_slots=enc.get("total_slots", 0),
|
|
populated_slots=enc.get("populated_slots", 0),
|
|
slots=slots_out,
|
|
health=enc_health,
|
|
))
|
|
|
|
# Host (non-enclosure) drives — fully precomputed by the poller.
|
|
host_drives_out: list[HostDrive] = []
|
|
for hd in await store.get_host_drives():
|
|
total_drives += 1
|
|
hs = hd.get("health_status", "healthy")
|
|
if hs == "error":
|
|
errors += 1
|
|
all_healthy = False
|
|
elif hs == "warning":
|
|
warnings += 1
|
|
for pd in hd.get("physical_drives", []):
|
|
total_drives += 1
|
|
pd_hs = pd.get("health_status", "healthy")
|
|
if pd_hs == "error":
|
|
errors += 1
|
|
all_healthy = False
|
|
elif pd_hs == "warning":
|
|
warnings += 1
|
|
host_drives_out.append(HostDrive(**hd))
|
|
|
|
# Surface poller freshness so stale data is visible.
|
|
meta = await store.get_meta()
|
|
if meta and meta.get("last_sweep_ts"):
|
|
response.headers["X-Poll-Age"] = str(int(store.now() - meta["last_sweep_ts"]))
|
|
|
|
return Overview(
|
|
healthy=all_healthy and errors == 0,
|
|
drive_count=total_drives,
|
|
warning_count=warnings,
|
|
error_count=errors,
|
|
enclosures=enc_results,
|
|
host_drives=host_drives_out,
|
|
)
|