adam
0f829e0380
Rearchitect so a single poller process is the sole owner of all SAS/SMART/ SES hardware I/O, serialized behind one gate and paced, writing results to Redis. The API/web and MQTT publisher become pure Redis readers — they no longer issue any subprocess and can restart freely without touching the bus. This addresses backplane/expander stress from concurrent + restart-triggered SMART/SES storms (the prior model re-ran a hardware sweep on every container start, and polled sg_ses 0x02+0x07 every 60s; 0x07 errored on the IOM6/ Xyratex expanders). - poller.py: paced sweep (inventory, SMART per-drive w/ gap, SES 0x02, ZFS, host/MegaRAID), startup jitter, single-instance Redis lock, LED-queue worker - services/hwgate.py: global serialization semaphore (POLL_CONCURRENCY=1) - services/store.py: Redis as the only producer<->consumer interface + LED queue - services/health.py: shared drive-health classifier (fixes overview double-count) - gate smartctl/sg_ses/zpool/ledctl; drop sg_ses 0x07 from the hot path - routers + temps read-only from the store; main.py drops the in-process poller - compose: 3 services (privileged poller + unprivileged app + redis) w/ pacing knobs
125 lines
4.5 KiB
Python
125 lines
4.5 KiB
Python
import asyncio
|
|
import json
|
|
import logging
|
|
|
|
from services.enclosure import discover_enclosures, list_slots
|
|
from services.health import compute_health_status
|
|
from services.hwgate import gate
|
|
from services.smart import _run_smartctl, scan_megaraid_drives
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def get_host_drives(pool_map: dict) -> list[dict]:
|
|
"""Discover non-enclosure block devices and return SMART data for each.
|
|
|
|
Poller-side producer: serialized through the hardware gate. ``pool_map``
|
|
is passed in (computed once per sweep) to avoid a second zpool call.
|
|
"""
|
|
# Get all block devices via lsblk
|
|
try:
|
|
async with gate():
|
|
proc = await asyncio.create_subprocess_exec(
|
|
"lsblk", "-d", "-o", "NAME,SIZE,TYPE,MODEL,ROTA,TRAN", "-J",
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE,
|
|
)
|
|
stdout, _ = await proc.communicate()
|
|
lsblk_data = json.loads(stdout)
|
|
except (FileNotFoundError, json.JSONDecodeError) as e:
|
|
logger.warning("lsblk failed: %s", e)
|
|
return []
|
|
|
|
# Collect all enclosure-mapped devices
|
|
enclosure_devices: set[str] = set()
|
|
for enc in discover_enclosures():
|
|
for slot in list_slots(enc["id"]):
|
|
if slot["device"]:
|
|
enclosure_devices.add(slot["device"])
|
|
|
|
# Filter to host-only disks
|
|
host_devices: list[dict] = []
|
|
for dev in lsblk_data.get("blockdevices", []):
|
|
name = dev.get("name", "")
|
|
dev_type = dev.get("type", "")
|
|
|
|
# Skip non-disk types and enclosure drives
|
|
if dev_type != "disk":
|
|
continue
|
|
if name in enclosure_devices:
|
|
continue
|
|
|
|
# Determine drive type from transport/model
|
|
tran = (dev.get("tran") or "").lower()
|
|
model = (dev.get("model") or "").lower()
|
|
rota = dev.get("rota")
|
|
|
|
if tran == "nvme" or name.startswith("nvme"):
|
|
drive_type = "nvme"
|
|
elif "perc" in model or "raid" in model or "megaraid" in model:
|
|
drive_type = "raid"
|
|
elif rota is False or rota == "0" or rota == 0:
|
|
drive_type = "ssd"
|
|
else:
|
|
drive_type = "hdd"
|
|
|
|
host_devices.append({"name": name, "drive_type": drive_type})
|
|
|
|
# Fetch SMART for each host disk (serialized by the gate inside _run_smartctl)
|
|
smart_results = await asyncio.gather(
|
|
*[_run_smartctl(d["name"]) for d in host_devices],
|
|
return_exceptions=True,
|
|
)
|
|
|
|
results: list[dict] = []
|
|
for dev_info, smart_result in zip(host_devices, smart_results):
|
|
name = dev_info["name"]
|
|
|
|
if isinstance(smart_result, Exception):
|
|
logger.warning("SMART query failed for host drive %s: %s", name, smart_result)
|
|
smart = {"device": name, "smart_supported": False}
|
|
else:
|
|
smart = smart_result
|
|
|
|
health_status = compute_health_status(smart)
|
|
zfs_info = pool_map.get(name, {})
|
|
|
|
results.append({
|
|
"device": name,
|
|
"drive_type": dev_info["drive_type"],
|
|
"model": smart.get("model"),
|
|
"serial": smart.get("serial"),
|
|
"wwn": smart.get("wwn"),
|
|
"firmware": smart.get("firmware"),
|
|
"capacity_bytes": smart.get("capacity_bytes"),
|
|
"smart_healthy": smart.get("smart_healthy"),
|
|
"smart_supported": smart.get("smart_supported", True),
|
|
"temperature_c": smart.get("temperature_c"),
|
|
"power_on_hours": smart.get("power_on_hours"),
|
|
"reallocated_sectors": smart.get("reallocated_sectors"),
|
|
"pending_sectors": smart.get("pending_sectors"),
|
|
"uncorrectable_errors": smart.get("uncorrectable_errors"),
|
|
"zfs_pool": zfs_info.get("pool"),
|
|
"zfs_vdev": zfs_info.get("vdev"),
|
|
"zfs_state": zfs_info.get("state"),
|
|
"health_status": health_status,
|
|
"physical_drives": [],
|
|
})
|
|
|
|
# Discover physical drives behind RAID controllers
|
|
has_raid = any(r["drive_type"] == "raid" and not r["smart_supported"] for r in results)
|
|
if has_raid:
|
|
megaraid_drives = await scan_megaraid_drives()
|
|
for pd in megaraid_drives:
|
|
pd["health_status"] = compute_health_status(pd)
|
|
pd["drive_type"] = "physical"
|
|
pd["physical_drives"] = []
|
|
|
|
# Attach to the first RAID host drive
|
|
for r in results:
|
|
if r["drive_type"] == "raid" and not r["smart_supported"]:
|
|
r["physical_drives"] = megaraid_drives
|
|
break
|
|
|
|
return results
|