adam
226f73851c
- poller.py: start lock_refresher() IMMEDIATELY after acquiring the lock, before the startup jitter and restart-storm deferral. Those sleeps can together exceed LOCK_TTL (restart defer is up to POLL_SWEEP_INTERVAL), so with the refresher started afterward the lock could expire mid-deferral and a second poller could begin sweeping concurrently — the exact hardware-storm risk this design prevents. - hwgate.py: clamp POLL_CONCURRENCY to 1 unless POLL_ALLOW_UNSAFE_CONCURRENCY is set (fragile SAS hardware; >1 can drop expanders). Warn either way. - build.sh: keep backward-compatible — a non-target first arg is treated as the commit message (./build.sh "msg" still works), known targets select web|poller|all.
62 lines
2.1 KiB
Python
62 lines
2.1 KiB
Python
"""Global hardware-access gate + pacing.
|
|
|
|
Every command that touches the SAS bus — smartctl, sg_ses, zpool, lsblk,
|
|
ledctl — must run inside this gate. It does two things:
|
|
|
|
1. Serializes hardware I/O (semaphore, default concurrency 1) so the
|
|
poller can never fire a thundering herd at fragile SAS expanders.
|
|
2. Paces it: after every hardware op it holds the gate for POLL_DRIVE_GAP
|
|
seconds, so back-to-back ops (gathered SMART, MegaRAID scans, SES,
|
|
ledctl) are always spaced — not just the per-drive loop.
|
|
|
|
This is a per-process gate; only the poller process calls hardware, so
|
|
per-process serialization is sufficient. Created lazily so it binds to the
|
|
running event loop.
|
|
"""
|
|
import asyncio
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
_sem: asyncio.Semaphore | None = None
|
|
|
|
|
|
def _semaphore() -> asyncio.Semaphore:
|
|
global _sem
|
|
if _sem is None:
|
|
n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1")))
|
|
# Serialized (1) is the safe default. Concurrent hardware I/O can drop
|
|
# fragile SAS expanders, so >1 is clamped unless explicitly overridden.
|
|
if n > 1:
|
|
override = os.environ.get("POLL_ALLOW_UNSAFE_CONCURRENCY", "").lower() in (
|
|
"1", "true", "yes",
|
|
)
|
|
if override:
|
|
logger.warning(
|
|
"POLL_CONCURRENCY=%d with unsafe override — concurrent "
|
|
"hardware I/O enabled; backplane must tolerate it", n,
|
|
)
|
|
else:
|
|
logger.warning(
|
|
"POLL_CONCURRENCY=%d ignored (clamped to 1 for hardware "
|
|
"safety); set POLL_ALLOW_UNSAFE_CONCURRENCY=1 to override", n,
|
|
)
|
|
n = 1
|
|
_sem = asyncio.Semaphore(n)
|
|
return _sem
|
|
|
|
|
|
@contextlib.asynccontextmanager
|
|
async def gate():
|
|
"""Acquire the hardware gate for one op, then hold it for the pacing gap."""
|
|
sem = _semaphore()
|
|
async with sem:
|
|
try:
|
|
yield
|
|
finally:
|
|
gap = float(os.environ.get("POLL_DRIVE_GAP", "0.5"))
|
|
if gap > 0:
|
|
await asyncio.sleep(gap)
|