Harden poller from Codex review: close residual storm/robustness gaps
Adversarial review (gpt-5.5) of the new architecture surfaced real gaps: Hardware-safety / storm prevention: - Restart-storm guard: on startup, defer the first sweep if a sweep ran < POLL_SWEEP_INTERVAL ago (persisted in Redis), so deploy-cycling can no longer trigger back-to-back full hardware sweeps (poller.py). - Centralize pacing in the hardware gate: POLL_DRIVE_GAP is now held after EVERY hardware op (SMART, SES, host, MegaRAID, ledctl), not just the enclosure-drive loop (hwgate.py); removed the now-redundant per-loop sleeps. - Single-instance lock made atomic (Lua compare-and-set / compare-and-expire) and the refresher is now fatal on any error + has a done-callback, so a dropped lock can never leave two pollers sweeping concurrently (store.py, poller.py). - Warn loudly when POLL_CONCURRENCY > 1. Correctness: - Heartbeat now actually writes: cache_set omits EX when ttl<=0 (Redis rejects EX 0), so poller meta/last_sweep_ts persists — this also enables the restart-storm guard and the poller_fresh health flag (cache.py). - Web image runs a single uvicorn worker so the MQTT publisher is a true singleton (two workers shared a client_id and flapped the broker) (Dockerfile). - LED enqueue catches Redis errors -> API returns 503, not 500 (store.py). Deploy independence: - build.sh takes a target (web|poller|all) so web iterations never rebuild or repush the poller image. Nits: drop dead SMART_CACHE_TTL constant.
This commit is contained in:
@@ -58,10 +58,13 @@ async def cache_get(key: str) -> Any | None:
|
||||
|
||||
|
||||
async def cache_set(key: str, value: Any, ttl: int = 120) -> None:
|
||||
"""SET key in Redis with expiry, silently catches errors."""
|
||||
"""SET key in Redis. ttl<=0 stores with no expiry (Redis rejects EX 0)."""
|
||||
if _redis is None:
|
||||
return
|
||||
try:
|
||||
await _redis.set(key, json.dumps(value), ex=ttl)
|
||||
if ttl and ttl > 0:
|
||||
await _redis.set(key, json.dumps(value), ex=ttl)
|
||||
else:
|
||||
await _redis.set(key, json.dumps(value))
|
||||
except Exception as e:
|
||||
logger.warning("Redis SET %s failed: %s", key, e)
|
||||
|
||||
@@ -1,27 +1,49 @@
|
||||
"""Global hardware-access gate.
|
||||
"""Global hardware-access gate + pacing.
|
||||
|
||||
Every command that touches the SAS bus — smartctl, sg_ses, zpool, ledctl —
|
||||
must run inside this gate. It serializes hardware I/O so the poller can
|
||||
never fire a thundering herd of subprocesses at fragile SAS expanders
|
||||
(which on some shelves drop out and suspend pools under concurrent load).
|
||||
Every command that touches the SAS bus — smartctl, sg_ses, zpool, lsblk,
|
||||
ledctl — must run inside this gate. It does two things:
|
||||
|
||||
Concurrency defaults to 1 (fully serialized). It can be raised via
|
||||
POLL_CONCURRENCY for healthier backplanes, but 1 is the safe default.
|
||||
1. Serializes hardware I/O (semaphore, default concurrency 1) so the
|
||||
poller can never fire a thundering herd at fragile SAS expanders.
|
||||
2. Paces it: after every hardware op it holds the gate for POLL_DRIVE_GAP
|
||||
seconds, so back-to-back ops (gathered SMART, MegaRAID scans, SES,
|
||||
ledctl) are always spaced — not just the per-drive loop.
|
||||
|
||||
The gate is a per-process asyncio.Semaphore — only the poller process
|
||||
calls hardware, so a per-process gate is sufficient. The semaphore is
|
||||
created lazily on first use so it binds to the running event loop.
|
||||
This is a per-process gate; only the poller process calls hardware, so
|
||||
per-process serialization is sufficient. Created lazily so it binds to the
|
||||
running event loop.
|
||||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import logging
|
||||
import os
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_sem: asyncio.Semaphore | None = None
|
||||
|
||||
|
||||
def gate() -> asyncio.Semaphore:
|
||||
"""Return the shared hardware semaphore (use as `async with gate():`)."""
|
||||
def _semaphore() -> asyncio.Semaphore:
|
||||
global _sem
|
||||
if _sem is None:
|
||||
n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1")))
|
||||
if n > 1:
|
||||
logger.warning(
|
||||
"POLL_CONCURRENCY=%d (>1) — hardware ops will run concurrently; "
|
||||
"only do this on backplanes known to tolerate it", n,
|
||||
)
|
||||
_sem = asyncio.Semaphore(n)
|
||||
return _sem
|
||||
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def gate():
|
||||
"""Acquire the hardware gate for one op, then hold it for the pacing gap."""
|
||||
sem = _semaphore()
|
||||
async with sem:
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
gap = float(os.environ.get("POLL_DRIVE_GAP", "0.5"))
|
||||
if gap > 0:
|
||||
await asyncio.sleep(gap)
|
||||
|
||||
@@ -18,8 +18,6 @@ ATTR_PENDING = 197
|
||||
ATTR_UNCORRECTABLE = 198
|
||||
ATTR_WEAR_LEVELING = 177 # SSD wear leveling
|
||||
|
||||
SMART_CACHE_TTL = int(os.environ.get("SMART_CACHE_TTL", "120"))
|
||||
|
||||
|
||||
def smartctl_available() -> bool:
|
||||
return shutil.which("smartctl") is not None
|
||||
|
||||
@@ -82,11 +82,15 @@ async def get_meta() -> dict | None:
|
||||
# ── locate-LED command queue ────────────────────────────────────────────
|
||||
async def enqueue_led(device: str, state: str) -> bool:
|
||||
"""API side: push a LED request for the poller to execute. Returns False
|
||||
if Redis is unavailable (so the API can surface 503)."""
|
||||
if Redis is unavailable or the push fails (so the API can surface 503)."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return False
|
||||
await client.rpush(K_LED_QUEUE, json.dumps({"device": device, "state": state}))
|
||||
try:
|
||||
await client.rpush(K_LED_QUEUE, json.dumps({"device": device, "state": state}))
|
||||
except Exception as e:
|
||||
logger.warning("LED enqueue failed: %s", e)
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@@ -105,29 +109,37 @@ async def pop_led(timeout: int = 5) -> dict | None:
|
||||
return None
|
||||
|
||||
|
||||
# ── poller single-instance lock ─────────────────────────────────────────
|
||||
# ── poller single-instance lock (atomic compare-and-set) ────────────────
|
||||
# Acquire if free OR already ours, refreshing the TTL — all in one round trip
|
||||
# so there's no GET/SET race that could let two pollers run concurrently.
|
||||
_ACQUIRE_LUA = (
|
||||
"local v = redis.call('get', KEYS[1]) "
|
||||
"if v == false or v == ARGV[1] then "
|
||||
" redis.call('set', KEYS[1], ARGV[1], 'EX', ARGV[2]) return 1 "
|
||||
"else return 0 end"
|
||||
)
|
||||
# Extend only if we still own it (atomic compare-and-expire).
|
||||
_REFRESH_LUA = (
|
||||
"if redis.call('get', KEYS[1]) == ARGV[1] then "
|
||||
" return redis.call('expire', KEYS[1], ARGV[2]) "
|
||||
"else return 0 end"
|
||||
)
|
||||
|
||||
|
||||
async def acquire_lock(token: str, ttl: int = 30) -> bool:
|
||||
"""Try to claim the poller lock. Returns True if acquired/owned."""
|
||||
"""Atomically claim the poller lock (or re-own it). True if held."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return True # no Redis → no coordination possible; run anyway
|
||||
got = await client.set(K_LOCK, token, nx=True, ex=ttl)
|
||||
if got:
|
||||
return True
|
||||
current = await client.get(K_LOCK)
|
||||
return current == token
|
||||
return bool(await client.eval(_ACQUIRE_LUA, 1, K_LOCK, token, ttl))
|
||||
|
||||
|
||||
async def refresh_lock(token: str, ttl: int = 30) -> bool:
|
||||
"""Extend the lock if we still own it."""
|
||||
"""Atomically extend the lock iff we still own it."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return True
|
||||
current = await client.get(K_LOCK)
|
||||
if current == token:
|
||||
await client.expire(K_LOCK, ttl)
|
||||
return True
|
||||
return False
|
||||
return bool(await client.eval(_REFRESH_LUA, 1, K_LOCK, token, ttl))
|
||||
|
||||
|
||||
def now() -> float:
|
||||
|
||||
Reference in New Issue
Block a user