Fix critical lock-expiry window found in Codex second pass

- poller.py: start lock_refresher() IMMEDIATELY after acquiring the lock,
  before the startup jitter and restart-storm deferral. Those sleeps can
  together exceed LOCK_TTL (restart defer is up to POLL_SWEEP_INTERVAL), so
  with the refresher started afterward the lock could expire mid-deferral and
  a second poller could begin sweeping concurrently — the exact hardware-storm
  risk this design prevents.
- hwgate.py: clamp POLL_CONCURRENCY to 1 unless POLL_ALLOW_UNSAFE_CONCURRENCY
  is set (fragile SAS hardware; >1 can drop expanders). Warn either way.
- build.sh: keep backward-compatible — a non-target first arg is treated as
  the commit message (./build.sh "msg" still works), known targets select
  web|poller|all.
This commit is contained in:
2026-06-16 16:26:36 +00:00
parent 7878e17e55
commit 226f73851c
3 changed files with 30 additions and 12 deletions

View File

@@ -13,14 +13,15 @@ REG="docker.adamksmith.xyz"
cd "$(dirname "$0")" cd "$(dirname "$0")"
TARGET="${1:-all}" # First arg is the target if it's a known one; otherwise it's treated as the
case "$TARGET" in # commit message (backward-compatible with `./build.sh "message"`).
poller) TARGETS="poller" ;; case "${1:-}" in
web) TARGETS="web" ;; poller) TARGETS="poller"; MSG="${2:-Build and push images}" ;;
all) TARGETS="poller web" ;; web) TARGETS="web"; MSG="${2:-Build and push images}" ;;
*) echo "usage: $0 [poller|web|all] [message]" >&2; exit 1 ;; all|"") TARGETS="poller web"; MSG="${2:-Build and push images}" ;;
*) TARGETS="poller web"; MSG="${1}" ;;
esac esac
MSG="${2:-Build and push images}" TARGET="${TARGETS// /+}"
# Stage and commit all changes # Stage and commit all changes
git add -A git add -A

View File

@@ -165,6 +165,13 @@ async def main() -> None:
await asyncio.sleep(LOCK_TTL // 2) await asyncio.sleep(LOCK_TTL // 2)
logger.info("poller lock acquired (%s)", TOKEN) logger.info("poller lock acquired (%s)", TOKEN)
# Start the refresher IMMEDIATELY — before any pre-sweep sleep. The jitter
# and restart-storm deferral below can together exceed LOCK_TTL, so without
# an active refresher the lock would expire and a second poller could begin
# sweeping concurrently (a hardware-storm risk).
refresher = asyncio.create_task(lock_refresher())
refresher.add_done_callback(_critical_task_done)
if not smartctl_available(): if not smartctl_available():
logger.warning("smartctl not found — install smartmontools") logger.warning("smartctl not found — install smartmontools")
if not sg_ses_available(): if not sg_ses_available():
@@ -188,8 +195,6 @@ async def main() -> None:
logger.info("last sweep %.0fs ago — deferring first sweep %.0fs", since, wait) logger.info("last sweep %.0fs ago — deferring first sweep %.0fs", since, wait)
await asyncio.sleep(wait) await asyncio.sleep(wait)
refresher = asyncio.create_task(lock_refresher())
refresher.add_done_callback(_critical_task_done)
leds = asyncio.create_task(led_consumer()) leds = asyncio.create_task(led_consumer())
try: try:
while True: while True:

View File

@@ -27,11 +27,23 @@ def _semaphore() -> asyncio.Semaphore:
global _sem global _sem
if _sem is None: if _sem is None:
n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1"))) n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1")))
# Serialized (1) is the safe default. Concurrent hardware I/O can drop
# fragile SAS expanders, so >1 is clamped unless explicitly overridden.
if n > 1: if n > 1:
logger.warning( override = os.environ.get("POLL_ALLOW_UNSAFE_CONCURRENCY", "").lower() in (
"POLL_CONCURRENCY=%d (>1) — hardware ops will run concurrently; " "1", "true", "yes",
"only do this on backplanes known to tolerate it", n,
) )
if override:
logger.warning(
"POLL_CONCURRENCY=%d with unsafe override — concurrent "
"hardware I/O enabled; backplane must tolerate it", n,
)
else:
logger.warning(
"POLL_CONCURRENCY=%d ignored (clamped to 1 for hardware "
"safety); set POLL_ALLOW_UNSAFE_CONCURRENCY=1 to override", n,
)
n = 1
_sem = asyncio.Semaphore(n) _sem = asyncio.Semaphore(n)
return _sem return _sem