Fix critical lock-expiry window found in Codex second pass
- poller.py: start lock_refresher() IMMEDIATELY after acquiring the lock, before the startup jitter and restart-storm deferral. Those sleeps can together exceed LOCK_TTL (restart defer is up to POLL_SWEEP_INTERVAL), so with the refresher started afterward the lock could expire mid-deferral and a second poller could begin sweeping concurrently — the exact hardware-storm risk this design prevents. - hwgate.py: clamp POLL_CONCURRENCY to 1 unless POLL_ALLOW_UNSAFE_CONCURRENCY is set (fragile SAS hardware; >1 can drop expanders). Warn either way. - build.sh: keep backward-compatible — a non-target first arg is treated as the commit message (./build.sh "msg" still works), known targets select web|poller|all.
This commit is contained in:
15
build.sh
15
build.sh
@@ -13,14 +13,15 @@ REG="docker.adamksmith.xyz"
|
|||||||
|
|
||||||
cd "$(dirname "$0")"
|
cd "$(dirname "$0")"
|
||||||
|
|
||||||
TARGET="${1:-all}"
|
# First arg is the target if it's a known one; otherwise it's treated as the
|
||||||
case "$TARGET" in
|
# commit message (backward-compatible with `./build.sh "message"`).
|
||||||
poller) TARGETS="poller" ;;
|
case "${1:-}" in
|
||||||
web) TARGETS="web" ;;
|
poller) TARGETS="poller"; MSG="${2:-Build and push images}" ;;
|
||||||
all) TARGETS="poller web" ;;
|
web) TARGETS="web"; MSG="${2:-Build and push images}" ;;
|
||||||
*) echo "usage: $0 [poller|web|all] [message]" >&2; exit 1 ;;
|
all|"") TARGETS="poller web"; MSG="${2:-Build and push images}" ;;
|
||||||
|
*) TARGETS="poller web"; MSG="${1}" ;;
|
||||||
esac
|
esac
|
||||||
MSG="${2:-Build and push images}"
|
TARGET="${TARGETS// /+}"
|
||||||
|
|
||||||
# Stage and commit all changes
|
# Stage and commit all changes
|
||||||
git add -A
|
git add -A
|
||||||
|
|||||||
@@ -165,6 +165,13 @@ async def main() -> None:
|
|||||||
await asyncio.sleep(LOCK_TTL // 2)
|
await asyncio.sleep(LOCK_TTL // 2)
|
||||||
logger.info("poller lock acquired (%s)", TOKEN)
|
logger.info("poller lock acquired (%s)", TOKEN)
|
||||||
|
|
||||||
|
# Start the refresher IMMEDIATELY — before any pre-sweep sleep. The jitter
|
||||||
|
# and restart-storm deferral below can together exceed LOCK_TTL, so without
|
||||||
|
# an active refresher the lock would expire and a second poller could begin
|
||||||
|
# sweeping concurrently (a hardware-storm risk).
|
||||||
|
refresher = asyncio.create_task(lock_refresher())
|
||||||
|
refresher.add_done_callback(_critical_task_done)
|
||||||
|
|
||||||
if not smartctl_available():
|
if not smartctl_available():
|
||||||
logger.warning("smartctl not found — install smartmontools")
|
logger.warning("smartctl not found — install smartmontools")
|
||||||
if not sg_ses_available():
|
if not sg_ses_available():
|
||||||
@@ -188,8 +195,6 @@ async def main() -> None:
|
|||||||
logger.info("last sweep %.0fs ago — deferring first sweep %.0fs", since, wait)
|
logger.info("last sweep %.0fs ago — deferring first sweep %.0fs", since, wait)
|
||||||
await asyncio.sleep(wait)
|
await asyncio.sleep(wait)
|
||||||
|
|
||||||
refresher = asyncio.create_task(lock_refresher())
|
|
||||||
refresher.add_done_callback(_critical_task_done)
|
|
||||||
leds = asyncio.create_task(led_consumer())
|
leds = asyncio.create_task(led_consumer())
|
||||||
try:
|
try:
|
||||||
while True:
|
while True:
|
||||||
|
|||||||
@@ -27,11 +27,23 @@ def _semaphore() -> asyncio.Semaphore:
|
|||||||
global _sem
|
global _sem
|
||||||
if _sem is None:
|
if _sem is None:
|
||||||
n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1")))
|
n = max(1, int(os.environ.get("POLL_CONCURRENCY", "1")))
|
||||||
|
# Serialized (1) is the safe default. Concurrent hardware I/O can drop
|
||||||
|
# fragile SAS expanders, so >1 is clamped unless explicitly overridden.
|
||||||
if n > 1:
|
if n > 1:
|
||||||
logger.warning(
|
override = os.environ.get("POLL_ALLOW_UNSAFE_CONCURRENCY", "").lower() in (
|
||||||
"POLL_CONCURRENCY=%d (>1) — hardware ops will run concurrently; "
|
"1", "true", "yes",
|
||||||
"only do this on backplanes known to tolerate it", n,
|
|
||||||
)
|
)
|
||||||
|
if override:
|
||||||
|
logger.warning(
|
||||||
|
"POLL_CONCURRENCY=%d with unsafe override — concurrent "
|
||||||
|
"hardware I/O enabled; backplane must tolerate it", n,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
logger.warning(
|
||||||
|
"POLL_CONCURRENCY=%d ignored (clamped to 1 for hardware "
|
||||||
|
"safety); set POLL_ALLOW_UNSAFE_CONCURRENCY=1 to override", n,
|
||||||
|
)
|
||||||
|
n = 1
|
||||||
_sem = asyncio.Semaphore(n)
|
_sem = asyncio.Semaphore(n)
|
||||||
return _sem
|
return _sem
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user