Add host-poller: chassis IPMI/iDRAC + CPU + on-metal drive temps

New third producer container (host-poller) owning the server chassis
subsystem, separate from the SAS-shelf poller:
- services/host_sensors.py: in-band IPMI (ipmitool sdr) for Inlet/Exhaust/etc.
  + CPU package temps from coretemp hwmon
- host_poller.py: own single-instance lock + heartbeat (host_poll keys),
  restart-storm guard, paced; writes jbod:host_sensors + jbod:host_drives
- move host/on-metal drive collection off the SAS poller to host-poller
  (reads jbod:zfs_map for annotation)
- store: generalized lock/meta with a key param; host_sensors + host-poll keys
- mqtt_publisher: publish env/CPU/host-drive temps as entities on the hub
  device (JBOD Monitor)
- Dockerfile host-poller target (ipmitool+smartmontools); compose.infra adds
  host-poller; build.sh gains host-poller/all
This commit is contained in:
2026-06-16 18:27:56 +00:00
parent 44a7824425
commit 1839e9d5f2
8 changed files with 381 additions and 21 deletions

95
services/host_sensors.py Normal file
View File

@@ -0,0 +1,95 @@
"""Host/chassis temperature collectors: IPMI (iDRAC) + CPU coretemp.
Used by the host-poller (separate process from the SAS poller). In-band IPMI
via ipmitool (/dev/ipmi0) gives environmental sensors (Inlet/Exhaust/…); the
kernel coretemp hwmon gives CPU package temps. IPMI is gated for serialization;
coretemp is a cheap sysfs read.
"""
import asyncio
import logging
import re
from pathlib import Path
from services.hwgate import gate
logger = logging.getLogger(__name__)
HWMON = Path("/sys/class/hwmon")
async def read_ipmi_temps() -> list[dict]:
"""`ipmitool sdr type temperature` → [{name, temperature_c, status, kind}]."""
try:
async with gate():
proc = await asyncio.create_subprocess_exec(
"ipmitool", "sdr", "type", "temperature",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
out, err = await proc.communicate()
except FileNotFoundError:
logger.warning("ipmitool not found")
return []
if proc.returncode != 0:
logger.warning("ipmitool failed: %s", err.decode(errors="replace").strip())
return []
return _parse_ipmi(out.decode(errors="replace"))
def _parse_ipmi(text: str) -> list[dict]:
# e.g. "Inlet Temp | 04h | ok | 7.1 | 23 degrees C"
# "Temp | 0Eh | ok | 3.1 | 40 degrees C" (CPU1)
sensors: list[dict] = []
for line in text.splitlines():
parts = [p.strip() for p in line.split("|")]
if len(parts) < 5:
continue
name, _sid, status, entity, reading = parts[:5]
m = re.search(r"(-?\d+(?:\.\d+)?)\s*degrees?\s*C", reading, re.IGNORECASE)
if not m: # "no reading", "disabled", "ns", etc.
continue
temp = float(m.group(1))
kind = "env"
# Dell reports CPU temps as "Temp" under entity 3.x — relabel them.
if name.lower() == "temp" and entity.startswith("3."):
name = f"CPU {entity.split('.')[-1]}"
kind = "cpu"
sensors.append({
"name": name,
"temperature_c": temp,
"status": status or "ok",
"kind": kind,
})
return sensors
def read_coretemp() -> list[dict]:
"""CPU package temps from coretemp hwmon → [{name, temperature_c, kind}]."""
cpus: list[dict] = []
if not HWMON.is_dir():
return cpus
for hw in sorted(HWMON.iterdir()):
try:
if (hw / "name").read_text().strip() != "coretemp":
continue
except OSError:
continue
for label_f in sorted(hw.glob("temp*_label")):
try:
label = label_f.read_text().strip()
except OSError:
continue
if not label.lower().startswith("package id"):
continue
input_f = label_f.with_name(label_f.name.replace("_label", "_input"))
try:
milli = int(input_f.read_text().strip())
except (OSError, ValueError):
continue
pkg = label.split("id")[-1].strip()
cpus.append({
"name": f"CPU {pkg}",
"temperature_c": round(milli / 1000),
"kind": "cpu",
})
return cpus