Add host-poller: chassis IPMI/iDRAC + CPU + on-metal drive temps
New third producer container (host-poller) owning the server chassis subsystem, separate from the SAS-shelf poller: - services/host_sensors.py: in-band IPMI (ipmitool sdr) for Inlet/Exhaust/etc. + CPU package temps from coretemp hwmon - host_poller.py: own single-instance lock + heartbeat (host_poll keys), restart-storm guard, paced; writes jbod:host_sensors + jbod:host_drives - move host/on-metal drive collection off the SAS poller to host-poller (reads jbod:zfs_map for annotation) - store: generalized lock/meta with a key param; host_sensors + host-poll keys - mqtt_publisher: publish env/CPU/host-drive temps as entities on the hub device (JBOD Monitor) - Dockerfile host-poller target (ipmitool+smartmontools); compose.infra adds host-poller; build.sh gains host-poller/all
This commit is contained in:
95
services/host_sensors.py
Normal file
95
services/host_sensors.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Host/chassis temperature collectors: IPMI (iDRAC) + CPU coretemp.
|
||||
|
||||
Used by the host-poller (separate process from the SAS poller). In-band IPMI
|
||||
via ipmitool (/dev/ipmi0) gives environmental sensors (Inlet/Exhaust/…); the
|
||||
kernel coretemp hwmon gives CPU package temps. IPMI is gated for serialization;
|
||||
coretemp is a cheap sysfs read.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from services.hwgate import gate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HWMON = Path("/sys/class/hwmon")
|
||||
|
||||
|
||||
async def read_ipmi_temps() -> list[dict]:
|
||||
"""`ipmitool sdr type temperature` → [{name, temperature_c, status, kind}]."""
|
||||
try:
|
||||
async with gate():
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ipmitool", "sdr", "type", "temperature",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
out, err = await proc.communicate()
|
||||
except FileNotFoundError:
|
||||
logger.warning("ipmitool not found")
|
||||
return []
|
||||
if proc.returncode != 0:
|
||||
logger.warning("ipmitool failed: %s", err.decode(errors="replace").strip())
|
||||
return []
|
||||
return _parse_ipmi(out.decode(errors="replace"))
|
||||
|
||||
|
||||
def _parse_ipmi(text: str) -> list[dict]:
|
||||
# e.g. "Inlet Temp | 04h | ok | 7.1 | 23 degrees C"
|
||||
# "Temp | 0Eh | ok | 3.1 | 40 degrees C" (CPU1)
|
||||
sensors: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
parts = [p.strip() for p in line.split("|")]
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
name, _sid, status, entity, reading = parts[:5]
|
||||
m = re.search(r"(-?\d+(?:\.\d+)?)\s*degrees?\s*C", reading, re.IGNORECASE)
|
||||
if not m: # "no reading", "disabled", "ns", etc.
|
||||
continue
|
||||
temp = float(m.group(1))
|
||||
kind = "env"
|
||||
# Dell reports CPU temps as "Temp" under entity 3.x — relabel them.
|
||||
if name.lower() == "temp" and entity.startswith("3."):
|
||||
name = f"CPU {entity.split('.')[-1]}"
|
||||
kind = "cpu"
|
||||
sensors.append({
|
||||
"name": name,
|
||||
"temperature_c": temp,
|
||||
"status": status or "ok",
|
||||
"kind": kind,
|
||||
})
|
||||
return sensors
|
||||
|
||||
|
||||
def read_coretemp() -> list[dict]:
|
||||
"""CPU package temps from coretemp hwmon → [{name, temperature_c, kind}]."""
|
||||
cpus: list[dict] = []
|
||||
if not HWMON.is_dir():
|
||||
return cpus
|
||||
for hw in sorted(HWMON.iterdir()):
|
||||
try:
|
||||
if (hw / "name").read_text().strip() != "coretemp":
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
for label_f in sorted(hw.glob("temp*_label")):
|
||||
try:
|
||||
label = label_f.read_text().strip()
|
||||
except OSError:
|
||||
continue
|
||||
if not label.lower().startswith("package id"):
|
||||
continue
|
||||
input_f = label_f.with_name(label_f.name.replace("_label", "_input"))
|
||||
try:
|
||||
milli = int(input_f.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
pkg = label.split("id")[-1].strip()
|
||||
cpus.append({
|
||||
"name": f"CPU {pkg}",
|
||||
"temperature_c": round(milli / 1000),
|
||||
"kind": "cpu",
|
||||
})
|
||||
return cpus
|
||||
Reference in New Issue
Block a user