Add host-poller: chassis IPMI/iDRAC + CPU + on-metal drive temps
New third producer container (host-poller) owning the server chassis subsystem, separate from the SAS-shelf poller: - services/host_sensors.py: in-band IPMI (ipmitool sdr) for Inlet/Exhaust/etc. + CPU package temps from coretemp hwmon - host_poller.py: own single-instance lock + heartbeat (host_poll keys), restart-storm guard, paced; writes jbod:host_sensors + jbod:host_drives - move host/on-metal drive collection off the SAS poller to host-poller (reads jbod:zfs_map for annotation) - store: generalized lock/meta with a key param; host_sensors + host-poll keys - mqtt_publisher: publish env/CPU/host-drive temps as entities on the hub device (JBOD Monitor) - Dockerfile host-poller target (ipmitool+smartmontools); compose.infra adds host-poller; build.sh gains host-poller/all
This commit is contained in:
95
services/host_sensors.py
Normal file
95
services/host_sensors.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""Host/chassis temperature collectors: IPMI (iDRAC) + CPU coretemp.
|
||||
|
||||
Used by the host-poller (separate process from the SAS poller). In-band IPMI
|
||||
via ipmitool (/dev/ipmi0) gives environmental sensors (Inlet/Exhaust/…); the
|
||||
kernel coretemp hwmon gives CPU package temps. IPMI is gated for serialization;
|
||||
coretemp is a cheap sysfs read.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from services.hwgate import gate
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
HWMON = Path("/sys/class/hwmon")
|
||||
|
||||
|
||||
async def read_ipmi_temps() -> list[dict]:
|
||||
"""`ipmitool sdr type temperature` → [{name, temperature_c, status, kind}]."""
|
||||
try:
|
||||
async with gate():
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"ipmitool", "sdr", "type", "temperature",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
out, err = await proc.communicate()
|
||||
except FileNotFoundError:
|
||||
logger.warning("ipmitool not found")
|
||||
return []
|
||||
if proc.returncode != 0:
|
||||
logger.warning("ipmitool failed: %s", err.decode(errors="replace").strip())
|
||||
return []
|
||||
return _parse_ipmi(out.decode(errors="replace"))
|
||||
|
||||
|
||||
def _parse_ipmi(text: str) -> list[dict]:
|
||||
# e.g. "Inlet Temp | 04h | ok | 7.1 | 23 degrees C"
|
||||
# "Temp | 0Eh | ok | 3.1 | 40 degrees C" (CPU1)
|
||||
sensors: list[dict] = []
|
||||
for line in text.splitlines():
|
||||
parts = [p.strip() for p in line.split("|")]
|
||||
if len(parts) < 5:
|
||||
continue
|
||||
name, _sid, status, entity, reading = parts[:5]
|
||||
m = re.search(r"(-?\d+(?:\.\d+)?)\s*degrees?\s*C", reading, re.IGNORECASE)
|
||||
if not m: # "no reading", "disabled", "ns", etc.
|
||||
continue
|
||||
temp = float(m.group(1))
|
||||
kind = "env"
|
||||
# Dell reports CPU temps as "Temp" under entity 3.x — relabel them.
|
||||
if name.lower() == "temp" and entity.startswith("3."):
|
||||
name = f"CPU {entity.split('.')[-1]}"
|
||||
kind = "cpu"
|
||||
sensors.append({
|
||||
"name": name,
|
||||
"temperature_c": temp,
|
||||
"status": status or "ok",
|
||||
"kind": kind,
|
||||
})
|
||||
return sensors
|
||||
|
||||
|
||||
def read_coretemp() -> list[dict]:
|
||||
"""CPU package temps from coretemp hwmon → [{name, temperature_c, kind}]."""
|
||||
cpus: list[dict] = []
|
||||
if not HWMON.is_dir():
|
||||
return cpus
|
||||
for hw in sorted(HWMON.iterdir()):
|
||||
try:
|
||||
if (hw / "name").read_text().strip() != "coretemp":
|
||||
continue
|
||||
except OSError:
|
||||
continue
|
||||
for label_f in sorted(hw.glob("temp*_label")):
|
||||
try:
|
||||
label = label_f.read_text().strip()
|
||||
except OSError:
|
||||
continue
|
||||
if not label.lower().startswith("package id"):
|
||||
continue
|
||||
input_f = label_f.with_name(label_f.name.replace("_label", "_input"))
|
||||
try:
|
||||
milli = int(input_f.read_text().strip())
|
||||
except (OSError, ValueError):
|
||||
continue
|
||||
pkg = label.split("id")[-1].strip()
|
||||
cpus.append({
|
||||
"name": f"CPU {pkg}",
|
||||
"temperature_c": round(milli / 1000),
|
||||
"kind": "cpu",
|
||||
})
|
||||
return cpus
|
||||
@@ -19,6 +19,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from services import store
|
||||
from services.secrets import read_kv2
|
||||
from services.temps import build_temp_snapshot, get_node_id
|
||||
|
||||
@@ -242,6 +243,61 @@ class MqttPublisher:
|
||||
self._publish_discovery(enc)
|
||||
self._publish_state(enc)
|
||||
|
||||
def publish_host(self, sensors: dict, drives: list) -> None:
|
||||
"""Publish chassis temps (IPMI env, CPU, on-metal drives) on the hub."""
|
||||
self._publish_hub()
|
||||
state_topic = f"{self.base_topic}/{self.node}/host/state"
|
||||
device = self._hub_device_block()
|
||||
payload: dict = {}
|
||||
|
||||
def announce(object_id: str, cfg: dict) -> None:
|
||||
uid = cfg["unique_id"]
|
||||
if uid not in self._discovered:
|
||||
topic = f"{self.discovery_prefix}/sensor/{self.hub_id}/{object_id}/config"
|
||||
self._client.publish(topic, json.dumps(cfg), qos=1, retain=True)
|
||||
self._discovered.add(uid)
|
||||
|
||||
common = {
|
||||
"device_class": "temperature",
|
||||
"unit_of_measurement": "°C",
|
||||
"state_class": "measurement",
|
||||
"state_topic": state_topic,
|
||||
"availability_topic": self.availability_topic,
|
||||
"device": device,
|
||||
}
|
||||
|
||||
def add(kind: str, name: str, temp, icon: str | None = None) -> None:
|
||||
if temp is None or not name:
|
||||
return
|
||||
key = f"{kind}_{_slug(name)}"
|
||||
payload[key] = temp
|
||||
cfg = {
|
||||
**common,
|
||||
"name": name,
|
||||
"unique_id": f"{self.hub_id}_{key}",
|
||||
"value_template": f"{{{{ value_json.{key} }}}}",
|
||||
}
|
||||
if icon:
|
||||
cfg["icon"] = icon
|
||||
announce(key, cfg)
|
||||
|
||||
for s in sensors.get("env", []):
|
||||
add("env", s.get("name"), s.get("temperature_c"))
|
||||
for c in sensors.get("cpu", []):
|
||||
add("cpu", c.get("name"), c.get("temperature_c"), icon="mdi:cpu-64-bit")
|
||||
|
||||
def add_drive(d: dict) -> None:
|
||||
dev = d.get("device")
|
||||
if dev:
|
||||
add("drive", f"Drive {dev}", d.get("temperature_c"), icon="mdi:harddisk")
|
||||
|
||||
for d in drives:
|
||||
add_drive(d)
|
||||
for pd in d.get("physical_drives", []):
|
||||
add_drive(pd)
|
||||
|
||||
self._client.publish(state_topic, json.dumps(payload), qos=0, retain=True)
|
||||
|
||||
async def run(self) -> None:
|
||||
"""Background loop: build a snapshot and publish on an interval."""
|
||||
await asyncio.sleep(3) # let the first SMART poll populate the cache
|
||||
@@ -249,9 +305,14 @@ class MqttPublisher:
|
||||
try:
|
||||
snapshot = await build_temp_snapshot()
|
||||
self.publish_snapshot(snapshot)
|
||||
host_sensors = await store.get_host_sensors() or {}
|
||||
host_drives = await store.get_host_drives() or []
|
||||
self.publish_host(host_sensors, host_drives)
|
||||
logger.info(
|
||||
"MQTT published temps for %d enclosure(s)",
|
||||
"MQTT published temps for %d enclosure(s) + host (%d env, %d cpu)",
|
||||
len(snapshot.get("enclosures", [])),
|
||||
len(host_sensors.get("env", [])),
|
||||
len(host_sensors.get("cpu", [])),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
@@ -23,9 +23,12 @@ K_SES = "jbod:ses:{}"
|
||||
K_ZFS = "jbod:zfs_map"
|
||||
K_INVENTORY = "jbod:inventory"
|
||||
K_HOST_DRIVES = "jbod:host_drives"
|
||||
K_HOST_SENSORS = "jbod:host_sensors"
|
||||
K_META = "jbod:poll:meta"
|
||||
K_HOST_META = "jbod:host_poll:meta"
|
||||
K_LED_QUEUE = "jbod:led:queue"
|
||||
K_LOCK = "jbod:poll:lock"
|
||||
K_HOST_LOCK = "jbod:host_poll:lock"
|
||||
|
||||
|
||||
# ── writes (poller) ─────────────────────────────────────────────────────
|
||||
@@ -49,9 +52,13 @@ async def set_host_drives(host_drives: list) -> None:
|
||||
await cache_set(K_HOST_DRIVES, host_drives, DATA_TTL)
|
||||
|
||||
|
||||
async def set_meta(meta: dict) -> None:
|
||||
async def set_host_sensors(sensors: dict) -> None:
|
||||
await cache_set(K_HOST_SENSORS, sensors, DATA_TTL)
|
||||
|
||||
|
||||
async def set_meta(meta: dict, key: str = K_META) -> None:
|
||||
# Meta has no TTL — it's the heartbeat; staleness is judged from its ts.
|
||||
await cache_set(K_META, meta, 0)
|
||||
await cache_set(key, meta, 0)
|
||||
|
||||
|
||||
# ── reads (consumers) ───────────────────────────────────────────────────
|
||||
@@ -75,8 +82,12 @@ async def get_host_drives() -> list:
|
||||
return await cache_get(K_HOST_DRIVES) or []
|
||||
|
||||
|
||||
async def get_meta() -> dict | None:
|
||||
return await cache_get(K_META)
|
||||
async def get_host_sensors() -> dict | None:
|
||||
return await cache_get(K_HOST_SENSORS)
|
||||
|
||||
|
||||
async def get_meta(key: str = K_META) -> dict | None:
|
||||
return await cache_get(key)
|
||||
|
||||
|
||||
# ── locate-LED command queue ────────────────────────────────────────────
|
||||
@@ -126,20 +137,20 @@ _REFRESH_LUA = (
|
||||
)
|
||||
|
||||
|
||||
async def acquire_lock(token: str, ttl: int = 30) -> bool:
|
||||
"""Atomically claim the poller lock (or re-own it). True if held."""
|
||||
async def acquire_lock(token: str, ttl: int = 30, key: str = K_LOCK) -> bool:
|
||||
"""Atomically claim a single-instance lock (or re-own it). True if held."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return True # no Redis → no coordination possible; run anyway
|
||||
return bool(await client.eval(_ACQUIRE_LUA, 1, K_LOCK, token, ttl))
|
||||
return bool(await client.eval(_ACQUIRE_LUA, 1, key, token, ttl))
|
||||
|
||||
|
||||
async def refresh_lock(token: str, ttl: int = 30) -> bool:
|
||||
async def refresh_lock(token: str, ttl: int = 30, key: str = K_LOCK) -> bool:
|
||||
"""Atomically extend the lock iff we still own it."""
|
||||
client = get_client()
|
||||
if client is None:
|
||||
return True
|
||||
return bool(await client.eval(_REFRESH_LUA, 1, K_LOCK, token, ttl))
|
||||
return bool(await client.eval(_REFRESH_LUA, 1, key, token, ttl))
|
||||
|
||||
|
||||
def now() -> float:
|
||||
|
||||
Reference in New Issue
Block a user