k8s: exclude passthrough source keys from jbod-mqtt secret

excludeRaw alone left all home_assistant source keys (api_key, vantage_*, etc.)
in the synced secret, so envFrom injected them into the web container. Add
transformation.excludes ['.*'] so only the templated MQTT_USERNAME/MQTT_PASSWORD
remain.

compose.infra.yml

@@ -52,7 +52,8 @@ services:
       - REDIS_HOST=127.0.0.1
       - REDIS_PORT=6379
       - REDIS_DB=0
-      - HOST_POLL_INTERVAL=120
+      - HOST_ENV_INTERVAL=60     # IPMI/iDRAC + CPU — responsive (inlet temp 1/min)
+      - HOST_DRIVE_INTERVAL=300  # on-metal SMART — gentle, isolated from env
       - POLL_DRIVE_GAP=0.5
       - POLL_CONCURRENCY=1
       - POLL_STARTUP_JITTER=10

host_poller.py

@@ -1,12 +1,16 @@
-"""Host-poller — server chassis temps (IPMI/iDRAC + CPU) and on-metal drives.
+"""Host-poller — server chassis temps, split into two independent loops.
 
-Separate process/container from the SAS poller: this owns the chassis
-subsystem (in-band IPMI, CPU coretemp, PERC/onboard drives), which is
-independent of the external SAS shelves and shouldn't share their hardware
-gate's bus. Writes to Redis; the web/MQTT consumer publishes these on the hub
-device. Own single-instance lock + heartbeat; paced via the per-process gate.
+Separate process/container from the SAS poller. Two concurrent loops in one
+process, sharing the single-instance lock AND the per-process hardware gate
+(so IPMI and SMART never hit hardware at the same instant), but on separate
+schedules:
 
-Host-drive ZFS annotation reuses jbod:zfs_map written by the SAS poller.
+  - env loop   (fast):  IPMI/iDRAC env + CPU coretemp  -> jbod:host_sensors
+  - drive loop (slow):  on-metal drive SMART           -> jbod:host_drives
+
+Keeping SMART on a gentler cadence than the responsive inlet/CPU reads. Each
+loop has its own heartbeat + restart-storm guard. Host-drive ZFS annotation
+reuses jbod:zfs_map written by the SAS poller.
 """
 import asyncio
 import logging
@@ -18,7 +22,7 @@ import time
 from services import store
 from services.cache import close_cache, init_cache, redis_available
 from services.host import get_host_drives
-from services.host_sensors import read_coretemp, read_ipmi_temps
+from services.host_sensors import read_coretemp, read_hwmon_env, read_ipmi_temps
 
 logging.basicConfig(
     level=logging.INFO,
@@ -26,25 +30,24 @@ logging.basicConfig(
 )
 logger = logging.getLogger("host-poller")
 
-SWEEP_INTERVAL = int(os.environ.get("HOST_POLL_INTERVAL", "120"))
+ENV_INTERVAL = int(os.environ.get("HOST_ENV_INTERVAL", "60"))
+DRIVE_INTERVAL = int(os.environ.get("HOST_DRIVE_INTERVAL", "300"))
 STARTUP_JITTER = float(os.environ.get("POLL_STARTUP_JITTER", "10"))
 LOCK_TTL = 30
 
 TOKEN = f"{socket.gethostname()}-hostpoll-{os.getpid()}"
 
 
-async def sweep() -> None:
+async def sweep_env() -> None:
+    """IPMI/iDRAC environmental + CPU temps (the responsive loop)."""
     t0 = time.monotonic()
     errors: list[str] = []
-
-    # 1. IPMI (iDRAC) environmental + CPU temps.
     try:
         ipmi = await read_ipmi_temps()
     except Exception as e:
         errors.append(f"ipmi:{e}")
         ipmi = []
-    env = [s for s in ipmi if s.get("kind") == "env"]
-    # Prefer kernel coretemp for CPU package temps; fall back to IPMI CPU temps.
+    env = [s for s in ipmi if s.get("kind") == "env"] + read_hwmon_env()
     cpu = read_coretemp() or [s for s in ipmi if s.get("kind") == "cpu"]
 
     await store.set_host_sensors({
@@ -53,30 +56,57 @@ async def sweep() -> None:
         "cpu": cpu,
         "ts": store.now(),
     })
-
-    # 2. On-metal (non-enclosure) drives — annotated from the SAS poller's zfs map.
-    try:
-        pool_map = await store.get_zfs_map()
-        host_drives = await get_host_drives(pool_map)
-        await store.set_host_drives(host_drives)
-    except Exception as e:
-        errors.append(f"hostdrives:{e}")
-        host_drives = []
-
-    duration = round(time.monotonic() - t0, 1)
     await store.set_meta({
         "last_sweep_ts": store.now(),
-        "last_sweep_duration": duration,
+        "last_sweep_duration": round(time.monotonic() - t0, 1),
         "env_count": len(env),
         "cpu_count": len(cpu),
-        "host_drive_count": len(host_drives),
         "errors": errors[:20],
-        "sweep_interval": SWEEP_INTERVAL,
-    }, key=store.K_HOST_META)
-    logger.info(
-        "host sweep done: %d env, %d cpu, %d drives, %.1fs, %d error(s)",
-        len(env), len(cpu), len(host_drives), duration, len(errors),
-    )
+        "interval": ENV_INTERVAL,
+    }, key=store.K_HOST_ENV_META)
+    logger.info("env sweep: %d env, %d cpu, %d error(s)", len(env), len(cpu), len(errors))
+
+
+async def sweep_drives() -> None:
+    """On-metal (non-enclosure) drive SMART (the gentle loop)."""
+    t0 = time.monotonic()
+    errors: list[str] = []
+    drives: list = []
+    try:
+        pool_map = await store.get_zfs_map()
+        drives = await get_host_drives(pool_map)
+        await store.set_host_drives(drives)
+    except Exception as e:
+        errors.append(f"hostdrives:{e}")
+    await store.set_meta({
+        "last_sweep_ts": store.now(),
+        "last_sweep_duration": round(time.monotonic() - t0, 1),
+        "host_drive_count": len(drives),
+        "errors": errors[:20],
+        "interval": DRIVE_INTERVAL,
+    }, key=store.K_HOST_DRIVE_META)
+    logger.info("drive sweep: %d drives, %.1fs, %d error(s)",
+                len(drives), round(time.monotonic() - t0, 1), len(errors))
+
+
+async def run_loop(name: str, fn, interval: int, meta_key: str) -> None:
+    """Run `fn` every `interval`s, with a restart-storm guard from its meta."""
+    meta = await store.get_meta(key=meta_key)
+    if meta and meta.get("last_sweep_ts"):
+        since = store.now() - meta["last_sweep_ts"]
+        if 0 <= since < interval:
+            wait = interval - since
+            logger.info("%s: last sweep %.0fs ago — deferring first %.0fs", name, since, wait)
+            await asyncio.sleep(wait)
+    while True:
+        t0 = time.monotonic()
+        try:
+            await fn()
+        except asyncio.CancelledError:
+            raise
+        except Exception as e:
+            logger.error("%s loop error: %s", name, e)
+        await asyncio.sleep(max(5, interval - (time.monotonic() - t0)))
 
 
 async def lock_refresher() -> None:
@@ -117,30 +147,17 @@ async def main() -> None:
     if os.geteuid() != 0:
         logger.warning("not running as root — ipmitool/smartctl may fail")
 
-    jitter = random.uniform(0, STARTUP_JITTER)
-    logger.info("startup jitter %.1fs", jitter)
-    await asyncio.sleep(jitter)
-
-    # Restart-storm guard (mirrors the SAS poller).
-    meta = await store.get_meta(key=store.K_HOST_META)
-    if meta and meta.get("last_sweep_ts"):
-        since = store.now() - meta["last_sweep_ts"]
-        if 0 <= since < SWEEP_INTERVAL:
-            wait = SWEEP_INTERVAL - since
-            logger.info("last host sweep %.0fs ago — deferring first sweep %.0fs", since, wait)
-            await asyncio.sleep(wait)
+    await asyncio.sleep(random.uniform(0, STARTUP_JITTER))
 
+    env_task = asyncio.create_task(
+        run_loop("env", sweep_env, ENV_INTERVAL, store.K_HOST_ENV_META))
+    drive_task = asyncio.create_task(
+        run_loop("drive", sweep_drives, DRIVE_INTERVAL, store.K_HOST_DRIVE_META))
     try:
-        while True:
-            t0 = time.monotonic()
-            try:
-                await sweep()
-            except Exception as e:
-                logger.error("host sweep error: %s", e)
-            elapsed = time.monotonic() - t0
-            await asyncio.sleep(max(5, SWEEP_INTERVAL - elapsed))
+        await asyncio.gather(env_task, drive_task)
     finally:
-        refresher.cancel()
+        for t in (refresher, env_task, drive_task):
+            t.cancel()
         await close_cache()
 
 

k8s/README.md

@@ -0,0 +1,66 @@
+# jbod-monitor on RKE2 (DaemonSet)
+
+Per-node self-contained deployment for Kubernetes worker nodes that can't run
+the docker-compose stack. Each opted-in node runs one pod (`host-poller` +
+`redis` + `web`) and publishes its own Home Assistant device
+(`JBOD Monitor (<node>)`) over MQTT.
+
+This mirrors the docker-compose stack with **no code changes** — each node is
+independent, exactly like the standalone hosts.
+
+## What it collects
+
+- **host-poller** (privileged, hostPath `/dev`+`/sys`): IPMI/iDRAC env, CPU
+  coretemp, on-metal drive SMART → node-local redis.
+- **web**: reads redis, publishes per-node MQTT discovery to the broker.
+- No SAS poller (no SES enclosures on these nodes). Consequence: host-drive
+  entities have no ZFS pool label (no zfs_map producer).
+
+## Prereqs — verified 2026-06-16
+
+1. **Registry pull**: registry requires auth. `registry-cred-vaultstaticsecret.yaml`
+   syncs `secret/registry/nexus` → `docker-registry-cred` (dockerconfigjson),
+   referenced as `imagePullSecrets` in the DaemonSet. (Mirrors the cluster's
+   existing `*-registry-cred` pattern.)
+2. **VSO**: v1.3.0; `openbao-kubernetes` VaultAuth healthy; `transformation.templates`
+   supported. No existing VSS in the cluster uses transformation, so watch the
+   `jbod-mqtt` VSS status on first apply for template errors.
+3. **MQTT egress**: confirmed — a cluster pod reached `10.5.30.3:1883`.
+4. **Nodes**: pascal/currie/fermi are untainted → no tolerations needed.
+
+## Apply order
+
+```bash
+# 1. Label ONLY the intended nodes (do not blanket-label all workers):
+kubectl label node usco-dc-pascal usco-dc-currie usco-dc-fermi jbod-monitor=enabled
+#    (canary: label just usco-dc-fermi first)
+
+# 2. Namespace + DaemonSet:
+kubectl apply -f k8s/daemonset.yaml
+
+# 3. Secrets (VSO → docker-registry-cred + jbod-mqtt). Apply, then confirm sync:
+kubectl apply -f k8s/registry-cred-vaultstaticsecret.yaml
+kubectl apply -f k8s/mqtt-vaultstaticsecret.yaml
+kubectl -n jbod-monitor get vaultstaticsecret           # SYNCED=True for both
+kubectl -n jbod-monitor get secret jbod-mqtt -o jsonpath='{.data.MQTT_USERNAME}' | base64 -d
+
+# 4. Roll the DaemonSet so pods pick up the secrets if they raced ahead:
+kubectl -n jbod-monitor rollout restart daemonset/jbod-monitor
+```
+
+## Verify
+
+```bash
+kubectl -n jbod-monitor get pods -o wide        # one per labeled node
+kubectl -n jbod-monitor logs <pod> -c host-poller | grep sweep
+kubectl -n jbod-monitor logs <pod> -c web | grep -i mqtt   # "MQTT connected"
+```
+
+Then check Home Assistant for `JBOD Monitor (usco-dc-pascal|currie|fermi)`.
+
+## Notes
+
+- pascal (R620) already has an iDRAC "System Board Inlet Temp" in HA via another
+  integration — its env temp will partially duplicate; drives + CPU are net-new.
+- Pin image tags by SHA (already done: host-poller `514502e`, web `1839e9d`).
+- Removal: `kubectl delete -f k8s/daemonset.yaml` and unlabel the nodes.

k8s/daemonset.yaml

@@ -0,0 +1,103 @@
+# jbod-monitor on RKE2 nodes — per-node self-contained DaemonSet (Model A).
+#
+# Each scheduled node runs one pod = host-poller + redis + web, sharing the
+# pod's localhost (no hostNetwork needed). Only host-poller is privileged and
+# mounts /dev + /sys to read IPMI / SMART / coretemp. web egresses to the MQTT
+# broker and publishes a per-node HA device (MQTT_NODE_ID = spec.nodeName).
+#
+# No SAS poller (these nodes have no SES enclosures). NOTE: without it there is
+# no zfs_map, so host-drive entities won't carry a ZFS pool label.
+#
+# Target nodes are chosen by label — see k8s/README.md (label only
+# pascal/currie/fermi). Apply order: namespace -> VaultStaticSecret -> this.
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: jbod-monitor
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: jbod-monitor
+  namespace: jbod-monitor
+  labels:
+    app: jbod-monitor
+spec:
+  selector:
+    matchLabels:
+      app: jbod-monitor
+  template:
+    metadata:
+      labels:
+        app: jbod-monitor
+    spec:
+      # Only nodes explicitly opted in (see README: kubectl label node ...).
+      nodeSelector:
+        jbod-monitor: "enabled"
+      imagePullSecrets:
+        - name: docker-registry-cred   # synced by registry-cred-vaultstaticsecret.yaml
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          args: ["redis-server", "--save", "60", "1", "--loglevel", "warning"]
+          volumeMounts:
+            - name: redis-data
+              mountPath: /data
+          resources:
+            requests: {cpu: 25m, memory: 32Mi}
+            limits: {memory: 128Mi}
+
+        - name: host-poller
+          image: docker.adamksmith.xyz/jbod-host-poller:514502e
+          securityContext:
+            privileged: true   # raw /dev access for ipmitool + smartctl
+          env:
+            - {name: TZ, value: "America/Denver"}
+            - {name: REDIS_HOST, value: "127.0.0.1"}
+            - {name: REDIS_PORT, value: "6379"}
+            - {name: HOST_ENV_INTERVAL, value: "60"}
+            - {name: HOST_DRIVE_INTERVAL, value: "300"}
+            - {name: POLL_CONCURRENCY, value: "1"}
+            - {name: POLL_DRIVE_GAP, value: "0.5"}
+            - {name: POLL_STARTUP_JITTER, value: "10"}
+            - {name: POLL_DATA_TTL, value: "900"}
+          volumeMounts:
+            - {name: dev, mountPath: /dev}
+            - {name: sys, mountPath: /sys, readOnly: true}
+            - {name: udev, mountPath: /run/udev, readOnly: true}
+          resources:
+            requests: {cpu: 25m, memory: 64Mi}
+            limits: {memory: 256Mi}
+
+        - name: web
+          image: docker.adamksmith.xyz/jbod-web:1839e9d
+          env:
+            - {name: TZ, value: "America/Denver"}
+            - {name: REDIS_HOST, value: "127.0.0.1"}
+            - {name: REDIS_PORT, value: "6379"}
+            - {name: MQTT_HOST, value: "10.5.30.3"}
+            - {name: MQTT_PORT, value: "1883"}
+            - {name: MQTT_DISCOVERY_PREFIX, value: "homeassistant"}
+            - {name: MQTT_BASE_TOPIC, value: "jbod-monitor"}
+            - {name: MQTT_PUBLISH_INTERVAL, value: "60"}
+            - name: MQTT_NODE_ID
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          envFrom:
+            - secretRef:
+                name: jbod-mqtt   # MQTT_USERNAME / MQTT_PASSWORD (see VaultStaticSecret)
+          resources:
+            requests: {cpu: 25m, memory: 64Mi}
+            limits: {memory: 256Mi}
+
+      volumes:
+        - name: redis-data
+          emptyDir: {}
+        - name: dev
+          hostPath: {path: /dev}
+        - name: sys
+          hostPath: {path: /sys}
+        - name: udev
+          hostPath: {path: /run/udev}

k8s/mqtt-vaultstaticsecret.yaml

@@ -0,0 +1,35 @@
+# MQTT broker creds for the web container, synced from OpenBao by VSO.
+#
+# Reads secret/home_assistant (keys mqtt_user / mqtt_pass) and renders a k8s
+# Secret `jbod-mqtt` with MQTT_USERNAME / MQTT_PASSWORD keys (consumed via
+# envFrom in the DaemonSet). VSO's `vso-read` policy can read secret/data/*,
+# so unlike the claude-read token it CAN read home_assistant — no creds ever
+# touch these manifests.
+#
+# Verify the transformation syntax against the installed VSO version.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: jbod-mqtt
+  namespace: jbod-monitor
+spec:
+  vaultAuthRef: vault-secrets-operator-system/openbao-kubernetes
+  mount: secret
+  type: kv-v2
+  path: home_assistant
+  refreshAfter: 1h
+  destination:
+    name: jbod-mqtt
+    create: true
+    transformation:
+      excludeRaw: true
+      # Drop ALL passthrough source keys (api_key, vantage_*, etc.) — keep only
+      # the templated MQTT_USERNAME/MQTT_PASSWORD below. Without this, envFrom
+      # would inject the entire home_assistant secret into the web container.
+      excludes:
+        - ".*"
+      templates:
+        MQTT_USERNAME:
+          text: '{{ .Secrets.mqtt_user }}'
+        MQTT_PASSWORD:
+          text: '{{ .Secrets.mqtt_pass }}'

k8s/registry-cred-vaultstaticsecret.yaml

@@ -0,0 +1,18 @@
+# Image pull secret for docker.adamksmith.xyz, synced by VSO from OpenBao.
+# Mirrors the cluster's existing *-registry-cred pattern (secret/registry/nexus
+# -> dockerconfigjson). Referenced as imagePullSecrets in the DaemonSet.
+apiVersion: secrets.hashicorp.com/v1beta1
+kind: VaultStaticSecret
+metadata:
+  name: docker-registry-cred
+  namespace: jbod-monitor
+spec:
+  vaultAuthRef: vault-secrets-operator-system/openbao-kubernetes
+  mount: secret
+  path: registry/nexus
+  type: kv-v2
+  refreshAfter: 1h
+  destination:
+    name: docker-registry-cred
+    create: true
+    type: kubernetes.io/dockerconfigjson

services/host_sensors.py

@@ -7,7 +7,9 @@ coretemp is a cheap sysfs read.
 """
 import asyncio
 import logging
+import os
 import re
+from collections import Counter
 from pathlib import Path
 
 from services.hwgate import gate
@@ -45,6 +47,10 @@ def _parse_ipmi(text: str) -> list[dict]:
         if len(parts) < 5:
             continue
         name, _sid, status, entity, reading = parts[:5]
+        # Skip relative/derived sensors (e.g. "P1 Therm Margin") — they read in
+        # "degrees C" but are a distance-to-throttle, not an absolute temp.
+        if "margin" in name.lower():
+            continue
         m = re.search(r"(-?\d+(?:\.\d+)?)\s*degrees?\s*C", reading, re.IGNORECASE)
         if not m:  # "no reading", "disabled", "ns", etc.
             continue
@@ -63,6 +69,46 @@ def _parse_ipmi(text: str) -> list[dict]:
     return sensors
 
 
+def read_hwmon_env() -> list[dict]:
+    """Read labelled temps from extra hwmon chips as env sensors.
+
+    For boxes without IPMI (e.g. Dell workstations), the BIOS SMM interface
+    (`dell_smm`) exposes CPU/ambient/SODIMM/board temps. Chips are configurable
+    via HOST_HWMON_CHIPS (comma list; default "dell_smm"); set to "" to disable.
+    Names are "<chip> <label>", with the temp index appended when a label
+    repeats within a chip (e.g. dell_smm's multiple "Other" sensors).
+    """
+    chips = [c.strip() for c in os.environ.get("HOST_HWMON_CHIPS", "dell_smm").split(",") if c.strip()]
+    if not chips or not HWMON.is_dir():
+        return []
+    out: list[dict] = []
+    for hw in sorted(HWMON.iterdir()):
+        try:
+            chip = (hw / "name").read_text().strip()
+        except OSError:
+            continue
+        if chip not in chips:
+            continue
+        entries: list[tuple[str, str, int]] = []  # (idx, label, temp)
+        for label_f in sorted(hw.glob("temp*_label")):
+            try:
+                label = label_f.read_text().strip()
+            except OSError:
+                continue
+            input_f = label_f.with_name(label_f.name.replace("_label", "_input"))
+            try:
+                temp = round(int(input_f.read_text().strip()) / 1000)
+            except (OSError, ValueError):
+                continue
+            idx = label_f.name[len("temp"):-len("_label")]
+            entries.append((idx, label, temp))
+        dup = {lbl for lbl, n in Counter(l for _, l, _ in entries).items() if n > 1}
+        for idx, label, temp in entries:
+            name = f"{chip} {label}" + (f" {idx}" if label in dup else "")
+            out.append({"name": name, "temperature_c": temp, "status": "ok", "kind": "env"})
+    return out
+
+
 def read_coretemp() -> list[dict]:
     """CPU package temps from coretemp hwmon → [{name, temperature_c, kind}]."""
     cpus: list[dict] = []
@@ -74,22 +120,28 @@ def read_coretemp() -> list[dict]:
                 continue
         except OSError:
             continue
+        packages: list[dict] = []
+        cores: list[int] = []
         for label_f in sorted(hw.glob("temp*_label")):
             try:
                 label = label_f.read_text().strip()
             except OSError:
                 continue
-            if not label.lower().startswith("package id"):
-                continue
             input_f = label_f.with_name(label_f.name.replace("_label", "_input"))
             try:
-                milli = int(input_f.read_text().strip())
+                temp = round(int(input_f.read_text().strip()) / 1000)
             except (OSError, ValueError):
                 continue
-            pkg = label.split("id")[-1].strip()
-            cpus.append({
-                "name": f"CPU {pkg}",
-                "temperature_c": round(milli / 1000),
-                "kind": "cpu",
-            })
+            low = label.lower()
+            if low.startswith("package id"):
+                pkg = label.split("id")[-1].strip()
+                packages.append({"name": f"CPU {pkg}", "temperature_c": temp, "kind": "cpu"})
+            elif low.startswith("core"):
+                cores.append(temp)
+        # Prefer the per-package sensor; older CPUs (no package sensor) fall back
+        # to the hottest core as the CPU temp.
+        if packages:
+            cpus.extend(packages)
+        elif cores:
+            cpus.append({"name": "CPU", "temperature_c": max(cores), "kind": "cpu"})
     return cpus

services/store.py

@@ -25,7 +25,8 @@ K_INVENTORY = "jbod:inventory"
 K_HOST_DRIVES = "jbod:host_drives"
 K_HOST_SENSORS = "jbod:host_sensors"
 K_META = "jbod:poll:meta"
-K_HOST_META = "jbod:host_poll:meta"
+K_HOST_ENV_META = "jbod:host_poll:env_meta"
+K_HOST_DRIVE_META = "jbod:host_poll:drive_meta"
 K_LED_QUEUE = "jbod:led:queue"
 K_LOCK = "jbod:poll:lock"
 K_HOST_LOCK = "jbod:host_poll:lock"