From 4c68bff9648ce95f2ccde56681560bd4861269d8 Mon Sep 17 00:00:00 2001 From: adam Date: Tue, 16 Jun 2026 19:41:51 +0000 Subject: [PATCH] Add k8s DaemonSet design for RKE2 nodes (Model A, per-node self-contained) Design artifact (NOT applied). Per-node pod = host-poller + redis + web sharing pod localhost; only host-poller privileged with hostPath /dev+/sys. No SAS poller (no enclosures). MQTT_NODE_ID from spec.nodeName; MQTT creds via VSO VaultStaticSecret reading secret/home_assistant (mqtt_user/pass -> MQTT_USERNAME/ PASSWORD). nodeSelector jbod-monitor=enabled to pin to pascal/currie/fermi. --- k8s/README.md | 62 +++++++++++++++++++ k8s/daemonset.yaml | 104 ++++++++++++++++++++++++++++++++ k8s/mqtt-vaultstaticsecret.yaml | 30 +++++++++ 3 files changed, 196 insertions(+) create mode 100644 k8s/README.md create mode 100644 k8s/daemonset.yaml create mode 100644 k8s/mqtt-vaultstaticsecret.yaml diff --git a/k8s/README.md b/k8s/README.md new file mode 100644 index 0000000..743af39 --- /dev/null +++ b/k8s/README.md @@ -0,0 +1,62 @@ +# jbod-monitor on RKE2 (DaemonSet) + +Per-node self-contained deployment for Kubernetes worker nodes that can't run +the docker-compose stack. Each opted-in node runs one pod (`host-poller` + +`redis` + `web`) and publishes its own Home Assistant device +(`JBOD Monitor ()`) over MQTT. + +This mirrors the docker-compose stack with **no code changes** — each node is +independent, exactly like the standalone hosts. + +## What it collects + +- **host-poller** (privileged, hostPath `/dev`+`/sys`): IPMI/iDRAC env, CPU + coretemp, on-metal drive SMART → node-local redis. +- **web**: reads redis, publishes per-node MQTT discovery to the broker. +- No SAS poller (no SES enclosures on these nodes). Consequence: host-drive + entities have no ZFS pool label (no zfs_map producer). + +## Prereqs to verify before applying + +1. **Registry pull**: confirm the cluster can pull `docker.adamksmith.xyz/*`. + If it needs auth, sync a `regcred` into the `jbod-monitor` namespace and + uncomment `imagePullSecrets` in `daemonset.yaml`. +2. **VSO**: `vault-secrets-operator-system/openbao-kubernetes` VaultAuth exists + (it does, per infra notes). Confirm the `transformation.templates` syntax + matches the installed VSO version. +3. **MQTT egress**: pods must reach `10.5.30.3:1883` (EMQX). + +## Apply order + +```bash +# 1. Label ONLY the intended nodes (do not blanket-label all workers): +kubectl label node usco-dc-pascal usco-dc-currie usco-dc-fermi jbod-monitor=enabled + +# 2. Namespace + DaemonSet: +kubectl apply -f k8s/daemonset.yaml + +# 3. MQTT creds (VSO → Secret jbod-mqtt). Apply, then confirm it synced: +kubectl apply -f k8s/mqtt-vaultstaticsecret.yaml +kubectl -n jbod-monitor get vaultstaticsecret jbod-mqtt # SYNCED=True +kubectl -n jbod-monitor get secret jbod-mqtt # has MQTT_USERNAME/PASSWORD + +# 4. Roll the web container so it picks up the secret (if it started first): +kubectl -n jbod-monitor rollout restart daemonset/jbod-monitor +``` + +## Verify + +```bash +kubectl -n jbod-monitor get pods -o wide # one per labeled node +kubectl -n jbod-monitor logs -c host-poller | grep sweep +kubectl -n jbod-monitor logs -c web | grep -i mqtt # "MQTT connected" +``` + +Then check Home Assistant for `JBOD Monitor (usco-dc-pascal|currie|fermi)`. + +## Notes + +- pascal (R620) already has an iDRAC "System Board Inlet Temp" in HA via another + integration — its env temp will partially duplicate; drives + CPU are net-new. +- Pin image tags by SHA (already done: host-poller `514502e`, web `1839e9d`). +- Removal: `kubectl delete -f k8s/daemonset.yaml` and unlabel the nodes. diff --git a/k8s/daemonset.yaml b/k8s/daemonset.yaml new file mode 100644 index 0000000..8f47d84 --- /dev/null +++ b/k8s/daemonset.yaml @@ -0,0 +1,104 @@ +# jbod-monitor on RKE2 nodes — per-node self-contained DaemonSet (Model A). +# +# Each scheduled node runs one pod = host-poller + redis + web, sharing the +# pod's localhost (no hostNetwork needed). Only host-poller is privileged and +# mounts /dev + /sys to read IPMI / SMART / coretemp. web egresses to the MQTT +# broker and publishes a per-node HA device (MQTT_NODE_ID = spec.nodeName). +# +# No SAS poller (these nodes have no SES enclosures). NOTE: without it there is +# no zfs_map, so host-drive entities won't carry a ZFS pool label. +# +# Target nodes are chosen by label — see k8s/README.md (label only +# pascal/currie/fermi). Apply order: namespace -> VaultStaticSecret -> this. +--- +apiVersion: v1 +kind: Namespace +metadata: + name: jbod-monitor +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: jbod-monitor + namespace: jbod-monitor + labels: + app: jbod-monitor +spec: + selector: + matchLabels: + app: jbod-monitor + template: + metadata: + labels: + app: jbod-monitor + spec: + # Only nodes explicitly opted in (see README: kubectl label node ...). + nodeSelector: + jbod-monitor: "enabled" + # If the registry needs auth, sync a regcred and uncomment: + # imagePullSecrets: + # - name: regcred + containers: + - name: redis + image: redis:7-alpine + args: ["redis-server", "--save", "60", "1", "--loglevel", "warning"] + volumeMounts: + - name: redis-data + mountPath: /data + resources: + requests: {cpu: 25m, memory: 32Mi} + limits: {memory: 128Mi} + + - name: host-poller + image: docker.adamksmith.xyz/jbod-host-poller:514502e + securityContext: + privileged: true # raw /dev access for ipmitool + smartctl + env: + - {name: TZ, value: "America/Denver"} + - {name: REDIS_HOST, value: "127.0.0.1"} + - {name: REDIS_PORT, value: "6379"} + - {name: HOST_ENV_INTERVAL, value: "60"} + - {name: HOST_DRIVE_INTERVAL, value: "300"} + - {name: POLL_CONCURRENCY, value: "1"} + - {name: POLL_DRIVE_GAP, value: "0.5"} + - {name: POLL_STARTUP_JITTER, value: "10"} + - {name: POLL_DATA_TTL, value: "900"} + volumeMounts: + - {name: dev, mountPath: /dev} + - {name: sys, mountPath: /sys, readOnly: true} + - {name: udev, mountPath: /run/udev, readOnly: true} + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {memory: 256Mi} + + - name: web + image: docker.adamksmith.xyz/jbod-web:1839e9d + env: + - {name: TZ, value: "America/Denver"} + - {name: REDIS_HOST, value: "127.0.0.1"} + - {name: REDIS_PORT, value: "6379"} + - {name: MQTT_HOST, value: "10.5.30.3"} + - {name: MQTT_PORT, value: "1883"} + - {name: MQTT_DISCOVERY_PREFIX, value: "homeassistant"} + - {name: MQTT_BASE_TOPIC, value: "jbod-monitor"} + - {name: MQTT_PUBLISH_INTERVAL, value: "60"} + - name: MQTT_NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + envFrom: + - secretRef: + name: jbod-mqtt # MQTT_USERNAME / MQTT_PASSWORD (see VaultStaticSecret) + resources: + requests: {cpu: 25m, memory: 64Mi} + limits: {memory: 256Mi} + + volumes: + - name: redis-data + emptyDir: {} + - name: dev + hostPath: {path: /dev} + - name: sys + hostPath: {path: /sys} + - name: udev + hostPath: {path: /run/udev} diff --git a/k8s/mqtt-vaultstaticsecret.yaml b/k8s/mqtt-vaultstaticsecret.yaml new file mode 100644 index 0000000..b1ac7de --- /dev/null +++ b/k8s/mqtt-vaultstaticsecret.yaml @@ -0,0 +1,30 @@ +# MQTT broker creds for the web container, synced from OpenBao by VSO. +# +# Reads secret/home_assistant (keys mqtt_user / mqtt_pass) and renders a k8s +# Secret `jbod-mqtt` with MQTT_USERNAME / MQTT_PASSWORD keys (consumed via +# envFrom in the DaemonSet). VSO's `vso-read` policy can read secret/data/*, +# so unlike the claude-read token it CAN read home_assistant — no creds ever +# touch these manifests. +# +# Verify the transformation syntax against the installed VSO version. +apiVersion: secrets.hashicorp.com/v1beta1 +kind: VaultStaticSecret +metadata: + name: jbod-mqtt + namespace: jbod-monitor +spec: + vaultAuthRef: vault-secrets-operator-system/openbao-kubernetes + mount: secret + type: kv-v2 + path: home_assistant + refreshAfter: 1h + destination: + name: jbod-mqtt + create: true + transformation: + excludeRaw: true + templates: + MQTT_USERNAME: + text: '{{ .Secrets.mqtt_user }}' + MQTT_PASSWORD: + text: '{{ .Secrets.mqtt_pass }}'