Add per-enclosure temperature metrics + Home Assistant MQTT publisher
- enclosure: fetch SES element descriptor page (0x07) and label temp/fan/ psu/voltage elements with human-readable names - temps service + /api/temps: per-enclosure hotspot (max across SES sensors and housed drive temps) plus named SES sensor list - mqtt_publisher: paho-mqtt with HA discovery (device per enclosure, hotspot + per-sensor entities), LWT availability, opt-in via MQTT_HOST - secrets: OpenBao KV v2 reader; MQTT creds sourced from secret/home_assistant with env fallback - compose/requirements/README updated
This commit is contained in:
@@ -152,29 +152,88 @@ def _parse_slot_number(entry: Path) -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
async def get_enclosure_status(sg_device: str) -> dict | None:
|
||||
"""Run sg_ses --page=0x02 and parse enclosure health data."""
|
||||
async def _run_sg_ses(sg_device: str, page: str) -> str | None:
|
||||
"""Run sg_ses for a given page, returning decoded stdout or None."""
|
||||
try:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"sg_ses", "--page=0x02", sg_device,
|
||||
"sg_ses", f"--page={page}", sg_device,
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
if proc.returncode != 0:
|
||||
logger.warning("sg_ses failed for %s: %s", sg_device, stderr.decode().strip())
|
||||
logger.warning(
|
||||
"sg_ses page %s failed for %s: %s",
|
||||
page, sg_device, stderr.decode().strip(),
|
||||
)
|
||||
return None
|
||||
return _parse_ses_page02(stdout.decode(errors="replace"))
|
||||
return stdout.decode(errors="replace")
|
||||
except FileNotFoundError:
|
||||
logger.warning("sg_ses not found")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("sg_ses error for %s: %s", sg_device, e)
|
||||
logger.warning("sg_ses page %s error for %s: %s", page, sg_device, e)
|
||||
return None
|
||||
|
||||
|
||||
def _parse_ses_page02(text: str) -> dict:
|
||||
"""Parse sg_ses --page=0x02 text output into structured health data."""
|
||||
async def get_enclosure_status(sg_device: str) -> dict | None:
|
||||
"""Run sg_ses status (0x02) + element descriptor (0x07) pages and parse.
|
||||
|
||||
The element descriptor page provides human-readable names for each
|
||||
element (e.g. "Temp Inlet", "Fan 1"); these are merged into the status
|
||||
elements so callers get labelled sensors. Descriptor lookup is
|
||||
best-effort — if page 0x07 is unsupported, elements fall back to bare
|
||||
indices.
|
||||
"""
|
||||
status_text, desc_text = await asyncio.gather(
|
||||
_run_sg_ses(sg_device, "0x02"),
|
||||
_run_sg_ses(sg_device, "0x07"),
|
||||
)
|
||||
if status_text is None:
|
||||
return None
|
||||
names = _parse_element_descriptors(desc_text) if desc_text else {}
|
||||
return _parse_ses_page02(status_text, names)
|
||||
|
||||
|
||||
def _norm_type(element_type: str) -> str:
|
||||
"""Normalize an SES element type line to a stable key.
|
||||
|
||||
sg_ses appends qualifiers like ", subenclosure id: 0 [ti=2]" to the
|
||||
type line; strip those so the same element type matches across the
|
||||
status (0x02) and element descriptor (0x07) pages.
|
||||
"""
|
||||
return element_type.split(",")[0].strip().lower()
|
||||
|
||||
|
||||
def _parse_element_descriptors(text: str) -> dict[tuple[str, int], str]:
|
||||
"""Parse sg_ses --page=0x07 into {(element_type, index): name}."""
|
||||
names: dict[tuple[str, int], str] = {}
|
||||
sections = re.split(r"(?=\s*Element type:)", text)
|
||||
for section in sections:
|
||||
type_match = re.match(r"\s*Element type:\s*(.+)", section)
|
||||
if not type_match:
|
||||
continue
|
||||
etype = _norm_type(type_match.group(1).strip().rstrip(","))
|
||||
for line in section.splitlines():
|
||||
m = re.match(r"\s*Element (\d+) descriptor:\s*(.*)", line)
|
||||
if not m:
|
||||
continue
|
||||
idx = int(m.group(1))
|
||||
name = m.group(2).strip()
|
||||
if name:
|
||||
names[(etype, idx)] = name
|
||||
return names
|
||||
|
||||
|
||||
def _parse_ses_page02(
|
||||
text: str, names: dict[tuple[str, int], str] | None = None
|
||||
) -> dict:
|
||||
"""Parse sg_ses --page=0x02 text output into structured health data.
|
||||
|
||||
``names`` maps (normalized element type, index) -> descriptor name from
|
||||
the element descriptor page (0x07); when present it labels each element.
|
||||
"""
|
||||
names = names or {}
|
||||
result = {
|
||||
"overall_status": "OK",
|
||||
"psus": [],
|
||||
@@ -192,6 +251,7 @@ def _parse_ses_page02(text: str) -> dict:
|
||||
if not type_match:
|
||||
continue
|
||||
element_type = type_match.group(1).strip().rstrip(",").lower()
|
||||
etype_key = _norm_type(element_type)
|
||||
|
||||
# Find individual element blocks (skip "Overall descriptor")
|
||||
elements = re.split(r"(?=\s*Element \d+ descriptor:)", section)
|
||||
@@ -209,12 +269,15 @@ def _parse_ses_page02(text: str) -> dict:
|
||||
if status.lower() == "not installed":
|
||||
continue
|
||||
|
||||
name = names.get((etype_key, idx))
|
||||
|
||||
if "power supply" in element_type:
|
||||
fail = "Fail=1" in elem_text
|
||||
ac_fail = "AC fail=1" in elem_text
|
||||
dc_fail = "DC fail=1" in elem_text
|
||||
result["psus"].append({
|
||||
"index": idx,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"fail": fail,
|
||||
"ac_fail": ac_fail,
|
||||
@@ -227,6 +290,7 @@ def _parse_ses_page02(text: str) -> dict:
|
||||
rpm = int(rpm_match.group(1)) if rpm_match else None
|
||||
result["fans"].append({
|
||||
"index": idx,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"rpm": rpm,
|
||||
"fail": fail,
|
||||
@@ -243,6 +307,7 @@ def _parse_ses_page02(text: str) -> dict:
|
||||
continue
|
||||
result["temps"].append({
|
||||
"index": idx,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"temperature_c": temp,
|
||||
})
|
||||
@@ -254,6 +319,7 @@ def _parse_ses_page02(text: str) -> dict:
|
||||
voltage = float(volt_match.group(1)) if volt_match else None
|
||||
result["voltages"].append({
|
||||
"index": idx,
|
||||
"name": name,
|
||||
"status": status,
|
||||
"voltage": voltage,
|
||||
})
|
||||
|
||||
226
services/mqtt_publisher.py
Normal file
226
services/mqtt_publisher.py
Normal file
@@ -0,0 +1,226 @@
|
||||
"""MQTT publisher with Home Assistant discovery for enclosure temperatures.
|
||||
|
||||
Opt-in: only starts when ``MQTT_HOST`` is set. Publishes one retained
|
||||
discovery config per HA entity (a device per enclosure, with a hotspot
|
||||
sensor plus one sensor per named SES temperature element) and pushes a
|
||||
single JSON state message per enclosure on each poll. Availability is
|
||||
tracked via an LWT so HA marks entities unavailable if the monitor dies.
|
||||
|
||||
Topic layout (defaults)::
|
||||
|
||||
homeassistant/sensor/<node>_enc<id>/hotspot/config (retained discovery)
|
||||
homeassistant/sensor/<node>_enc<id>/temp<n>/config (retained discovery)
|
||||
jbod-monitor/<node>/status online | offline (LWT)
|
||||
jbod-monitor/<node>/enclosure/<id>/state {"hotspot_c":.., "sensor_0":..}
|
||||
"""
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
|
||||
from services.secrets import read_kv2
|
||||
from services.temps import build_temp_snapshot, get_node_id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
import paho.mqtt.client as mqtt
|
||||
from paho.mqtt.enums import CallbackAPIVersion
|
||||
|
||||
_PAHO_AVAILABLE = True
|
||||
except ImportError: # pragma: no cover - exercised only without the dep
|
||||
mqtt = None
|
||||
CallbackAPIVersion = None
|
||||
_PAHO_AVAILABLE = False
|
||||
|
||||
|
||||
def mqtt_enabled() -> bool:
|
||||
"""MQTT publishing is opt-in via MQTT_HOST."""
|
||||
return bool(os.environ.get("MQTT_HOST"))
|
||||
|
||||
|
||||
def _slug(value: str) -> str:
|
||||
"""Sanitize a string for use in MQTT topics / HA unique_ids."""
|
||||
return re.sub(r"[^a-zA-Z0-9_-]", "_", value).strip("_") or "x"
|
||||
|
||||
|
||||
def _resolve_credentials() -> tuple[str | None, str | None]:
|
||||
"""Resolve broker credentials, preferring OpenBao over plaintext env.
|
||||
|
||||
If ``MQTT_SECRET_PATH`` is set, read username/password from that OpenBao
|
||||
KV v2 path (keys ``username``/``password``). Falls back to the
|
||||
``MQTT_USERNAME``/``MQTT_PASSWORD`` env vars when unset or on failure.
|
||||
"""
|
||||
username = os.environ.get("MQTT_USERNAME") or None
|
||||
password = os.environ.get("MQTT_PASSWORD") or None
|
||||
|
||||
secret_path = os.environ.get("MQTT_SECRET_PATH")
|
||||
if secret_path:
|
||||
user_key = os.environ.get("MQTT_SECRET_USER_KEY", "username")
|
||||
pass_key = os.environ.get("MQTT_SECRET_PASS_KEY", "password")
|
||||
data = read_kv2(secret_path)
|
||||
if data:
|
||||
username = data.get(user_key, username)
|
||||
password = data.get(pass_key, password)
|
||||
logger.info("Loaded MQTT credentials from OpenBao (%s)", secret_path)
|
||||
else:
|
||||
logger.warning(
|
||||
"MQTT_SECRET_PATH set (%s) but OpenBao read failed — "
|
||||
"falling back to env credentials", secret_path,
|
||||
)
|
||||
return username, password
|
||||
|
||||
|
||||
class MqttPublisher:
|
||||
def __init__(self) -> None:
|
||||
self.host = os.environ["MQTT_HOST"]
|
||||
self.port = int(os.environ.get("MQTT_PORT", "1883"))
|
||||
self.username, self.password = _resolve_credentials()
|
||||
self.discovery_prefix = os.environ.get("MQTT_DISCOVERY_PREFIX", "homeassistant")
|
||||
self.base_topic = os.environ.get("MQTT_BASE_TOPIC", "jbod-monitor")
|
||||
self.interval = int(os.environ.get("MQTT_PUBLISH_INTERVAL", "60"))
|
||||
self.node = _slug(get_node_id())
|
||||
|
||||
self.availability_topic = f"{self.base_topic}/{self.node}/status"
|
||||
# unique_ids for which we've already published discovery this connection.
|
||||
self._discovered: set[str] = set()
|
||||
|
||||
self._client = mqtt.Client(
|
||||
CallbackAPIVersion.VERSION2,
|
||||
client_id=os.environ.get("MQTT_CLIENT_ID", f"jbod-monitor-{self.node}"),
|
||||
)
|
||||
if self.username:
|
||||
self._client.username_pw_set(self.username, self.password)
|
||||
self._client.will_set(self.availability_topic, "offline", qos=1, retain=True)
|
||||
self._client.reconnect_delay_set(min_delay=1, max_delay=60)
|
||||
self._client.on_connect = self._on_connect
|
||||
self._client.on_disconnect = self._on_disconnect
|
||||
|
||||
def start(self) -> None:
|
||||
logger.info("MQTT connecting to %s:%d", self.host, self.port)
|
||||
self._client.connect_async(self.host, self.port, keepalive=60)
|
||||
self._client.loop_start()
|
||||
|
||||
def stop(self) -> None:
|
||||
try:
|
||||
self._client.publish(self.availability_topic, "offline", qos=1, retain=True)
|
||||
except Exception:
|
||||
pass
|
||||
self._client.loop_stop()
|
||||
try:
|
||||
self._client.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# paho callbacks (run in the network thread) -----------------------------
|
||||
def _on_connect(self, client, userdata, flags, reason_code, properties=None):
|
||||
if reason_code != 0:
|
||||
logger.warning("MQTT connect failed: %s", reason_code)
|
||||
return
|
||||
logger.info("MQTT connected")
|
||||
# Re-announce discovery + availability on every (re)connect.
|
||||
self._discovered.clear()
|
||||
client.publish(self.availability_topic, "online", qos=1, retain=True)
|
||||
|
||||
def _on_disconnect(self, client, userdata, flags, reason_code, properties=None):
|
||||
logger.warning("MQTT disconnected: %s", reason_code)
|
||||
|
||||
# discovery / state ------------------------------------------------------
|
||||
def _device_block(self, enc: dict) -> dict:
|
||||
enc_id = enc["id"]
|
||||
vendor = enc.get("vendor", "").strip()
|
||||
model = enc.get("model", "").strip()
|
||||
name = " ".join(p for p in [vendor, model] if p) or "JBOD enclosure"
|
||||
return {
|
||||
"identifiers": [f"{self.node}_enc_{_slug(enc_id)}"],
|
||||
"name": f"JBOD {name} ({enc_id})",
|
||||
"manufacturer": vendor or None,
|
||||
"model": model or None,
|
||||
"via_device": f"{self.node}_jbod_monitor",
|
||||
}
|
||||
|
||||
def _publish_discovery(self, enc: dict) -> None:
|
||||
enc_id = enc["id"]
|
||||
enc_slug = _slug(enc_id)
|
||||
state_topic = f"{self.base_topic}/{self.node}/enclosure/{enc_slug}/state"
|
||||
device = self._device_block(enc)
|
||||
|
||||
def announce(object_id: str, payload: dict) -> None:
|
||||
uid = payload["unique_id"]
|
||||
if uid in self._discovered:
|
||||
return
|
||||
topic = (
|
||||
f"{self.discovery_prefix}/sensor/"
|
||||
f"{self.node}_enc{enc_slug}/{object_id}/config"
|
||||
)
|
||||
self._client.publish(topic, json.dumps(payload), qos=1, retain=True)
|
||||
self._discovered.add(uid)
|
||||
|
||||
common = {
|
||||
"device_class": "temperature",
|
||||
"unit_of_measurement": "°C",
|
||||
"state_class": "measurement",
|
||||
"state_topic": state_topic,
|
||||
"availability_topic": self.availability_topic,
|
||||
"device": device,
|
||||
}
|
||||
|
||||
# Hotspot (max across SES sensors + drive temps).
|
||||
announce("hotspot", {
|
||||
**common,
|
||||
"name": "Hotspot",
|
||||
"unique_id": f"{self.node}_enc{enc_slug}_hotspot",
|
||||
"value_template": "{{ value_json.hotspot_c }}",
|
||||
"icon": "mdi:thermometer-high",
|
||||
"json_attributes_topic": state_topic,
|
||||
"json_attributes_template":
|
||||
'{"drive_max_c": {{ value_json.drive_max_c | default("null") }}, '
|
||||
'"drive_temp_count": {{ value_json.drive_temp_count | default(0) }}}',
|
||||
})
|
||||
|
||||
# One sensor per named SES temperature element.
|
||||
for s in enc.get("sensors", []):
|
||||
idx = s["index"]
|
||||
label = s.get("name") or f"Temp {idx}"
|
||||
announce(f"temp{idx}", {
|
||||
**common,
|
||||
"name": label,
|
||||
"unique_id": f"{self.node}_enc{enc_slug}_temp{idx}",
|
||||
"value_template": f"{{{{ value_json.sensor_{idx} }}}}",
|
||||
})
|
||||
|
||||
def _publish_state(self, enc: dict) -> None:
|
||||
enc_slug = _slug(enc["id"])
|
||||
state_topic = f"{self.base_topic}/{self.node}/enclosure/{enc_slug}/state"
|
||||
payload: dict = {
|
||||
"hotspot_c": enc.get("hotspot_c"),
|
||||
"drive_max_c": enc.get("drive_max_c"),
|
||||
"drive_temp_count": enc.get("drive_temp_count", 0),
|
||||
}
|
||||
for s in enc.get("sensors", []):
|
||||
payload[f"sensor_{s['index']}"] = s.get("temperature_c")
|
||||
payload[f"sensor_{s['index']}_status"] = s.get("status")
|
||||
self._client.publish(state_topic, json.dumps(payload), qos=0, retain=True)
|
||||
|
||||
def publish_snapshot(self, snapshot: dict) -> None:
|
||||
for enc in snapshot.get("enclosures", []):
|
||||
self._publish_discovery(enc)
|
||||
self._publish_state(enc)
|
||||
|
||||
async def run(self) -> None:
|
||||
"""Background loop: build a snapshot and publish on an interval."""
|
||||
await asyncio.sleep(3) # let the first SMART poll populate the cache
|
||||
while True:
|
||||
try:
|
||||
snapshot = await build_temp_snapshot()
|
||||
self.publish_snapshot(snapshot)
|
||||
logger.info(
|
||||
"MQTT published temps for %d enclosure(s)",
|
||||
len(snapshot.get("enclosures", [])),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
except Exception as e:
|
||||
logger.error("MQTT publish loop error: %s", e)
|
||||
await asyncio.sleep(self.interval)
|
||||
58
services/secrets.py
Normal file
58
services/secrets.py
Normal file
@@ -0,0 +1,58 @@
|
||||
"""Minimal OpenBao (Vault) KV v2 reader for runtime secret injection.
|
||||
|
||||
Mirrors the homelab pattern used by other non-k8s services: read
|
||||
``<addr>/v1/<mount>/data/<path>`` with an ``X-Vault-Token``. Stdlib only —
|
||||
no extra dependency. Token comes from ``OPENBAO_TOKEN`` or, preferably, a
|
||||
mounted token file via ``OPENBAO_TOKEN_FILE`` (Docker/host secret).
|
||||
"""
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_ADDR = "https://vault.adamksmith.xyz"
|
||||
DEFAULT_MOUNT = "secret"
|
||||
|
||||
|
||||
def _resolve_token() -> str | None:
|
||||
token = os.environ.get("OPENBAO_TOKEN")
|
||||
if token:
|
||||
return token.strip()
|
||||
token_file = os.environ.get("OPENBAO_TOKEN_FILE")
|
||||
if token_file:
|
||||
try:
|
||||
return open(token_file).read().strip()
|
||||
except OSError as e:
|
||||
logger.warning("Could not read OPENBAO_TOKEN_FILE %s: %s", token_file, e)
|
||||
return None
|
||||
|
||||
|
||||
def read_kv2(path: str) -> dict | None:
|
||||
"""Read a KV v2 secret. ``path`` is relative to the mount (e.g.
|
||||
"jbod-monitor/mqtt"). Returns the inner data dict, or None on failure.
|
||||
"""
|
||||
token = _resolve_token()
|
||||
if not token:
|
||||
logger.warning("OpenBao read requested for %s but no token configured", path)
|
||||
return None
|
||||
|
||||
addr = os.environ.get("OPENBAO_ADDR", DEFAULT_ADDR).rstrip("/")
|
||||
mount = os.environ.get("OPENBAO_MOUNT", DEFAULT_MOUNT).strip("/")
|
||||
url = f"{addr}/v1/{mount}/data/{path.strip('/')}"
|
||||
|
||||
req = urllib.request.Request(url, headers={"X-Vault-Token": token})
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=10) as resp:
|
||||
body = json.loads(resp.read())
|
||||
except urllib.error.HTTPError as e:
|
||||
logger.warning("OpenBao read %s failed: HTTP %s", path, e.code)
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.warning("OpenBao read %s failed: %s", path, e)
|
||||
return None
|
||||
|
||||
# KV v2 wraps the secret in data.data.
|
||||
return body.get("data", {}).get("data")
|
||||
84
services/temps.py
Normal file
84
services/temps.py
Normal file
@@ -0,0 +1,84 @@
|
||||
"""Per-enclosure temperature aggregation.
|
||||
|
||||
Builds a compact snapshot of enclosure temperatures suitable for Home
|
||||
Assistant: each named SES temperature sensor plus a derived per-enclosure
|
||||
hotspot (the hottest reading across SES sensors *and* the drives housed in
|
||||
that enclosure — drive temps are usually the real hotspot signal).
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import socket
|
||||
|
||||
from services.enclosure import discover_enclosures, get_enclosure_status, list_slots
|
||||
from services.smart import get_smart_data
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def get_node_id() -> str:
|
||||
"""Stable identifier for this monitor instance (defaults to hostname)."""
|
||||
import os
|
||||
|
||||
return os.environ.get("MQTT_NODE_ID") or socket.gethostname() or "jbod-monitor"
|
||||
|
||||
|
||||
async def build_temp_snapshot() -> dict:
|
||||
"""Collect per-enclosure temperature metrics for all enclosures."""
|
||||
enclosures = discover_enclosures()
|
||||
|
||||
async def _health(enc):
|
||||
if enc.get("sg_device"):
|
||||
return await get_enclosure_status(enc["sg_device"])
|
||||
return None
|
||||
|
||||
health_results = await asyncio.gather(
|
||||
*[_health(enc) for enc in enclosures], return_exceptions=True
|
||||
)
|
||||
|
||||
out_enclosures: list[dict] = []
|
||||
for enc, health in zip(enclosures, health_results):
|
||||
if isinstance(health, Exception):
|
||||
logger.warning("SES health failed for %s: %s", enc["id"], health)
|
||||
health = None
|
||||
|
||||
sensors: list[dict] = []
|
||||
sensor_temps: list[float] = []
|
||||
if isinstance(health, dict):
|
||||
for t in health.get("temps", []):
|
||||
temp = t.get("temperature_c")
|
||||
sensors.append({
|
||||
"index": t["index"],
|
||||
"name": t.get("name"),
|
||||
"status": t.get("status", "Unknown"),
|
||||
"temperature_c": temp,
|
||||
})
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
sensor_temps.append(float(temp))
|
||||
|
||||
# Drive temperatures for drives housed in this enclosure.
|
||||
drive_temps: list[float] = []
|
||||
devices = [s["device"] for s in list_slots(enc["id"]) if s.get("device")]
|
||||
if devices:
|
||||
smart_results = await asyncio.gather(
|
||||
*[get_smart_data(d) for d in devices], return_exceptions=True
|
||||
)
|
||||
for res in smart_results:
|
||||
if isinstance(res, Exception):
|
||||
continue
|
||||
data, _hit = res
|
||||
temp = data.get("temperature_c")
|
||||
if isinstance(temp, (int, float)) and temp > 0:
|
||||
drive_temps.append(float(temp))
|
||||
|
||||
all_temps = sensor_temps + drive_temps
|
||||
out_enclosures.append({
|
||||
"id": enc["id"],
|
||||
"vendor": enc.get("vendor", ""),
|
||||
"model": enc.get("model", ""),
|
||||
"hotspot_c": max(all_temps) if all_temps else None,
|
||||
"drive_max_c": max(drive_temps) if drive_temps else None,
|
||||
"drive_temp_count": len(drive_temps),
|
||||
"sensors": sensors,
|
||||
})
|
||||
|
||||
return {"node": get_node_id(), "enclosures": out_enclosures}
|
||||
Reference in New Issue
Block a user