diff --git a/test/test_system_status_available_memory.py b/test/test_system_status_available_memory.py new file mode 100644 index 00000000..c08bde22 --- /dev/null +++ b/test/test_system_status_available_memory.py @@ -0,0 +1,92 @@ +"""/api/v3/system/status must report MemAvailable, not just used/total. + +"Memory used %" cannot tell a healthy board from one about to fail. Page cache +counts as used and is reclaimable on demand, so a Pi can read 70% used and be +perfectly fine, or read the same and be minutes from trouble. MemAvailable is +the kernel's own estimate of what a new allocation can actually obtain, and it +is the number that tracked the failure on a 1GB Pi 3B+: healthy running sat at +500MB+, the crash happened at 73MB, and by then fork() was failing -- sshd +could not spawn a session and systemd could not respawn the display, while the +kernel carried on answering pings. + +psutil.virtual_memory().available is MemAvailable on Linux. total - used is not +a substitute: they diverge exactly when unreclaimable memory (shmem, tmpfs) is +in play, which is when the distinction matters. +""" + +import json +import sys +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from flask import Flask + +sys.path.insert(0, str(Path(__file__).parent.parent)) + +MB = 1024 * 1024 + + +@pytest.fixture +def client(): + pytest.importorskip("psutil") + app = Flask(__name__) + app.config["TESTING"] = True + from web_interface.blueprints.api_v3 import api_v3 + for attr in ("config_manager", "plugin_manager", "cache_manager"): + setattr(api_v3, attr, MagicMock()) + if "api_v3" not in app.blueprints: + app.register_blueprint(api_v3, url_prefix="/api/v3") + return app.test_client() + + +def _memory(total_mb, used_mb, available_mb): + m = MagicMock() + m.total = total_mb * MB + m.used = used_mb * MB + m.available = available_mb * MB + m.percent = round(used_mb / total_mb * 100, 1) + return m + + +def _get_status(client, memory): + # The endpoint caches for 10s; bypass so each case is measured fresh. + with patch("web_interface.cache.get_cached", return_value=None), \ + patch("psutil.virtual_memory", return_value=memory), \ + patch("psutil.cpu_percent", return_value=5.0), \ + patch("psutil.boot_time", return_value=0.0): + resp = client.get("/api/v3/system/status") + assert resp.status_code == 200, resp.data + return json.loads(resp.data)["data"] + + +def test_available_memory_is_reported(client): + data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284)) + assert "memory_available_mb" in data + assert data["memory_available_mb"] == pytest.approx(284, abs=0.5) + + +def test_available_is_not_total_minus_used(client): + # The case the readout exists for: 600MB is "not used", but only 300MB can + # actually be allocated. Reporting used% alone would call this healthy. + data = _get_status(client, _memory(total_mb=1000, used_mb=400, available_mb=300)) + + derived = data["memory_total_mb"] - data["memory_used_mb"] + assert derived == pytest.approx(600, abs=1) + assert data["memory_available_mb"] == pytest.approx(300, abs=0.5) + assert data["memory_available_mb"] != pytest.approx(derived, abs=1), \ + "available must come from MemAvailable, not be derived from used" + + +def test_existing_memory_fields_are_unchanged(client): + data = _get_status(client, _memory(total_mb=905, used_mb=620, available_mb=284)) + assert data["memory_total_mb"] == pytest.approx(905, abs=0.5) + assert data["memory_used_mb"] == pytest.approx(620, abs=0.5) + assert "memory_used_percent" in data + + +def test_a_nearly_exhausted_board_reports_a_small_number(client): + # 73MB available is what the board actually read when it stopped being able + # to fork. The readout has to surface that rather than round it away. + data = _get_status(client, _memory(total_mb=905, used_mb=800, available_mb=73)) + assert data["memory_available_mb"] == pytest.approx(73, abs=0.5) diff --git a/web_interface/blueprints/api_v3.py b/web_interface/blueprints/api_v3.py index a5a2979f..35dfd885 100644 --- a/web_interface/blueprints/api_v3.py +++ b/web_interface/blueprints/api_v3.py @@ -1563,6 +1563,11 @@ def get_system_status(): 'memory_used_percent': round(memory_percent, 1), 'memory_total_mb': round(memory.total / (1024 * 1024), 1), 'memory_used_mb': round(memory.used / (1024 * 1024), 1), + # MemAvailable, not total-minus-used: it accounts for reclaimable + # page cache, so it is what actually predicts memory trouble. A + # board can read 70% "used" and be fine, or read the same and be + # about to fail fork(), and only this number tells them apart. + 'memory_available_mb': round(memory.available / (1024 * 1024), 1), 'cpu_temp': round(cpu_temp, 1) if cpu_temp is not None else None, 'disk_used_percent': round(disk_percent, 1), 'disk_total_gb': round(disk.total / (1024 * 1024 * 1024), 1), diff --git a/web_interface/templates/v3/partials/tools.html b/web_interface/templates/v3/partials/tools.html index f4f7cfdd..1d0fb104 100644 --- a/web_interface/templates/v3/partials/tools.html +++ b/web_interface/templates/v3/partials/tools.html @@ -687,12 +687,35 @@

Plugin Health

const mUsedGb = d.memory_used_mb != null ? (d.memory_used_mb / 1024).toFixed(1) : null; const mTotGb = d.memory_total_mb != null ? (d.memory_total_mb / 1024).toFixed(1) : null; const temp = d.cpu_temp != null ? d.cpu_temp + '°C' : 'N/A'; + // Available memory is the number that predicts trouble. When it + // runs out the board does not fail cleanly: fork() starts + // returning ENOMEM, so sshd cannot spawn a session and systemd + // cannot respawn the display, while the kernel keeps answering + // pings. Thresholds are drawn from that failure -- it was + // measured at 73MB free, and healthy running sits well above. + // Round ONCE, then colour and label off the same number. + // The API sends one decimal place, so classifying the raw + // value and displaying the rounded one disagreed at the + // boundaries: 149.6 rendered as "150 MB" in red, and 299.6 as + // "300 MB" in amber, both contradicting the threshold the + // colour claims to apply. Which side of the line a spare + // 0.4MB falls on does not matter; the tile agreeing with + // itself does. + const availMb = d.memory_available_mb == null + ? null : Math.round(d.memory_available_mb); + const availColor = availMb == null ? 'text-gray-400' + : availMb < 150 ? 'text-red-600' + : availMb < 300 ? 'text-amber-500' + : 'text-green-600'; panel.innerHTML = diagTile('fa-microchip', 'text-blue-600', 'CPU Usage', (d.cpu_percent != null ? d.cpu_percent : '--') + '%', null) + diagTile('fa-memory', 'text-green-600', 'Memory', (d.memory_used_percent != null ? d.memory_used_percent : '--') + '%', (mUsedGb && mTotGb) ? `${mUsedGb} / ${mTotGb} GB` : null) + + diagTile('fa-memory', availColor, 'Available Memory', + availMb != null ? `${availMb} MB` : '--', + mTotGb ? `of ${mTotGb} GB total` : null) + diagTile('fa-thermometer-half', 'text-red-600', 'CPU Temp', temp, null) + diagTile('fa-hdd', 'text-indigo-600', 'Disk', (d.disk_used_percent != null ? d.disk_used_percent : '--') + '%',