From 3c23527ddd77335e7a2ef65585b46560b3401b4d Mon Sep 17 00:00:00 2001 From: JarbasAi Date: Sat, 12 Sep 2026 12:28:13 +0100 Subject: [PATCH] feat: add ovos-logs containers subcommand bridging Docker/Podman stdout Implements OpenVoiceOS/ovos-utils#410: on an ovos-docker install with "logs": {"path": "stdout"} there are no host log files, so ovos-logs has nothing to read. New ovos_utils.container_logs module discovers running OVOS/HiveMind containers (docker or podman), maps each to the same per-category name a file-based install uses (skills/audio/voice/bus/ phal/gui/other, ported from andlo's tested ovos-tui-client reference), and appends `docker logs -f --tail 0` output to shared per-category files so every existing ovos-logs consumer works unchanged. The new `ovos-logs containers` subcommand runs the bridge in the foreground until interrupted; point any other ovos-logs command's -p at the printed directory to read it. Co-Authored-By: Claude Code --- ovos_utils/container_logs.py | 173 +++++++++++++++++++++++++ ovos_utils/log_parser.py | 62 +++++++++ test/unittests/test_container_logs.py | 178 ++++++++++++++++++++++++++ 3 files changed, 413 insertions(+) create mode 100644 ovos_utils/container_logs.py create mode 100644 test/unittests/test_container_logs.py diff --git a/ovos_utils/container_logs.py b/ovos_utils/container_logs.py new file mode 100644 index 00000000..e1af4c85 --- /dev/null +++ b/ovos_utils/container_logs.py @@ -0,0 +1,173 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Bridges Docker/Podman container stdout into the small, familiar set of +per-category log files a file-based install already produces (skills.log, +audio.log, voice.log, bus.log, phal.log, gui.log, plus other.log for +anything uncategorized), so ``ovos-logs`` and every other consumer of +:func:`ovos_utils.log.get_available_logs` can read a Docker/Podman install +the same way it reads a systemd/venv one. + +On a Docker/Podman install following ``ovos-docker``'s own documented example +config (``"logs": {"path": "stdout"}``), there are no log files on the host +at all - only container stdout - so tooling built around log files has +nothing to work with there. This module makes container stdout look like an +ordinary log directory instead of teaching every consumer a second, parallel +way to receive log lines. + +Ported from a working, tested reference implementation contributed by +andlo (https://github.com/andlo/ovos-tui-client/blob/main/ovos_tui_client/services.py), +confirmed there against a real, running ovos-docker install (26 containers). +""" +import subprocess +from pathlib import Path +from typing import List, Optional + + +def find_container_binary() -> Optional[str]: + """Returns "docker" or "podman", whichever actually works, or None.""" + for binary in ("docker", "podman"): + try: + result = subprocess.run([binary, "ps"], capture_output=True, + timeout=5) + except (subprocess.SubprocessError, FileNotFoundError, OSError): + continue + if result.returncode == 0: + return binary + return None + + +def list_container_names(binary: Optional[str] = None) -> List[str]: + """Returns a sorted list of container names that look OVOS-related + (containing "ovos" or "hivemind", case-insensitive, matching + ovos-docker's own naming convention), or [] if no runtime is available + or none match. + + @param binary: "docker" or "podman"; auto-detected if not given + """ + binary = binary or find_container_binary() + if binary is None: + return [] + try: + result = subprocess.run([binary, "ps", "--format", "{{.Names}}"], + capture_output=True, text=True, timeout=5) + except (subprocess.SubprocessError, FileNotFoundError, OSError): + return [] + if result.returncode != 0: + return [] + names = [n.strip() for n in result.stdout.splitlines() if n.strip()] + matching = [n for n in names if "ovos" in n.lower() or "hivemind" in n.lower()] + return sorted(matching) + + +def categorize_container_name(name: str) -> str: + """Maps a Docker/Podman container name to the same category names a + file-based install already uses - "skills", "audio", "voice", "bus", + "phal", "gui" - or "other" if it doesn't recognizably fit one of those. + + Pattern-matched, not an exhaustive lookup table: "ovos_skill_" (any + suffix) always maps to "skills", matching how every individual skill + already shares one skills.log file on a normal install, not one file + per skill. "ovos_core" also maps to "skills" specifically because its + own log content (intent-service/pipeline handling) is exactly what + already lands in skills.log on a normal install. + + @param name: container name, e.g. "ovos_skill_alarm" or "ovos_audio" + @return: one of "skills", "audio", "voice", "bus", "phal", "gui", "other" + """ + n = name.lower() + if n.startswith("ovos_skill") or n == "ovos_core": + return "skills" + if n == "ovos_audio": + return "audio" + if n == "ovos_listener": + return "voice" + if n == "ovos_messagebus": + return "bus" + if n.startswith("ovos_phal"): + return "phal" + if "gui" in n: + return "gui" + return "other" + + +def start_container_log_bridges(container_names: List[str], + target_dir: Path) -> List[subprocess.Popen]: + """Bridges Docker/Podman container stdout into the same small set of + per-category log files a file-based install already produces, under + ``target_dir`` - not one file per container. That directory can then be + passed as-is to any ``ovos-logs`` command's ``--paths``/``-p`` option (or + to :func:`ovos_utils.log.get_available_logs`), reusing 100% of the + existing file-tailing/coloring/filtering machinery. + + Multiple containers sharing a category (most commonly "skills" - every + ``ovos_skill_*`` container) each get their own ``docker logs -f`` + subprocess, but all of them append to the same shared file - concurrent + appends from separate processes are safe here without explicit locking, + since POSIX guarantees a single write() to a file opened with O_APPEND is + atomic as long as it is smaller than PIPE_BUF (4096 bytes on Linux), + which holds for any normal single log line. + + Returns a list of ``subprocess.Popen`` handles - the caller owns their + lifecycle and MUST terminate them (see :func:`stop_container_log_bridges`); + they are not cleaned up automatically here. Returns ``[]`` immediately + (no processes started) if neither docker nor podman is available. + + @param container_names: container names to bridge, e.g. from + :func:`list_container_names` + @param target_dir: directory the per-category ``.log`` files are + appended to; created if missing + """ + binary = find_container_binary() + if binary is None: + return [] + target_dir = Path(target_dir) + target_dir.mkdir(parents=True, exist_ok=True) + handles = [] + for name in container_names: + category = categorize_container_name(name) + log_path = target_dir / f"{category}.log" + log_file = open(log_path, "a") + try: + proc = subprocess.Popen( + [binary, "logs", "-f", "--tail", "0", name], + stdout=log_file, stderr=subprocess.STDOUT, + ) + except (subprocess.SubprocessError, FileNotFoundError, OSError): + log_file.close() + continue + handles.append(proc) + return handles + + +def stop_container_log_bridges(handles: List[subprocess.Popen]) -> None: + """Terminates every subprocess started by + :func:`start_container_log_bridges`. Gives each a moment to exit + cleanly before force-killing, and never raises even if a process + already exited on its own (e.g. the container itself stopped). + + @param handles: the list returned by :func:`start_container_log_bridges` + """ + for proc in handles: + if proc.poll() is not None: + continue # already exited + try: + proc.terminate() + except OSError: + continue # process died between poll() and terminate() + for proc in handles: + try: + proc.wait(timeout=3) + except subprocess.TimeoutExpired: + proc.kill() + except OSError: + continue # process already gone diff --git a/ovos_utils/log_parser.py b/ovos_utils/log_parser.py index 4aa3cb8f..2979226d 100644 --- a/ovos_utils/log_parser.py +++ b/ovos_utils/log_parser.py @@ -1,6 +1,8 @@ import re import os +import time from datetime import datetime +from pathlib import Path from traceback import FrameSummary from dataclasses import dataclass from typing import Any, Tuple, List, Generator, Dict, Union, Optional @@ -22,6 +24,9 @@ date_format = "DMY" from ovos_utils.log import get_log_path, get_log_paths, get_available_logs +from ovos_utils.container_logs import (list_container_names, + start_container_log_bridges, + stop_container_log_bridges) TIME_FORMAT = '%Y-%m-%d %H:%M:%S.%f' @@ -672,3 +677,60 @@ def reduce(size, date, logs, paths): if reduced: console.print(f"{service} log reduced") + + +@ovos_logs.command() +@click.option("--paths", "-p", type=click.Path(), default=None, + help="directory to bridge container logs into " + "[default: a per-run directory under the xdg cache home]") +@click.option("--container", "-c", "containers", multiple=True, default=None, + help="container name to bridge; may be repeated " + "[default: autodetect every ovos/hivemind container]") +def containers(paths, containers): + """\b + Bridge Docker/Podman container stdout into the same small set of + per-category log files (skills.log, audio.log, voice.log, bus.log, + phal.log, gui.log, other.log) a file-based install already produces. + \b + Every OVOS/HiveMind container is discovered automatically unless one or + more `-c` are given. Runs in the foreground until interrupted (Ctrl+C), + bridging container stdout the whole time; point any other `ovos-logs` + command's `-p` at the printed directory (in another terminal) to read it + with the existing tailing/coloring/filtering machinery. + \b + Does nothing and exits with an error if neither `docker` nor `podman` is + on PATH, or if no matching container is running. + \b + > Examples: + > ovos-logs containers # bridge every ovos/hivemind container, autodetected + > ovos-logs containers -c ovos_core -c ovos_audio # bridge only the named containers + > ovos-logs containers -p /tmp/ovos-container-logs # bridge into a chosen directory + """ + from ovos_utils.xdg_utils import xdg_cache_home + + target_dir = Path(paths) if paths else \ + Path(xdg_cache_home()) / "ovos_container_logs" + names = list(containers) if containers else list_container_names() + if not names: + console = Console() + console.print("[red]No docker/podman OVOS/HiveMind containers found[/red]") + raise SystemExit(1) + + handles = start_container_log_bridges(names, target_dir) + if not handles: + console = Console() + console.print("[red]Neither docker nor podman is available, or no " + "bridge could be started[/red]") + raise SystemExit(1) + + console = Console() + console.print(f"Bridging {len(handles)} container(s) into {target_dir}") + console.print(f"Point another `ovos-logs` command at it with: -p {target_dir}") + console.print("Press Ctrl+C to stop") + try: + while True: + time.sleep(1) + except KeyboardInterrupt: + pass + finally: + stop_container_log_bridges(handles) diff --git a/test/unittests/test_container_logs.py b/test/unittests/test_container_logs.py new file mode 100644 index 00000000..6a8fa44f --- /dev/null +++ b/test/unittests/test_container_logs.py @@ -0,0 +1,178 @@ +# Copyright 2024, OpenVoiceOS +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for ovos_utils.container_logs module.""" + +import subprocess +import tempfile +import unittest +import unittest.mock +from pathlib import Path +from unittest.mock import patch + +from ovos_utils.container_logs import (categorize_container_name, + find_container_binary, + list_container_names, + start_container_log_bridges, + stop_container_log_bridges) + + +class TestCategorizeContainerName(unittest.TestCase): + """Tests for the container name to log category mapping.""" + + def test_skill_containers_map_to_skills(self) -> None: + """Every ovos_skill_* container maps to the shared skills category, + matching how every skill shares one skills.log on a normal install.""" + for name in ("ovos_skill_alarm_ovos_skill", "ovos_skill-podcast", + "Ovos_Skill_Count_OpenVoiceOS"): + self.assertEqual(categorize_container_name(name), "skills") + + def test_core_container_maps_to_skills(self) -> None: + """The core container's content (intent service handling) is what + already lands in skills.log on a normal install.""" + self.assertEqual(categorize_container_name("ovos_core"), "skills") + + def test_fixed_service_containers(self) -> None: + self.assertEqual(categorize_container_name("ovos_audio"), "audio") + self.assertEqual(categorize_container_name("ovos_listener"), "voice") + self.assertEqual(categorize_container_name("ovos_messagebus"), "bus") + + def test_phal_containers_map_to_phal(self) -> None: + for name in ("ovos_phal", "ovos_phal_plugin_bluetooth"): + self.assertEqual(categorize_container_name(name), "phal") + + def test_gui_containers_map_to_gui(self) -> None: + self.assertEqual(categorize_container_name("ovos_gui"), "gui") + + def test_skill_prefix_wins_over_later_rules(self) -> None: + """The skill prefix rule fires first, matching andlo's tested + reference: a skill container goes to the shared skills category + even when its name also contains another marker.""" + self.assertEqual(categorize_container_name("ovos_skill_gui"), "skills") + + def test_unknown_containers_map_to_other(self) -> None: + for name in ("hivemind_bridge", "nginx", "redis"): + self.assertEqual(categorize_container_name(name), "other") + + +class TestFindContainerBinary(unittest.TestCase): + """Tests for docker/podman binary detection.""" + + def test_docker_found(self) -> None: + ok = subprocess.CompletedProcess(["docker", "ps"], 0) + with patch("ovos_utils.container_logs.subprocess.run", + return_value=ok): + self.assertEqual(find_container_binary(), "docker") + + def test_podman_used_when_docker_fails(self) -> None: + fail = subprocess.CompletedProcess(["docker", "ps"], 1) + ok = subprocess.CompletedProcess(["podman", "ps"], 0) + with patch("ovos_utils.container_logs.subprocess.run", + side_effect=[fail, ok]): + self.assertEqual(find_container_binary(), "podman") + + def test_none_when_neither_works(self) -> None: + fail = subprocess.CompletedProcess([], 1) + missing = FileNotFoundError("nope") + with patch("ovos_utils.container_logs.subprocess.run", + side_effect=[missing, fail]): + self.assertIsNone(find_container_binary()) + + +class TestListContainerNames(unittest.TestCase): + """Tests for OVOS/HiveMind container discovery.""" + + def test_filters_to_ovos_and_hivemind(self) -> None: + ok = subprocess.CompletedProcess( + [], 0, + stdout="ovos_core\novos_audio\nnginx\nredis\nHiveMind-Bridge\n\n") + with patch("ovos_utils.container_logs.find_container_binary", + return_value="docker"), \ + patch("ovos_utils.container_logs.subprocess.run", + return_value=ok): + self.assertEqual(list_container_names(), + ["HiveMind-Bridge", "ovos_audio", "ovos_core"]) + + def test_no_runtime_returns_empty(self) -> None: + with patch("ovos_utils.container_logs.find_container_binary", + return_value=None): + self.assertEqual(list_container_names(), []) + + def test_runtime_error_returns_empty(self) -> None: + fail = subprocess.CompletedProcess([], 1) + with patch("ovos_utils.container_logs.find_container_binary", + return_value="docker"), \ + patch("ovos_utils.container_logs.subprocess.run", + return_value=fail): + self.assertEqual(list_container_names(), []) + + +class TestContainerLogBridges(unittest.TestCase): + """Tests for starting and stopping the log bridge subprocesses.""" + + def test_start_returns_empty_without_runtime(self) -> None: + with patch("ovos_utils.container_logs.find_container_binary", + return_value=None): + self.assertEqual( + start_container_log_bridges(["ovos_core"], Path("/tmp")), []) + + def test_start_bridges_append_to_shared_category_files(self) -> None: + """Multiple containers of one category each get their own process, + but all append to the same shared file.""" + procs = [self._fake_proc(), self._fake_proc()] + with tempfile.TemporaryDirectory() as tmp: + target = Path(tmp) + with patch("ovos_utils.container_logs.find_container_binary", + return_value="docker"), \ + patch("ovos_utils.container_logs.subprocess.Popen", + side_effect=procs) as popen: + handles = start_container_log_bridges( + ["ovos_skill_alarm", "ovos_skill_volume"], target) + self.assertEqual(handles, procs) + self.assertEqual(popen.call_count, 2) + for call, name in zip(popen.call_args_list, + ["ovos_skill_alarm", "ovos_skill_volume"]): + self.assertEqual(call.args[0], + ["docker", "logs", "-f", "--tail", "0", name]) + self.assertEqual(call.kwargs["stderr"], subprocess.STDOUT) + self.assertTrue((target / "skills.log").exists()) + + def test_stop_terminates_then_kills_stuck_process(self) -> None: + proc = self._fake_proc() + proc.wait.side_effect = subprocess.TimeoutExpired("docker", 3) + stop_container_log_bridges([proc]) + proc.terminate.assert_called_once_with() + proc.kill.assert_called_once_with() + + def test_stop_skips_already_exited_process(self) -> None: + proc = self._fake_proc() + proc.poll.return_value = 0 + stop_container_log_bridges([proc]) + proc.terminate.assert_not_called() + + def test_stop_never_raises_on_dead_process(self) -> None: + proc = self._fake_proc() + proc.terminate.side_effect = OSError("already dead") + proc.wait.side_effect = OSError("already dead") + stop_container_log_bridges([proc]) + + @staticmethod + def _fake_proc(): + proc = unittest.mock.MagicMock(spec=subprocess.Popen) + proc.poll.return_value = None + return proc + + +if __name__ == "__main__": + unittest.main()