From 8f254155c12d63b07e912b389dd62157cbc951b2 Mon Sep 17 00:00:00 2001 From: Mateo Date: Sat, 12 Sep 2026 11:55:01 +0200 Subject: [PATCH 1/3] fix(engine): replace the heartbeat SIGALRM cutoff with a requests timeout --- pyproject.toml | 2 +- pyroengine/engine.py | 29 +++++++++++++---------------- requirements-git.txt | 2 +- tests/test_engine.py | 14 ++++++++++++++ uv.lock | 4 ++-- 5 files changed, 31 insertions(+), 20 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c9737a69..62d131a0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "requests>=2.33.0,<3", "huggingface_hub==0.23.1", "python-dotenv==1.1.0", - "pyroclient @ git+https://github.com/pyronear/pyro-api.git@main#subdirectory=client", + "pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client", "pyro_camera_api_client", "pyro_predictor", ] diff --git a/pyroengine/engine.py b/pyroengine/engine.py index c4440138..b14215d9 100644 --- a/pyroengine/engine.py +++ b/pyroengine/engine.py @@ -6,13 +6,12 @@ import io import logging import shutil -import signal import time from collections import deque from dataclasses import dataclass from datetime import datetime from pathlib import Path -from typing import Any, Dict, List, Never, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple import numpy as np from PIL import Image @@ -21,6 +20,7 @@ from pyroclient import client from requests.exceptions import ConnectionError as RequestsConnectionError from requests.exceptions import RequestException +from requests.exceptions import Timeout as RequestsTimeout from requests.models import Response __all__ = ["ContextCrop", "Engine"] @@ -72,21 +72,18 @@ class ContextCrop: full_h: int -def handler(_signum: int, _frame: object) -> Never: - raise TimeoutError("Heartbeat check timed out") +def heartbeat_with_timeout(api_instance: Any, cam_id: str, timeout: int = 3) -> None: # noqa: ANN401 + """Send a heartbeat and give up after `timeout` seconds so a slow API never stalls the loop. - -def heartbeat_with_timeout(api_instance: Any, cam_id: str, timeout: int = 1) -> None: # noqa: ANN401 - signal.signal(signal.SIGALRM, handler) - signal.alarm(timeout) + This must stay a plain requests timeout: an asynchronous cutoff (signal.alarm raising inside the + HTTP call) can interrupt urllib3 while it holds a pool lock and deadlock the whole process. + """ try: - api_instance.heartbeat(cam_id) - except TimeoutError: + api_instance.heartbeat(cam_id, timeout=timeout) + except RequestsTimeout: logger.warning(f"Heartbeat check timed out for {cam_id}") except RequestsConnectionError: logger.warning(f"Unable to reach the pyro-api with {cam_id}") - finally: - signal.alarm(0) class Engine(Predictor): @@ -219,10 +216,10 @@ def _end_event(self, cam_key: str) -> None: if entry[4]: # is_staged: belongs to the event that just ended window[i] = (entry[0], entry[1], [], entry[3], True, entry[5]) - def heartbeat(self, cam_id: str) -> Response: - """Updates last ping of device""" + def heartbeat(self, cam_id: str, timeout: Optional[int] = None) -> Response: + """Updates last ping of device; `timeout` overrides the client timeout for this request only""" ip = cam_id.split("_")[0] - return self.api_client[ip].heartbeat() + return self.api_client[ip].heartbeat(timeout=timeout) def predict( self, @@ -266,7 +263,7 @@ def predict( # Heartbeat if len(self.api_client) > 0 and isinstance(cam_id, str): - heartbeat_with_timeout(self, cam_id, timeout=1) + heartbeat_with_timeout(self, cam_id) if ( self._states[cam_key]["last_image_sent"] is None or time.time() - self._states[cam_key]["last_image_sent"] > self.send_last_image_period diff --git a/requirements-git.txt b/requirements-git.txt index f85813b3..d75eea3c 100644 --- a/requirements-git.txt +++ b/requirements-git.txt @@ -1 +1 @@ -pyroclient @ git+https://github.com/pyronear/pyro-api.git@5c73e72ec86b9a84bc088dded28f19b2cb58282a#subdirectory=client +pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client diff --git a/tests/test_engine.py b/tests/test_engine.py index 5928a169..56049036 100644 --- a/tests/test_engine.py +++ b/tests/test_engine.py @@ -11,7 +11,9 @@ import pytest from dotenv import load_dotenv from PIL import Image +from requests.exceptions import Timeout +import pyroengine.engine as engine_module from pyroengine.engine import CONTEXT_MAX_SIDE, ContextCrop, Engine @@ -614,6 +616,18 @@ def now(cls, tz=None) -> datetime: engine.predict(image, cam_id) +def test_predict_survives_heartbeat_timeout(tmp_path, mock_forest_image): + """A slow heartbeat is dropped with a warning; predict() must neither stall nor raise.""" + engine, fake_client, cam_id = _build_engine_with_pose_stub(tmp_path, datetime(2026, 5, 1, 9, 0, 0)) + fake_client.heartbeat.side_effect = Timeout("slow api") + + _run_predict_at(engine, cam_id, mock_forest_image, datetime(2026, 5, 1, 9, 0, 1)) + + # The cutoff is a requests timeout forwarded to pyroclient, never a signal.alarm. + fake_client.heartbeat.assert_called_once_with(timeout=3) + assert not hasattr(engine_module, "signal") + + def test_pose_image_skipped_when_engine_starts_after_noon(tmp_path, mock_forest_image): init_clock = datetime(2026, 5, 1, 14, 0, 0) engine, fake_client, cam_id = _build_engine_with_pose_stub(tmp_path, init_clock) diff --git a/uv.lock b/uv.lock index e8ab51f7..b40f72dc 100644 --- a/uv.lock +++ b/uv.lock @@ -1158,7 +1158,7 @@ requires-dist = [ [[package]] name = "pyroclient" version = "0.2.0.dev0" -source = { git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=main#5c73e72ec86b9a84bc088dded28f19b2cb58282a" } +source = { git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=361d7a61d75ee7fd7256b4aa118291f20619a2c7#361d7a61d75ee7fd7256b4aa118291f20619a2c7" } dependencies = [ { name = "requests" }, ] @@ -1208,7 +1208,7 @@ requires-dist = [ { name = "huggingface-hub", specifier = "==0.23.1" }, { name = "pyro-camera-api-client", editable = "pyro_camera_api/client" }, { name = "pyro-predictor", editable = "pyro-predictor" }, - { name = "pyroclient", git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=main" }, + { name = "pyroclient", git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=361d7a61d75ee7fd7256b4aa118291f20619a2c7" }, { name = "python-dotenv", specifier = "==1.1.0" }, { name = "requests", specifier = ">=2.33.0,<3" }, ] From a38b545f22f0fb46ff024a357e622c7d025190ba Mon Sep 17 00:00:00 2001 From: Mateo Date: Sat, 12 Sep 2026 11:55:01 +0200 Subject: [PATCH 2/3] fix(compose): restart the engine when its log goes stale --- docker-compose.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 28bd1265..196d7dec 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -49,7 +49,10 @@ services: restart: always network_mode: host healthcheck: - test: ["CMD-SHELL", "grep -q 'confidence\\|Nighttime detected' engine.log || exit 1"] + # The loop logs every pass and the night sleep is 60 min, so a log untouched for 70 min means a + # stuck process. Docker never restarts an unhealthy container by itself: kill everything but + # PID 1 so the pipeline exits and `restart: always` brings the engine back. + test: ["CMD-SHELL", "find engine.log -mmin -70 | grep -q . || kill -9 -1"] interval: 30s retries: 3 start_period: 20s From 53de9c1bac0bf15549ced205ae708f038dfe0899 Mon Sep 17 00:00:00 2001 From: Mateo Date: Wed, 21 Oct 2026 10:41:50 +0200 Subject: [PATCH 3/3] build(deps): pin pyroclient back to main now that pyro-api#691 is merged --- pyproject.toml | 2 +- requirements-git.txt | 2 +- uv.lock | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 62d131a0..c9737a69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,7 +30,7 @@ dependencies = [ "requests>=2.33.0,<3", "huggingface_hub==0.23.1", "python-dotenv==1.1.0", - "pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client", + "pyroclient @ git+https://github.com/pyronear/pyro-api.git@main#subdirectory=client", "pyro_camera_api_client", "pyro_predictor", ] diff --git a/requirements-git.txt b/requirements-git.txt index d75eea3c..557e6dd6 100644 --- a/requirements-git.txt +++ b/requirements-git.txt @@ -1 +1 @@ -pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client +pyroclient @ git+https://github.com/pyronear/pyro-api.git@5d965ddd22750e6aafa4e6fc45b629a5374861c4#subdirectory=client diff --git a/uv.lock b/uv.lock index b40f72dc..dd7339f3 100644 --- a/uv.lock +++ b/uv.lock @@ -1158,7 +1158,7 @@ requires-dist = [ [[package]] name = "pyroclient" version = "0.2.0.dev0" -source = { git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=361d7a61d75ee7fd7256b4aa118291f20619a2c7#361d7a61d75ee7fd7256b4aa118291f20619a2c7" } +source = { git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=main#5d965ddd22750e6aafa4e6fc45b629a5374861c4" } dependencies = [ { name = "requests" }, ] @@ -1208,7 +1208,7 @@ requires-dist = [ { name = "huggingface-hub", specifier = "==0.23.1" }, { name = "pyro-camera-api-client", editable = "pyro_camera_api/client" }, { name = "pyro-predictor", editable = "pyro-predictor" }, - { name = "pyroclient", git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=361d7a61d75ee7fd7256b4aa118291f20619a2c7" }, + { name = "pyroclient", git = "https://github.com/pyronear/pyro-api.git?subdirectory=client&rev=main" }, { name = "python-dotenv", specifier = "==1.1.0" }, { name = "requests", specifier = ">=2.33.0,<3" }, ]