fix(engine): drop the SIGALRM heartbeat cutoff that can deadlock the loop

docker-compose.yml

@@ -49,7 +49,10 @@ services:
     restart: always
     network_mode: host
     healthcheck:
-      test: ["CMD-SHELL", "grep -q 'confidence\\|Nighttime detected' engine.log || exit 1"]
+      # The loop logs every pass and the night sleep is 60 min, so a log untouched for 70 min means a
+      # stuck process. Docker never restarts an unhealthy container by itself: kill everything but
+      # PID 1 so the pipeline exits and `restart: always` brings the engine back.
+      test: ["CMD-SHELL", "find engine.log -mmin -70 | grep -q . || kill -9 -1"]
       interval: 30s
       retries: 3
       start_period: 20s

pyproject.toml

@@ -30,7 +30,7 @@ dependencies = [
     "requests>=2.33.0,<3",
     "huggingface_hub==0.23.1",
     "python-dotenv==1.1.0",
-    "pyroclient @ git+https://github.com/pyronear/pyro-api.git@main#subdirectory=client",
+    "pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client",
     "pyro_camera_api_client",
     "pyro_predictor",
 ]

pyroengine/engine.py

@@ -6,13 +6,12 @@
 import io
 import logging
 import shutil
-import signal
 import time
 from collections import deque
 from dataclasses import dataclass
 from datetime import datetime
 from pathlib import Path
-from typing import Any, Dict, List, Never, Optional, Tuple
+from typing import Any, Dict, List, Optional, Tuple
 
 import numpy as np
 from PIL import Image
@@ -21,6 +20,7 @@
 from pyroclient import client
 from requests.exceptions import ConnectionError as RequestsConnectionError
 from requests.exceptions import RequestException
+from requests.exceptions import Timeout as RequestsTimeout
 from requests.models import Response
 
 __all__ = ["ContextCrop", "Engine"]
@@ -72,21 +72,18 @@ class ContextCrop:
     full_h: int
 
 
-def handler(_signum: int, _frame: object) -> Never:
-    raise TimeoutError("Heartbeat check timed out")
+def heartbeat_with_timeout(api_instance: Any, cam_id: str, timeout: int = 3) -> None:  # noqa: ANN401
+    """Send a heartbeat and give up after `timeout` seconds so a slow API never stalls the loop.
 
-
-def heartbeat_with_timeout(api_instance: Any, cam_id: str, timeout: int = 1) -> None:  # noqa: ANN401
-    signal.signal(signal.SIGALRM, handler)
-    signal.alarm(timeout)
+    This must stay a plain requests timeout: an asynchronous cutoff (signal.alarm raising inside the
+    HTTP call) can interrupt urllib3 while it holds a pool lock and deadlock the whole process.
+    """
     try:
-        api_instance.heartbeat(cam_id)
-    except TimeoutError:
+        api_instance.heartbeat(cam_id, timeout=timeout)
+    except RequestsTimeout:
         logger.warning(f"Heartbeat check timed out for {cam_id}")
     except RequestsConnectionError:
         logger.warning(f"Unable to reach the pyro-api with {cam_id}")
-    finally:
-        signal.alarm(0)
 
 
 class Engine(Predictor):
@@ -219,10 +216,10 @@ def _end_event(self, cam_key: str) -> None:
             if entry[4]:  # is_staged: belongs to the event that just ended
                 window[i] = (entry[0], entry[1], [], entry[3], True, entry[5])
 
-    def heartbeat(self, cam_id: str) -> Response:
-        """Updates last ping of device"""
+    def heartbeat(self, cam_id: str, timeout: Optional[int] = None) -> Response:
+        """Updates last ping of device; `timeout` overrides the client timeout for this request only"""
         ip = cam_id.split("_")[0]
-        return self.api_client[ip].heartbeat()
+        return self.api_client[ip].heartbeat(timeout=timeout)
 
     def predict(
         self,
@@ -266,7 +263,7 @@ def predict(
 
         # Heartbeat
         if len(self.api_client) > 0 and isinstance(cam_id, str):
-            heartbeat_with_timeout(self, cam_id, timeout=1)
+            heartbeat_with_timeout(self, cam_id)
             if (
                 self._states[cam_key]["last_image_sent"] is None
                 or time.time() - self._states[cam_key]["last_image_sent"] > self.send_last_image_period

requirements-git.txt

@@ -1 +1 @@
-pyroclient @ git+https://github.com/pyronear/pyro-api.git@5c73e72ec86b9a84bc088dded28f19b2cb58282a#subdirectory=client
+pyroclient @ git+https://github.com/pyronear/pyro-api.git@361d7a61d75ee7fd7256b4aa118291f20619a2c7#subdirectory=client

tests/test_engine.py

@@ -11,7 +11,9 @@
 import pytest
 from dotenv import load_dotenv
 from PIL import Image
+from requests.exceptions import Timeout
 
+import pyroengine.engine as engine_module
 from pyroengine.engine import CONTEXT_MAX_SIDE, ContextCrop, Engine
 
 
@@ -614,6 +616,18 @@ def now(cls, tz=None) -> datetime:
         engine.predict(image, cam_id)
 
 
+def test_predict_survives_heartbeat_timeout(tmp_path, mock_forest_image):
+    """A slow heartbeat is dropped with a warning; predict() must neither stall nor raise."""
+    engine, fake_client, cam_id = _build_engine_with_pose_stub(tmp_path, datetime(2026, 5, 1, 9, 0, 0))
+    fake_client.heartbeat.side_effect = Timeout("slow api")
+
+    _run_predict_at(engine, cam_id, mock_forest_image, datetime(2026, 5, 1, 9, 0, 1))
+
+    # The cutoff is a requests timeout forwarded to pyroclient, never a signal.alarm.
+    fake_client.heartbeat.assert_called_once_with(timeout=3)
+    assert not hasattr(engine_module, "signal")
+
+
 def test_pose_image_skipped_when_engine_starts_after_noon(tmp_path, mock_forest_image):
     init_clock = datetime(2026, 5, 1, 14, 0, 0)
     engine, fake_client, cam_id = _build_engine_with_pose_stub(tmp_path, init_clock)