fix(flights): stop a dead tile server freezing the scroll

plugins/ledmatrix-flights/manager.py

@@ -15,6 +15,17 @@
 from typing import Any, Dict, List, Optional, Tuple
 
 import requests
+
+# Tile fetching happens on the render thread, so these bound how long the
+# scroll can freeze on an unreachable tile server.
+#
+# The timeout was 10s, tried against each of two URLs, for every tile in the
+# grid -- profiled on a live rig the loop sat 5.3s in a single getaddrinfo.
+# Three seconds is generous for a tile that normally arrives in tens of
+# milliseconds, and the cooldown means a server that is down costs one such
+# wait every five minutes rather than one per URL per tile.
+_TILE_TIMEOUT_SECONDS = 3
+_TILE_FAILURE_COOLDOWN = 300.0
 from PIL import Image, ImageDraw, ImageFont, ImageEnhance
 
 # Import base plugin class
@@ -213,6 +224,14 @@ def __init__(self, plugin_id: str, config: Dict[str, Any], display_manager, cach
         self._last_raw_payload = None
         self._last_raw_payload_at = 0.0
 
+        # Set when a tile fetch fails; until it passes, tiles come from cache
+        # only. Tile fetching runs on the render thread -- get_vegas_content()
+        # composites the map inline -- so an unreachable tile server froze the
+        # scroll for as long as it took to give up. Profiled on a live rig:
+        # 5.3s stuck in a single getaddrinfo, and the loop below pays that per
+        # URL per tile, across a whole grid of them.
+        self._tile_network_blocked_until = 0.0
+
         # Runtime data
         self.aircraft_data = {}  # ICAO -> aircraft dict (within map_radius_miles)
         self.all_aircraft_data = {}  # ICAO -> aircraft dict (all with position, for stats)
@@ -1970,14 +1989,20 @@ def _fetch_tile(self, x: int, y: int, zoom: int) -> Optional[Image.Image]:
             except Exception as e:
                 self.logger.warning(f"[Flight Tracker] Failed to load cached tile {x},{y},{zoom}: {e}")
 
+        # Uncached, so the only way to get it is the network -- which this
+        # code path cannot afford to wait on. Give up immediately while the
+        # server is known bad; the map renders with the tiles it does have.
+        if time.time() < self._tile_network_blocked_until:
+            return None
+
         # Fetch from server - try multiple URLs
         urls = self._get_tile_urls(x, y, zoom)
 
         for i, url in enumerate(urls):
             try:
                 self.logger.debug(f"[Flight Tracker] Fetching tile {x},{y} at zoom {zoom} from: {url}")
 
-                response = requests.get(url, timeout=10)
+                response = requests.get(url, timeout=_TILE_TIMEOUT_SECONDS)
                 response.raise_for_status()
 
                 # Check if we got an error page instead of a tile
@@ -2057,11 +2082,27 @@ def _fetch_tile(self, x: int, y: int, zoom: int) -> Optional[Image.Image]:
             except Exception as e:
                 self.logger.warning(f"[Flight Tracker] Failed to fetch tile from {url}: {e}")
                 if i == len(urls) - 1:  # Last URL failed
+                    self._block_tile_network()
                     return None
                 continue  # Try next URL
-        
+
         # If we get here, all URLs failed
+        self._block_tile_network()
         return None
+
+    def _block_tile_network(self) -> None:
+        """Serve tiles from cache alone for a while after a failure.
+
+        One tile failing means the server or the network is unavailable, and
+        every other tile in the grid is about to discover the same thing at
+        the same price. Since this runs on the render thread, paying it once
+        is a frozen scroll and paying it per tile is a stopped one.
+        """
+        self._tile_network_blocked_until = time.time() + _TILE_FAILURE_COOLDOWN
+        self.logger.warning(
+            "[Flight Tracker] Tile fetch failed; using cached tiles only for "
+            "%.0fs so the scroll does not stall on every remaining tile",
+            _TILE_FAILURE_COOLDOWN)
 
     def _get_map_background(self, center_lat: float, center_lon: float) -> Optional[Image.Image]:
         """Get the map background for the current view."""

plugins/ledmatrix-flights/manifest.json

@@ -1,7 +1,7 @@
 {
   "id": "ledmatrix-flights",
   "name": "Flight Tracker",
-  "version": "1.12.8",
+  "version": "1.12.9",
   "description": "Real-time aircraft tracking with ADS-B/FlightRadar24/OpenSky/adsb.fi/adsb.lol data, map backgrounds, area mode, flight tracking, anchor airport, flight records, and optional airport weather (METAR/TAF/PIREP/SIGMET via the free NOAA Aviation Weather Center API)",
   "author": "ChuckBuilds",
   "entry_point": "manager.py",
@@ -37,6 +37,12 @@
   "min_ledmatrix_version": "2.0.0",
   "max_ledmatrix_version": "3.0.0",
   "versions": [
+    {
+      "released": "2026-08-12",
+      "version": "1.12.9",
+      "ledmatrix_min_version": "2.0.0",
+      "notes": "Stop an unreachable tile server freezing the Vegas scroll. Tiles are fetched from inside get_vegas_content(), which runs on the render thread, with a 10s timeout tried against each of two URLs for every tile in the grid and nothing bounding the total. Profiled with py-spy on a live rig, the render thread sat 5.30s in a single getaddrinfo. The timeout is now 3s and one failure switches tiles to cache-only for five minutes, so a dead server costs one short wait rather than one per URL per tile; the map simply draws with the tiles it already has."
+    },
     {
       "released": "2026-08-11",
       "version": "1.12.8",

plugins/ledmatrix-flights/test_tile_network_bound.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python3
+"""Tests that an unreachable tile server cannot freeze the scroll.
+
+Map tiles are fetched from inside get_vegas_content(), which the Vegas
+coordinator calls on the render thread -- so every second spent waiting on a
+tile is a second the marquee is stopped. Profiled on a live rig with py-spy,
+the main thread was found stuck 5.30s in a single getaddrinfo:
+
+    coordinator.run_iteration -> stream_manager._fetch_plugin_content
+      -> plugin_adapter._get_native_content
+        -> ledmatrix-flights get_vegas_content
+          -> _render_map_image -> _get_map_background -> _fetch_tile
+            -> requests.get -> socket.getaddrinfo
+
+The old timeout was 10s, tried against each of two URLs, for every tile in
+the grid. Nothing bounded the total.
+
+Run: python3 plugins/ledmatrix-flights/test_tile_network_bound.py
+"""
+
+import sys
+import time
+from pathlib import Path
+
+PLUGIN_DIR = Path(__file__).resolve().parent
+sys.path.insert(0, str(PLUGIN_DIR))
+
+# manager.py imports the core's BasePlugin at module scope, so the core tree
+# has to be importable. LEDMATRIX_CORE points at a checkout; without one there
+# is nothing to test against.
+import os
+_core = os.environ.get('LEDMATRIX_CORE', '')
+for _candidate in (_core, str(PLUGIN_DIR.parents[2] / 'LEDMatrix')):
+    if _candidate and (Path(_candidate) / 'src' / 'plugin_system').is_dir():
+        sys.path.insert(0, _candidate)
+        break
+else:
+    print("SKIP: no LEDMatrix core checkout found (set LEDMATRIX_CORE)")
+    sys.exit(2)
+
+try:
+    import requests  # noqa: F401
+except ImportError:
+    print("SKIP: requests not installed")
+    sys.exit(2)
+
+import manager as fm  # noqa: E402
+
+failures = []
+
+
+def check(name, cond, detail=""):
+    if cond:
+        print("  PASS  %s" % name)
+    else:
+        print("  FAIL  %s%s" % (name, (": " + detail) if detail else ""))
+        failures.append(name)
+
+
+class FakePlugin:
+    """Just enough of the plugin to exercise _fetch_tile's network policy."""
+
+    _fetch_tile = fm.FlightTrackerPlugin._fetch_tile
+    _block_tile_network = fm.FlightTrackerPlugin._block_tile_network
+
+    def __init__(self, tmp):
+        self._tile_network_blocked_until = 0.0
+        self.cache_error_count = 0
+        self._tmp = tmp
+        self.attempts = 0
+        self.logger = type("L", (), {m: (lambda *a, **k: None) for m in
+                                     ("debug", "info", "warning", "error")})()
+
+    def _get_tile_cache_path(self, x, y, z):
+        return Path(self._tmp, "%d_%d_%d.png" % (z, x, y))
+
+    def _is_tile_cached(self, x, y, z):
+        return self._get_tile_cache_path(x, y, z).exists()
+
+    def _get_tile_urls(self, x, y, z):
+        return ["http://a.invalid/%d/%d/%d.png" % (z, x, y),
+                "http://b.invalid/%d/%d/%d.png" % (z, x, y)]
+
+
+def main():
+    import tempfile
+    tmp = tempfile.mkdtemp()
+
+    print("the per-request timeout is short enough for a render thread")
+    check("timeout is bounded", fm._TILE_TIMEOUT_SECONDS <= 5,
+          "%ss" % fm._TILE_TIMEOUT_SECONDS)
+    # Two URLs per tile, and a grid has many tiles. A whole grid must not be
+    # able to outlast the plugin executor's 30s budget on its own.
+    worst_one_tile = fm._TILE_TIMEOUT_SECONDS * 2
+    check("one tile cannot exhaust a 30s update budget", worst_one_tile < 30,
+          "%ss" % worst_one_tile)
+
+    print("\na failing tile stops the rest of the grid trying")
+    p = FakePlugin(tmp)
+    calls = {"n": 0}
+    real_get = fm.requests.get
+
+    def boom(*a, **k):
+        calls["n"] += 1
+        raise fm.requests.exceptions.ConnectionError("no route")
+
+    fm.requests.get = boom
+    try:
+        first = p._fetch_tile(1, 1, 10)
+        after_first = calls["n"]
+        for i in range(40):                      # the rest of a tile grid
+            p._fetch_tile(2 + i, 1, 10)
+        check("first tile returns nothing", first is None)
+        check("it tried both URLs once", after_first == 2, "%d" % after_first)
+        check("and no further tile touched the network",
+              calls["n"] == after_first, "%d total attempts" % calls["n"])
+    finally:
+        fm.requests.get = real_get
+
+    print("\nthe block is time-bounded, not permanent")
+    check("a cooldown is set", p._tile_network_blocked_until > time.time())
+    check("and it expires",
+          p._tile_network_blocked_until - time.time() <= fm._TILE_FAILURE_COOLDOWN + 1)
+    p._tile_network_blocked_until = 0.0
+    fm.requests.get = boom
+    try:
+        before = calls["n"]
+        p._fetch_tile(99, 99, 10)
+        check("the network is retried once the cooldown lapses",
+              calls["n"] > before)
+    finally:
+        fm.requests.get = real_get
+
+    print("\ncached tiles are still served while the network is blocked")
+    from PIL import Image
+    p2 = FakePlugin(tmp)
+    p2._tile_network_blocked_until = time.time() + 999
+    cached = p2._get_tile_cache_path(7, 7, 10)
+    Image.new('RGB', (256, 256), (10, 20, 30)).save(cached)
+    before_cached = calls["n"]
+    fm.requests.get = boom
+    try:
+        got = p2._fetch_tile(7, 7, 10)
+    finally:
+        fm.requests.get = real_get
+    check("a cached tile comes back", got is not None)
+    check("without any network call", calls["n"] == before_cached,
+          "%d attempts" % (calls["n"] - before_cached))
+
+    print("\nand an uncached one simply yields nothing, quickly")
+    t = time.perf_counter()
+    missing = p2._fetch_tile(8, 8, 10)
+    elapsed = time.perf_counter() - t
+    check("returns None", missing is None)
+    check("and returns at once", elapsed < 0.05, "%.3fs" % elapsed)
+
+    print("\n%s" % ("FAILED: %d" % len(failures) if failures
+                    else "All checks passed"))
+    return 1 if failures else 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())