diff --git a/plugins/ledmatrix-flights/manager.py b/plugins/ledmatrix-flights/manager.py index 83b6aa83..8cb2fdda 100644 --- a/plugins/ledmatrix-flights/manager.py +++ b/plugins/ledmatrix-flights/manager.py @@ -15,6 +15,17 @@ from typing import Any, Dict, List, Optional, Tuple import requests + +# Tile fetching happens on the render thread, so these bound how long the +# scroll can freeze on an unreachable tile server. +# +# The timeout was 10s, tried against each of two URLs, for every tile in the +# grid -- profiled on a live rig the loop sat 5.3s in a single getaddrinfo. +# Three seconds is generous for a tile that normally arrives in tens of +# milliseconds, and the cooldown means a server that is down costs one such +# wait every five minutes rather than one per URL per tile. +_TILE_TIMEOUT_SECONDS = 3 +_TILE_FAILURE_COOLDOWN = 300.0 from PIL import Image, ImageDraw, ImageFont, ImageEnhance # Import base plugin class @@ -213,6 +224,14 @@ def __init__(self, plugin_id: str, config: Dict[str, Any], display_manager, cach self._last_raw_payload = None self._last_raw_payload_at = 0.0 + # Set when a tile fetch fails; until it passes, tiles come from cache + # only. Tile fetching runs on the render thread -- get_vegas_content() + # composites the map inline -- so an unreachable tile server froze the + # scroll for as long as it took to give up. Profiled on a live rig: + # 5.3s stuck in a single getaddrinfo, and the loop below pays that per + # URL per tile, across a whole grid of them. + self._tile_network_blocked_until = 0.0 + # Runtime data self.aircraft_data = {} # ICAO -> aircraft dict (within map_radius_miles) self.all_aircraft_data = {} # ICAO -> aircraft dict (all with position, for stats) @@ -1970,6 +1989,12 @@ def _fetch_tile(self, x: int, y: int, zoom: int) -> Optional[Image.Image]: except Exception as e: self.logger.warning(f"[Flight Tracker] Failed to load cached tile {x},{y},{zoom}: {e}") + # Uncached, so the only way to get it is the network -- which this + # code path cannot afford to wait on. Give up immediately while the + # server is known bad; the map renders with the tiles it does have. + if time.time() < self._tile_network_blocked_until: + return None + # Fetch from server - try multiple URLs urls = self._get_tile_urls(x, y, zoom) @@ -1977,7 +2002,7 @@ def _fetch_tile(self, x: int, y: int, zoom: int) -> Optional[Image.Image]: try: self.logger.debug(f"[Flight Tracker] Fetching tile {x},{y} at zoom {zoom} from: {url}") - response = requests.get(url, timeout=10) + response = requests.get(url, timeout=_TILE_TIMEOUT_SECONDS) response.raise_for_status() # Check if we got an error page instead of a tile @@ -2057,11 +2082,27 @@ def _fetch_tile(self, x: int, y: int, zoom: int) -> Optional[Image.Image]: except Exception as e: self.logger.warning(f"[Flight Tracker] Failed to fetch tile from {url}: {e}") if i == len(urls) - 1: # Last URL failed + self._block_tile_network() return None continue # Try next URL - + # If we get here, all URLs failed + self._block_tile_network() return None + + def _block_tile_network(self) -> None: + """Serve tiles from cache alone for a while after a failure. + + One tile failing means the server or the network is unavailable, and + every other tile in the grid is about to discover the same thing at + the same price. Since this runs on the render thread, paying it once + is a frozen scroll and paying it per tile is a stopped one. + """ + self._tile_network_blocked_until = time.time() + _TILE_FAILURE_COOLDOWN + self.logger.warning( + "[Flight Tracker] Tile fetch failed; using cached tiles only for " + "%.0fs so the scroll does not stall on every remaining tile", + _TILE_FAILURE_COOLDOWN) def _get_map_background(self, center_lat: float, center_lon: float) -> Optional[Image.Image]: """Get the map background for the current view.""" diff --git a/plugins/ledmatrix-flights/manifest.json b/plugins/ledmatrix-flights/manifest.json index 2af1eaa5..39fd15a8 100644 --- a/plugins/ledmatrix-flights/manifest.json +++ b/plugins/ledmatrix-flights/manifest.json @@ -1,7 +1,7 @@ { "id": "ledmatrix-flights", "name": "Flight Tracker", - "version": "1.12.8", + "version": "1.12.9", "description": "Real-time aircraft tracking with ADS-B/FlightRadar24/OpenSky/adsb.fi/adsb.lol data, map backgrounds, area mode, flight tracking, anchor airport, flight records, and optional airport weather (METAR/TAF/PIREP/SIGMET via the free NOAA Aviation Weather Center API)", "author": "ChuckBuilds", "entry_point": "manager.py", @@ -37,6 +37,12 @@ "min_ledmatrix_version": "2.0.0", "max_ledmatrix_version": "3.0.0", "versions": [ + { + "released": "2026-08-12", + "version": "1.12.9", + "ledmatrix_min_version": "2.0.0", + "notes": "Stop an unreachable tile server freezing the Vegas scroll. Tiles are fetched from inside get_vegas_content(), which runs on the render thread, with a 10s timeout tried against each of two URLs for every tile in the grid and nothing bounding the total. Profiled with py-spy on a live rig, the render thread sat 5.30s in a single getaddrinfo. The timeout is now 3s and one failure switches tiles to cache-only for five minutes, so a dead server costs one short wait rather than one per URL per tile; the map simply draws with the tiles it already has." + }, { "released": "2026-08-11", "version": "1.12.8", diff --git a/plugins/ledmatrix-flights/test_tile_network_bound.py b/plugins/ledmatrix-flights/test_tile_network_bound.py new file mode 100644 index 00000000..f93a8061 --- /dev/null +++ b/plugins/ledmatrix-flights/test_tile_network_bound.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Tests that an unreachable tile server cannot freeze the scroll. + +Map tiles are fetched from inside get_vegas_content(), which the Vegas +coordinator calls on the render thread -- so every second spent waiting on a +tile is a second the marquee is stopped. Profiled on a live rig with py-spy, +the main thread was found stuck 5.30s in a single getaddrinfo: + + coordinator.run_iteration -> stream_manager._fetch_plugin_content + -> plugin_adapter._get_native_content + -> ledmatrix-flights get_vegas_content + -> _render_map_image -> _get_map_background -> _fetch_tile + -> requests.get -> socket.getaddrinfo + +The old timeout was 10s, tried against each of two URLs, for every tile in +the grid. Nothing bounded the total. + +Run: python3 plugins/ledmatrix-flights/test_tile_network_bound.py +""" + +import sys +import time +from pathlib import Path + +PLUGIN_DIR = Path(__file__).resolve().parent +sys.path.insert(0, str(PLUGIN_DIR)) + +# manager.py imports the core's BasePlugin at module scope, so the core tree +# has to be importable. LEDMATRIX_CORE points at a checkout; without one there +# is nothing to test against. +import os +_core = os.environ.get('LEDMATRIX_CORE', '') +for _candidate in (_core, str(PLUGIN_DIR.parents[2] / 'LEDMatrix')): + if _candidate and (Path(_candidate) / 'src' / 'plugin_system').is_dir(): + sys.path.insert(0, _candidate) + break +else: + print("SKIP: no LEDMatrix core checkout found (set LEDMATRIX_CORE)") + sys.exit(2) + +try: + import requests # noqa: F401 +except ImportError: + print("SKIP: requests not installed") + sys.exit(2) + +import manager as fm # noqa: E402 + +failures = [] + + +def check(name, cond, detail=""): + if cond: + print(" PASS %s" % name) + else: + print(" FAIL %s%s" % (name, (": " + detail) if detail else "")) + failures.append(name) + + +class FakePlugin: + """Just enough of the plugin to exercise _fetch_tile's network policy.""" + + _fetch_tile = fm.FlightTrackerPlugin._fetch_tile + _block_tile_network = fm.FlightTrackerPlugin._block_tile_network + + def __init__(self, tmp): + self._tile_network_blocked_until = 0.0 + self.cache_error_count = 0 + self._tmp = tmp + self.attempts = 0 + self.logger = type("L", (), {m: (lambda *a, **k: None) for m in + ("debug", "info", "warning", "error")})() + + def _get_tile_cache_path(self, x, y, z): + return Path(self._tmp, "%d_%d_%d.png" % (z, x, y)) + + def _is_tile_cached(self, x, y, z): + return self._get_tile_cache_path(x, y, z).exists() + + def _get_tile_urls(self, x, y, z): + return ["http://a.invalid/%d/%d/%d.png" % (z, x, y), + "http://b.invalid/%d/%d/%d.png" % (z, x, y)] + + +def main(): + import tempfile + tmp = tempfile.mkdtemp() + + print("the per-request timeout is short enough for a render thread") + check("timeout is bounded", fm._TILE_TIMEOUT_SECONDS <= 5, + "%ss" % fm._TILE_TIMEOUT_SECONDS) + # Two URLs per tile, and a grid has many tiles. A whole grid must not be + # able to outlast the plugin executor's 30s budget on its own. + worst_one_tile = fm._TILE_TIMEOUT_SECONDS * 2 + check("one tile cannot exhaust a 30s update budget", worst_one_tile < 30, + "%ss" % worst_one_tile) + + print("\na failing tile stops the rest of the grid trying") + p = FakePlugin(tmp) + calls = {"n": 0} + real_get = fm.requests.get + + def boom(*a, **k): + calls["n"] += 1 + raise fm.requests.exceptions.ConnectionError("no route") + + fm.requests.get = boom + try: + first = p._fetch_tile(1, 1, 10) + after_first = calls["n"] + for i in range(40): # the rest of a tile grid + p._fetch_tile(2 + i, 1, 10) + check("first tile returns nothing", first is None) + check("it tried both URLs once", after_first == 2, "%d" % after_first) + check("and no further tile touched the network", + calls["n"] == after_first, "%d total attempts" % calls["n"]) + finally: + fm.requests.get = real_get + + print("\nthe block is time-bounded, not permanent") + check("a cooldown is set", p._tile_network_blocked_until > time.time()) + check("and it expires", + p._tile_network_blocked_until - time.time() <= fm._TILE_FAILURE_COOLDOWN + 1) + p._tile_network_blocked_until = 0.0 + fm.requests.get = boom + try: + before = calls["n"] + p._fetch_tile(99, 99, 10) + check("the network is retried once the cooldown lapses", + calls["n"] > before) + finally: + fm.requests.get = real_get + + print("\ncached tiles are still served while the network is blocked") + from PIL import Image + p2 = FakePlugin(tmp) + p2._tile_network_blocked_until = time.time() + 999 + cached = p2._get_tile_cache_path(7, 7, 10) + Image.new('RGB', (256, 256), (10, 20, 30)).save(cached) + before_cached = calls["n"] + fm.requests.get = boom + try: + got = p2._fetch_tile(7, 7, 10) + finally: + fm.requests.get = real_get + check("a cached tile comes back", got is not None) + check("without any network call", calls["n"] == before_cached, + "%d attempts" % (calls["n"] - before_cached)) + + print("\nand an uncached one simply yields nothing, quickly") + t = time.perf_counter() + missing = p2._fetch_tile(8, 8, 10) + elapsed = time.perf_counter() - t + check("returns None", missing is None) + check("and returns at once", elapsed < 0.05, "%.3fs" % elapsed) + + print("\n%s" % ("FAILED: %d" % len(failures) if failures + else "All checks passed")) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())