From 563ddd962ba6a1e7b3f6f3b858d468d362ca5e0e Mon Sep 17 00:00:00 2001 From: Hector van der Aa Date: Fri, 22 May 2026 18:41:42 +0200 Subject: [PATCH] step 4: add async tiles and persistent cache --- AGENTS.md | 21 +- README.md | 14 +- examples/basic_map.py | 2 + examples/cache_stress.py | 43 +++ src/dpg_map/api.py | 17 +- src/dpg_map/cache.py | 50 ++++ src/dpg_map/renderer.py | 116 ++++++-- src/dpg_map/state.py | 19 +- src/dpg_map/tiles.py | 556 ++++++++++++++++++++++++++++++++++++++- src/dpg_map/widget.py | 1 + tests/test_tiles.py | 87 ++++++ 11 files changed, 880 insertions(+), 46 deletions(-) create mode 100644 examples/cache_stress.py create mode 100644 tests/test_tiles.py diff --git a/AGENTS.md b/AGENTS.md index 6826117..d7ec201 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,17 +2,18 @@ ## Current status -Step 3 complete. +Step 4 complete. ## Completed steps Step 1 - Public API contract and pure core. Step 2 - Thread-safe state, commands, overlays, and cache model. Step 3 - Widget shell, sizing system, and GUI-thread frame pump. +Step 4 - Tile manager, persistent cache, and asynchronous loading. ## Current step -Step 4 - Tile manager, persistent cache, and asynchronous loading. +Step 5 - Interaction: pan, zoom, and view commands. ## Design decisions @@ -45,15 +46,29 @@ None yet. - Implemented GUI-thread renderer frame pump that schedules frame callbacks, drains command queues, measures size, resizes the drawlist, and draws a placeholder background/attribution. - Implemented sizing helpers for measured size, last non-zero size preservation, visibility transitions, effective draw size, and resize dirty flags. - Implemented map interaction hit-rectangle calculation. +- Implemented TileID, TileStatus, Tile, visible tile calculation, and TileManager. +- Implemented asynchronous tile worker queue for disk reads, HTTP fetches, and image decoding. +- Implemented provider-namespaced persistent cache writes, access metadata updates, clearing, and LRU pruning. +- Implemented memory tile cache with visible-tile protection and deferred GUI-thread texture deletion. +- Integrated tile result processing, stale generation/provider rejection, texture creation, and tile drawing into the GUI-thread renderer. +- Added OpenStreetMap User-Agent warning/fallback and configured examples with example user agents. - Added Step 3 examples for basic map, window sizing, child-window sizing, table sizing, and hidden-tab sizing. +- Added Step 4 cache stress example. - Added Step 3 tests for sizing transitions, zero-size preservation, resize dirty flags, command drain ordering, and hit rectangles. +- Added Step 4 tests for visible tile calculation, stale result rejection, protected memory eviction, and tile image decoding. - Ran a Dear PyGui context smoke check for `map_widget` child-window/drawlist creation. - Ran `uv run pytest`. - Ran `uv run ruff check .`. - Ran `uv run ruff format .`. - Ran `uv run ruff format --check .`. - Ran `uv run pyright`. +- Ran `uv run pytest`. +- Ran `uv run ruff check .`. +- Ran `uv run ruff format .`. +- Ran `uv run ruff format --check .`. +- Ran `uv run pyright`. +- Ran a Dear PyGui context smoke check for `map_widget` child-window/drawlist/texture-registry creation. ## Next action -Implement Step 4. +Implement Step 5. diff --git a/README.md b/README.md index 3e444a6..0a6558e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ `dpg-map` is a Dear PyGui map widget package under rebuild. -The Step 3 widget shell is in place: +The Step 4 tile manager is in place: ```python import dpg_map as dpgm @@ -28,15 +28,16 @@ Implemented so far: - thread-safe logical map state and map registry - command queue with coalescing for overlay and view updates - logical marker, polyline, trajectory, and layer models -- persistent disk cache path, metadata, scanning, and prune planning +- persistent disk cache paths, metadata, scanning, pruning, and clearing - Dear PyGui `child_window` + measured-size `drawlist` widget shell -- GUI-thread frame pump that drains commands and redraws a placeholder background +- GUI-thread frame pump that drains commands, manages textures, and draws raster tiles - sizing helpers that preserve the last non-zero size across hidden layouts - interaction hit-rectangle calculation for the measured map area +- asynchronous tile workers that read disk cache, fetch HTTP, and decode images +- memory cache with visible-tile protection and GUI-thread texture deletion -Real tile loading and overlay drawing are not implemented yet. Step 4 will add -the asynchronous tile manager, persistent tile loading, and GUI-thread texture -creation. +Overlay drawing is not implemented yet. Step 5 will add pan/zoom interaction and +view command projection. Examples: @@ -46,4 +47,5 @@ uv run python examples/sizing_window.py uv run python examples/sizing_child.py uv run python examples/sizing_table.py uv run python examples/hidden_tab.py +uv run python examples/cache_stress.py ``` diff --git a/examples/basic_map.py b/examples/basic_map.py index 9f76d5c..4d3ef4f 100644 --- a/examples/basic_map.py +++ b/examples/basic_map.py @@ -8,6 +8,8 @@ dpg: Any = _dpg def main() -> None: + dpgm.configure(user_agent="dpg-map basic example") + dpg.create_context() dpg.create_viewport(title="dpg-map basic", width=900, height=600) diff --git a/examples/cache_stress.py b/examples/cache_stress.py new file mode 100644 index 0000000..e1c1c58 --- /dev/null +++ b/examples/cache_stress.py @@ -0,0 +1,43 @@ +from pathlib import Path +from typing import Any + +import dearpygui.dearpygui as _dpg + +import dpg_map as dpgm + +dpg: Any = _dpg + + +def main() -> None: + cache_dir = Path(__file__).resolve().parent / ".tile-cache" + dpgm.configure( + cache_dir=cache_dir, + memory_cache_max_tiles=32, + disk_cache_max_bytes=30_000_000, + prefetch_margin_tiles=1, + user_agent="dpg-map cache_stress example", + ) + + dpg.create_context() + dpg.create_viewport(title="dpg-map cache stress", width=1000, height=700) + + with ( + dpg.window(label="Cache Stress", width=-1, height=-1), + dpgm.map_widget( + tag="cache-map", + center=(47.9029, 1.9093), + zoom=14, + width=-1, + height=-1, + ), + ): + dpgm.add_marker("start", lat=47.9029, lon=1.9093, label="Orleans") + + dpg.setup_dearpygui() + dpg.show_viewport() + dpg.start_dearpygui() + dpg.destroy_context() + + +if __name__ == "__main__": + main() diff --git a/src/dpg_map/api.py b/src/dpg_map/api.py index 3ae097a..05e132c 100644 --- a/src/dpg_map/api.py +++ b/src/dpg_map/api.py @@ -8,7 +8,7 @@ from math import isfinite from pathlib import Path from typing import Any -from .cache import CacheStats, disk_cache_size_bytes +from .cache import CacheStats, clear_disk_cache_path, disk_cache_root, disk_cache_size_bytes from .commands import CommandKind, MapCommand from .exceptions import CoordinateError, OverlayNotFoundError from .overlays import LayerState, MarkerOverlay, Overlay, PolylineOverlay, TrajectoryOverlay @@ -469,14 +469,17 @@ def set_provider(provider: str | TileProvider, *, map_tag: Tag | None = None) -> def clear_memory_cache(*, map_tag: Tag | None = None) -> None: state = get_map_state(map_tag) with state.lock: + mark_dirty(state, DirtyFlags.TILES) _queue(state, CommandKind.CLEAR_MEMORY_CACHE, {}) def clear_disk_cache(*, map_tag: Tag | None = None) -> None: if map_tag is None: + clear_disk_cache_path(get_config().cache_dir) return state = get_map_state(map_tag) with state.lock: + mark_dirty(state, DirtyFlags.TILES) _queue(state, CommandKind.CLEAR_DISK_CACHE, {}) @@ -488,16 +491,21 @@ def get_cache_stats(*, map_tag: Tag | None = None) -> CacheStats: memory_max_tiles=config.memory_cache_max_tiles, disk_bytes=disk_cache_size_bytes(cache_dir), disk_max_bytes=config.disk_cache_max_bytes, - disk_path=cache_dir, + disk_path=disk_cache_root(cache_dir), ) state = get_map_state(map_tag) with state.lock: + tile_snapshot = state.tile_manager.snapshot() return CacheStats( - memory_tiles=0, + memory_tiles=tile_snapshot.memory_tiles, memory_max_tiles=config.memory_cache_max_tiles, + memory_hits=tile_snapshot.memory_hits, + memory_misses=tile_snapshot.memory_misses, disk_bytes=disk_cache_size_bytes(state.cache_dir), disk_max_bytes=config.disk_cache_max_bytes, - disk_path=state.cache_dir, + disk_hits=tile_snapshot.disk_hits, + disk_misses=tile_snapshot.disk_misses, + disk_path=disk_cache_root(state.cache_dir), ) @@ -525,4 +533,5 @@ def get_map_debug_state(*, map_tag: Tag | None = None) -> dict[str, Any]: "pending_command_count": len(state.command_queue), "generation": state.generation, "active_drag": state.interaction.active_drag, + "tiles": asdict(state.tile_manager.snapshot()), } diff --git a/src/dpg_map/cache.py b/src/dpg_map/cache.py index 6455d80..78b291c 100644 --- a/src/dpg_map/cache.py +++ b/src/dpg_map/cache.py @@ -3,6 +3,7 @@ from __future__ import annotations import json +import shutil from dataclasses import asdict, dataclass, field from pathlib import Path from time import time @@ -179,6 +180,24 @@ def write_disk_metadata(path: Path, metadata: DiskCacheMetadata) -> None: raise CacheError(f"could not write cache metadata: {path}") from exc +def touch_disk_metadata(path: Path, *, accessed_at: float | None = None) -> None: + """Update only the last access timestamp for a metadata file.""" + + metadata = read_disk_metadata(path) + write_disk_metadata( + path, + DiskCacheMetadata( + url=metadata.url, + etag=metadata.etag, + last_modified=metadata.last_modified, + expires=metadata.expires, + downloaded_at=metadata.downloaded_at, + last_accessed_at=time() if accessed_at is None else accessed_at, + size_bytes=metadata.size_bytes, + ), + ) + + def scan_disk_cache(cache_dir: str | Path | None) -> list[DiskCacheEntry]: """Scan tile files under a disk cache root.""" @@ -237,3 +256,34 @@ def plan_disk_prune( prune.append(entry.tile_path) total -= entry.metadata.size_bytes return prune + + +def prune_disk_cache( + cache_dir: str | Path | None, + max_bytes: int | None, + *, + protected_paths: set[Path] | None = None, +) -> list[Path]: + """Delete LRU tile files until the cache fits the configured limit.""" + + planned = plan_disk_prune(cache_dir, max_bytes, protected_paths=protected_paths) + for path in planned: + metadata_path = tile_metadata_path(path) + try: + path.unlink(missing_ok=True) + metadata_path.unlink(missing_ok=True) + except OSError as exc: + raise CacheError(f"could not prune cached tile: {path}") from exc + return planned + + +def clear_disk_cache_path(cache_dir: str | Path | None) -> None: + """Remove all persistent tile cache files under a cache root.""" + + root = disk_cache_root(cache_dir) + if not root.exists(): + return + try: + shutil.rmtree(root) + except OSError as exc: + raise CacheError(f"could not clear disk cache: {root}") from exc diff --git a/src/dpg_map/renderer.py b/src/dpg_map/renderer.py index e8f5c2c..fd3892a 100644 --- a/src/dpg_map/renderer.py +++ b/src/dpg_map/renderer.py @@ -9,16 +9,16 @@ from .commands import CommandKind, MapCommand from .interaction import HitRect, calculate_hit_rect from .sizing import SizeMeasurement, apply_size_measurement from .state import DirtyFlags, MapState +from .tiles import Tile, VisibleTile class MapRenderer: - """GUI-thread renderer for the Step 3 widget shell.""" + """GUI-thread renderer for the map widget shell and tile layer.""" def __init__(self, state: MapState, dpg: Any) -> None: self.state = state self._dpg = dpg self._background_tag = f"{state.tag}##background" - self._title_tag = f"{state.tag}##placeholder-title" self._attribution_tag = f"{state.tag}##attribution" self.last_drained_commands: tuple[MapCommand, ...] = () self.last_hit_rect: HitRect | None = None @@ -43,7 +43,7 @@ class MapRenderer: self.schedule_next_frame() def render_frame(self) -> None: - """Drain pending commands, refresh size, and draw the placeholder shell.""" + """Drain pending commands, refresh size, process tiles, and redraw.""" commands = drain_renderer_commands(self.state) self.last_drained_commands = tuple(commands) @@ -51,15 +51,47 @@ class MapRenderer: with self.state.lock: dirty = self.state.dirty - should_draw = bool(dirty & (DirtyFlags.FULL | DirtyFlags.SIZE)) + should_draw = bool(dirty & (DirtyFlags.FULL | DirtyFlags.SIZE | DirtyFlags.TILES)) visible = self.state.is_visible width = self.state.measured_width or self.state.last_nonzero_width height = self.state.measured_height or self.state.last_nonzero_height provider_attribution = self.state.provider.attribution + provider = self.state.provider + center = self.state.center + zoom = self.state.zoom + generation = self.state.generation + cache_dir = self.state.cache_dir self.state.dirty = DirtyFlags.NONE - if should_draw and visible: - self._draw_placeholder(width, height, provider_attribution) + accepted_tiles = self.state.tile_manager.drain_results( + generation=generation, + provider_name=provider.name, + ) + self._delete_evicted_textures() + for tile in accepted_tiles: + self._ensure_texture(tile) + + visible_tiles: list[VisibleTile] = [] + if visible and width > 0 and height > 0: + visible_tiles = self.state.tile_manager.request_visible_tiles( + center=center, + zoom=zoom, + width=width, + height=height, + provider=provider, + generation=generation, + cache_dir=cache_dir, + margin=self._prefetch_margin(), + ) + + if visible and (should_draw or accepted_tiles): + self._draw_tile_layer( + visible_tiles=visible_tiles, + width=width, + height=height, + attribution=provider_attribution, + tile_size=provider.tile_size, + ) def _update_size_from_dpg(self) -> None: width, height = self._measure_child_content() @@ -83,7 +115,15 @@ class MapRenderer: return (0, 0) return (max(0, int(width)), max(0, int(height))) - def _draw_placeholder(self, width: int, height: int, attribution: str) -> None: + def _draw_tile_layer( + self, + *, + visible_tiles: list[VisibleTile], + width: int, + height: int, + attribution: str, + tile_size: int, + ) -> None: width = max(1, int(width)) height = max(1, int(height)) self._dpg.delete_item(self.state.drawlist_tag, children_only=True) @@ -95,15 +135,19 @@ class MapRenderer: color=(54, 68, 78, 255), fill=(29, 38, 45, 255), ) - self._dpg.draw_text( - (12, 12), - "dpg-map", - parent=self.state.drawlist_tag, - tag=self._title_tag, - color=(232, 238, 242, 255), - size=18, - ) - label = attribution or "Map tiles load in Step 4" + for visible_tile in visible_tiles: + tile = self.state.tile_manager.get_ready_tile(visible_tile.tile_id) + if tile is None or tile.texture_tag is None: + continue + screen_x = visible_tile.screen_x + screen_y = visible_tile.screen_y + self._dpg.draw_image( + tile.texture_tag, + (screen_x, screen_y), + (screen_x + tile_size, screen_y + tile_size), + parent=self.state.drawlist_tag, + ) + label = attribution or "Map tiles" text_y = max(28, height - 24) self._dpg.draw_text( (12, text_y), @@ -114,9 +158,36 @@ class MapRenderer: size=12, ) + def _ensure_texture(self, tile: Tile) -> None: + if tile.texture_tag is not None: + return + texture_tag = ( + f"{self.state.tag}##tile-{tile.tile_id.provider_name}-" + f"{tile.tile_id.z}-{tile.tile_id.x}-{tile.tile_id.y}" + ) + if not self._dpg.does_item_exist(texture_tag): + self._dpg.add_static_texture( + tile.width, + tile.height, + tile.pixels, + tag=texture_tag, + parent=self.state.texture_registry_tag, + ) + self.state.tile_manager.set_texture_tag(tile.tile_id, texture_tag) + + def _delete_evicted_textures(self) -> None: + for texture_tag in self.state.tile_manager.take_texture_deletions(): + if self._dpg.does_item_exist(texture_tag): + self._dpg.delete_item(texture_tag) + + def _prefetch_margin(self) -> int: + from .state import get_config + + return get_config().prefetch_margin_tiles + def drain_renderer_commands(state: MapState) -> list[MapCommand]: - """Drain and apply GUI-thread command side effects that exist in Step 3.""" + """Drain and apply GUI-thread command side effects.""" commands = state.command_queue.drain() if not commands: @@ -127,6 +198,7 @@ def drain_renderer_commands(state: MapState) -> list[MapCommand]: if command.kind is CommandKind.SET_VIEW: state.dirty |= DirtyFlags.VIEW | DirtyFlags.TILES | DirtyFlags.OVERLAYS elif command.kind is CommandKind.SET_PROVIDER: + state.tile_manager.clear_memory_cache() state.dirty |= DirtyFlags.PROVIDER | DirtyFlags.TILES elif command.kind in { CommandKind.ADD_OVERLAY, @@ -138,11 +210,13 @@ def drain_renderer_commands(state: MapState) -> list[MapCommand]: }: state.dirty |= DirtyFlags.OVERLAYS elif command.kind is CommandKind.CLEAR_MAP: + state.tile_manager.clear_memory_cache() state.dirty |= DirtyFlags.TILES | DirtyFlags.OVERLAYS - elif command.kind in { - CommandKind.CLEAR_MEMORY_CACHE, - CommandKind.CLEAR_DISK_CACHE, - }: + elif command.kind is CommandKind.CLEAR_MEMORY_CACHE: + state.tile_manager.clear_memory_cache() + state.dirty |= DirtyFlags.TILES + elif command.kind is CommandKind.CLEAR_DISK_CACHE: + state.tile_manager.clear_disk_cache(state.cache_dir) state.dirty |= DirtyFlags.TILES return commands diff --git a/src/dpg_map/state.py b/src/dpg_map/state.py index 9de41c0..da5a6fa 100644 --- a/src/dpg_map/state.py +++ b/src/dpg_map/state.py @@ -14,6 +14,7 @@ from .commands import MapCommandQueue from .exceptions import InvalidProviderError, MapNotFoundError from .overlays import LayerState, Overlay from .providers import TileProvider, get_default_provider, get_provider +from .tiles import TileManager from .types import LatLon, Tag @@ -44,16 +45,6 @@ class DpgMapConfig: debug: bool = False -@dataclass(slots=True) -class TileManagerState: - """Logical tile/cache counters until the tile manager is implemented.""" - - queued_tiles: int = 0 - loading_tiles: int = 0 - failed_tiles: int = 0 - visible_tile_count: int = 0 - - @dataclass(slots=True) class InteractionState: """Logical interaction state until GUI interaction is implemented.""" @@ -103,7 +94,7 @@ class MapState: overlays: dict[Tag, Overlay] = field(default_factory=dict) layers: dict[str, LayerState] = field(default_factory=default_layers) command_queue: MapCommandQueue = field(default_factory=MapCommandQueue) - tile_manager: TileManagerState = field(default_factory=TileManagerState) + tile_manager: TileManager = field(default_factory=TileManager) renderer: object | None = None interaction: InteractionState = field(default_factory=InteractionState) lock: RLock = field(default_factory=RLock) @@ -238,6 +229,12 @@ def create_map_state( cache_dir=resolved_cache_dir, user_agent=user_agent if user_agent is not None else config.user_agent, ) + state.tile_manager = TileManager( + memory_cache_max_tiles=config.memory_cache_max_tiles, + disk_cache_max_bytes=config.disk_cache_max_bytes, + worker_count=config.tile_worker_count, + user_agent=state.user_agent, + ) with _maps_lock: _maps[map_tag] = state return state diff --git a/src/dpg_map/tiles.py b/src/dpg_map/tiles.py index 4db1a8a..2dcd307 100644 --- a/src/dpg_map/tiles.py +++ b/src/dpg_map/tiles.py @@ -1 +1,555 @@ -"""Tile identity, lifecycle, and worker coordination.""" +"""Tile identity, lifecycle, cache, and worker coordination.""" + +from __future__ import annotations + +import warnings +from dataclasses import dataclass, field +from enum import Enum +from io import BytesIO +from pathlib import Path +from queue import Empty, Queue +from threading import Lock, Thread +from time import time +from typing import Literal, cast + +import requests +from PIL import Image, UnidentifiedImageError + +from .cache import ( + DiskCacheMetadata, + MemoryCacheEntry, + MemoryCacheModel, + clear_disk_cache_path, + prune_disk_cache, + tile_cache_path, + tile_metadata_path, + touch_disk_metadata, + write_disk_metadata, +) +from .exceptions import CacheError +from .projection import latlon_to_world, map_size +from .providers import TileProvider +from .types import LatLon + + +class TileStatus(Enum): + """Lifecycle status for a tile.""" + + QUEUED = "queued" + LOADING = "loading" + READY = "ready" + FAILED = "failed" + + +@dataclass(frozen=True, slots=True) +class TileID: + """Provider-namespaced XYZ tile identity.""" + + provider_name: str + z: int + x: int + y: int + + +@dataclass(slots=True) +class Tile: + """Decoded tile data plus GUI-thread texture metadata.""" + + tile_id: TileID + status: TileStatus + generation: int + width: int = 0 + height: int = 0 + pixels: tuple[float, ...] = () + texture_tag: object | None = None + source: Literal["memory", "disk", "network"] | None = None + error: str | None = None + last_accessed_at: float = field(default_factory=time) + + +@dataclass(frozen=True, slots=True) +class VisibleTile: + """A visible tile and its screen-space top-left point.""" + + tile_id: TileID + screen_x: float + screen_y: float + + +@dataclass(frozen=True, slots=True) +class TileRequest: + """Worker-thread tile request.""" + + tile_id: TileID + generation: int + url: str + path: Path + headers: dict[str, str] + disk_cache_max_bytes: int | None + protected_paths: frozenset[Path] = frozenset() + + +@dataclass(frozen=True, slots=True) +class TileResult: + """Worker-thread tile result consumed by the GUI thread.""" + + tile_id: TileID + generation: int + status: TileStatus + width: int = 0 + height: int = 0 + pixels: tuple[float, ...] = () + source: Literal["disk", "network"] | None = None + error: str | None = None + + +@dataclass(frozen=True, slots=True) +class TileManagerSnapshot: + """Thread-safe counters for diagnostics.""" + + queued_tiles: int + loading_tiles: int + failed_tiles: int + visible_tile_count: int + memory_tiles: int + memory_hits: int + memory_misses: int + disk_hits: int + disk_misses: int + stale_results: int + + +def calculate_visible_tiles( + *, + center: LatLon, + zoom: int, + width: int, + height: int, + provider: TileProvider, + margin: int = 0, +) -> list[VisibleTile]: + """Return visible XYZ tiles plus a margin, with wrapped X and clamped Y.""" + + if width <= 0 or height <= 0: + return [] + tile_size = provider.tile_size + center_x, center_y = latlon_to_world(center[0], center[1], zoom, tile_size) + left = center_x - width / 2.0 + top = center_y - height / 2.0 + right = center_x + width / 2.0 + bottom = center_y + height / 2.0 + + max_tile = (2**zoom) - 1 + start_x = int(left // tile_size) - margin + end_x = int(right // tile_size) + margin + start_y = max(0, int(top // tile_size) - margin) + end_y = min(max_tile, int(bottom // tile_size) + margin) + world_size = map_size(zoom, tile_size) + + tiles: list[VisibleTile] = [] + seen: set[TileID] = set() + for y in range(start_y, end_y + 1): + for raw_x in range(start_x, end_x + 1): + x = raw_x % (max_tile + 1) + tile_id = TileID(provider.name, zoom, x, y) + if tile_id in seen: + continue + seen.add(tile_id) + tile_left = raw_x * tile_size + if tile_left < -tile_size: + tile_left += world_size + screen_x = tile_left - left + screen_y = y * tile_size - top + tiles.append(VisibleTile(tile_id, screen_x, screen_y)) + return tiles + + +class TileManager: + """Asynchronous tile loader with memory and persistent disk caches.""" + + def __init__( + self, + *, + memory_cache_max_tiles: int = 512, + disk_cache_max_bytes: int | None = 2_000_000_000, + worker_count: int = 4, + user_agent: str | None = None, + ) -> None: + self.memory = MemoryCacheModel(max_tiles=memory_cache_max_tiles) + self.disk_cache_max_bytes = disk_cache_max_bytes + self.worker_count = worker_count + self.user_agent = user_agent + self._tiles: dict[TileID, Tile] = {} + self._visible_tile_ids: set[TileID] = set() + self._queued: set[TileID] = set() + self._loading: set[TileID] = set() + self._failed: set[TileID] = set() + self._request_queue: Queue[TileRequest | None] = Queue() + self._result_queue: Queue[TileResult] = Queue() + self._threads: list[Thread] = [] + self._lock = Lock() + self._delete_texture_tags: list[object] = [] + self._disk_hits = 0 + self._disk_misses = 0 + self._stale_results = 0 + self._warned_osm_user_agent = False + + def start(self) -> None: + """Start worker threads once.""" + + with self._lock: + if self._threads: + return + count = max(1, self.worker_count) + for index in range(count): + thread = Thread( + target=self._worker_loop, + name=f"dpg-map-tile-worker-{index + 1}", + daemon=True, + ) + self._threads.append(thread) + thread.start() + + def stop(self) -> None: + """Ask workers to exit.""" + + with self._lock: + threads = list(self._threads) + self._threads.clear() + for _ in threads: + self._request_queue.put(None) + + def request_visible_tiles( + self, + *, + center: LatLon, + zoom: int, + width: int, + height: int, + provider: TileProvider, + generation: int, + cache_dir: str | Path | None, + margin: int, + ) -> list[VisibleTile]: + """Queue missing visible tiles and return their screen positions.""" + + visible = calculate_visible_tiles( + center=center, + zoom=zoom, + width=width, + height=height, + provider=provider, + margin=margin, + ) + visible_ids = {tile.tile_id for tile in visible} + with self._lock: + self._visible_tile_ids = visible_ids + for entry in self.memory.entries.values(): + entry.protected = entry.tile_id in visible_ids + + self.start() + for visible_tile in visible: + self._queue_tile( + visible_tile.tile_id, + provider=provider, + generation=generation, + cache_dir=cache_dir, + ) + return visible + + def _queue_tile( + self, + tile_id: TileID, + *, + provider: TileProvider, + generation: int, + cache_dir: str | Path | None, + ) -> None: + with self._lock: + tile = self._tiles.get(tile_id) + if tile is not None and tile.status is TileStatus.READY: + self.memory.record_access(tile_id) + tile.last_accessed_at = time() + return + entry = self.memory.record_access(tile_id) + if entry is not None: + return + if tile_id in self._queued or tile_id in self._loading: + return + self._queued.add(tile_id) + self._tiles[tile_id] = Tile(tile_id, TileStatus.QUEUED, generation=generation) + + headers = dict(provider.headers) + if provider.name == "osm": + headers = self._headers_with_osm_user_agent(headers) + path = tile_cache_path( + cache_dir, + provider.name, + tile_id.z, + tile_id.x, + tile_id.y, + provider.file_extension or "png", + ) + request = TileRequest( + tile_id=tile_id, + generation=generation, + url=provider.build_url(x=tile_id.x, y=tile_id.y, z=tile_id.z), + path=path, + headers=headers, + disk_cache_max_bytes=self.disk_cache_max_bytes, + protected_paths=frozenset(path for path in self._visible_disk_paths(cache_dir)), + ) + self._request_queue.put(request) + + def _headers_with_osm_user_agent(self, headers: dict[str, str]) -> dict[str, str]: + if any(key.lower() == "user-agent" for key in headers): + return headers + if self.user_agent: + headers["User-Agent"] = self.user_agent + return headers + with self._lock: + should_warn = not self._warned_osm_user_agent + self._warned_osm_user_agent = True + if should_warn: + warnings.warn( + "OpenStreetMap tile usage should configure an application-specific user_agent", + RuntimeWarning, + stacklevel=3, + ) + headers["User-Agent"] = "dpg-map/0.1" + return headers + + def _visible_disk_paths(self, cache_dir: str | Path | None) -> list[Path]: + paths: list[Path] = [] + with self._lock: + visible = tuple(self._visible_tile_ids) + for tile_id in visible: + paths.append( + tile_cache_path( + cache_dir, + tile_id.provider_name, + tile_id.z, + tile_id.x, + tile_id.y, + ) + ) + return paths + + def get_ready_tile(self, tile_id: TileID) -> Tile | None: + """Return a ready tile, updating memory LRU metadata.""" + + with self._lock: + tile = self._tiles.get(tile_id) + if tile is None or tile.status is not TileStatus.READY: + return None + tile.last_accessed_at = time() + self.memory.record_access(tile_id) + return tile + + def drain_results(self, *, generation: int, provider_name: str) -> list[Tile]: + """Accept current-generation results and return ready tiles.""" + + accepted: list[Tile] = [] + while True: + try: + result = self._result_queue.get_nowait() + except Empty: + break + with self._lock: + self._queued.discard(result.tile_id) + self._loading.discard(result.tile_id) + if result.generation != generation or result.tile_id.provider_name != provider_name: + self._stale_results += 1 + continue + if result.status is TileStatus.FAILED: + self._failed.add(result.tile_id) + self._tiles[result.tile_id] = Tile( + result.tile_id, + TileStatus.FAILED, + generation=result.generation, + error=result.error, + ) + continue + tile = Tile( + result.tile_id, + TileStatus.READY, + generation=result.generation, + width=result.width, + height=result.height, + pixels=result.pixels, + source=result.source, + ) + self._tiles[result.tile_id] = tile + self.memory.put( + MemoryCacheEntry( + tile_id=result.tile_id, + size_bytes=len(result.pixels) * 4, + protected=result.tile_id in self._visible_tile_ids, + texture_tag=None, + ) + ) + accepted.append(tile) + self._evict_memory_if_needed() + return accepted + + def set_texture_tag(self, tile_id: TileID, texture_tag: object) -> None: + """Record a GUI-thread texture tag for a ready tile.""" + + with self._lock: + tile = self._tiles.get(tile_id) + if tile is not None: + tile.texture_tag = texture_tag + entry = self.memory.entries.get(tile_id) + if entry is not None: + entry.texture_tag = texture_tag + + def take_texture_deletions(self) -> list[object]: + """Return texture tags that must be deleted by the GUI thread.""" + + with self._lock: + tags = list(self._delete_texture_tags) + self._delete_texture_tags.clear() + return tags + + def clear_memory_cache(self) -> list[object]: + """Clear decoded memory tiles and return texture tags for GUI deletion.""" + + with self._lock: + tags = [ + entry.texture_tag + for entry in self.memory.entries.values() + if entry.texture_tag is not None + ] + self._delete_texture_tags.extend(tags) + self.memory.entries.clear() + self._tiles.clear() + self._queued.clear() + self._loading.clear() + self._failed.clear() + return tags + + def clear_disk_cache(self, cache_dir: str | Path | None) -> None: + """Clear the persistent cache root.""" + + clear_disk_cache_path(cache_dir) + + def snapshot(self) -> TileManagerSnapshot: + """Return diagnostic counters.""" + + with self._lock: + return TileManagerSnapshot( + queued_tiles=len(self._queued), + loading_tiles=len(self._loading), + failed_tiles=len(self._failed), + visible_tile_count=len(self._visible_tile_ids), + memory_tiles=len(self.memory.entries), + memory_hits=self.memory.hits, + memory_misses=self.memory.misses, + disk_hits=self._disk_hits, + disk_misses=self._disk_misses, + stale_results=self._stale_results, + ) + + def _evict_memory_if_needed(self) -> None: + with self._lock: + evict_ids = self.memory.plan_evictions() + for raw_tile_id in evict_ids: + tile_id = cast(TileID, raw_tile_id) + entry = self.memory.entries.pop(tile_id, None) + tile = self._tiles.pop(tile_id, None) + texture_tag = None + if entry is not None: + texture_tag = entry.texture_tag + if texture_tag is None and tile is not None: + texture_tag = tile.texture_tag + if texture_tag is not None: + self._delete_texture_tags.append(texture_tag) + + def _worker_loop(self) -> None: + while True: + request = self._request_queue.get() + if request is None: + return + with self._lock: + self._queued.discard(request.tile_id) + self._loading.add(request.tile_id) + tile = self._tiles.get(request.tile_id) + if tile is not None: + tile.status = TileStatus.LOADING + result = self._load_tile(request) + self._result_queue.put(result) + + def _load_tile(self, request: TileRequest) -> TileResult: + try: + raw, source = self._read_or_fetch(request) + width, height, pixels = decode_tile_image(raw) + return TileResult( + request.tile_id, + request.generation, + TileStatus.READY, + width=width, + height=height, + pixels=pixels, + source=source, + ) + except Exception as exc: + return TileResult( + request.tile_id, + request.generation, + TileStatus.FAILED, + error=str(exc), + ) + + def _read_or_fetch(self, request: TileRequest) -> tuple[bytes, Literal["disk", "network"]]: + if request.path.exists(): + try: + raw = request.path.read_bytes() + touch_disk_metadata(tile_metadata_path(request.path)) + except (OSError, CacheError): + raw = b"" + if raw: + with self._lock: + self._disk_hits += 1 + return raw, "disk" + + with self._lock: + self._disk_misses += 1 + response = requests.get(request.url, headers=request.headers, timeout=20) + response.raise_for_status() + raw = response.content + downloaded_at = time() + try: + request.path.parent.mkdir(parents=True, exist_ok=True) + request.path.write_bytes(raw) + write_disk_metadata( + tile_metadata_path(request.path), + DiskCacheMetadata( + url=request.url, + etag=response.headers.get("ETag"), + last_modified=response.headers.get("Last-Modified"), + expires=response.headers.get("Expires"), + downloaded_at=downloaded_at, + last_accessed_at=downloaded_at, + size_bytes=len(raw), + ), + ) + prune_disk_cache( + request.path.parents[3], + request.disk_cache_max_bytes, + protected_paths=set(request.protected_paths), + ) + except (OSError, CacheError): + pass + return raw, "network" + + +def decode_tile_image(raw: bytes) -> tuple[int, int, tuple[float, ...]]: + """Decode an image into Dear PyGui-compatible RGBA float data.""" + + try: + image = Image.open(BytesIO(raw)).convert("RGBA") + except UnidentifiedImageError as exc: + raise ValueError("tile image data could not be decoded") from exc + width, height = image.size + pixels = tuple(channel / 255.0 for channel in image.tobytes()) + return width, height, pixels diff --git a/src/dpg_map/widget.py b/src/dpg_map/widget.py index 60acc92..870a9f7 100644 --- a/src/dpg_map/widget.py +++ b/src/dpg_map/widget.py @@ -61,6 +61,7 @@ def map_widget( autosize_y=autosize_y, **child_kwargs, ) + dpg.add_texture_registry(tag=state.texture_registry_tag) dpg.add_drawlist(1, 1, tag=state.drawlist_tag, parent=state.child_window_tag) renderer = MapRenderer(state, dpg) diff --git a/tests/test_tiles.py b/tests/test_tiles.py new file mode 100644 index 0000000..3471856 --- /dev/null +++ b/tests/test_tiles.py @@ -0,0 +1,87 @@ +from __future__ import annotations + +from io import BytesIO + +from PIL import Image + +from dpg_map.providers import OSM +from dpg_map.tiles import ( + TileID, + TileManager, + TileResult, + TileStatus, + calculate_visible_tiles, + decode_tile_image, +) + + +def test_visible_tile_calculation_uses_provider_namespace() -> None: + tiles = calculate_visible_tiles( + center=(0.0, 0.0), + zoom=2, + width=256, + height=256, + provider=OSM, + margin=0, + ) + + assert tiles + assert {tile.tile_id.provider_name for tile in tiles} == {"osm"} + assert all(0 <= tile.tile_id.x <= 3 for tile in tiles) + assert all(0 <= tile.tile_id.y <= 3 for tile in tiles) + + +def test_tile_manager_ignores_stale_generation_results() -> None: + manager = TileManager() + stale = TileResult( + TileID("osm", 1, 0, 0), + generation=1, + status=TileStatus.READY, + width=1, + height=1, + pixels=(1.0, 1.0, 1.0, 1.0), + source="disk", + ) + manager._result_queue.put(stale) + + accepted = manager.drain_results(generation=2, provider_name="osm") + + assert accepted == [] + assert manager.snapshot().stale_results == 1 + + +def test_memory_eviction_protects_visible_tiles() -> None: + manager = TileManager(memory_cache_max_tiles=1) + protected = TileID("osm", 1, 0, 0) + evictable = TileID("osm", 1, 0, 1) + with manager._lock: + manager._visible_tile_ids = {protected} + + for tile_id in (protected, evictable): + manager._result_queue.put( + TileResult( + tile_id, + generation=1, + status=TileStatus.READY, + width=1, + height=1, + pixels=(1.0, 1.0, 1.0, 1.0), + source="disk", + ) + ) + + manager.drain_results(generation=1, provider_name="osm") + + assert manager.get_ready_tile(protected) is not None + assert manager.get_ready_tile(evictable) is None + + +def test_decode_png_tile_image() -> None: + image = Image.new("RGBA", (1, 1), (255, 0, 128, 255)) + buffer = BytesIO() + image.save(buffer, format="PNG") + + width, height, pixels = decode_tile_image(buffer.getvalue()) + + assert (width, height) == (1, 1) + assert pixels == (1.0, 0.0, 128 / 255, 1.0)