Load navigation basemaps without a comma account

The imported Api wrapper has no dongle_id attribute, so the original tile
loader failed before its first HTTP request and mapsd supplied black frames.
Replace that account-dependent lookup with supported public Protomaps build
discovery and bounded HTTP byte ranges. Pin archive URL, size and strong ETag,
isolate cache identities and retain strict integrity/freshness checks.

Show the specific model-side waiting reason for the matching signal and route,
and display OpenStreetMap attribution over the preview without changing model
pixels. Keep the selected navigation model, steering, cooperative steering,
action conversion and all route ownership gates unchanged.

Validation includes real public index/range requests and a saved-route raster
accepted by the unchanged navigation input buffer, dedicated public transport
tests, navigation lifecycle and original routing regressions, and UI reason
tests. This is transport/input validation; target-device consumption and driven
turn quality still require verification.
This commit is contained in:
Isaac Barham
2026-11-03 15:07:18 -05:00
parent c2588f3958
commit cff1a7c1ea
6 changed files with 524 additions and 82 deletions
@@ -80,7 +80,16 @@ inputs are zeroed. A route preview alone never establishes active model input.
The feature is opt-in under **Signal Navigation (Experimental)**. It needs
online Directions requests, a Mapbox public token, and automatically fetched
road-map tiles. Existing steering, cooperative steering, action conversion and
road-map tiles. Directions use the configured Mapbox token. The basemap is fetched
from public Protomaps builds, without comma authentication or a Prime subscription.
Only the required archive byte ranges are downloaded and cached; the entire
planet archive is not downloaded. The map data is derived from
[OpenStreetMap](https://www.openstreetmap.org/copyright); attribution is displayed
over the map preview without changing the model raster. The public basemap is not
claimed to be byte-identical to comma's training archive. See
[Protomaps downloads](https://docs.protomaps.com/basemaps/downloads) for distribution
and attribution information; a wider deployment should serve its own archive mirror.
Existing steering, cooperative steering, action conversion and
the restored 1.6 m/s low-speed handling are preserved.
Recorded-camera model replay can test whether real route inputs change the
@@ -1,4 +1,5 @@
import gzip
import json
import os
import re
import struct
@@ -18,7 +19,7 @@ import requests
import openpilot.cereal.messaging as messaging
from openpilot.common.prefix import OpenpilotPrefix
from openpilot.selfdrive.navigation.mapsd import MapsD
from openpilot.selfdrive.navigation.navd import REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner
from openpilot.selfdrive.navigation.navd import RECOMPUTE_HOLD, REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner
from openpilot.selfdrive.navigation.pmtiles import PMTiles, decode_directory, tile_id
from openpilot.selfdrive.navigation.renderer import MAJOR, MEDIUM, MINOR, SIZE, STREET, TRACK, MapRenderer, draw_segments, mercator, project, stroke_widths
from openpilot.selfdrive.navigation.route import RouteTracker
@@ -101,6 +102,16 @@ def route_engine(sm, pm, values):
class RangeHandler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == '/builds.json':
self.server.index_requests += 1
entries = {} if self.server.invalid_index else [
{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.server.archive)}]
data = json.dumps(entries).encode()
self.send_response(200)
self.send_header('Content-Length', str(len(data)))
self.end_headers()
self.wfile.write(data)
return
self.server.requests.append(dict(self.headers))
if self.headers.get('If-Match', self.server.etag) != self.server.etag:
self.send_error(412)
@@ -123,7 +134,7 @@ class RangeHandler(BaseHTTPRequestHandler):
class TileServerCase(unittest.TestCase):
"""Serves a leaf-directory archive over HTTP range requests, with the comma API and tile cache stubbed."""
"""Serves a real public-index/leaf-directory archive protocol over local HTTP."""
def setUp(self):
self.directory = tempfile.TemporaryDirectory()
@@ -137,11 +148,10 @@ class TileServerCase(unittest.TestCase):
threading.Thread(target=self.server.serve_forever, kwargs={'poll_interval': 0.01}, daemon=True).start()
self.addCleanup(self.server.server_close)
self.addCleanup(self.server.shutdown)
self.api = MagicMock(dongle_id='test-dongle')
self.api.get_token.return_value = 'device-token'
self.url = f'http://127.0.0.1:{self.server.server_port}/planet.pmtiles?sig=secret'
self.api.get.return_value.json.return_value = {'url': self.url, 'expires': time.time() + 86400} # noqa: TID251
for name, value in [('Api', MagicMock(return_value=self.api)), ('Params', MagicMock()), ('PC', True)]:
self.server.index_requests = 0
self.server.invalid_index = False
self.url = f'http://127.0.0.1:{self.server.server_port}'
for name, value in [('PC', True), ('BUILD_INDEX_URL', self.url + '/builds.json'), ('BUILD_BASE_URL', self.url + '/')]:
self.enterContext(patch(f'openpilot.selfdrive.navigation.tiles.{name}', value))
self.enterContext(patch('openpilot.selfdrive.navigation.tiles.Paths.comma_home', return_value=self.directory.name))
@@ -168,6 +178,15 @@ class TestEndToEnd(TileServerCase):
sm.update(0)
return mapsd.update(sm)
def restored_route_frame():
# Destination cancellation retains the existing recompute cooldown.
# Advance its bounded update count before testing tile-cache reuse.
for _ in range((RECOMPUTE_HOLD + 1) * navd.update_frequency):
result = frame()
if navd.route is not None:
return result
self.fail('Destination route was not restored within the existing recompute cooldown')
image, timestamp = frame()
# The route is centered over the white road, with the ego arrow three quarters down.
self.assertEqual((image[128, 128], image[128, 131], image[188, 128], image[195, 128]), (128, 255, 255, 0))
@@ -180,39 +199,39 @@ class TestEndToEnd(TileServerCase):
values.pop('NavDestination')
self.assertFalse(frame()[0].any()) # black without a route
values['NavDestination'] = dict(destination)
self.assertEqual(frame()[0][128, 128], 128)
self.assertEqual(restored_route_frame()[0][128, 128], 128)
self.assertEqual(len(self.server.requests), fetched) # the second route is served from the cache
image, timestamp = frame(fix=False)
self.assertFalse(image.any() or timestamp) # black, not stale, without a fix
self.api.get.assert_called_once()
self.assertEqual(self.server.index_requests, 1)
self.assertTrue(all('Range' in headers for headers in self.server.requests))
self.assertTrue(all(headers['If-Match'] == self.server.etag for headers in self.server.requests[1:]))
self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.iterdir()))
self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.rglob('*') if p.is_file()))
# The archive is replaced on the server and the cache evicted: the next fetch fails, the map goes black,
# and after a delay a fresh reader starts over on the new version.
self.server.etag = '"new-version"'
for cached in self.cache.iterdir():
cached.unlink()
for cached in self.cache.rglob('*'):
if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name):
cached.unlink()
values.pop('NavDestination')
frame()
values['NavDestination'] = dict(destination)
image, timestamp = frame()
image, timestamp = restored_route_frame()
self.assertFalse(image.any() or timestamp)
fetched = len(self.server.requests)
self.assertFalse(frame()[1]) # not retried yet
self.assertEqual(len(self.server.requests), fetched)
mapsd.retry_at = 0
self.assertEqual(frame()[0][128, 128], 128)
self.assertEqual((self.cache / 'version').read_text().split()[0], '"new-version"')
self.assertEqual(json.loads((self.cache / 'archive.json').read_text())['etag'], '"new-version"')
class TestRemoteTiles(TileServerCase):
def test_shared_cache_and_refresh(self):
with closing(PMTiles(RemoteTiles())) as archive:
archive.source.expires = 0 # the signed URL is refreshed before the next fetch
self.assertEqual(archive.get(0, 0, 0), road_tile())
self.assertEqual(archive.get(0, 0, 0), road_tile())
self.assertEqual(self.api.get.call_count, 2)
self.assertEqual(self.server.index_requests, 1)
count = len(self.server.requests)
with closing(PMTiles(RemoteTiles())) as archive:
self.assertEqual(archive.get(0, 0, 0), road_tile())
@@ -220,7 +239,9 @@ class TestRemoteTiles(TileServerCase):
self.assertEqual(len(self.server.requests), count + 1)
with patch('openpilot.selfdrive.navigation.tiles.CACHE_LIMIT', 1), closing(RemoteTiles()) as source:
source.read(1, 2) # an uncached range is downloaded, then the cache is trimmed to the limit
self.assertEqual([p.name for p in self.cache.iterdir()], ['version'])
retained = [p.name for p in self.cache.rglob('*') if p.is_file() and re.fullmatch(r'\d+-\d+', p.name)]
self.assertEqual(retained, ['0-127']) # current header is retained for offline startup
self.assertTrue((self.cache / 'archive.json').exists())
def test_prefetch_merges_nearby_ranges(self):
with patch('openpilot.selfdrive.navigation.tiles.MERGE_GAP', 16), closing(RemoteTiles()) as source:
@@ -235,15 +256,16 @@ class TestRemoteTiles(TileServerCase):
def test_failures(self):
with closing(PMTiles(RemoteTiles())) as archive:
self.server.etag = '"new-version"'
with self.assertRaisesRegex(OSError, '412'): # the archive changed under an open reader
with self.assertRaises(OSError): # the archive changed under an open reader
archive.get(0, 0, 0)
with closing(PMTiles(RemoteTiles())) as archive: # a new reader starts over on the new version
self.assertEqual(archive.get(0, 0, 0), road_tile())
self.server.ignore_range = True
with self.assertRaisesRegex(OSError, '200'):
with self.assertRaises(OSError):
RemoteTiles()
self.api.get.return_value.raise_for_status.side_effect = requests.HTTPError(self.url)
with self.assertRaisesRegex(OSError, '^Unable to fetch map tiles$'): # the signed URL stays out of the error
self.server.ignore_range = False
self.server.invalid_index = True
with self.assertRaisesRegex(OSError, '^Unable to load public Protomaps basemap$'):
RemoteTiles()
@@ -0,0 +1,235 @@
"""Actual public tile client requests and cache behavior, without accounts/network."""
import ast
import io
import importlib.util
import json
from pathlib import Path
import re
import sys
import tempfile
from types import SimpleNamespace as NS
import unittest
from unittest.mock import patch
import requests
from requests.adapters import BaseAdapter
from requests.structures import CaseInsensitiveDict
SOURCE = Path(__file__).resolve().parents[1]
REAL_SESSION = requests.Session
class PublicServer(BaseAdapter):
def __init__(self):
self.archive = b'PMTiles\x03' + bytes(range(256)) * 64
self.entries = [{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.archive)}]
self.etag = '"public-build"'
self.calls = []
self.offline = False
self.index_bytes = None
self.index_headers = {}
self.range_headers = {}
self.range_status = 206
self.range_body = None
def send(self, request, **_kwargs):
self.calls.append(request)
if self.offline:
raise requests.ConnectionError('fixture network unavailable')
response = requests.Response()
response.request, response.url = request, request.url
if request.url == 'https://build-metadata.protomaps.dev/builds.json':
body = self.index_bytes if self.index_bytes is not None else json.dumps(self.entries).encode()
response.status_code = 200
headers = {'Content-Length': str(len(body))} | self.index_headers
else:
if not re.fullmatch(r'https://build\.protomaps\.com/\d{8}\.pmtiles', request.url):
raise AssertionError('Client must never contact an account or arbitrary endpoint')
match = re.fullmatch(r'bytes=(\d+)-(\d+)', request.headers['Range'])
start, end = map(int, match.groups())
body = self.archive[start:end + 1] if self.range_body is None else self.range_body
response.status_code = self.range_status
headers = {'Content-Range': f'bytes {start}-{end}/{len(self.archive)}', 'ETag': self.etag,
'Content-Length': str(len(body))} | self.range_headers
response.headers = CaseInsensitiveDict(headers)
response.raw = io.BytesIO(body)
return response
def close(self):
pass
class PublicTilesTests(unittest.TestCase):
def setUp(self):
self.temp = tempfile.TemporaryDirectory()
self.addCleanup(self.temp.cleanup)
self.home = Path(self.temp.name)
dependencies = {'openpilot.common.hardware': NS(PC=True),
'openpilot.common.hardware.hw': NS(Paths=NS(comma_home=lambda: self.temp.name)),
'openpilot.selfdrive.navigation.pmtiles': NS(HEADER_LENGTH=127)}
spec = importlib.util.spec_from_file_location('actual_public_tiles', SOURCE / 'tiles.py')
self.tiles = importlib.util.module_from_spec(spec)
with patch.dict(sys.modules, dependencies):
spec.loader.exec_module(self.tiles)
self.server = PublicServer()
def session():
value = REAL_SESSION()
value.trust_env = False
value.mount('https://', self.server)
return value
self.enterContext(patch.object(self.tiles.requests, 'Session', side_effect=session))
def client(self):
value = self.tiles.RemoteTiles()
self.addCleanup(value.close)
return value
def ranges(self):
return [call for call in self.server.calls if 'Range' in call.headers]
def test_latest_compatible_public_build_has_no_account_or_token_dependency(self):
self.server.entries += [{'key': '20261104.pmtiles', 'version': '5.0.0', 'size': len(self.server.archive)},
{'key': '20261102.pmtiles', 'version': '4.99.1', 'size': len(self.server.archive)}]
client = self.client()
self.assertEqual(client.url, 'https://build.protomaps.com/20261103.pmtiles')
self.assertEqual(client.read(300, 100), self.server.archive[300:400])
self.assertEqual(self.ranges()[-1].headers['If-Match'], self.server.etag)
for call in self.server.calls:
self.assertNotIn('Authorization', call.headers)
self.assertNotIn('access_token', call.url)
self.assertNotIn('comma', call.url)
imported = [ast.unparse(node) for node in ast.parse((SOURCE / 'tiles.py').read_text()).body if isinstance(node, (ast.Import, ast.ImportFrom))]
self.assertFalse(any('api' in value.lower() or 'params' in value.lower() for value in imported))
def test_invalid_index_or_unsupported_schema_fails_before_archive_request(self):
invalid = ({'url': 'https://untrusted.invalid/archive'}, [],
[{'key': '../20261103.pmtiles', 'version': '4.15.2', 'size': 200}],
[{'key': '20260230.pmtiles', 'version': '4.15.2', 'size': 200}],
[{'key': '20261103.pmtiles', 'version': '5.0.0', 'size': 200}],
[{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': True}])
for value in invalid:
with self.subTest(value=value):
self.server.entries = value
with self.assertRaises(OSError):
self.client()
self.assertEqual(self.ranges(), [])
self.server.index_bytes = b'not JSON'
with self.assertRaises(OSError):
self.client()
def test_index_body_and_declared_length_are_bounded(self):
self.server.index_headers = {'Content-Length': str(self.tiles.INDEX_LIMIT + 1)}
with self.assertRaises(OSError):
self.client()
self.server.index_headers = {}
self.server.index_bytes = b' ' * (self.tiles.INDEX_LIMIT + 1)
with self.assertRaises(OSError):
self.client()
self.assertEqual(self.ranges(), [])
def test_range_status_identity_length_and_content_range_fail_closed(self):
cases = [({'status': 200}), ({'headers': {'Content-Range': 'bytes 1-127/999'}}),
({'headers': {'ETag': ''}}), ({'headers': {'ETag': 'W/"weak"'}}),
({'headers': {'Content-Length': '99999999999999'}}),
({'headers': {'Content-Encoding': 'gzip'}}),
({'body': b'x' * 126, 'headers': {'Content-Length': '127'}}),
({'body': b'x' * 128, 'headers': {'Content-Length': '127'}})]
for case in cases:
with self.subTest(case=case):
self.server.range_status = case.get('status', 206)
self.server.range_headers = case.get('headers', {})
self.server.range_body = case.get('body')
with self.assertRaises(OSError):
self.client()
def test_archive_cannot_change_etag_or_total_size_mid_reader(self):
client = self.client()
self.server.etag = '"changed"'
with self.assertRaisesRegex(OSError, 'changed'):
client.read(200, 100)
self.assertFalse((client.cache / '200-100').exists())
self.server.etag = client.etag
self.server.range_headers = {'Content-Range': f'bytes 200-299/{client.size + 1}'}
with self.assertRaises(OSError):
client.read(200, 100)
def test_restart_revalidates_header_and_reuses_only_same_archive_ranges(self):
first = self.client()
first.read(200, 100)
count = len(self.server.calls)
second = self.client()
self.assertEqual(second.cache, first.cache)
self.assertEqual(second.read(200, 100), self.server.archive[200:300])
self.assertEqual(len(self.server.calls), count + 2) # build index and pinned header only
self.server.entries[0]['key'] = '20261104.pmtiles'
third = self.client()
self.assertNotEqual(third.cache, first.cache)
count = len(self.server.calls)
third.read(200, 100)
self.assertEqual(len(self.server.calls), count + 1)
def test_offline_restart_reuses_pinned_header_and_cached_ranges_only(self):
first = self.client()
first.read(200, 100)
self.server.offline = True
second = self.client()
self.assertEqual(second.url, first.url)
self.assertEqual(second.etag, first.etag)
self.assertEqual(second.read(200, 100), self.server.archive[200:300])
with self.assertRaises(OSError):
second.read(500, 100)
def test_malformed_fresh_index_does_not_fall_back_or_relabel_old_bytes(self):
first = self.client()
first.read(200, 100)
self.server.entries = []
with self.assertRaises(OSError):
self.client()
self.assertEqual(first.read(200, 100), self.server.archive[200:300])
def test_prefetch_merges_nearby_ranges_but_caps_every_request(self):
client = self.client()
with patch.object(self.tiles, 'MAX_RANGE', 64):
count = len(self.ranges())
client.prefetch([(200, 32), (244, 32), (400, 16), (400, 16)])
added = self.ranges()[count:]
self.assertEqual(len(added), 3)
for call in added:
start, end = map(int, call.headers['Range'][6:].split('-'))
self.assertLessEqual(end - start + 1, 64)
count = len(self.server.calls)
self.assertEqual(client.read(244, 32), self.server.archive[244:276])
self.assertEqual(len(self.server.calls), count)
count = len(self.ranges())
client.prefetch([(600, 10), (620, 10)])
self.assertEqual(len(self.ranges()), count + 1)
def test_cache_limit_counts_ranges_from_all_archive_identities(self):
client = self.client()
with patch.object(self.tiles, 'CACHE_LIMIT', 160):
client.read(200, 100)
total = sum(path.stat().st_size for path in client.base.rglob('*') if path.is_file() and re.fullmatch(r'\d+-\d+', path.name))
self.assertLessEqual(total, 160)
def test_cache_trim_retains_current_header_for_offline_restart(self):
first = self.client()
with patch.object(self.tiles, 'CACHE_LIMIT', 228):
first.read(200, 100)
first.read(400, 100)
self.assertTrue((first.cache / '0-127').exists())
self.assertFalse((first.cache / '200-100').exists())
self.server.offline = True
second = self.client()
self.assertEqual(second.read(400, 100), self.server.archive[400:500])
def test_invalid_or_excessive_ranges_do_not_make_requests(self):
client = self.client()
count = len(self.server.calls)
for offset, length in ((-1, 1), (0, 0), (0, self.tiles.MAX_RANGE + 1), (client.size, 1), (True, 1)):
with self.subTest(offset=offset, length=length), self.assertRaises(ValueError):
client.read(offset, length)
self.assertEqual(len(self.server.calls), count)
if __name__ == '__main__':
unittest.main()
@@ -92,6 +92,58 @@ class TestSignalTurnStatus(unittest.TestCase):
widget = load_widget(ui_state).NavigationPanel()
self.assertEqual(widget.signal_status['primary'], 'Navigation active')
def test_ready_route_explains_fresh_matched_model_map_wait(self):
ui_state, values = state('ready')
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'}
module = load_widget(ui_state)
widget = module.NavigationPanel()
self.assertEqual(widget.signal_status['primary'], 'Route ready')
self.assertEqual(widget.signal_status['secondary'], 'Waiting for model map')
widget._render(module.rl.Rectangle(0, 0, 261, 400))
self.assertIn('Waiting for model map', widget._eta_label.text)
def test_fresh_matched_wait_reasons_are_specific_without_claiming_active(self):
expected = {'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid',
'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map',
'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable'}
for reason, text in expected.items():
with self.subTest(reason=reason):
ui_state, values = state('ready')
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
'session': 'one', 'routeGeneration': 10, 'reason': reason}
result = load_widget(ui_state).NavigationPanel().signal_status
self.assertEqual(result['primary'], 'Route ready')
self.assertEqual(result['secondary'], text)
def test_stale_wrong_route_or_unsupported_model_reason_is_not_reused(self):
for change in ({'updatedAt': 99.}, {'updatedAt': 101.}, {'updatedAt': float('nan')}, {'updatedAt': True},
{'session': 'other'}, {'routeGeneration': 11}, {'supported': False}, {'active': 0}):
with self.subTest(change=change):
ui_state, values = state('ready')
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
'session': 'one', 'routeGeneration': 10, 'reason': 'map_invalid', **change}
result = load_widget(ui_state).NavigationPanel().signal_status
self.assertEqual(result['primary'], 'Route ready')
self.assertEqual(result['secondary'], 'Waiting for map and model')
def test_active_and_pending_route_labels_ignore_old_wait_reason(self):
for mode, active, primary in (('turn', True, 'Navigation active'), ('pending', False, 'Turn candidate')):
ui_state, values = state('ready')
values['MapboxTurnIntent']['mode'] = mode
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': active,
'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'}
result = load_widget(ui_state).NavigationPanel().signal_status
self.assertEqual(result['primary'], primary)
self.assertEqual(result['secondary'], 'Right onto Example Street')
def test_unknown_or_malformed_wait_reason_keeps_generic_message(self):
for reason in (None, 'unknown_future_reason', [], {}):
ui_state, values = state('ready')
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
'session': 'one', 'routeGeneration': 10, 'reason': reason}
self.assertEqual(load_widget(ui_state).NavigationPanel().signal_status['secondary'], 'Waiting for map and model')
def test_explicit_destination_keeps_normal_arrow_and_shows_blocked_footer(self):
ui_state, _ = state('blocked', destination=True)
module = load_widget(ui_state)
+160 -59
View File
@@ -1,114 +1,215 @@
"""Byte ranges of the hosted PMTiles basemap, cached on disk."""
"""Bounded byte ranges of a public Protomaps v4 basemap; no device account.
Experimental single-device transport. Public daily-build hosting is not a fleet
backend: deployments should mirror the archive while preserving its identity.
"""
import hashlib
import json
import re
import time
from datetime import date
from pathlib import Path
import requests
from openpilot.common.api import Api
from openpilot.common.hardware import PC
from openpilot.common.hardware.hw import Paths
from openpilot.common.params import Params
from openpilot.selfdrive.navigation.pmtiles import HEADER_LENGTH
BUILD_INDEX_URL = 'https://build-metadata.protomaps.dev/builds.json'
BUILD_BASE_URL = 'https://build.protomaps.com/'
INDEX_LIMIT = 512 * 1024
MAX_RANGE = 4 * 1024 * 1024
CACHE_LIMIT = 64 * 1024 * 1024
MERGE_GAP = 256 * 1024 # tiles this close together in the archive are fetched in one range request
MERGE_GAP = 256 * 1024
TIMEOUT = (5, 10)
def _build(entry):
"""Only supported official dated archives, never a URL supplied by metadata."""
if not isinstance(entry, dict):
return None
key, size, version = (entry.get(name) for name in ('key', 'size', 'version'))
if (not isinstance(key, str) or not re.fullmatch(r'\d{8}\.pmtiles', key) or
isinstance(size, bool) or not isinstance(size, int) or not HEADER_LENGTH <= size <= 1024 ** 4 or
not isinstance(version, str) or not re.fullmatch(r'4\.\d+\.\d+', version)):
return None
try:
date(int(key[:4]), int(key[4:6]), int(key[6:8]))
except ValueError:
return None
return {'key': key, 'size': size, 'version': version, 'url': BUILD_BASE_URL + key}
def _strong_etag(value):
return isinstance(value, str) and re.fullmatch(r'"[\x21\x23-\x7e]{1,510}"', value) is not None
class RemoteTiles:
def __init__(self):
self.cache = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles'
self.cache.mkdir(parents=True, exist_ok=True, mode=0o700)
self.base = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles'
self.base.mkdir(parents=True, exist_ok=True, mode=0o700)
self.session = requests.Session()
self.api = Api(Params().get('DongleId'))
self.url = ''
self.expires = 0
self.etag = None
self.size = 0
version = self.cache / 'version'
self.url, self.etag, self.size = '', None, 0
self.cache = None
saved = self._saved_identity()
try:
header = self._fetch(0, HEADER_LENGTH) # learns the archive's ETag and size
if not version.exists() or version.read_text() != f'{self.etag} {self.size}':
for cached in self.cache.iterdir(): # a new archive has new offsets
cached.unlink()
version.write_text(f'{self.etag} {self.size}')
try:
build = self._latest_build()
self.url, self.size = build['url'], build['size']
header = self._fetch(0, HEADER_LENGTH)
identity = build | {'etag': self.etag}
except requests.RequestException:
# An offline reader may reuse only a previously pinned archive with its
# exact cached header. Missing ranges still fail; no route-only raster.
if saved is None:
raise
identity = saved
self.url, self.etag, self.size = identity['url'], identity['etag'], identity['size']
self.cache = self._cache_for(identity)
header = (self.cache / f'0-{HEADER_LENGTH}').read_bytes()
if len(header) != HEADER_LENGTH or header[:8] != b'PMTiles\x03':
raise ValueError('Invalid public PMTiles header')
self.cache = self._cache_for(identity)
self.cache.mkdir(parents=True, exist_ok=True)
self._store(0, header)
except Exception:
temporary = self.base / 'archive.tmp'
temporary.write_text(json.dumps(identity))
temporary.replace(self.base / 'archive.json')
self._trim()
except (requests.RequestException, OSError, ValueError, TypeError, KeyError):
self.close()
raise
raise OSError('Unable to load public Protomaps basemap') from None
def _cache_for(self, identity):
key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:32]
return self.base / 'archives' / key
def _saved_identity(self):
try:
path = self.base / 'archive.json'
if path.stat().st_size > 4096:
return None
saved = json.loads(path.read_text())
build = _build(saved)
if build is not None and saved.get('url') == build['url'] and _strong_etag(saved.get('etag')):
return build | {'etag': saved['etag']}
except (OSError, ValueError, TypeError):
pass
return None
def _latest_build(self):
with self.session.get(BUILD_INDEX_URL, headers={'Accept-Encoding': 'identity'}, timeout=TIMEOUT, stream=True,
allow_redirects=False) as response:
response.raise_for_status()
if response.status_code != 200:
raise ValueError('Unexpected public build index response')
declared = response.headers.get('Content-Length')
if declared is not None and not 0 <= int(declared) <= INDEX_LIMIT:
raise ValueError('Oversized public build index')
data = bytearray()
for chunk in response.iter_content(chunk_size=65536):
if len(data) + len(chunk) > INDEX_LIMIT:
raise ValueError('Oversized public build index')
data.extend(chunk)
entries = json.loads(data)
if not isinstance(entries, list) or len(entries) > 4096:
raise ValueError('Invalid public build index')
builds = [build for entry in entries if (build := _build(entry)) is not None]
if not builds:
raise ValueError('No compatible public Protomaps v4 build')
return max(builds, key=lambda build: build['key'])
def _range(self, offset, length):
if (isinstance(offset, bool) or isinstance(length, bool) or not isinstance(offset, int) or not isinstance(length, int) or
offset < 0 or not 0 < length <= MAX_RANGE or offset + length > self.size):
raise ValueError('Invalid or oversized public map range')
def _fetch(self, offset: int, length: int) -> bytes:
try:
if time.time() >= self.expires - 60: # noqa: TID251 - SAS expiry is a Unix timestamp
response = self.api.get(f'v1/{self.api.dongle_id}/nav/tiles', access_token=self.api.get_token(), timeout=10)
response.raise_for_status()
metadata = response.json()
self.url, self.expires = metadata['url'], metadata['expires']
headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag is not None:
headers['If-Match'] = self.etag
with self.session.get(self.url, headers=headers, timeout=10, stream=True) as response:
if response.status_code != 206:
raise OSError(f'Map tile range request failed ({response.status_code})')
match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', ''))
if match is None or tuple(map(int, match.groups()[:2])) != (offset, offset + length - 1):
raise OSError('Invalid map tile range response')
size = int(match[3])
etag = response.headers.get('ETag')
if not etag or (self.etag is not None and (etag != self.etag or size != self.size)):
raise OSError('Map archive changed during read')
data = bytearray()
for chunk in response.iter_content(chunk_size=min(length + 1, 65536)):
data.extend(chunk)
if len(data) > length:
raise OSError('Oversized map tile range response')
if len(data) != length:
raise OSError('Truncated map tile range response')
self.size, self.etag = size, etag
return bytes(data)
except (requests.RequestException, ValueError, KeyError, TypeError, RuntimeError):
# requests exceptions contain the signed URL; keep it out of logs.
raise OSError('Unable to fetch map tiles') from None
self._range(offset, length)
headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag is not None:
headers['If-Match'] = self.etag
with self.session.get(self.url, headers=headers, timeout=TIMEOUT, stream=True, allow_redirects=False) as response:
response.raise_for_status()
if response.status_code != 206:
raise OSError(f'Public map range request failed ({response.status_code})')
match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', ''))
if match is None or tuple(map(int, match.groups())) != (offset, offset + length - 1, self.size):
raise OSError('Invalid public map range response')
etag = response.headers.get('ETag')
if not _strong_etag(etag) or (self.etag is not None and etag != self.etag):
raise OSError('Public map archive changed during read')
declared = response.headers.get('Content-Length')
if declared is not None and int(declared) != length:
raise OSError('Invalid public map range length')
if response.headers.get('Content-Encoding', 'identity') != 'identity':
raise OSError('Encoded public map range response')
data = bytearray()
for chunk in response.iter_content(chunk_size=min(length + 1, 65536)):
if len(data) + len(chunk) > length:
raise OSError('Oversized public map range response')
data.extend(chunk)
if len(data) != length:
raise OSError('Truncated public map range response')
self.etag = etag
return bytes(data)
def _store(self, offset: int, data: bytes):
(self.cache / f'{offset}-{len(data)}').write_bytes(data)
path = self.cache / f'{offset}-{len(data)}'
temporary = path.with_suffix('.tmp')
temporary.write_bytes(data)
temporary.replace(path)
def _trim(self):
files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.cache.iterdir() if re.fullmatch(r'\d+-\d+', cached.name)]
files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.base.rglob('*')
if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name)]
size = sum(cached_size for _, cached_size, _ in files)
for _, cached_size, cached in sorted(files):
if size <= CACHE_LIMIT:
break
if cached == self.cache / f'0-{HEADER_LENGTH}':
continue # Required for an offline restart of the current pinned archive.
cached.unlink(missing_ok=True)
size -= cached_size
def read(self, offset: int, length: int) -> bytes:
self._range(offset, length)
try:
data = (self.cache / f'{offset}-{length}').read_bytes()
if len(data) == length:
return data
except FileNotFoundError:
pass
data = self._fetch(offset, length)
try:
data = self._fetch(offset, length)
except (requests.RequestException, ValueError):
raise OSError('Unable to fetch public map tiles') from None
self._store(offset, data)
self._trim()
return data
def prefetch(self, ranges):
"""Cache (offset, length) ranges ahead of reads, fetching neighboring ones in a single request."""
missing = sorted((offset, length) for offset, length in set(ranges) if not (self.cache / f'{offset}-{length}').exists())
missing = []
for offset, length in sorted(set(ranges)):
self._range(offset, length)
if not (self.cache / f'{offset}-{length}').exists():
missing.append((offset, length))
if not missing:
return
groups = [[missing[0]]]
for offset, length in missing[1:]:
start = groups[-1][0][0]
last_offset, last_length = groups[-1][-1]
if offset - (last_offset + last_length) <= MERGE_GAP:
if offset - (last_offset + last_length) <= MERGE_GAP and offset + length - start <= MAX_RANGE:
groups[-1].append((offset, length))
else:
groups.append([(offset, length)])
for group in groups:
start, end = group[0][0], group[-1][0] + group[-1][1]
data = self._fetch(start, end - start)
start, end = group[0][0], max(offset + length for offset, length in group)
try:
data = self._fetch(start, end - start)
except (requests.RequestException, ValueError):
raise OSError('Unable to prefetch public map tiles') from None
for offset, length in group:
self._store(offset, data[offset - start:offset - start + length])
self._trim()
@@ -53,10 +53,28 @@ def signal_turn_status(enabled, intent, context, now, left_blinker, right_blinke
model_state.get('session') == intent['session'] and
isinstance(context.get('routeGeneration'), int) and context['routeGeneration'] > 0 and
model_state.get('routeGeneration') == context['routeGeneration'])
waiting = 'Waiting for map and model'
matched_wait = (isinstance(model_updated, (int, float)) and not isinstance(model_updated, bool) and
np.isfinite(model_updated) and 0. <= now-model_updated <= .5 and
model_state.get('supported') is True and model_state.get('active') is False and
model_state.get('session') == intent['session'] and
isinstance(context.get('routeGeneration'), int) and not isinstance(context['routeGeneration'], bool) and
context['routeGeneration'] > 0 and model_state.get('routeGeneration') == context['routeGeneration'])
model_reason = model_state.get('reason') if matched_wait else None
if isinstance(model_reason, str):
waiting = {
'map_missing': 'Waiting for model map', 'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid',
'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map',
'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable',
'signal_navigation_disabled': 'Signal navigation is inactive', 'route_context_missing': 'Waiting for route context',
'route_context_unavailable': 'Route context unavailable', 'signal_session_mismatch': 'Waiting for current signal',
'destination_route_unavailable': 'Waiting for destination route', 'waiting_for_navigation': 'Waiting for navigation inputs',
'ready': 'Waiting for model response',
}.get(model_reason, waiting)
pending = intent.get('mode') == 'pending'
return {'state': 'ready', 'primary': 'Turn candidate' if pending else 'Navigation active' if active else 'Route ready',
'direction': direction, 'distanceM': distance,
'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else 'Waiting for map and model')}
'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else waiting)}
if status == 'searching':
return {'state': status, 'primary': f'Finding next {direction}', 'secondary': 'Checking mapped streets'}
if status == 'passed':
@@ -187,6 +205,11 @@ class NavigationPanel(Widget):
source = rl.Rectangle((SIZE - width) // 2, min(SIZE - height, max(0, EGO[1] + MARKER_MARGIN - height)), width, height)
target = rl.Rectangle(map_rect.x + (map_rect.width - width) / 2, map_rect.y + map_rect.height - height, width, height)
rl.draw_texture_pro(self._texture, source, target, rl.Vector2(0, 0), 0, rl.WHITE)
# Attribution is a display overlay; the model receives the original raster.
credit = rl.Rectangle(map_rect.x+2, map_rect.y+map_rect.height-17, map_rect.width-4, 16)
rl.draw_rectangle_rec(credit, rl.Color(7, 13, 18, 220))
gui_label(credit, '© OpenStreetMap contributors', 12, self._muted, FontWeight.MEDIUM,
alignment_vertical=TextAlignmentVertical.TOP)
else:
self._draw_status(map_rect)
if signal is not None and (notification or ui_state.nav_destination_set):