diff --git a/openpilot/selfdrive/navigation/SIGNAL_NAVIGATION.md b/openpilot/selfdrive/navigation/SIGNAL_NAVIGATION.md index 1bdc552ad3..b1ee72c1cc 100644 --- a/openpilot/selfdrive/navigation/SIGNAL_NAVIGATION.md +++ b/openpilot/selfdrive/navigation/SIGNAL_NAVIGATION.md @@ -80,7 +80,16 @@ inputs are zeroed. A route preview alone never establishes active model input. The feature is opt-in under **Signal Navigation (Experimental)**. It needs online Directions requests, a Mapbox public token, and automatically fetched -road-map tiles. Existing steering, cooperative steering, action conversion and +road-map tiles. Directions use the configured Mapbox token. The basemap is fetched +from public Protomaps builds, without comma authentication or a Prime subscription. +Only the required archive byte ranges are downloaded and cached; the entire +planet archive is not downloaded. The map data is derived from +[OpenStreetMap](https://www.openstreetmap.org/copyright); attribution is displayed +over the map preview without changing the model raster. The public basemap is not +claimed to be byte-identical to comma's training archive. See +[Protomaps downloads](https://docs.protomaps.com/basemaps/downloads) for distribution +and attribution information; a wider deployment should serve its own archive mirror. +Existing steering, cooperative steering, action conversion and the restored 1.6 m/s low-speed handling are preserved. Recorded-camera model replay can test whether real route inputs change the diff --git a/openpilot/selfdrive/navigation/tests/test_navigation.py b/openpilot/selfdrive/navigation/tests/test_navigation.py index 84d2beee03..dae29f3d12 100644 --- a/openpilot/selfdrive/navigation/tests/test_navigation.py +++ b/openpilot/selfdrive/navigation/tests/test_navigation.py @@ -1,4 +1,5 @@ import gzip +import json import os import re import struct @@ -18,7 +19,7 @@ import requests import openpilot.cereal.messaging as messaging from openpilot.common.prefix import OpenpilotPrefix from openpilot.selfdrive.navigation.mapsd import MapsD -from openpilot.selfdrive.navigation.navd import REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner +from openpilot.selfdrive.navigation.navd import RECOMPUTE_HOLD, REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner from openpilot.selfdrive.navigation.pmtiles import PMTiles, decode_directory, tile_id from openpilot.selfdrive.navigation.renderer import MAJOR, MEDIUM, MINOR, SIZE, STREET, TRACK, MapRenderer, draw_segments, mercator, project, stroke_widths from openpilot.selfdrive.navigation.route import RouteTracker @@ -101,6 +102,16 @@ def route_engine(sm, pm, values): class RangeHandler(BaseHTTPRequestHandler): def do_GET(self): + if self.path == '/builds.json': + self.server.index_requests += 1 + entries = {} if self.server.invalid_index else [ + {'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.server.archive)}] + data = json.dumps(entries).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + return self.server.requests.append(dict(self.headers)) if self.headers.get('If-Match', self.server.etag) != self.server.etag: self.send_error(412) @@ -123,7 +134,7 @@ class RangeHandler(BaseHTTPRequestHandler): class TileServerCase(unittest.TestCase): - """Serves a leaf-directory archive over HTTP range requests, with the comma API and tile cache stubbed.""" + """Serves a real public-index/leaf-directory archive protocol over local HTTP.""" def setUp(self): self.directory = tempfile.TemporaryDirectory() @@ -137,11 +148,10 @@ class TileServerCase(unittest.TestCase): threading.Thread(target=self.server.serve_forever, kwargs={'poll_interval': 0.01}, daemon=True).start() self.addCleanup(self.server.server_close) self.addCleanup(self.server.shutdown) - self.api = MagicMock(dongle_id='test-dongle') - self.api.get_token.return_value = 'device-token' - self.url = f'http://127.0.0.1:{self.server.server_port}/planet.pmtiles?sig=secret' - self.api.get.return_value.json.return_value = {'url': self.url, 'expires': time.time() + 86400} # noqa: TID251 - for name, value in [('Api', MagicMock(return_value=self.api)), ('Params', MagicMock()), ('PC', True)]: + self.server.index_requests = 0 + self.server.invalid_index = False + self.url = f'http://127.0.0.1:{self.server.server_port}' + for name, value in [('PC', True), ('BUILD_INDEX_URL', self.url + '/builds.json'), ('BUILD_BASE_URL', self.url + '/')]: self.enterContext(patch(f'openpilot.selfdrive.navigation.tiles.{name}', value)) self.enterContext(patch('openpilot.selfdrive.navigation.tiles.Paths.comma_home', return_value=self.directory.name)) @@ -168,6 +178,15 @@ class TestEndToEnd(TileServerCase): sm.update(0) return mapsd.update(sm) + def restored_route_frame(): + # Destination cancellation retains the existing recompute cooldown. + # Advance its bounded update count before testing tile-cache reuse. + for _ in range((RECOMPUTE_HOLD + 1) * navd.update_frequency): + result = frame() + if navd.route is not None: + return result + self.fail('Destination route was not restored within the existing recompute cooldown') + image, timestamp = frame() # The route is centered over the white road, with the ego arrow three quarters down. self.assertEqual((image[128, 128], image[128, 131], image[188, 128], image[195, 128]), (128, 255, 255, 0)) @@ -180,39 +199,39 @@ class TestEndToEnd(TileServerCase): values.pop('NavDestination') self.assertFalse(frame()[0].any()) # black without a route values['NavDestination'] = dict(destination) - self.assertEqual(frame()[0][128, 128], 128) + self.assertEqual(restored_route_frame()[0][128, 128], 128) self.assertEqual(len(self.server.requests), fetched) # the second route is served from the cache image, timestamp = frame(fix=False) self.assertFalse(image.any() or timestamp) # black, not stale, without a fix - self.api.get.assert_called_once() + self.assertEqual(self.server.index_requests, 1) self.assertTrue(all('Range' in headers for headers in self.server.requests)) self.assertTrue(all(headers['If-Match'] == self.server.etag for headers in self.server.requests[1:])) - self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.iterdir())) + self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.rglob('*') if p.is_file())) # The archive is replaced on the server and the cache evicted: the next fetch fails, the map goes black, # and after a delay a fresh reader starts over on the new version. self.server.etag = '"new-version"' - for cached in self.cache.iterdir(): - cached.unlink() + for cached in self.cache.rglob('*'): + if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name): + cached.unlink() values.pop('NavDestination') frame() values['NavDestination'] = dict(destination) - image, timestamp = frame() + image, timestamp = restored_route_frame() self.assertFalse(image.any() or timestamp) fetched = len(self.server.requests) self.assertFalse(frame()[1]) # not retried yet self.assertEqual(len(self.server.requests), fetched) mapsd.retry_at = 0 self.assertEqual(frame()[0][128, 128], 128) - self.assertEqual((self.cache / 'version').read_text().split()[0], '"new-version"') + self.assertEqual(json.loads((self.cache / 'archive.json').read_text())['etag'], '"new-version"') class TestRemoteTiles(TileServerCase): def test_shared_cache_and_refresh(self): with closing(PMTiles(RemoteTiles())) as archive: - archive.source.expires = 0 # the signed URL is refreshed before the next fetch self.assertEqual(archive.get(0, 0, 0), road_tile()) self.assertEqual(archive.get(0, 0, 0), road_tile()) - self.assertEqual(self.api.get.call_count, 2) + self.assertEqual(self.server.index_requests, 1) count = len(self.server.requests) with closing(PMTiles(RemoteTiles())) as archive: self.assertEqual(archive.get(0, 0, 0), road_tile()) @@ -220,7 +239,9 @@ class TestRemoteTiles(TileServerCase): self.assertEqual(len(self.server.requests), count + 1) with patch('openpilot.selfdrive.navigation.tiles.CACHE_LIMIT', 1), closing(RemoteTiles()) as source: source.read(1, 2) # an uncached range is downloaded, then the cache is trimmed to the limit - self.assertEqual([p.name for p in self.cache.iterdir()], ['version']) + retained = [p.name for p in self.cache.rglob('*') if p.is_file() and re.fullmatch(r'\d+-\d+', p.name)] + self.assertEqual(retained, ['0-127']) # current header is retained for offline startup + self.assertTrue((self.cache / 'archive.json').exists()) def test_prefetch_merges_nearby_ranges(self): with patch('openpilot.selfdrive.navigation.tiles.MERGE_GAP', 16), closing(RemoteTiles()) as source: @@ -235,15 +256,16 @@ class TestRemoteTiles(TileServerCase): def test_failures(self): with closing(PMTiles(RemoteTiles())) as archive: self.server.etag = '"new-version"' - with self.assertRaisesRegex(OSError, '412'): # the archive changed under an open reader + with self.assertRaises(OSError): # the archive changed under an open reader archive.get(0, 0, 0) with closing(PMTiles(RemoteTiles())) as archive: # a new reader starts over on the new version self.assertEqual(archive.get(0, 0, 0), road_tile()) self.server.ignore_range = True - with self.assertRaisesRegex(OSError, '200'): + with self.assertRaises(OSError): RemoteTiles() - self.api.get.return_value.raise_for_status.side_effect = requests.HTTPError(self.url) - with self.assertRaisesRegex(OSError, '^Unable to fetch map tiles$'): # the signed URL stays out of the error + self.server.ignore_range = False + self.server.invalid_index = True + with self.assertRaisesRegex(OSError, '^Unable to load public Protomaps basemap$'): RemoteTiles() diff --git a/openpilot/selfdrive/navigation/tests/test_public_tiles.py b/openpilot/selfdrive/navigation/tests/test_public_tiles.py new file mode 100644 index 0000000000..45e7770e6b --- /dev/null +++ b/openpilot/selfdrive/navigation/tests/test_public_tiles.py @@ -0,0 +1,235 @@ +"""Actual public tile client requests and cache behavior, without accounts/network.""" +import ast +import io +import importlib.util +import json +from pathlib import Path +import re +import sys +import tempfile +from types import SimpleNamespace as NS +import unittest +from unittest.mock import patch + +import requests +from requests.adapters import BaseAdapter +from requests.structures import CaseInsensitiveDict + +SOURCE = Path(__file__).resolve().parents[1] +REAL_SESSION = requests.Session + + +class PublicServer(BaseAdapter): + def __init__(self): + self.archive = b'PMTiles\x03' + bytes(range(256)) * 64 + self.entries = [{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.archive)}] + self.etag = '"public-build"' + self.calls = [] + self.offline = False + self.index_bytes = None + self.index_headers = {} + self.range_headers = {} + self.range_status = 206 + self.range_body = None + + def send(self, request, **_kwargs): + self.calls.append(request) + if self.offline: + raise requests.ConnectionError('fixture network unavailable') + response = requests.Response() + response.request, response.url = request, request.url + if request.url == 'https://build-metadata.protomaps.dev/builds.json': + body = self.index_bytes if self.index_bytes is not None else json.dumps(self.entries).encode() + response.status_code = 200 + headers = {'Content-Length': str(len(body))} | self.index_headers + else: + if not re.fullmatch(r'https://build\.protomaps\.com/\d{8}\.pmtiles', request.url): + raise AssertionError('Client must never contact an account or arbitrary endpoint') + match = re.fullmatch(r'bytes=(\d+)-(\d+)', request.headers['Range']) + start, end = map(int, match.groups()) + body = self.archive[start:end + 1] if self.range_body is None else self.range_body + response.status_code = self.range_status + headers = {'Content-Range': f'bytes {start}-{end}/{len(self.archive)}', 'ETag': self.etag, + 'Content-Length': str(len(body))} | self.range_headers + response.headers = CaseInsensitiveDict(headers) + response.raw = io.BytesIO(body) + return response + + def close(self): + pass + + +class PublicTilesTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.home = Path(self.temp.name) + dependencies = {'openpilot.common.hardware': NS(PC=True), + 'openpilot.common.hardware.hw': NS(Paths=NS(comma_home=lambda: self.temp.name)), + 'openpilot.selfdrive.navigation.pmtiles': NS(HEADER_LENGTH=127)} + spec = importlib.util.spec_from_file_location('actual_public_tiles', SOURCE / 'tiles.py') + self.tiles = importlib.util.module_from_spec(spec) + with patch.dict(sys.modules, dependencies): + spec.loader.exec_module(self.tiles) + self.server = PublicServer() + def session(): + value = REAL_SESSION() + value.trust_env = False + value.mount('https://', self.server) + return value + self.enterContext(patch.object(self.tiles.requests, 'Session', side_effect=session)) + + def client(self): + value = self.tiles.RemoteTiles() + self.addCleanup(value.close) + return value + + def ranges(self): + return [call for call in self.server.calls if 'Range' in call.headers] + + def test_latest_compatible_public_build_has_no_account_or_token_dependency(self): + self.server.entries += [{'key': '20261104.pmtiles', 'version': '5.0.0', 'size': len(self.server.archive)}, + {'key': '20261102.pmtiles', 'version': '4.99.1', 'size': len(self.server.archive)}] + client = self.client() + self.assertEqual(client.url, 'https://build.protomaps.com/20261103.pmtiles') + self.assertEqual(client.read(300, 100), self.server.archive[300:400]) + self.assertEqual(self.ranges()[-1].headers['If-Match'], self.server.etag) + for call in self.server.calls: + self.assertNotIn('Authorization', call.headers) + self.assertNotIn('access_token', call.url) + self.assertNotIn('comma', call.url) + imported = [ast.unparse(node) for node in ast.parse((SOURCE / 'tiles.py').read_text()).body if isinstance(node, (ast.Import, ast.ImportFrom))] + self.assertFalse(any('api' in value.lower() or 'params' in value.lower() for value in imported)) + + def test_invalid_index_or_unsupported_schema_fails_before_archive_request(self): + invalid = ({'url': 'https://untrusted.invalid/archive'}, [], + [{'key': '../20261103.pmtiles', 'version': '4.15.2', 'size': 200}], + [{'key': '20260230.pmtiles', 'version': '4.15.2', 'size': 200}], + [{'key': '20261103.pmtiles', 'version': '5.0.0', 'size': 200}], + [{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': True}]) + for value in invalid: + with self.subTest(value=value): + self.server.entries = value + with self.assertRaises(OSError): + self.client() + self.assertEqual(self.ranges(), []) + self.server.index_bytes = b'not JSON' + with self.assertRaises(OSError): + self.client() + + def test_index_body_and_declared_length_are_bounded(self): + self.server.index_headers = {'Content-Length': str(self.tiles.INDEX_LIMIT + 1)} + with self.assertRaises(OSError): + self.client() + self.server.index_headers = {} + self.server.index_bytes = b' ' * (self.tiles.INDEX_LIMIT + 1) + with self.assertRaises(OSError): + self.client() + self.assertEqual(self.ranges(), []) + + def test_range_status_identity_length_and_content_range_fail_closed(self): + cases = [({'status': 200}), ({'headers': {'Content-Range': 'bytes 1-127/999'}}), + ({'headers': {'ETag': ''}}), ({'headers': {'ETag': 'W/"weak"'}}), + ({'headers': {'Content-Length': '99999999999999'}}), + ({'headers': {'Content-Encoding': 'gzip'}}), + ({'body': b'x' * 126, 'headers': {'Content-Length': '127'}}), + ({'body': b'x' * 128, 'headers': {'Content-Length': '127'}})] + for case in cases: + with self.subTest(case=case): + self.server.range_status = case.get('status', 206) + self.server.range_headers = case.get('headers', {}) + self.server.range_body = case.get('body') + with self.assertRaises(OSError): + self.client() + + def test_archive_cannot_change_etag_or_total_size_mid_reader(self): + client = self.client() + self.server.etag = '"changed"' + with self.assertRaisesRegex(OSError, 'changed'): + client.read(200, 100) + self.assertFalse((client.cache / '200-100').exists()) + self.server.etag = client.etag + self.server.range_headers = {'Content-Range': f'bytes 200-299/{client.size + 1}'} + with self.assertRaises(OSError): + client.read(200, 100) + + def test_restart_revalidates_header_and_reuses_only_same_archive_ranges(self): + first = self.client() + first.read(200, 100) + count = len(self.server.calls) + second = self.client() + self.assertEqual(second.cache, first.cache) + self.assertEqual(second.read(200, 100), self.server.archive[200:300]) + self.assertEqual(len(self.server.calls), count + 2) # build index and pinned header only + self.server.entries[0]['key'] = '20261104.pmtiles' + third = self.client() + self.assertNotEqual(third.cache, first.cache) + count = len(self.server.calls) + third.read(200, 100) + self.assertEqual(len(self.server.calls), count + 1) + + def test_offline_restart_reuses_pinned_header_and_cached_ranges_only(self): + first = self.client() + first.read(200, 100) + self.server.offline = True + second = self.client() + self.assertEqual(second.url, first.url) + self.assertEqual(second.etag, first.etag) + self.assertEqual(second.read(200, 100), self.server.archive[200:300]) + with self.assertRaises(OSError): + second.read(500, 100) + + def test_malformed_fresh_index_does_not_fall_back_or_relabel_old_bytes(self): + first = self.client() + first.read(200, 100) + self.server.entries = [] + with self.assertRaises(OSError): + self.client() + self.assertEqual(first.read(200, 100), self.server.archive[200:300]) + + def test_prefetch_merges_nearby_ranges_but_caps_every_request(self): + client = self.client() + with patch.object(self.tiles, 'MAX_RANGE', 64): + count = len(self.ranges()) + client.prefetch([(200, 32), (244, 32), (400, 16), (400, 16)]) + added = self.ranges()[count:] + self.assertEqual(len(added), 3) + for call in added: + start, end = map(int, call.headers['Range'][6:].split('-')) + self.assertLessEqual(end - start + 1, 64) + count = len(self.server.calls) + self.assertEqual(client.read(244, 32), self.server.archive[244:276]) + self.assertEqual(len(self.server.calls), count) + count = len(self.ranges()) + client.prefetch([(600, 10), (620, 10)]) + self.assertEqual(len(self.ranges()), count + 1) + + def test_cache_limit_counts_ranges_from_all_archive_identities(self): + client = self.client() + with patch.object(self.tiles, 'CACHE_LIMIT', 160): + client.read(200, 100) + total = sum(path.stat().st_size for path in client.base.rglob('*') if path.is_file() and re.fullmatch(r'\d+-\d+', path.name)) + self.assertLessEqual(total, 160) + + def test_cache_trim_retains_current_header_for_offline_restart(self): + first = self.client() + with patch.object(self.tiles, 'CACHE_LIMIT', 228): + first.read(200, 100) + first.read(400, 100) + self.assertTrue((first.cache / '0-127').exists()) + self.assertFalse((first.cache / '200-100').exists()) + self.server.offline = True + second = self.client() + self.assertEqual(second.read(400, 100), self.server.archive[400:500]) + + def test_invalid_or_excessive_ranges_do_not_make_requests(self): + client = self.client() + count = len(self.server.calls) + for offset, length in ((-1, 1), (0, 0), (0, self.tiles.MAX_RANGE + 1), (client.size, 1), (True, 1)): + with self.subTest(offset=offset, length=length), self.assertRaises(ValueError): + client.read(offset, length) + self.assertEqual(len(self.server.calls), count) + + +if __name__ == '__main__': + unittest.main() diff --git a/openpilot/selfdrive/navigation/tests/test_signal_turn_status_ui.py b/openpilot/selfdrive/navigation/tests/test_signal_turn_status_ui.py index 64f3c3e41b..7f45d7529c 100644 --- a/openpilot/selfdrive/navigation/tests/test_signal_turn_status_ui.py +++ b/openpilot/selfdrive/navigation/tests/test_signal_turn_status_ui.py @@ -92,6 +92,58 @@ class TestSignalTurnStatus(unittest.TestCase): widget = load_widget(ui_state).NavigationPanel() self.assertEqual(widget.signal_status['primary'], 'Navigation active') + def test_ready_route_explains_fresh_matched_model_map_wait(self): + ui_state, values = state('ready') + values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False, + 'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'} + module = load_widget(ui_state) + widget = module.NavigationPanel() + self.assertEqual(widget.signal_status['primary'], 'Route ready') + self.assertEqual(widget.signal_status['secondary'], 'Waiting for model map') + widget._render(module.rl.Rectangle(0, 0, 261, 400)) + self.assertIn('Waiting for model map', widget._eta_label.text) + + def test_fresh_matched_wait_reasons_are_specific_without_claiming_active(self): + expected = {'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid', + 'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map', + 'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable'} + for reason, text in expected.items(): + with self.subTest(reason=reason): + ui_state, values = state('ready') + values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False, + 'session': 'one', 'routeGeneration': 10, 'reason': reason} + result = load_widget(ui_state).NavigationPanel().signal_status + self.assertEqual(result['primary'], 'Route ready') + self.assertEqual(result['secondary'], text) + + def test_stale_wrong_route_or_unsupported_model_reason_is_not_reused(self): + for change in ({'updatedAt': 99.}, {'updatedAt': 101.}, {'updatedAt': float('nan')}, {'updatedAt': True}, + {'session': 'other'}, {'routeGeneration': 11}, {'supported': False}, {'active': 0}): + with self.subTest(change=change): + ui_state, values = state('ready') + values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False, + 'session': 'one', 'routeGeneration': 10, 'reason': 'map_invalid', **change} + result = load_widget(ui_state).NavigationPanel().signal_status + self.assertEqual(result['primary'], 'Route ready') + self.assertEqual(result['secondary'], 'Waiting for map and model') + + def test_active_and_pending_route_labels_ignore_old_wait_reason(self): + for mode, active, primary in (('turn', True, 'Navigation active'), ('pending', False, 'Turn candidate')): + ui_state, values = state('ready') + values['MapboxTurnIntent']['mode'] = mode + values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': active, + 'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'} + result = load_widget(ui_state).NavigationPanel().signal_status + self.assertEqual(result['primary'], primary) + self.assertEqual(result['secondary'], 'Right onto Example Street') + + def test_unknown_or_malformed_wait_reason_keeps_generic_message(self): + for reason in (None, 'unknown_future_reason', [], {}): + ui_state, values = state('ready') + values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False, + 'session': 'one', 'routeGeneration': 10, 'reason': reason} + self.assertEqual(load_widget(ui_state).NavigationPanel().signal_status['secondary'], 'Waiting for map and model') + def test_explicit_destination_keeps_normal_arrow_and_shows_blocked_footer(self): ui_state, _ = state('blocked', destination=True) module = load_widget(ui_state) diff --git a/openpilot/selfdrive/navigation/tiles.py b/openpilot/selfdrive/navigation/tiles.py index c1196e3328..33c1717582 100644 --- a/openpilot/selfdrive/navigation/tiles.py +++ b/openpilot/selfdrive/navigation/tiles.py @@ -1,114 +1,215 @@ -"""Byte ranges of the hosted PMTiles basemap, cached on disk.""" +"""Bounded byte ranges of a public Protomaps v4 basemap; no device account. + +Experimental single-device transport. Public daily-build hosting is not a fleet +backend: deployments should mirror the archive while preserving its identity. +""" +import hashlib +import json import re -import time +from datetime import date from pathlib import Path import requests -from openpilot.common.api import Api from openpilot.common.hardware import PC from openpilot.common.hardware.hw import Paths -from openpilot.common.params import Params from openpilot.selfdrive.navigation.pmtiles import HEADER_LENGTH +BUILD_INDEX_URL = 'https://build-metadata.protomaps.dev/builds.json' +BUILD_BASE_URL = 'https://build.protomaps.com/' +INDEX_LIMIT = 512 * 1024 +MAX_RANGE = 4 * 1024 * 1024 CACHE_LIMIT = 64 * 1024 * 1024 -MERGE_GAP = 256 * 1024 # tiles this close together in the archive are fetched in one range request +MERGE_GAP = 256 * 1024 +TIMEOUT = (5, 10) + + +def _build(entry): + """Only supported official dated archives, never a URL supplied by metadata.""" + if not isinstance(entry, dict): + return None + key, size, version = (entry.get(name) for name in ('key', 'size', 'version')) + if (not isinstance(key, str) or not re.fullmatch(r'\d{8}\.pmtiles', key) or + isinstance(size, bool) or not isinstance(size, int) or not HEADER_LENGTH <= size <= 1024 ** 4 or + not isinstance(version, str) or not re.fullmatch(r'4\.\d+\.\d+', version)): + return None + try: + date(int(key[:4]), int(key[4:6]), int(key[6:8])) + except ValueError: + return None + return {'key': key, 'size': size, 'version': version, 'url': BUILD_BASE_URL + key} + + +def _strong_etag(value): + return isinstance(value, str) and re.fullmatch(r'"[\x21\x23-\x7e]{1,510}"', value) is not None class RemoteTiles: def __init__(self): - self.cache = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles' - self.cache.mkdir(parents=True, exist_ok=True, mode=0o700) + self.base = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles' + self.base.mkdir(parents=True, exist_ok=True, mode=0o700) self.session = requests.Session() - self.api = Api(Params().get('DongleId')) - self.url = '' - self.expires = 0 - self.etag = None - self.size = 0 - version = self.cache / 'version' + self.url, self.etag, self.size = '', None, 0 + self.cache = None + saved = self._saved_identity() try: - header = self._fetch(0, HEADER_LENGTH) # learns the archive's ETag and size - if not version.exists() or version.read_text() != f'{self.etag} {self.size}': - for cached in self.cache.iterdir(): # a new archive has new offsets - cached.unlink() - version.write_text(f'{self.etag} {self.size}') + try: + build = self._latest_build() + self.url, self.size = build['url'], build['size'] + header = self._fetch(0, HEADER_LENGTH) + identity = build | {'etag': self.etag} + except requests.RequestException: + # An offline reader may reuse only a previously pinned archive with its + # exact cached header. Missing ranges still fail; no route-only raster. + if saved is None: + raise + identity = saved + self.url, self.etag, self.size = identity['url'], identity['etag'], identity['size'] + self.cache = self._cache_for(identity) + header = (self.cache / f'0-{HEADER_LENGTH}').read_bytes() + if len(header) != HEADER_LENGTH or header[:8] != b'PMTiles\x03': + raise ValueError('Invalid public PMTiles header') + self.cache = self._cache_for(identity) + self.cache.mkdir(parents=True, exist_ok=True) self._store(0, header) - except Exception: + temporary = self.base / 'archive.tmp' + temporary.write_text(json.dumps(identity)) + temporary.replace(self.base / 'archive.json') + self._trim() + except (requests.RequestException, OSError, ValueError, TypeError, KeyError): self.close() - raise + raise OSError('Unable to load public Protomaps basemap') from None + + def _cache_for(self, identity): + key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:32] + return self.base / 'archives' / key + + def _saved_identity(self): + try: + path = self.base / 'archive.json' + if path.stat().st_size > 4096: + return None + saved = json.loads(path.read_text()) + build = _build(saved) + if build is not None and saved.get('url') == build['url'] and _strong_etag(saved.get('etag')): + return build | {'etag': saved['etag']} + except (OSError, ValueError, TypeError): + pass + return None + + def _latest_build(self): + with self.session.get(BUILD_INDEX_URL, headers={'Accept-Encoding': 'identity'}, timeout=TIMEOUT, stream=True, + allow_redirects=False) as response: + response.raise_for_status() + if response.status_code != 200: + raise ValueError('Unexpected public build index response') + declared = response.headers.get('Content-Length') + if declared is not None and not 0 <= int(declared) <= INDEX_LIMIT: + raise ValueError('Oversized public build index') + data = bytearray() + for chunk in response.iter_content(chunk_size=65536): + if len(data) + len(chunk) > INDEX_LIMIT: + raise ValueError('Oversized public build index') + data.extend(chunk) + entries = json.loads(data) + if not isinstance(entries, list) or len(entries) > 4096: + raise ValueError('Invalid public build index') + builds = [build for entry in entries if (build := _build(entry)) is not None] + if not builds: + raise ValueError('No compatible public Protomaps v4 build') + return max(builds, key=lambda build: build['key']) + + def _range(self, offset, length): + if (isinstance(offset, bool) or isinstance(length, bool) or not isinstance(offset, int) or not isinstance(length, int) or + offset < 0 or not 0 < length <= MAX_RANGE or offset + length > self.size): + raise ValueError('Invalid or oversized public map range') def _fetch(self, offset: int, length: int) -> bytes: - try: - if time.time() >= self.expires - 60: # noqa: TID251 - SAS expiry is a Unix timestamp - response = self.api.get(f'v1/{self.api.dongle_id}/nav/tiles', access_token=self.api.get_token(), timeout=10) - response.raise_for_status() - metadata = response.json() - self.url, self.expires = metadata['url'], metadata['expires'] - headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'} - if self.etag is not None: - headers['If-Match'] = self.etag - with self.session.get(self.url, headers=headers, timeout=10, stream=True) as response: - if response.status_code != 206: - raise OSError(f'Map tile range request failed ({response.status_code})') - match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', '')) - if match is None or tuple(map(int, match.groups()[:2])) != (offset, offset + length - 1): - raise OSError('Invalid map tile range response') - size = int(match[3]) - etag = response.headers.get('ETag') - if not etag or (self.etag is not None and (etag != self.etag or size != self.size)): - raise OSError('Map archive changed during read') - data = bytearray() - for chunk in response.iter_content(chunk_size=min(length + 1, 65536)): - data.extend(chunk) - if len(data) > length: - raise OSError('Oversized map tile range response') - if len(data) != length: - raise OSError('Truncated map tile range response') - self.size, self.etag = size, etag - return bytes(data) - except (requests.RequestException, ValueError, KeyError, TypeError, RuntimeError): - # requests exceptions contain the signed URL; keep it out of logs. - raise OSError('Unable to fetch map tiles') from None + self._range(offset, length) + headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'} + if self.etag is not None: + headers['If-Match'] = self.etag + with self.session.get(self.url, headers=headers, timeout=TIMEOUT, stream=True, allow_redirects=False) as response: + response.raise_for_status() + if response.status_code != 206: + raise OSError(f'Public map range request failed ({response.status_code})') + match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', '')) + if match is None or tuple(map(int, match.groups())) != (offset, offset + length - 1, self.size): + raise OSError('Invalid public map range response') + etag = response.headers.get('ETag') + if not _strong_etag(etag) or (self.etag is not None and etag != self.etag): + raise OSError('Public map archive changed during read') + declared = response.headers.get('Content-Length') + if declared is not None and int(declared) != length: + raise OSError('Invalid public map range length') + if response.headers.get('Content-Encoding', 'identity') != 'identity': + raise OSError('Encoded public map range response') + data = bytearray() + for chunk in response.iter_content(chunk_size=min(length + 1, 65536)): + if len(data) + len(chunk) > length: + raise OSError('Oversized public map range response') + data.extend(chunk) + if len(data) != length: + raise OSError('Truncated public map range response') + self.etag = etag + return bytes(data) def _store(self, offset: int, data: bytes): - (self.cache / f'{offset}-{len(data)}').write_bytes(data) + path = self.cache / f'{offset}-{len(data)}' + temporary = path.with_suffix('.tmp') + temporary.write_bytes(data) + temporary.replace(path) def _trim(self): - files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.cache.iterdir() if re.fullmatch(r'\d+-\d+', cached.name)] + files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.base.rglob('*') + if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name)] size = sum(cached_size for _, cached_size, _ in files) for _, cached_size, cached in sorted(files): if size <= CACHE_LIMIT: break + if cached == self.cache / f'0-{HEADER_LENGTH}': + continue # Required for an offline restart of the current pinned archive. cached.unlink(missing_ok=True) size -= cached_size def read(self, offset: int, length: int) -> bytes: + self._range(offset, length) try: data = (self.cache / f'{offset}-{length}').read_bytes() if len(data) == length: return data except FileNotFoundError: pass - data = self._fetch(offset, length) + try: + data = self._fetch(offset, length) + except (requests.RequestException, ValueError): + raise OSError('Unable to fetch public map tiles') from None self._store(offset, data) self._trim() return data def prefetch(self, ranges): - """Cache (offset, length) ranges ahead of reads, fetching neighboring ones in a single request.""" - missing = sorted((offset, length) for offset, length in set(ranges) if not (self.cache / f'{offset}-{length}').exists()) + missing = [] + for offset, length in sorted(set(ranges)): + self._range(offset, length) + if not (self.cache / f'{offset}-{length}').exists(): + missing.append((offset, length)) if not missing: return groups = [[missing[0]]] for offset, length in missing[1:]: + start = groups[-1][0][0] last_offset, last_length = groups[-1][-1] - if offset - (last_offset + last_length) <= MERGE_GAP: + if offset - (last_offset + last_length) <= MERGE_GAP and offset + length - start <= MAX_RANGE: groups[-1].append((offset, length)) else: groups.append([(offset, length)]) for group in groups: - start, end = group[0][0], group[-1][0] + group[-1][1] - data = self._fetch(start, end - start) + start, end = group[0][0], max(offset + length for offset, length in group) + try: + data = self._fetch(start, end - start) + except (requests.RequestException, ValueError): + raise OSError('Unable to prefetch public map tiles') from None for offset, length in group: self._store(offset, data[offset - start:offset - start + length]) self._trim() diff --git a/openpilot/selfdrive/ui/mici/onroad/navigation.py b/openpilot/selfdrive/ui/mici/onroad/navigation.py index ad9aa583b8..c972a3a4aa 100644 --- a/openpilot/selfdrive/ui/mici/onroad/navigation.py +++ b/openpilot/selfdrive/ui/mici/onroad/navigation.py @@ -53,10 +53,28 @@ def signal_turn_status(enabled, intent, context, now, left_blinker, right_blinke model_state.get('session') == intent['session'] and isinstance(context.get('routeGeneration'), int) and context['routeGeneration'] > 0 and model_state.get('routeGeneration') == context['routeGeneration']) + waiting = 'Waiting for map and model' + matched_wait = (isinstance(model_updated, (int, float)) and not isinstance(model_updated, bool) and + np.isfinite(model_updated) and 0. <= now-model_updated <= .5 and + model_state.get('supported') is True and model_state.get('active') is False and + model_state.get('session') == intent['session'] and + isinstance(context.get('routeGeneration'), int) and not isinstance(context['routeGeneration'], bool) and + context['routeGeneration'] > 0 and model_state.get('routeGeneration') == context['routeGeneration']) + model_reason = model_state.get('reason') if matched_wait else None + if isinstance(model_reason, str): + waiting = { + 'map_missing': 'Waiting for model map', 'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid', + 'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map', + 'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable', + 'signal_navigation_disabled': 'Signal navigation is inactive', 'route_context_missing': 'Waiting for route context', + 'route_context_unavailable': 'Route context unavailable', 'signal_session_mismatch': 'Waiting for current signal', + 'destination_route_unavailable': 'Waiting for destination route', 'waiting_for_navigation': 'Waiting for navigation inputs', + 'ready': 'Waiting for model response', + }.get(model_reason, waiting) pending = intent.get('mode') == 'pending' return {'state': 'ready', 'primary': 'Turn candidate' if pending else 'Navigation active' if active else 'Route ready', 'direction': direction, 'distanceM': distance, - 'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else 'Waiting for map and model')} + 'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else waiting)} if status == 'searching': return {'state': status, 'primary': f'Finding next {direction}', 'secondary': 'Checking mapped streets'} if status == 'passed': @@ -187,6 +205,11 @@ class NavigationPanel(Widget): source = rl.Rectangle((SIZE - width) // 2, min(SIZE - height, max(0, EGO[1] + MARKER_MARGIN - height)), width, height) target = rl.Rectangle(map_rect.x + (map_rect.width - width) / 2, map_rect.y + map_rect.height - height, width, height) rl.draw_texture_pro(self._texture, source, target, rl.Vector2(0, 0), 0, rl.WHITE) + # Attribution is a display overlay; the model receives the original raster. + credit = rl.Rectangle(map_rect.x+2, map_rect.y+map_rect.height-17, map_rect.width-4, 16) + rl.draw_rectangle_rec(credit, rl.Color(7, 13, 18, 220)) + gui_label(credit, '© OpenStreetMap contributors', 12, self._muted, FontWeight.MEDIUM, + alignment_vertical=TextAlignmentVertical.TOP) else: self._draw_status(map_rect) if signal is not None and (notification or ui_state.nav_destination_set):