mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-11-12 20:03:43 +08:00
Load navigation basemaps without a comma account
The imported Api wrapper has no dongle_id attribute, so the original tile loader failed before its first HTTP request and mapsd supplied black frames. Replace that account-dependent lookup with supported public Protomaps build discovery and bounded HTTP byte ranges. Pin archive URL, size and strong ETag, isolate cache identities and retain strict integrity/freshness checks. Show the specific model-side waiting reason for the matching signal and route, and display OpenStreetMap attribution over the preview without changing model pixels. Keep the selected navigation model, steering, cooperative steering, action conversion and all route ownership gates unchanged. Validation includes real public index/range requests and a saved-route raster accepted by the unchanged navigation input buffer, dedicated public transport tests, navigation lifecycle and original routing regressions, and UI reason tests. This is transport/input validation; target-device consumption and driven turn quality still require verification.
This commit is contained in:
@@ -80,7 +80,16 @@ inputs are zeroed. A route preview alone never establishes active model input.
|
||||
|
||||
The feature is opt-in under **Signal Navigation (Experimental)**. It needs
|
||||
online Directions requests, a Mapbox public token, and automatically fetched
|
||||
road-map tiles. Existing steering, cooperative steering, action conversion and
|
||||
road-map tiles. Directions use the configured Mapbox token. The basemap is fetched
|
||||
from public Protomaps builds, without comma authentication or a Prime subscription.
|
||||
Only the required archive byte ranges are downloaded and cached; the entire
|
||||
planet archive is not downloaded. The map data is derived from
|
||||
[OpenStreetMap](https://www.openstreetmap.org/copyright); attribution is displayed
|
||||
over the map preview without changing the model raster. The public basemap is not
|
||||
claimed to be byte-identical to comma's training archive. See
|
||||
[Protomaps downloads](https://docs.protomaps.com/basemaps/downloads) for distribution
|
||||
and attribution information; a wider deployment should serve its own archive mirror.
|
||||
Existing steering, cooperative steering, action conversion and
|
||||
the restored 1.6 m/s low-speed handling are preserved.
|
||||
|
||||
Recorded-camera model replay can test whether real route inputs change the
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import gzip
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import struct
|
||||
@@ -18,7 +19,7 @@ import requests
|
||||
import openpilot.cereal.messaging as messaging
|
||||
from openpilot.common.prefix import OpenpilotPrefix
|
||||
from openpilot.selfdrive.navigation.mapsd import MapsD
|
||||
from openpilot.selfdrive.navigation.navd import REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner
|
||||
from openpilot.selfdrive.navigation.navd import RECOMPUTE_HOLD, REROUTE_COUNTER_MIN, ROUTE_RESEND_PERIOD, RouteEngine, set_banner
|
||||
from openpilot.selfdrive.navigation.pmtiles import PMTiles, decode_directory, tile_id
|
||||
from openpilot.selfdrive.navigation.renderer import MAJOR, MEDIUM, MINOR, SIZE, STREET, TRACK, MapRenderer, draw_segments, mercator, project, stroke_widths
|
||||
from openpilot.selfdrive.navigation.route import RouteTracker
|
||||
@@ -101,6 +102,16 @@ def route_engine(sm, pm, values):
|
||||
|
||||
class RangeHandler(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
if self.path == '/builds.json':
|
||||
self.server.index_requests += 1
|
||||
entries = {} if self.server.invalid_index else [
|
||||
{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.server.archive)}]
|
||||
data = json.dumps(entries).encode()
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
return
|
||||
self.server.requests.append(dict(self.headers))
|
||||
if self.headers.get('If-Match', self.server.etag) != self.server.etag:
|
||||
self.send_error(412)
|
||||
@@ -123,7 +134,7 @@ class RangeHandler(BaseHTTPRequestHandler):
|
||||
|
||||
|
||||
class TileServerCase(unittest.TestCase):
|
||||
"""Serves a leaf-directory archive over HTTP range requests, with the comma API and tile cache stubbed."""
|
||||
"""Serves a real public-index/leaf-directory archive protocol over local HTTP."""
|
||||
|
||||
def setUp(self):
|
||||
self.directory = tempfile.TemporaryDirectory()
|
||||
@@ -137,11 +148,10 @@ class TileServerCase(unittest.TestCase):
|
||||
threading.Thread(target=self.server.serve_forever, kwargs={'poll_interval': 0.01}, daemon=True).start()
|
||||
self.addCleanup(self.server.server_close)
|
||||
self.addCleanup(self.server.shutdown)
|
||||
self.api = MagicMock(dongle_id='test-dongle')
|
||||
self.api.get_token.return_value = 'device-token'
|
||||
self.url = f'http://127.0.0.1:{self.server.server_port}/planet.pmtiles?sig=secret'
|
||||
self.api.get.return_value.json.return_value = {'url': self.url, 'expires': time.time() + 86400} # noqa: TID251
|
||||
for name, value in [('Api', MagicMock(return_value=self.api)), ('Params', MagicMock()), ('PC', True)]:
|
||||
self.server.index_requests = 0
|
||||
self.server.invalid_index = False
|
||||
self.url = f'http://127.0.0.1:{self.server.server_port}'
|
||||
for name, value in [('PC', True), ('BUILD_INDEX_URL', self.url + '/builds.json'), ('BUILD_BASE_URL', self.url + '/')]:
|
||||
self.enterContext(patch(f'openpilot.selfdrive.navigation.tiles.{name}', value))
|
||||
self.enterContext(patch('openpilot.selfdrive.navigation.tiles.Paths.comma_home', return_value=self.directory.name))
|
||||
|
||||
@@ -168,6 +178,15 @@ class TestEndToEnd(TileServerCase):
|
||||
sm.update(0)
|
||||
return mapsd.update(sm)
|
||||
|
||||
def restored_route_frame():
|
||||
# Destination cancellation retains the existing recompute cooldown.
|
||||
# Advance its bounded update count before testing tile-cache reuse.
|
||||
for _ in range((RECOMPUTE_HOLD + 1) * navd.update_frequency):
|
||||
result = frame()
|
||||
if navd.route is not None:
|
||||
return result
|
||||
self.fail('Destination route was not restored within the existing recompute cooldown')
|
||||
|
||||
image, timestamp = frame()
|
||||
# The route is centered over the white road, with the ego arrow three quarters down.
|
||||
self.assertEqual((image[128, 128], image[128, 131], image[188, 128], image[195, 128]), (128, 255, 255, 0))
|
||||
@@ -180,39 +199,39 @@ class TestEndToEnd(TileServerCase):
|
||||
values.pop('NavDestination')
|
||||
self.assertFalse(frame()[0].any()) # black without a route
|
||||
values['NavDestination'] = dict(destination)
|
||||
self.assertEqual(frame()[0][128, 128], 128)
|
||||
self.assertEqual(restored_route_frame()[0][128, 128], 128)
|
||||
self.assertEqual(len(self.server.requests), fetched) # the second route is served from the cache
|
||||
image, timestamp = frame(fix=False)
|
||||
self.assertFalse(image.any() or timestamp) # black, not stale, without a fix
|
||||
self.api.get.assert_called_once()
|
||||
self.assertEqual(self.server.index_requests, 1)
|
||||
self.assertTrue(all('Range' in headers for headers in self.server.requests))
|
||||
self.assertTrue(all(headers['If-Match'] == self.server.etag for headers in self.server.requests[1:]))
|
||||
self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.iterdir()))
|
||||
self.assertTrue(all(b'secret' not in p.read_bytes() for p in self.cache.rglob('*') if p.is_file()))
|
||||
# The archive is replaced on the server and the cache evicted: the next fetch fails, the map goes black,
|
||||
# and after a delay a fresh reader starts over on the new version.
|
||||
self.server.etag = '"new-version"'
|
||||
for cached in self.cache.iterdir():
|
||||
cached.unlink()
|
||||
for cached in self.cache.rglob('*'):
|
||||
if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name):
|
||||
cached.unlink()
|
||||
values.pop('NavDestination')
|
||||
frame()
|
||||
values['NavDestination'] = dict(destination)
|
||||
image, timestamp = frame()
|
||||
image, timestamp = restored_route_frame()
|
||||
self.assertFalse(image.any() or timestamp)
|
||||
fetched = len(self.server.requests)
|
||||
self.assertFalse(frame()[1]) # not retried yet
|
||||
self.assertEqual(len(self.server.requests), fetched)
|
||||
mapsd.retry_at = 0
|
||||
self.assertEqual(frame()[0][128, 128], 128)
|
||||
self.assertEqual((self.cache / 'version').read_text().split()[0], '"new-version"')
|
||||
self.assertEqual(json.loads((self.cache / 'archive.json').read_text())['etag'], '"new-version"')
|
||||
|
||||
|
||||
class TestRemoteTiles(TileServerCase):
|
||||
def test_shared_cache_and_refresh(self):
|
||||
with closing(PMTiles(RemoteTiles())) as archive:
|
||||
archive.source.expires = 0 # the signed URL is refreshed before the next fetch
|
||||
self.assertEqual(archive.get(0, 0, 0), road_tile())
|
||||
self.assertEqual(archive.get(0, 0, 0), road_tile())
|
||||
self.assertEqual(self.api.get.call_count, 2)
|
||||
self.assertEqual(self.server.index_requests, 1)
|
||||
count = len(self.server.requests)
|
||||
with closing(PMTiles(RemoteTiles())) as archive:
|
||||
self.assertEqual(archive.get(0, 0, 0), road_tile())
|
||||
@@ -220,7 +239,9 @@ class TestRemoteTiles(TileServerCase):
|
||||
self.assertEqual(len(self.server.requests), count + 1)
|
||||
with patch('openpilot.selfdrive.navigation.tiles.CACHE_LIMIT', 1), closing(RemoteTiles()) as source:
|
||||
source.read(1, 2) # an uncached range is downloaded, then the cache is trimmed to the limit
|
||||
self.assertEqual([p.name for p in self.cache.iterdir()], ['version'])
|
||||
retained = [p.name for p in self.cache.rglob('*') if p.is_file() and re.fullmatch(r'\d+-\d+', p.name)]
|
||||
self.assertEqual(retained, ['0-127']) # current header is retained for offline startup
|
||||
self.assertTrue((self.cache / 'archive.json').exists())
|
||||
|
||||
def test_prefetch_merges_nearby_ranges(self):
|
||||
with patch('openpilot.selfdrive.navigation.tiles.MERGE_GAP', 16), closing(RemoteTiles()) as source:
|
||||
@@ -235,15 +256,16 @@ class TestRemoteTiles(TileServerCase):
|
||||
def test_failures(self):
|
||||
with closing(PMTiles(RemoteTiles())) as archive:
|
||||
self.server.etag = '"new-version"'
|
||||
with self.assertRaisesRegex(OSError, '412'): # the archive changed under an open reader
|
||||
with self.assertRaises(OSError): # the archive changed under an open reader
|
||||
archive.get(0, 0, 0)
|
||||
with closing(PMTiles(RemoteTiles())) as archive: # a new reader starts over on the new version
|
||||
self.assertEqual(archive.get(0, 0, 0), road_tile())
|
||||
self.server.ignore_range = True
|
||||
with self.assertRaisesRegex(OSError, '200'):
|
||||
with self.assertRaises(OSError):
|
||||
RemoteTiles()
|
||||
self.api.get.return_value.raise_for_status.side_effect = requests.HTTPError(self.url)
|
||||
with self.assertRaisesRegex(OSError, '^Unable to fetch map tiles$'): # the signed URL stays out of the error
|
||||
self.server.ignore_range = False
|
||||
self.server.invalid_index = True
|
||||
with self.assertRaisesRegex(OSError, '^Unable to load public Protomaps basemap$'):
|
||||
RemoteTiles()
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Actual public tile client requests and cache behavior, without accounts/network."""
|
||||
import ast
|
||||
import io
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
import tempfile
|
||||
from types import SimpleNamespace as NS
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import requests
|
||||
from requests.adapters import BaseAdapter
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
|
||||
SOURCE = Path(__file__).resolve().parents[1]
|
||||
REAL_SESSION = requests.Session
|
||||
|
||||
|
||||
class PublicServer(BaseAdapter):
|
||||
def __init__(self):
|
||||
self.archive = b'PMTiles\x03' + bytes(range(256)) * 64
|
||||
self.entries = [{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': len(self.archive)}]
|
||||
self.etag = '"public-build"'
|
||||
self.calls = []
|
||||
self.offline = False
|
||||
self.index_bytes = None
|
||||
self.index_headers = {}
|
||||
self.range_headers = {}
|
||||
self.range_status = 206
|
||||
self.range_body = None
|
||||
|
||||
def send(self, request, **_kwargs):
|
||||
self.calls.append(request)
|
||||
if self.offline:
|
||||
raise requests.ConnectionError('fixture network unavailable')
|
||||
response = requests.Response()
|
||||
response.request, response.url = request, request.url
|
||||
if request.url == 'https://build-metadata.protomaps.dev/builds.json':
|
||||
body = self.index_bytes if self.index_bytes is not None else json.dumps(self.entries).encode()
|
||||
response.status_code = 200
|
||||
headers = {'Content-Length': str(len(body))} | self.index_headers
|
||||
else:
|
||||
if not re.fullmatch(r'https://build\.protomaps\.com/\d{8}\.pmtiles', request.url):
|
||||
raise AssertionError('Client must never contact an account or arbitrary endpoint')
|
||||
match = re.fullmatch(r'bytes=(\d+)-(\d+)', request.headers['Range'])
|
||||
start, end = map(int, match.groups())
|
||||
body = self.archive[start:end + 1] if self.range_body is None else self.range_body
|
||||
response.status_code = self.range_status
|
||||
headers = {'Content-Range': f'bytes {start}-{end}/{len(self.archive)}', 'ETag': self.etag,
|
||||
'Content-Length': str(len(body))} | self.range_headers
|
||||
response.headers = CaseInsensitiveDict(headers)
|
||||
response.raw = io.BytesIO(body)
|
||||
return response
|
||||
|
||||
def close(self):
|
||||
pass
|
||||
|
||||
|
||||
class PublicTilesTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.temp = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.temp.cleanup)
|
||||
self.home = Path(self.temp.name)
|
||||
dependencies = {'openpilot.common.hardware': NS(PC=True),
|
||||
'openpilot.common.hardware.hw': NS(Paths=NS(comma_home=lambda: self.temp.name)),
|
||||
'openpilot.selfdrive.navigation.pmtiles': NS(HEADER_LENGTH=127)}
|
||||
spec = importlib.util.spec_from_file_location('actual_public_tiles', SOURCE / 'tiles.py')
|
||||
self.tiles = importlib.util.module_from_spec(spec)
|
||||
with patch.dict(sys.modules, dependencies):
|
||||
spec.loader.exec_module(self.tiles)
|
||||
self.server = PublicServer()
|
||||
def session():
|
||||
value = REAL_SESSION()
|
||||
value.trust_env = False
|
||||
value.mount('https://', self.server)
|
||||
return value
|
||||
self.enterContext(patch.object(self.tiles.requests, 'Session', side_effect=session))
|
||||
|
||||
def client(self):
|
||||
value = self.tiles.RemoteTiles()
|
||||
self.addCleanup(value.close)
|
||||
return value
|
||||
|
||||
def ranges(self):
|
||||
return [call for call in self.server.calls if 'Range' in call.headers]
|
||||
|
||||
def test_latest_compatible_public_build_has_no_account_or_token_dependency(self):
|
||||
self.server.entries += [{'key': '20261104.pmtiles', 'version': '5.0.0', 'size': len(self.server.archive)},
|
||||
{'key': '20261102.pmtiles', 'version': '4.99.1', 'size': len(self.server.archive)}]
|
||||
client = self.client()
|
||||
self.assertEqual(client.url, 'https://build.protomaps.com/20261103.pmtiles')
|
||||
self.assertEqual(client.read(300, 100), self.server.archive[300:400])
|
||||
self.assertEqual(self.ranges()[-1].headers['If-Match'], self.server.etag)
|
||||
for call in self.server.calls:
|
||||
self.assertNotIn('Authorization', call.headers)
|
||||
self.assertNotIn('access_token', call.url)
|
||||
self.assertNotIn('comma', call.url)
|
||||
imported = [ast.unparse(node) for node in ast.parse((SOURCE / 'tiles.py').read_text()).body if isinstance(node, (ast.Import, ast.ImportFrom))]
|
||||
self.assertFalse(any('api' in value.lower() or 'params' in value.lower() for value in imported))
|
||||
|
||||
def test_invalid_index_or_unsupported_schema_fails_before_archive_request(self):
|
||||
invalid = ({'url': 'https://untrusted.invalid/archive'}, [],
|
||||
[{'key': '../20261103.pmtiles', 'version': '4.15.2', 'size': 200}],
|
||||
[{'key': '20260230.pmtiles', 'version': '4.15.2', 'size': 200}],
|
||||
[{'key': '20261103.pmtiles', 'version': '5.0.0', 'size': 200}],
|
||||
[{'key': '20261103.pmtiles', 'version': '4.15.2', 'size': True}])
|
||||
for value in invalid:
|
||||
with self.subTest(value=value):
|
||||
self.server.entries = value
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
self.assertEqual(self.ranges(), [])
|
||||
self.server.index_bytes = b'not JSON'
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
|
||||
def test_index_body_and_declared_length_are_bounded(self):
|
||||
self.server.index_headers = {'Content-Length': str(self.tiles.INDEX_LIMIT + 1)}
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
self.server.index_headers = {}
|
||||
self.server.index_bytes = b' ' * (self.tiles.INDEX_LIMIT + 1)
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
self.assertEqual(self.ranges(), [])
|
||||
|
||||
def test_range_status_identity_length_and_content_range_fail_closed(self):
|
||||
cases = [({'status': 200}), ({'headers': {'Content-Range': 'bytes 1-127/999'}}),
|
||||
({'headers': {'ETag': ''}}), ({'headers': {'ETag': 'W/"weak"'}}),
|
||||
({'headers': {'Content-Length': '99999999999999'}}),
|
||||
({'headers': {'Content-Encoding': 'gzip'}}),
|
||||
({'body': b'x' * 126, 'headers': {'Content-Length': '127'}}),
|
||||
({'body': b'x' * 128, 'headers': {'Content-Length': '127'}})]
|
||||
for case in cases:
|
||||
with self.subTest(case=case):
|
||||
self.server.range_status = case.get('status', 206)
|
||||
self.server.range_headers = case.get('headers', {})
|
||||
self.server.range_body = case.get('body')
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
|
||||
def test_archive_cannot_change_etag_or_total_size_mid_reader(self):
|
||||
client = self.client()
|
||||
self.server.etag = '"changed"'
|
||||
with self.assertRaisesRegex(OSError, 'changed'):
|
||||
client.read(200, 100)
|
||||
self.assertFalse((client.cache / '200-100').exists())
|
||||
self.server.etag = client.etag
|
||||
self.server.range_headers = {'Content-Range': f'bytes 200-299/{client.size + 1}'}
|
||||
with self.assertRaises(OSError):
|
||||
client.read(200, 100)
|
||||
|
||||
def test_restart_revalidates_header_and_reuses_only_same_archive_ranges(self):
|
||||
first = self.client()
|
||||
first.read(200, 100)
|
||||
count = len(self.server.calls)
|
||||
second = self.client()
|
||||
self.assertEqual(second.cache, first.cache)
|
||||
self.assertEqual(second.read(200, 100), self.server.archive[200:300])
|
||||
self.assertEqual(len(self.server.calls), count + 2) # build index and pinned header only
|
||||
self.server.entries[0]['key'] = '20261104.pmtiles'
|
||||
third = self.client()
|
||||
self.assertNotEqual(third.cache, first.cache)
|
||||
count = len(self.server.calls)
|
||||
third.read(200, 100)
|
||||
self.assertEqual(len(self.server.calls), count + 1)
|
||||
|
||||
def test_offline_restart_reuses_pinned_header_and_cached_ranges_only(self):
|
||||
first = self.client()
|
||||
first.read(200, 100)
|
||||
self.server.offline = True
|
||||
second = self.client()
|
||||
self.assertEqual(second.url, first.url)
|
||||
self.assertEqual(second.etag, first.etag)
|
||||
self.assertEqual(second.read(200, 100), self.server.archive[200:300])
|
||||
with self.assertRaises(OSError):
|
||||
second.read(500, 100)
|
||||
|
||||
def test_malformed_fresh_index_does_not_fall_back_or_relabel_old_bytes(self):
|
||||
first = self.client()
|
||||
first.read(200, 100)
|
||||
self.server.entries = []
|
||||
with self.assertRaises(OSError):
|
||||
self.client()
|
||||
self.assertEqual(first.read(200, 100), self.server.archive[200:300])
|
||||
|
||||
def test_prefetch_merges_nearby_ranges_but_caps_every_request(self):
|
||||
client = self.client()
|
||||
with patch.object(self.tiles, 'MAX_RANGE', 64):
|
||||
count = len(self.ranges())
|
||||
client.prefetch([(200, 32), (244, 32), (400, 16), (400, 16)])
|
||||
added = self.ranges()[count:]
|
||||
self.assertEqual(len(added), 3)
|
||||
for call in added:
|
||||
start, end = map(int, call.headers['Range'][6:].split('-'))
|
||||
self.assertLessEqual(end - start + 1, 64)
|
||||
count = len(self.server.calls)
|
||||
self.assertEqual(client.read(244, 32), self.server.archive[244:276])
|
||||
self.assertEqual(len(self.server.calls), count)
|
||||
count = len(self.ranges())
|
||||
client.prefetch([(600, 10), (620, 10)])
|
||||
self.assertEqual(len(self.ranges()), count + 1)
|
||||
|
||||
def test_cache_limit_counts_ranges_from_all_archive_identities(self):
|
||||
client = self.client()
|
||||
with patch.object(self.tiles, 'CACHE_LIMIT', 160):
|
||||
client.read(200, 100)
|
||||
total = sum(path.stat().st_size for path in client.base.rglob('*') if path.is_file() and re.fullmatch(r'\d+-\d+', path.name))
|
||||
self.assertLessEqual(total, 160)
|
||||
|
||||
def test_cache_trim_retains_current_header_for_offline_restart(self):
|
||||
first = self.client()
|
||||
with patch.object(self.tiles, 'CACHE_LIMIT', 228):
|
||||
first.read(200, 100)
|
||||
first.read(400, 100)
|
||||
self.assertTrue((first.cache / '0-127').exists())
|
||||
self.assertFalse((first.cache / '200-100').exists())
|
||||
self.server.offline = True
|
||||
second = self.client()
|
||||
self.assertEqual(second.read(400, 100), self.server.archive[400:500])
|
||||
|
||||
def test_invalid_or_excessive_ranges_do_not_make_requests(self):
|
||||
client = self.client()
|
||||
count = len(self.server.calls)
|
||||
for offset, length in ((-1, 1), (0, 0), (0, self.tiles.MAX_RANGE + 1), (client.size, 1), (True, 1)):
|
||||
with self.subTest(offset=offset, length=length), self.assertRaises(ValueError):
|
||||
client.read(offset, length)
|
||||
self.assertEqual(len(self.server.calls), count)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -92,6 +92,58 @@ class TestSignalTurnStatus(unittest.TestCase):
|
||||
widget = load_widget(ui_state).NavigationPanel()
|
||||
self.assertEqual(widget.signal_status['primary'], 'Navigation active')
|
||||
|
||||
def test_ready_route_explains_fresh_matched_model_map_wait(self):
|
||||
ui_state, values = state('ready')
|
||||
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
|
||||
'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'}
|
||||
module = load_widget(ui_state)
|
||||
widget = module.NavigationPanel()
|
||||
self.assertEqual(widget.signal_status['primary'], 'Route ready')
|
||||
self.assertEqual(widget.signal_status['secondary'], 'Waiting for model map')
|
||||
widget._render(module.rl.Rectangle(0, 0, 261, 400))
|
||||
self.assertIn('Waiting for model map', widget._eta_label.text)
|
||||
|
||||
def test_fresh_matched_wait_reasons_are_specific_without_claiming_active(self):
|
||||
expected = {'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid',
|
||||
'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map',
|
||||
'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable'}
|
||||
for reason, text in expected.items():
|
||||
with self.subTest(reason=reason):
|
||||
ui_state, values = state('ready')
|
||||
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
|
||||
'session': 'one', 'routeGeneration': 10, 'reason': reason}
|
||||
result = load_widget(ui_state).NavigationPanel().signal_status
|
||||
self.assertEqual(result['primary'], 'Route ready')
|
||||
self.assertEqual(result['secondary'], text)
|
||||
|
||||
def test_stale_wrong_route_or_unsupported_model_reason_is_not_reused(self):
|
||||
for change in ({'updatedAt': 99.}, {'updatedAt': 101.}, {'updatedAt': float('nan')}, {'updatedAt': True},
|
||||
{'session': 'other'}, {'routeGeneration': 11}, {'supported': False}, {'active': 0}):
|
||||
with self.subTest(change=change):
|
||||
ui_state, values = state('ready')
|
||||
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
|
||||
'session': 'one', 'routeGeneration': 10, 'reason': 'map_invalid', **change}
|
||||
result = load_widget(ui_state).NavigationPanel().signal_status
|
||||
self.assertEqual(result['primary'], 'Route ready')
|
||||
self.assertEqual(result['secondary'], 'Waiting for map and model')
|
||||
|
||||
def test_active_and_pending_route_labels_ignore_old_wait_reason(self):
|
||||
for mode, active, primary in (('turn', True, 'Navigation active'), ('pending', False, 'Turn candidate')):
|
||||
ui_state, values = state('ready')
|
||||
values['MapboxTurnIntent']['mode'] = mode
|
||||
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': active,
|
||||
'session': 'one', 'routeGeneration': 10, 'reason': 'map_missing'}
|
||||
result = load_widget(ui_state).NavigationPanel().signal_status
|
||||
self.assertEqual(result['primary'], primary)
|
||||
self.assertEqual(result['secondary'], 'Right onto Example Street')
|
||||
|
||||
def test_unknown_or_malformed_wait_reason_keeps_generic_message(self):
|
||||
for reason in (None, 'unknown_future_reason', [], {}):
|
||||
ui_state, values = state('ready')
|
||||
values['NavModelState'] = {'updatedAt': 100., 'supported': True, 'active': False,
|
||||
'session': 'one', 'routeGeneration': 10, 'reason': reason}
|
||||
self.assertEqual(load_widget(ui_state).NavigationPanel().signal_status['secondary'], 'Waiting for map and model')
|
||||
|
||||
def test_explicit_destination_keeps_normal_arrow_and_shows_blocked_footer(self):
|
||||
ui_state, _ = state('blocked', destination=True)
|
||||
module = load_widget(ui_state)
|
||||
|
||||
@@ -1,114 +1,215 @@
|
||||
"""Byte ranges of the hosted PMTiles basemap, cached on disk."""
|
||||
"""Bounded byte ranges of a public Protomaps v4 basemap; no device account.
|
||||
|
||||
Experimental single-device transport. Public daily-build hosting is not a fleet
|
||||
backend: deployments should mirror the archive while preserving its identity.
|
||||
"""
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from datetime import date
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
from openpilot.common.api import Api
|
||||
from openpilot.common.hardware import PC
|
||||
from openpilot.common.hardware.hw import Paths
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.selfdrive.navigation.pmtiles import HEADER_LENGTH
|
||||
|
||||
BUILD_INDEX_URL = 'https://build-metadata.protomaps.dev/builds.json'
|
||||
BUILD_BASE_URL = 'https://build.protomaps.com/'
|
||||
INDEX_LIMIT = 512 * 1024
|
||||
MAX_RANGE = 4 * 1024 * 1024
|
||||
CACHE_LIMIT = 64 * 1024 * 1024
|
||||
MERGE_GAP = 256 * 1024 # tiles this close together in the archive are fetched in one range request
|
||||
MERGE_GAP = 256 * 1024
|
||||
TIMEOUT = (5, 10)
|
||||
|
||||
|
||||
def _build(entry):
|
||||
"""Only supported official dated archives, never a URL supplied by metadata."""
|
||||
if not isinstance(entry, dict):
|
||||
return None
|
||||
key, size, version = (entry.get(name) for name in ('key', 'size', 'version'))
|
||||
if (not isinstance(key, str) or not re.fullmatch(r'\d{8}\.pmtiles', key) or
|
||||
isinstance(size, bool) or not isinstance(size, int) or not HEADER_LENGTH <= size <= 1024 ** 4 or
|
||||
not isinstance(version, str) or not re.fullmatch(r'4\.\d+\.\d+', version)):
|
||||
return None
|
||||
try:
|
||||
date(int(key[:4]), int(key[4:6]), int(key[6:8]))
|
||||
except ValueError:
|
||||
return None
|
||||
return {'key': key, 'size': size, 'version': version, 'url': BUILD_BASE_URL + key}
|
||||
|
||||
|
||||
def _strong_etag(value):
|
||||
return isinstance(value, str) and re.fullmatch(r'"[\x21\x23-\x7e]{1,510}"', value) is not None
|
||||
|
||||
|
||||
class RemoteTiles:
|
||||
def __init__(self):
|
||||
self.cache = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles'
|
||||
self.cache.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.base = Path(Paths.comma_home() if PC else '/data') / 'navigation' / 'tiles'
|
||||
self.base.mkdir(parents=True, exist_ok=True, mode=0o700)
|
||||
self.session = requests.Session()
|
||||
self.api = Api(Params().get('DongleId'))
|
||||
self.url = ''
|
||||
self.expires = 0
|
||||
self.etag = None
|
||||
self.size = 0
|
||||
version = self.cache / 'version'
|
||||
self.url, self.etag, self.size = '', None, 0
|
||||
self.cache = None
|
||||
saved = self._saved_identity()
|
||||
try:
|
||||
header = self._fetch(0, HEADER_LENGTH) # learns the archive's ETag and size
|
||||
if not version.exists() or version.read_text() != f'{self.etag} {self.size}':
|
||||
for cached in self.cache.iterdir(): # a new archive has new offsets
|
||||
cached.unlink()
|
||||
version.write_text(f'{self.etag} {self.size}')
|
||||
try:
|
||||
build = self._latest_build()
|
||||
self.url, self.size = build['url'], build['size']
|
||||
header = self._fetch(0, HEADER_LENGTH)
|
||||
identity = build | {'etag': self.etag}
|
||||
except requests.RequestException:
|
||||
# An offline reader may reuse only a previously pinned archive with its
|
||||
# exact cached header. Missing ranges still fail; no route-only raster.
|
||||
if saved is None:
|
||||
raise
|
||||
identity = saved
|
||||
self.url, self.etag, self.size = identity['url'], identity['etag'], identity['size']
|
||||
self.cache = self._cache_for(identity)
|
||||
header = (self.cache / f'0-{HEADER_LENGTH}').read_bytes()
|
||||
if len(header) != HEADER_LENGTH or header[:8] != b'PMTiles\x03':
|
||||
raise ValueError('Invalid public PMTiles header')
|
||||
self.cache = self._cache_for(identity)
|
||||
self.cache.mkdir(parents=True, exist_ok=True)
|
||||
self._store(0, header)
|
||||
except Exception:
|
||||
temporary = self.base / 'archive.tmp'
|
||||
temporary.write_text(json.dumps(identity))
|
||||
temporary.replace(self.base / 'archive.json')
|
||||
self._trim()
|
||||
except (requests.RequestException, OSError, ValueError, TypeError, KeyError):
|
||||
self.close()
|
||||
raise
|
||||
raise OSError('Unable to load public Protomaps basemap') from None
|
||||
|
||||
def _cache_for(self, identity):
|
||||
key = hashlib.sha256(json.dumps(identity, sort_keys=True).encode()).hexdigest()[:32]
|
||||
return self.base / 'archives' / key
|
||||
|
||||
def _saved_identity(self):
|
||||
try:
|
||||
path = self.base / 'archive.json'
|
||||
if path.stat().st_size > 4096:
|
||||
return None
|
||||
saved = json.loads(path.read_text())
|
||||
build = _build(saved)
|
||||
if build is not None and saved.get('url') == build['url'] and _strong_etag(saved.get('etag')):
|
||||
return build | {'etag': saved['etag']}
|
||||
except (OSError, ValueError, TypeError):
|
||||
pass
|
||||
return None
|
||||
|
||||
def _latest_build(self):
|
||||
with self.session.get(BUILD_INDEX_URL, headers={'Accept-Encoding': 'identity'}, timeout=TIMEOUT, stream=True,
|
||||
allow_redirects=False) as response:
|
||||
response.raise_for_status()
|
||||
if response.status_code != 200:
|
||||
raise ValueError('Unexpected public build index response')
|
||||
declared = response.headers.get('Content-Length')
|
||||
if declared is not None and not 0 <= int(declared) <= INDEX_LIMIT:
|
||||
raise ValueError('Oversized public build index')
|
||||
data = bytearray()
|
||||
for chunk in response.iter_content(chunk_size=65536):
|
||||
if len(data) + len(chunk) > INDEX_LIMIT:
|
||||
raise ValueError('Oversized public build index')
|
||||
data.extend(chunk)
|
||||
entries = json.loads(data)
|
||||
if not isinstance(entries, list) or len(entries) > 4096:
|
||||
raise ValueError('Invalid public build index')
|
||||
builds = [build for entry in entries if (build := _build(entry)) is not None]
|
||||
if not builds:
|
||||
raise ValueError('No compatible public Protomaps v4 build')
|
||||
return max(builds, key=lambda build: build['key'])
|
||||
|
||||
def _range(self, offset, length):
|
||||
if (isinstance(offset, bool) or isinstance(length, bool) or not isinstance(offset, int) or not isinstance(length, int) or
|
||||
offset < 0 or not 0 < length <= MAX_RANGE or offset + length > self.size):
|
||||
raise ValueError('Invalid or oversized public map range')
|
||||
|
||||
def _fetch(self, offset: int, length: int) -> bytes:
|
||||
try:
|
||||
if time.time() >= self.expires - 60: # noqa: TID251 - SAS expiry is a Unix timestamp
|
||||
response = self.api.get(f'v1/{self.api.dongle_id}/nav/tiles', access_token=self.api.get_token(), timeout=10)
|
||||
response.raise_for_status()
|
||||
metadata = response.json()
|
||||
self.url, self.expires = metadata['url'], metadata['expires']
|
||||
headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'}
|
||||
if self.etag is not None:
|
||||
headers['If-Match'] = self.etag
|
||||
with self.session.get(self.url, headers=headers, timeout=10, stream=True) as response:
|
||||
if response.status_code != 206:
|
||||
raise OSError(f'Map tile range request failed ({response.status_code})')
|
||||
match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', ''))
|
||||
if match is None or tuple(map(int, match.groups()[:2])) != (offset, offset + length - 1):
|
||||
raise OSError('Invalid map tile range response')
|
||||
size = int(match[3])
|
||||
etag = response.headers.get('ETag')
|
||||
if not etag or (self.etag is not None and (etag != self.etag or size != self.size)):
|
||||
raise OSError('Map archive changed during read')
|
||||
data = bytearray()
|
||||
for chunk in response.iter_content(chunk_size=min(length + 1, 65536)):
|
||||
data.extend(chunk)
|
||||
if len(data) > length:
|
||||
raise OSError('Oversized map tile range response')
|
||||
if len(data) != length:
|
||||
raise OSError('Truncated map tile range response')
|
||||
self.size, self.etag = size, etag
|
||||
return bytes(data)
|
||||
except (requests.RequestException, ValueError, KeyError, TypeError, RuntimeError):
|
||||
# requests exceptions contain the signed URL; keep it out of logs.
|
||||
raise OSError('Unable to fetch map tiles') from None
|
||||
self._range(offset, length)
|
||||
headers = {'Range': f'bytes={offset}-{offset + length - 1}', 'Accept-Encoding': 'identity'}
|
||||
if self.etag is not None:
|
||||
headers['If-Match'] = self.etag
|
||||
with self.session.get(self.url, headers=headers, timeout=TIMEOUT, stream=True, allow_redirects=False) as response:
|
||||
response.raise_for_status()
|
||||
if response.status_code != 206:
|
||||
raise OSError(f'Public map range request failed ({response.status_code})')
|
||||
match = re.fullmatch(r'bytes (\d+)-(\d+)/(\d+)', response.headers.get('Content-Range', ''))
|
||||
if match is None or tuple(map(int, match.groups())) != (offset, offset + length - 1, self.size):
|
||||
raise OSError('Invalid public map range response')
|
||||
etag = response.headers.get('ETag')
|
||||
if not _strong_etag(etag) or (self.etag is not None and etag != self.etag):
|
||||
raise OSError('Public map archive changed during read')
|
||||
declared = response.headers.get('Content-Length')
|
||||
if declared is not None and int(declared) != length:
|
||||
raise OSError('Invalid public map range length')
|
||||
if response.headers.get('Content-Encoding', 'identity') != 'identity':
|
||||
raise OSError('Encoded public map range response')
|
||||
data = bytearray()
|
||||
for chunk in response.iter_content(chunk_size=min(length + 1, 65536)):
|
||||
if len(data) + len(chunk) > length:
|
||||
raise OSError('Oversized public map range response')
|
||||
data.extend(chunk)
|
||||
if len(data) != length:
|
||||
raise OSError('Truncated public map range response')
|
||||
self.etag = etag
|
||||
return bytes(data)
|
||||
|
||||
def _store(self, offset: int, data: bytes):
|
||||
(self.cache / f'{offset}-{len(data)}').write_bytes(data)
|
||||
path = self.cache / f'{offset}-{len(data)}'
|
||||
temporary = path.with_suffix('.tmp')
|
||||
temporary.write_bytes(data)
|
||||
temporary.replace(path)
|
||||
|
||||
def _trim(self):
|
||||
files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.cache.iterdir() if re.fullmatch(r'\d+-\d+', cached.name)]
|
||||
files = [(cached.stat().st_mtime, cached.stat().st_size, cached) for cached in self.base.rglob('*')
|
||||
if cached.is_file() and re.fullmatch(r'\d+-\d+', cached.name)]
|
||||
size = sum(cached_size for _, cached_size, _ in files)
|
||||
for _, cached_size, cached in sorted(files):
|
||||
if size <= CACHE_LIMIT:
|
||||
break
|
||||
if cached == self.cache / f'0-{HEADER_LENGTH}':
|
||||
continue # Required for an offline restart of the current pinned archive.
|
||||
cached.unlink(missing_ok=True)
|
||||
size -= cached_size
|
||||
|
||||
def read(self, offset: int, length: int) -> bytes:
|
||||
self._range(offset, length)
|
||||
try:
|
||||
data = (self.cache / f'{offset}-{length}').read_bytes()
|
||||
if len(data) == length:
|
||||
return data
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
data = self._fetch(offset, length)
|
||||
try:
|
||||
data = self._fetch(offset, length)
|
||||
except (requests.RequestException, ValueError):
|
||||
raise OSError('Unable to fetch public map tiles') from None
|
||||
self._store(offset, data)
|
||||
self._trim()
|
||||
return data
|
||||
|
||||
def prefetch(self, ranges):
|
||||
"""Cache (offset, length) ranges ahead of reads, fetching neighboring ones in a single request."""
|
||||
missing = sorted((offset, length) for offset, length in set(ranges) if not (self.cache / f'{offset}-{length}').exists())
|
||||
missing = []
|
||||
for offset, length in sorted(set(ranges)):
|
||||
self._range(offset, length)
|
||||
if not (self.cache / f'{offset}-{length}').exists():
|
||||
missing.append((offset, length))
|
||||
if not missing:
|
||||
return
|
||||
groups = [[missing[0]]]
|
||||
for offset, length in missing[1:]:
|
||||
start = groups[-1][0][0]
|
||||
last_offset, last_length = groups[-1][-1]
|
||||
if offset - (last_offset + last_length) <= MERGE_GAP:
|
||||
if offset - (last_offset + last_length) <= MERGE_GAP and offset + length - start <= MAX_RANGE:
|
||||
groups[-1].append((offset, length))
|
||||
else:
|
||||
groups.append([(offset, length)])
|
||||
for group in groups:
|
||||
start, end = group[0][0], group[-1][0] + group[-1][1]
|
||||
data = self._fetch(start, end - start)
|
||||
start, end = group[0][0], max(offset + length for offset, length in group)
|
||||
try:
|
||||
data = self._fetch(start, end - start)
|
||||
except (requests.RequestException, ValueError):
|
||||
raise OSError('Unable to prefetch public map tiles') from None
|
||||
for offset, length in group:
|
||||
self._store(offset, data[offset - start:offset - start + length])
|
||||
self._trim()
|
||||
|
||||
@@ -53,10 +53,28 @@ def signal_turn_status(enabled, intent, context, now, left_blinker, right_blinke
|
||||
model_state.get('session') == intent['session'] and
|
||||
isinstance(context.get('routeGeneration'), int) and context['routeGeneration'] > 0 and
|
||||
model_state.get('routeGeneration') == context['routeGeneration'])
|
||||
waiting = 'Waiting for map and model'
|
||||
matched_wait = (isinstance(model_updated, (int, float)) and not isinstance(model_updated, bool) and
|
||||
np.isfinite(model_updated) and 0. <= now-model_updated <= .5 and
|
||||
model_state.get('supported') is True and model_state.get('active') is False and
|
||||
model_state.get('session') == intent['session'] and
|
||||
isinstance(context.get('routeGeneration'), int) and not isinstance(context['routeGeneration'], bool) and
|
||||
context['routeGeneration'] > 0 and model_state.get('routeGeneration') == context['routeGeneration'])
|
||||
model_reason = model_state.get('reason') if matched_wait else None
|
||||
if isinstance(model_reason, str):
|
||||
waiting = {
|
||||
'map_missing': 'Waiting for model map', 'map_stale': 'Model map is stale', 'map_invalid': 'Model map is invalid',
|
||||
'instruction_stale': 'Route instructions are stale', 'route_generation_mismatch': 'Waiting for current route map',
|
||||
'disabled': 'Navigation inputs disabled', 'invalid_time_or_speed': 'Vehicle data unavailable',
|
||||
'signal_navigation_disabled': 'Signal navigation is inactive', 'route_context_missing': 'Waiting for route context',
|
||||
'route_context_unavailable': 'Route context unavailable', 'signal_session_mismatch': 'Waiting for current signal',
|
||||
'destination_route_unavailable': 'Waiting for destination route', 'waiting_for_navigation': 'Waiting for navigation inputs',
|
||||
'ready': 'Waiting for model response',
|
||||
}.get(model_reason, waiting)
|
||||
pending = intent.get('mode') == 'pending'
|
||||
return {'state': 'ready', 'primary': 'Turn candidate' if pending else 'Navigation active' if active else 'Route ready',
|
||||
'direction': direction, 'distanceM': distance,
|
||||
'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else 'Waiting for map and model')}
|
||||
'secondary': (f"{direction.capitalize()} onto {context.get('name', '')}".strip() if active or pending else waiting)}
|
||||
if status == 'searching':
|
||||
return {'state': status, 'primary': f'Finding next {direction}', 'secondary': 'Checking mapped streets'}
|
||||
if status == 'passed':
|
||||
@@ -187,6 +205,11 @@ class NavigationPanel(Widget):
|
||||
source = rl.Rectangle((SIZE - width) // 2, min(SIZE - height, max(0, EGO[1] + MARKER_MARGIN - height)), width, height)
|
||||
target = rl.Rectangle(map_rect.x + (map_rect.width - width) / 2, map_rect.y + map_rect.height - height, width, height)
|
||||
rl.draw_texture_pro(self._texture, source, target, rl.Vector2(0, 0), 0, rl.WHITE)
|
||||
# Attribution is a display overlay; the model receives the original raster.
|
||||
credit = rl.Rectangle(map_rect.x+2, map_rect.y+map_rect.height-17, map_rect.width-4, 16)
|
||||
rl.draw_rectangle_rec(credit, rl.Color(7, 13, 18, 220))
|
||||
gui_label(credit, '© OpenStreetMap contributors', 12, self._muted, FontWeight.MEDIUM,
|
||||
alignment_vertical=TextAlignmentVertical.TOP)
|
||||
else:
|
||||
self._draw_status(map_rect)
|
||||
if signal is not None and (notification or ui_state.nav_destination_set):
|
||||
|
||||
Reference in New Issue
Block a user