Ship the offline map provider

build

@@ -126,6 +126,8 @@ if [[ "${#shortcut_targets[@]}" -gt 0 ]]; then
   exit $?
 fi
 
+python3 "${ROOT_DIR}/tools/release/stage_mapd_provider.py" --source "${ROOT_DIR}" --require-tracked
+
 if [[ "${#original_args[@]}" -gt 0 ]]; then
   exec "${build_helper}" build "${original_args[@]}"
 fi

openpilot/starpilot/maps/provider/.gitignore

@@ -1,2 +0,0 @@
-mapd
-manifest.json

openpilot/starpilot/maps/provider/manifest.json

@@ -0,0 +1 @@
+{"binarySha256":"c7dd6cb333774675464818f52fe60e12a3650aee30a808cd0052326b8f5636f9","goVersion":"go1.25.1","schemaVersion":1,"sourceDigest":"aa19b0de0f7fc40fde1fc97109ae7810dccfd3468b9b1d056d72229b7df6b5bc","sourceRevision":"0ebfc2a4bebc4914d7aac35723dc047d9d34faa8","target":"linux-arm64-static","upstreamRevision":"7201c6b4b4ec1b0b9ea21daa8c05b80fdd7e01ee"}

tools/ci/run_predeploy.py

@@ -17,7 +17,8 @@ STAGES = ('build', 'source', 'host', 'native', 'vehicles', 'recorded', 'replay')
 
 def commands(output, cache, download=False):
   python = sys.executable
-  source = [('lint', ['bash', 'scripts/lint/lint.sh']),
+  source = [('mapd-package', [python, 'tools/release/stage_mapd_provider.py', '--source', str(ROOT), '--require-tracked']),
+            ('lint', ['bash', 'scripts/lint/lint.sh']),
             ('dependencies', [python, 'tools/vendor/check.py', '--revision', 'HEAD']),
             ('schemas', [python, 'tools/ci/schema_policy.py'])]
   for name, path in (('vendor', 'tools/vendor/tests'), ('packaging', 'tools/release/tests'),

tools/release/release_files.py

@@ -22,9 +22,10 @@ blacklist = [
 
 # Dependency sources are ordinary tracked files. A gitlink cannot be packaged
 # without a separate checkout, so fail rather than silently omitting its source.
-def release_files(root: str, include_big_model: bool = False) -> list[bytes]:
+def release_files(root: str, include_big_model: bool = False, *, require_provider: bool = False) -> list[bytes]:
   entries = subprocess.check_output(["git", "ls-files", "--stage", "-z"], cwd=root).split(b"\0")
   files = []
+  modes = {}
   for entry in entries:
     if not entry:
       continue
@@ -44,7 +45,13 @@ def release_files(root: str, include_big_model: bool = False) -> list[bytes]:
     if blacklisted:
       continue
 
+    modes[tracked_file] = mode
     files.append(tracked_file)
+  if require_provider or b"openpilot/starpilot/maps/shadow_lifecycle.py" in files:
+    for name, mode in ((b"openpilot/starpilot/maps/provider/mapd", b"100755"),
+                       (b"openpilot/starpilot/maps/provider/manifest.json", b"100644")):
+      if modes.get(name) != mode:
+        raise ValueError(f"Mapd Git package missing or invalid mode: {os.fsdecode(name)}; track the verified provider package")
   return files
 
 

tools/release/stage_mapd_provider.py

@@ -100,6 +100,11 @@ def stage_provider(source: Path, destination: Path) -> dict:
     _directories(destination)
   else:
     provider.mkdir(mode=0o755)
+  existing = [destination / PROVIDER / name for name in ('mapd', 'manifest.json')]
+  if any(path.exists() or path.is_symlink() for path in existing):
+    if validate_provider(destination) != manifest:
+      raise ValueError('release Mapd package differs from the validated package')
+    return manifest
   outputs = []
   try:
     for name, mode in (('mapd', 0o755), ('manifest.json', 0o644)):
@@ -127,8 +132,12 @@ def main() -> int:
   parser = argparse.ArgumentParser(description=__doc__)
   parser.add_argument('--source', type=Path, required=True)
   parser.add_argument('--destination', type=Path)
+  parser.add_argument('--require-tracked', action='store_true')
   args = parser.parse_args()
   try:
+    if args.require_tracked:
+      from tools.release.release_files import release_files
+      release_files(str(args.source), require_provider=True)
     manifest = stage_provider(args.source, args.destination) if args.destination else validate_provider(args.source)
   except (OSError, ValueError, KeyError, TypeError, StopIteration) as error:
     parser.exit(1, f'Mapd release package unavailable: {error}\nBuild it in the source checkout with ./build --mapd.\n')

tools/release/tests/test_stage_mapd_provider.py

@@ -97,6 +97,46 @@ class TestStageMapdProvider(unittest.TestCase):
     with self.assertRaisesRegex(ValueError, 'release Mapd source differs'):
       stage_provider(self.source, self.destination)
 
+  def test_tracked_archive_and_release_copy_include_valid_provider(self):
+    lifecycle = self.source / 'openpilot/starpilot/maps/shadow_lifecycle.py'
+    lifecycle.write_text('# fixture lifecycle\n')
+    self.git('add', 'openpilot/starpilot/maps')
+    self.git('-c', 'user.name=Test', '-c', 'user.email=test@example.invalid',
+             'commit', '--quiet', '-m', 'tracked provider')
+    archive = self.base / 'source.tar'
+    self.git('archive', '--format=tar', '--output', str(archive), 'HEAD')
+    extracted = self.base / 'archive'
+    extracted.mkdir()
+    shutil.unpack_archive(archive, extracted)
+    self.assertEqual(validate_provider(extracted), validate_provider(self.source))
+    for raw in release_files(str(self.source)):
+      target = self.destination / raw.decode()
+      target.parent.mkdir(parents=True, exist_ok=True)
+      shutil.copy2(self.source / raw.decode(), target)
+    binary = self.destination / PROVIDER / 'mapd'
+    before = binary.stat()
+    self.assertEqual(stage_provider(self.source, self.destination), validate_provider(self.source))
+    self.assertEqual(binary.stat().st_ino, before.st_ino)
+    self.assertEqual(binary.stat().st_mode, before.st_mode)
+
+  def test_ignored_local_provider_cannot_mask_missing_git_package(self):
+    self.assertIsInstance(validate_provider(self.source), dict)
+    with self.assertRaisesRegex(ValueError, 'Mapd Git package missing'):
+      release_files(str(self.source), require_provider=True)
+
+  def test_existing_partial_tampered_or_symlink_package_denied(self):
+    stage_provider(self.source, self.destination)
+    binary = self.destination / PROVIDER / 'mapd'
+    binary.write_bytes(binary.read_bytes() + b'tampered')
+    with self.assertRaisesRegex(ValueError, 'hash or embedded'):
+      stage_provider(self.source, self.destination)
+    binary.unlink()
+    with self.assertRaises(OSError):
+      stage_provider(self.source, self.destination)
+    binary.symlink_to(self.source / PROVIDER / 'mapd')
+    with self.assertRaises(ValueError):
+      stage_provider(self.source, self.destination)
+
   def test_finder_metadata_does_not_change_source_attestation(self):
     manifest = self.make_package()
     metadata = self.source / 'mapd_repo/.DS_Store'