StarPilot/openpilot/starpilot/tests/test_first_start_migration.py

import importlib.util
import json
from pathlib import Path
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch

SOURCE = Path(__file__).resolve().parents[1] / 'state_migration.py'
spec = importlib.util.spec_from_file_location('isolated_state_migration', SOURCE)
assert spec is not None
migration = importlib.util.module_from_spec(spec)
spec.loader.exec_module(migration)


class TestFirstStartMigration(unittest.TestCase):
  def setUp(self):
    self.temporary = tempfile.TemporaryDirectory()
    self.addCleanup(self.temporary.cleanup)
    self.root = Path(self.temporary.name)
    self.namespace = self.root / 'params' / 'd'
    self.namespace.mkdir(parents=True)
    self.storage = self.root / 'recovery'
    self.types = {'IsMetric': 'BOOL', 'ForceStops': 'BOOL', 'OpenpilotEnabledToggle': 'BOOL', 'SteerFriction': 'FLOAT',
                  'CalibrationParams': 'BYTES', 'CarParamsCache': 'BYTES', 'CarParamsPersistent': 'BYTES', 'CarParamsPrevRoute': 'BYTES',
                  'LiveParametersV2': 'BYTES', 'LiveDelay': 'BYTES',
                  'LiveTorqueParameters': 'BYTES', 'SecOCKey': 'BYTES', 'DongleId': 'STRING',
                  'BluetoothEnabled': 'BOOL', 'GsmMetered': 'BOOL', 'HasAcceptedTerms': 'STRING'}
    self.params = SimpleNamespace(get_param_path=lambda: str(self.namespace), all_keys=lambda: list(self.types),
                                  get_type=lambda key: SimpleNamespace(name=self.types[key]))
    self.cache = SimpleNamespace(CACHE_KEYS={'CalibrationParams', 'CarParamsCache', 'CarParamsPersistent', 'CarParamsPrevRoute',
                                            'LiveParametersV2', 'LiveDelay', 'LiveTorqueParameters'},
                                 inspect_cache=lambda key, raw: SimpleNamespace(status='valid' if raw.startswith(b'envelope:') else 'legacy'))
    self.converter = SimpleNamespace(migrate_legacy_cache=lambda key, raw: b'envelope:' + raw if key == 'CalibrationParams' else None)
    self.addCleanup(patch.stopall)
    patch.dict('sys.modules', {'openpilot.starpilot.schema_cache': self.cache,
                              'openpilot.starpilot.legacy_cache_migration': self.converter}).start()

  def write(self, values):
    for key, value in values.items():
      (self.namespace / key).write_bytes(value)

  def values(self):
    return migration._read_namespace(self.namespace)

  def start(self, **kwargs):
    migration.prepare_manager_start(self.params, self.storage, auto_migrate=True, **kwargs)

  def test_epoch_two_archives_settings_and_calibration_retains_only_operational_state(self):
    original = {'IsMetric': b'1', 'ForceStops': b'0', 'SteerFriction': b'0.123',
                'SecOCKey': b'private\x00bytes', 'CalibrationParams': b'legacy calibration',
                'CarParamsPersistent': b'old identity', 'OldDomSetting': b'9',
                'BluetoothEnabled': b'1', 'DongleId': b'operational-identity', 'HasAcceptedTerms': b'accepted'}
    self.write(original)
    self.start()
    expected = {key: value for key, value in original.items() if key in migration.OPERATIONAL_KEYS}
    self.assertEqual(self.values(), expected)
    snapshots = list((self.storage / 'snapshots').iterdir())
    self.assertEqual(len(snapshots), 1)
    self.assertEqual(migration.load_snapshot(snapshots[0]), original)
    receipt = json.loads((snapshots[0] / 'migration.json').read_bytes())
    self.assertEqual(receipt['status'], 'migrated')
    before = {str(path): path.read_bytes() for path in self.storage.rglob('*') if path.is_file()}
    self.start()
    self.assertEqual({str(path): path.read_bytes() for path in self.storage.rglob('*') if path.is_file()}, before)

  def test_invalid_operational_bool_archived_all_old_preferences_removed(self):
    original = {'IsMetric': b'True', 'ForceStops': b'1', 'OpenpilotEnabledToggle': b'true',
                'BluetoothEnabled': b'true', 'GsmMetered': b'1'}
    self.write(original)
    self.start()
    self.assertEqual(self.values(), {'GsmMetered': b'1'})
    self.assertEqual(migration.load_snapshot(next((self.storage / 'snapshots').iterdir())), original)

  def test_fresh_driving_settings_leave_galaxy_tunnel_pairing_untouched(self):
    from openpilot.starpilot.galaxy.remote import RemotePairing
    pairing = RemotePairing(self.storage / 'galaxy')
    assert pairing.pair('a' * 64) is not None
    path = pairing.root / pairing.FILE
    saved = path.read_bytes()
    self.write({'CalibrationParams': b'old calibration', 'SteerFriction': b'0.2', 'DongleId': b'identity'})
    self.start()
    self.assertEqual(self.values(), {'DongleId': b'identity'})
    self.assertEqual(path.read_bytes(), saved)
    assert pairing.read() is not None

  def test_default_and_dry_run_remain_strict_without_mutation(self):
    self.write({'SteerFriction': b'0.2'})
    with self.assertRaises(migration.MigrationRequired):
      self.start(dry_run=True)
    self.assertFalse(self.storage.exists())
    with self.assertRaises(migration.MigrationRequired):
      migration.prepare_manager_start(self.params, self.storage)
    self.assertEqual(self.values(), {'SteerFriction': b'0.2'})

  def test_archive_readback_failure_preserves_all_bytes_and_does_not_qualify(self):
    original = {'CalibrationParams': b'bad calibration', 'OldDomSetting': b'1'}
    self.write(original)
    with patch.object(migration, 'load_snapshot', return_value={}):
      with self.assertRaises(migration.MigrationRequired):
        self.start()
    self.assertEqual(self.values(), original)
    self.assertEqual(list((self.storage / 'profiles').iterdir()), [])
    self.assertEqual(migration.load_snapshot(next((self.storage / 'snapshots').iterdir())), original)

  def test_partial_failure_snapshot_survives_retry_and_marker_is_last(self):
    original = {'CarParamsPersistent': b'old car', 'OldDomSetting': b'2', 'SecOCKey': b'credential'}
    self.write(original)
    actual_write = migration._atomic_write
    def fail_receipt(path, raw):
      if path.name == 'migration.json' and json.loads(raw).get('attempted') == ['CarParamsPersistent', 'OldDomSetting']:
        raise OSError('injected')
      return actual_write(path, raw)
    with patch.object(migration, '_atomic_write', side_effect=fail_receipt):
      with self.assertRaises((OSError, migration.MigrationRequired)):
        self.start()
    self.assertEqual(list((self.storage / 'profiles').iterdir()), [])
    self.assertIn(original, [migration.load_snapshot(path) for path in (self.storage / 'snapshots').iterdir()])
    self.start()
    self.assertEqual(self.values(), {'SecOCKey': b'credential'})
    self.assertIn(original, [migration.load_snapshot(path) for path in (self.storage / 'snapshots').iterdir()])

  def test_initialized_vehicle_cache_contract_upgrade_preserves_every_other_byte(self):
    self.start()
    original = {'CarParamsPersistent': b'legacy car', 'CarParamsPrevRoute': b'legacy route',
                'CalibrationParams': b'envelope:calibration', 'LiveTorqueParameters': b'envelope:learner',
                'IsMetric': b'1', 'ForceStops': b'1', 'SecOCKey': b'private\x00bytes'}
    self.write(original)
    marker = next((self.storage / 'profiles').iterdir())
    marker_bytes = marker.read_bytes()
    with self.assertRaises(migration.MigrationRequired):
      self.start(dry_run=True)
    self.assertEqual(self.values(), original)
    self.start()
    expected = {key: raw for key, raw in original.items() if key not in ('CarParamsPersistent', 'CarParamsPrevRoute')}
    self.assertEqual(self.values(), expected)
    self.assertEqual(marker.read_bytes(), marker_bytes)
    snapshot = next((self.storage / 'snapshots').iterdir())
    self.assertEqual(migration.load_snapshot(snapshot), original)
    self.assertEqual(set(json.loads((snapshot / 'migration.json').read_bytes())['actions']),
                     {'CarParamsPersistent', 'CarParamsPrevRoute'})
    self.start()
    self.assertEqual(self.values(), expected)

  def test_initialized_three_stale_car_caches_preserve_valid_state_and_restart(self):
    self.start()
    original = {'CarParamsCache': b'incompatible schema cache', 'CarParamsPersistent': b'incompatible persistent',
                'CarParamsPrevRoute': b'incompatible previous route', 'CalibrationParams': b'envelope:calibration',
                'LiveParametersV2': b'envelope:vehicle learner', 'LiveDelay': b'envelope:delay',
                'IsMetric': b'1', 'ForceStops': b'1', 'SteerFriction': b'0.123', 'DongleId': b'identity'}
    self.write(original)
    marker = next((self.storage / 'profiles').iterdir())
    marker_bytes = marker.read_bytes()
    self.start()
    retired = {'CarParamsCache', 'CarParamsPersistent', 'CarParamsPrevRoute'}
    expected = {key: value for key, value in original.items() if key not in retired}
    self.assertEqual(self.values(), expected)
    self.assertEqual(marker.read_bytes(), marker_bytes)
    snapshot = next((self.storage / 'snapshots').iterdir())
    self.assertEqual(migration.load_snapshot(snapshot), original)
    self.assertEqual(set(json.loads((snapshot / 'migration.json').read_bytes())['actions']), retired)
    before = {str(path): path.read_bytes() for path in self.storage.rglob('*') if path.is_file()}
    self.start()
    self.assertEqual(self.values(), expected)
    self.assertEqual({str(path): path.read_bytes() for path in self.storage.rglob('*') if path.is_file()}, before)

  def test_initialized_isolated_stale_car_params_cache_is_reconstructible(self):
    self.start()
    original = {'CarParamsCache': b'incompatible cache', 'CalibrationParams': b'envelope:calibration',
                'IsMetric': b'1', 'ForceStops': b'0'}
    self.write(original)
    with self.assertRaises(migration.MigrationRequired):
      self.start(dry_run=True)
    self.assertEqual(self.values(), original)
    self.start()
    self.assertEqual(self.values(), {key: value for key, value in original.items() if key != 'CarParamsCache'})
    self.assertEqual(migration.load_snapshot(next((self.storage / 'snapshots').iterdir())), original)

  def test_initialized_retired_setting_is_archived_and_other_state_kept(self):
    self.start()
    original = {'ShareUsageStats': b'1', 'IsMetric': b'1', 'ForceStops': b'0'}
    self.write(original)
    self.start()
    self.assertEqual(self.values(), {'IsMetric': b'1', 'ForceStops': b'0'})
    self.assertEqual(migration.load_snapshot(next((self.storage / 'snapshots').iterdir())), original)

  def test_initialized_valid_car_caches_are_retained_without_archive(self):
    self.start()
    original = {key: b'envelope:' + key.encode() for key in self.cache.CACHE_KEYS}
    original.update(IsMetric=b'1', ForceStops=b'1', SteerFriction=b'0.321')
    self.write(original)
    self.start()
    self.start()
    self.assertEqual(self.values(), original)
    self.assertFalse((self.storage / 'snapshots').exists())

  def test_initialized_cache_retirement_receipt_failure_preserves_raw_snapshot_and_all_state(self):
    self.start()
    original = {'CarParamsCache': b'incompatible cache', 'CarParamsPersistent': b'incompatible persistent',
                'CarParamsPrevRoute': b'incompatible route', 'CalibrationParams': b'envelope:calibration',
                'LiveParametersV2': b'envelope:learner', 'LiveDelay': b'envelope:delay', 'ForceStops': b'1'}
    self.write(original)
    marker = next((self.storage / 'profiles').iterdir())
    marker_bytes = marker.read_bytes()
    actual_write = migration._atomic_write
    def fail_receipt(path, raw):
      if path.name == 'migration.json':
        raise OSError('receipt storage failure')
      return actual_write(path, raw)
    with patch.object(migration, '_atomic_write', side_effect=fail_receipt):
      with self.assertRaises(OSError):
        self.start()
    self.assertEqual(self.values(), original)
    self.assertEqual(marker.read_bytes(), marker_bytes)
    snapshot = next((self.storage / 'snapshots').iterdir())
    self.assertEqual(migration.load_snapshot(snapshot), original)

  def test_initialized_critical_cache_corruption_still_rejected(self):
    self.start()
    original = {'CarParamsPersistent': b'legacy car', 'CalibrationParams': b'bad calibration'}
    self.write(original)
    with self.assertRaises(migration.MigrationRequired):
      self.start()
    self.assertEqual(self.values(), original)

  def test_initialized_vehicle_cache_upgrade_does_not_admit_other_invalid_state(self):
    self.start()
    for key, raw in (('OldDomSetting', b'1'), ('IsMetric', b'true'), ('ForceStops', b'true'),
                     ('LiveTorqueParameters', b'bad learner')):
      with self.subTest(key=key):
        original: dict = {'CarParamsPersistent': b'legacy car', key: raw}
        self.write(original)
        with self.assertRaises(migration.MigrationRequired):
          self.start()
        self.assertEqual(self.values(), original)
        for entry in self.namespace.iterdir():
          entry.unlink()

  def test_snapshot_failure_prevents_every_mutation(self):
    original = {'OldDomSetting': b'2', 'IsMetric': b'1'}
    self.write(original)
    with patch.object(migration, '_save_snapshot', side_effect=OSError('storage unavailable')):
      with self.assertRaises(OSError):
        self.start()
    self.assertEqual(self.values(), original)
    self.assertEqual(list((self.storage / 'profiles').iterdir()), [])

  def test_old_epoch_resets_once_then_new_choices_survive(self):
    self.start()
    marker = next((self.storage / 'profiles').iterdir())
    old = json.loads(marker.read_bytes())
    old.update(version=1, schema_epoch=1)
    marker.write_bytes(migration.canonical_json(old))
    self.write({'CalibrationParams': b'envelope:old calibration', 'ForceStops': b'1', 'SecOCKey': b'auth'})
    self.start()
    self.assertEqual(self.values(), {'SecOCKey': b'auth'})
    self.assertEqual(json.loads(marker.read_bytes())['schema_epoch'], 2)
    self.write({'ForceStops': b'1', 'IsMetric': b'1'})
    self.start()
    self.assertEqual(self.values(), {'SecOCKey': b'auth', 'ForceStops': b'1', 'IsMetric': b'1'})

  def test_marker_failure_happens_only_after_verified_archive_and_reset(self):
    original = {'CalibrationParams': b'legacy calibration', 'DongleId': b'identity'}
    self.write(original)
    actual_write = migration._atomic_write
    def fail_marker(path, raw):
      if path.parent.name == 'profiles':
        raise OSError('marker failed')
      return actual_write(path, raw)
    with patch.object(migration, '_atomic_write', side_effect=fail_marker):
      with self.assertRaises(OSError):
        self.start()
    self.assertEqual(self.values(), {'DongleId': b'identity'})
    self.assertEqual(list((self.storage / 'profiles').iterdir()), [])
    self.assertEqual(migration.load_snapshot(next((self.storage / 'snapshots').iterdir())), original)
    self.start()
    self.assertEqual(self.values(), {'DongleId': b'identity'})


if __name__ == '__main__':
  unittest.main()