StarPilot/openpilot/starpilot/state_migration.py

"""Offline settings recovery and staging. Archived bytes are never an import policy."""
import base64
import binascii
import fcntl
import hashlib
import json
import os
import re
import shutil
import stat
import tempfile
from contextlib import contextmanager
from pathlib import Path


PREFERENCES = frozenset({
  "AlwaysOnLateral", "IsMetric", "RecordAudio", "RecordFront", "RecordFrontLock",
  "ShowDebugInfo", "GsmMetered", "OpenpilotEnabledToggle",
})
MAX_VALUE_BYTES = 32 * 1024 * 1024
MAX_SNAPSHOT_BYTES = 256 * 1024 * 1024
MAX_KEYS = 4096


class MigrationRequired(RuntimeError):
  """Existing state has not been qualified for this runtime."""


def canonical_json(value):
  return json.dumps(value, sort_keys=True, separators=(',', ':'), ensure_ascii=True).encode('utf-8')


def digest(data):
  return hashlib.sha256(data).hexdigest()


def _key(value):
  if not isinstance(value, str) or not value or value in ('.', '..') or '/' in value or '\0' in value:
    raise ValueError("Invalid settings key")
  return value


def _hash(value, length=64):
  if not isinstance(value, str) or re.fullmatch(f'[0-9a-f]{{{length}}}', value) is None:
    raise ValueError("Invalid source digest")


def _private_dir(path):
  path = Path(path)
  path.mkdir(mode=0o700, parents=True, exist_ok=True)
  info = path.lstat()
  if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077:
    raise ValueError("Recovery storage must be a private, owned directory")
  return path


def _fsync_dir(path):
  fd = os.open(path, os.O_RDONLY | os.O_DIRECTORY)
  try:
    os.fsync(fd)
  finally:
    os.close(fd)


def _write(path, data):
  fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
  with os.fdopen(fd, 'wb') as output:
    output.write(data)
    output.flush()
    os.fsync(output.fileno())


def _atomic_write(path, data):
  fd, temporary = tempfile.mkstemp(prefix='.pending-', dir=path.parent)
  try:
    with os.fdopen(fd, 'wb') as output:
      output.write(data)
      output.flush()
      os.fsync(output.fileno())
    os.replace(temporary, path)
    _fsync_dir(path.parent)
  finally:
    Path(temporary).unlink(missing_ok=True)


@contextmanager
def _params_lock(namespace, *, read_only=False):
  # Params writers use this same lock in the parent of the namespace symlink.
  flags = os.O_RDONLY | os.O_NOFOLLOW if read_only else os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW
  try:
    fd = os.open(namespace.parent / '.lock', flags, 0o600)
  except FileNotFoundError:
    if not read_only:
      raise
    yield
    return
  try:
    if not stat.S_ISREG(os.fstat(fd).st_mode):
      raise ValueError("Invalid Params lock")
    fcntl.flock(fd, fcntl.LOCK_EX)
    yield
  finally:
    os.close(fd)


def _read_value(path):
  fd = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK)
  with os.fdopen(fd, 'rb') as source:
    if not stat.S_ISREG(os.fstat(source.fileno()).st_mode):
      raise ValueError("Settings namespace must contain only regular files")
    value = source.read(MAX_VALUE_BYTES + 1)
  if len(value) > MAX_VALUE_BYTES:
    raise ValueError("Settings value exceeds recovery limit")
  return value


def _read_namespace(namespace):
  actual = namespace.resolve(strict=True)
  values = {}
  total = 0
  for entry in sorted(actual.iterdir()):
    key = _key(entry.name)
    value = _read_value(entry)
    total += len(value)
    values[key] = value
    if total > MAX_SNAPSHOT_BYTES or len(values) > MAX_KEYS:
      raise ValueError("Settings namespace exceeds recovery limit")
  return values


def _inventory(values):
  return [{'key': key, 'sha256': digest(value), 'size': len(value)} for key, value in sorted(values.items())]


def _outside_params(namespace, storage):
  if storage.resolve().is_relative_to(namespace.parent.resolve()) or storage.resolve().is_relative_to(namespace.resolve()):
    raise ValueError("Recovery storage must be outside Params")


def _save_snapshot(values, storage):
  inventory = _inventory(values)
  snapshot_id = digest(canonical_json(inventory))
  storage = _private_dir(storage)
  destination = storage / snapshot_id
  if destination.exists():
    if load_snapshot(destination) != values:
      raise ValueError("Existing recovery snapshot does not match")
    return destination
  temporary = Path(tempfile.mkdtemp(prefix='.pending-', dir=storage))
  try:
    data_dir = _private_dir(temporary / 'values')
    for key, value in values.items():
      _write(data_dir / key, value)
    _write(temporary / 'manifest.json', canonical_json({'format': 'starpilot-raw-settings', 'version': 1, 'entries': inventory}))
    _fsync_dir(data_dir)
    _fsync_dir(temporary)
    os.rename(temporary, destination)
    _fsync_dir(storage)
  finally:
    if temporary.exists():
      shutil.rmtree(temporary)
  return destination


def snapshot_settings(namespace, storage):
  """Capture all raw values, including empty files, unknown keys and credentials.

  Call with services stopped. The native Params lock coordinates cooperating
  writers, but cannot coordinate an older runtime's separate settings cache.
  """
  namespace, storage = Path(namespace).absolute(), Path(storage).absolute()
  _outside_params(namespace, storage)
  with _params_lock(namespace):
    return _save_snapshot(_read_namespace(namespace), storage)


def load_snapshot(snapshot):
  snapshot = Path(snapshot)
  manifest = json.loads(_read_value(snapshot / 'manifest.json'))
  if not isinstance(manifest, dict) or set(manifest) != {'format', 'version', 'entries'}:
    raise ValueError("Invalid recovery snapshot fields")
  if manifest['format'] != 'starpilot-raw-settings' or type(manifest['version']) is not int or manifest['version'] != 1:
    raise ValueError("Unsupported recovery snapshot")
  values = _read_namespace(snapshot / 'values')
  inventory = _inventory(values)
  if inventory != manifest['entries'] or digest(canonical_json(inventory)) != snapshot.name:
    raise ValueError("Recovery snapshot integrity failure")
  return values


def restore_snapshot(snapshot, namespace):
  """Restore exact bytes to a NEW, unused namespace; never overwrite live state."""
  values = load_snapshot(snapshot)
  namespace = Path(namespace).absolute()
  namespace.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
  with _params_lock(namespace):
    if namespace.exists() or namespace.is_symlink():
      raise FileExistsError("Recovery requires an unused destination namespace")
    temporary = Path(tempfile.mkdtemp(prefix='.restored-', dir=namespace.parent))
    try:
      for key, value in values.items():
        _write(temporary / key, value)
      _fsync_dir(temporary)
      namespace.symlink_to(temporary, target_is_directory=True)
      _fsync_dir(namespace.parent)
    except BaseException:
      if not namespace.is_symlink():
        shutil.rmtree(temporary)
      raise
  return namespace


def validate_bundle(bundle):
  if not isinstance(bundle, dict) or set(bundle) != {'format', 'version', 'source', 'preferences', 'pending', 'omitted'}:
    raise ValueError("Invalid settings bundle fields")
  if bundle['format'] != 'starpilot-settings' or type(bundle['version']) is not int or bundle['version'] != 1:
    raise ValueError("Unsupported settings bundle")
  source = bundle['source']
  required = {'revision', 'registry_sha256', 'snapshot_sha256'}
  if not isinstance(source, dict) or not required <= source.keys() or source.keys() - required - {'cache_snapshot_sha256'}:
    raise ValueError("Invalid settings source")
  for key, value in source.items():
    _hash(value, 40 if key == 'revision' else 64)
  preferences = bundle['preferences']
  if not isinstance(preferences, dict) or not preferences.keys() <= PREFERENCES or 'AlwaysOnLateral' not in preferences:
    raise ValueError("Unreviewed active preference")
  if any(type(value) is not bool for value in preferences.values()):
    raise ValueError("Preferences require canonical booleans")
  seen = set(preferences)
  for group in ('pending', 'omitted'):
    if not isinstance(bundle[group], list) or len(bundle[group]) > MAX_KEYS:
      raise ValueError("Invalid settings disposition list")
    for entry in bundle[group]:
      expected = {'key', 'type', 'raw_base64', 'sha256'} if group == 'pending' else {'key', 'reason'}
      if not isinstance(entry, dict) or set(entry) != expected:
        raise ValueError("Invalid settings disposition fields")
      key = _key(entry['key'])
      if key in seen:
        raise ValueError("Duplicate settings disposition")
      seen.add(key)
      if group == 'omitted':
        if not isinstance(entry['reason'], str) or not entry['reason']:
          raise ValueError("Missing settings omission reason")
        continue
      if not isinstance(entry['type'], str) or entry['type'] not in {'BOOL', 'INT', 'FLOAT', 'TIME', 'JSON', 'STRING'}:
        raise ValueError("Invalid pending settings type")
      if not isinstance(entry['raw_base64'], str) or len(entry['raw_base64']) > 4 * ((MAX_VALUE_BYTES + 2) // 3):
        raise ValueError("Invalid pending settings encoding size")
      try:
        value = base64.b64decode(entry['raw_base64'], validate=True)
      except (ValueError, binascii.Error) as error:
        raise ValueError("Invalid pending settings encoding") from error
      _hash(entry['sha256'])
      if len(value) > MAX_VALUE_BYTES or digest(value) != entry['sha256']:
        raise ValueError("Pending settings integrity failure")
  if len(canonical_json(bundle)) > MAX_SNAPSHOT_BYTES:
    raise ValueError("Settings bundle exceeds recovery limit")


def decode_bundle(encoded):
  """Parse serialized input without silently accepting duplicate JSON members."""
  if not isinstance(encoded, bytes) or len(encoded) > MAX_SNAPSHOT_BYTES:
    raise ValueError("Invalid settings bundle size or encoding")

  def unique_object(pairs):
    result = {}
    for key, value in pairs:
      if key in result:
        raise ValueError("Duplicate settings JSON member")
      result[key] = value
    return result

  bundle = json.loads(encoded, object_pairs_hook=unique_object)
  validate_bundle(bundle)
  return bundle


def stage_settings(bundle, storage, namespace):
  """Persist a reviewed-format bundle outside Params. Nothing is activated."""
  validate_bundle(bundle)
  _outside_params(Path(namespace).absolute(), Path(storage).absolute())
  encoded = canonical_json(bundle)
  destination = _private_dir(Path(storage) / 'staged') / f'{digest(encoded)}.json'
  _atomic_write(destination, encoded)
  return destination


def prepare_manager_start(params, storage, *, dry_run=False):
  """Refuse first boot over unqualified state before Params cleanup can erase it.

  A fresh namespace, or only the reviewed canonical preferences, can initialize
  this development runtime. Staging a bundle does NOT qualify a migrated profile.
  Dry-run checks admission without creating recovery storage or qualifying state.
  It is a read-only point-in-time check, not a cutover guarantee.
  """
  from openpilot.starpilot.schema_cache import CACHE_KEYS, inspect_cache

  namespace, storage = Path(params.get_param_path()).absolute(), Path(storage).absolute()
  _outside_params(namespace, storage)
  marker_dir = storage / 'profiles' if dry_run else _private_dir(storage / 'profiles')
  if dry_run and (marker_dir.exists() or marker_dir.is_symlink()):
    info = marker_dir.lstat()
    if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or info.st_mode & 0o077:
      raise ValueError("Recovery storage must be a private, owned directory")
  marker = marker_dir / f'{digest(str(namespace).encode())}.json'
  with _params_lock(namespace, read_only=dry_run):
    values = _read_namespace(namespace)
    identity = {'version': 1, 'schema_epoch': 1, 'namespace': str(namespace), 'target': str(namespace.resolve())}
    try:
      # This marker has one canonical encoding. Truncation, duplicate fields or
      # wrong JSON types cannot qualify a profile or bypass its recovery archive.
      initialized = _read_value(marker) == canonical_json(identity)
    except (OSError, ValueError):
      initialized = False
    known = {key.decode() if isinstance(key, bytes) else key for key in params.all_keys()}
    unknown = set(values) - known
    invalid_preferences = any(key in values and values[key] not in (b'0', b'1') for key in PREFERENCES)
    incompatible_cache = any(inspect_cache(key, values[key]).status != 'valid' for key in CACHE_KEYS if key in values)
    # This retired cache has no compatible producer or conversion.
    incompatible_cache |= 'LocationFilterInitialState' in values
    unqualified = unknown or invalid_preferences or incompatible_cache or (not initialized and bool(set(values) - PREFERENCES))
    if unqualified:
      if dry_run:
        reasons = []
        if unknown:
          reasons.append('unknown saved keys')
        if invalid_preferences:
          reasons.append('invalid saved preferences')
        if incompatible_cache:
          reasons.append('incompatible retained cache')
        if not initialized and bool(set(values) - PREFERENCES):
          reasons.append('unqualified existing namespace')
        raise MigrationRequired('Settings require migration before startup: ' + ', '.join(reasons))
      snapshot = _save_snapshot(values, storage / 'snapshots')
      raise MigrationRequired(f"Settings require migration before startup; raw recovery snapshot: {snapshot}")
    if not initialized and not dry_run:
      _atomic_write(marker, canonical_json(identity))