StarPilot/openpilot/starpilot/galaxy/drive_stats.py

"""Durable, bounded statistics for analyzed local Galaxy drives."""

from __future__ import annotations

from collections import defaultdict
from collections.abc import Callable
from datetime import UTC, datetime, timedelta, tzinfo
import hashlib
import json
import math
import os
from pathlib import Path
import stat
import threading
import time

from openpilot.starpilot.galaxy.drive_history import DriveHistory, MAX_ROOT_ENTRIES, MAX_STATS_SEGMENTS_PER_ROUTE
from openpilot.starpilot.galaxy.drive_analysis import ANALYSIS_VERSION
from openpilot.starpilot.storage import galaxy_storage_root

SCHEMA_VERSION = 1
MAX_CACHED_ROUTES = 2048
MAX_CACHE_BYTES = 4_000_000
MAX_SCAN_ROUTES = 100
MAX_ANALYSES_PER_SCAN = 2
SCAN_INTERVAL_SECONDS = 60
RETRY_SECONDS = 300
MAX_RETRY_ATTEMPTS = 5
MAX_RETRY_AGE_SECONDS = 86400
MAX_RETRY_DELAY_SECONDS = 3600
RECENT_LIMIT = 5


def _finite(value, minimum=0.0):
  if type(value) not in (int, float):
    return None
  try:
    if not math.isfinite(value) or value < minimum:
      return None
  except (OverflowError, ValueError):
    return None
  return float(value)


def _valid_record(value: object) -> bool:
  if not isinstance(value, dict) or type(value.get('routeId')) is not str or len(value['routeId']) > 180:
    return False
  if type(value.get('complete')) is not bool or type(value.get('ignored')) is not bool:
    return False
  if type(value.get('segmentCount')) is not int or not 0 <= value['segmentCount'] <= MAX_STATS_SEGMENTS_PER_ROUTE:
    return False
  for key in ('startTime', 'endTime'):
    if value.get(key) is not None and (_finite(value[key]) is None or value[key] > 253402300799):
      return False
  for key in ('distanceMeters', 'durationSeconds', 'engagedSeconds', 'aolSeconds'):
    if value.get(key) is not None and _finite(value[key]) is None:
      return False
  for key in ('distractedMoments', 'unresponsiveMoments'):
    if value.get(key) is not None and (type(value[key]) is not int or value[key] < 0):
      return False
  if value.get('model') is not None and (type(value['model']) is not str or len(value['model']) > 120):
    return False
  if value.get('reason') is not None and (type(value['reason']) is not str or len(value['reason']) > 200):
    return False
  if value.get('engagedPercent') is not None and (_finite(value['engagedPercent']) is None or value['engagedPercent'] > 100):
    return False
  if value.get('aolPercent') is not None and (_finite(value['aolPercent']) is None or value['aolPercent'] > 100):
    return False
  if (value.get('aolSeconds') is None) != (value.get('aolPercent') is None):
    return False
  if value.get('aolSeconds') is not None and (value.get('durationSeconds') is None or value['aolSeconds'] > value['durationSeconds']):
    return False
  if type(value.get('sourceSignature')) is not str or len(value['sourceSignature']) > 128:
    return False
  if type(value.get('analysisVersion', 0)) is not int or not 0 <= value.get('analysisVersion', 0) < 2**31:
    return False
  manifest = value.get('sourceManifest')
  if manifest is not None and (not isinstance(manifest, dict) or len(manifest) > MAX_STATS_SEGMENTS_PER_ROUTE * 2 or
      any(type(key) is not str or len(key) > 220 or not isinstance(revision, list) or
          len(revision) != 5 or any(type(part) is not int or part < 0 for part in revision)
          for key, revision in manifest.items())):
    return False
  if value['complete'] and (value.get('startTime') is None or value.get('endTime') is None or
      any(value.get(key) is None for key in ('distanceMeters', 'durationSeconds', 'engagedSeconds')) or
      value['endTime'] < value['startTime'] or value['engagedSeconds'] > value['durationSeconds']):
    return False
  return True


def _route_signature(root: Path, route: dict) -> tuple[str, dict]:
  parts = []
  manifest = {}
  for segment in route.get('segments', []):
    name = segment.get('segmentName', '')
    files = segment.get('files', {})
    for filename in ('rlog.zst', 'rlog.bz2', 'qlog.zst', 'qlog.bz2'):
      try:
        info = (root / name / filename).stat(follow_symlinks=False)
        if stat.S_ISREG(info.st_mode):
          manifest[f'{name}/{filename}'] = [info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns]
      except OSError:
        pass
    parts.append((segment.get('number'), name, files))
  encoded = json.dumps(parts, sort_keys=True, separators=(',', ':')).encode()
  source = hashlib.sha256(encoded + json.dumps(manifest, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
  return source, manifest


def _only_deleted_sources(previous: dict, current: dict) -> bool:
  old = previous.get('sourceManifest')
  return isinstance(old, dict) and all(old.get(key) == revision for key, revision in current.items())


def _personal_records(drives: list[dict], zone: tzinfo) -> list[dict]:
  if not drives:
    return []
  ordered = sorted(drives, key=lambda r: (r['startTime'], r['routeId']))
  daily = defaultdict(lambda: {'duration': 0.0, 'engaged': 0.0})
  weekly = defaultdict(float)
  longest = max(drives, key=lambda r: r['distanceMeters'])
  undistracted = []
  clean_streak = best_clean_streak = 0
  for drive in ordered:
    day = datetime.fromtimestamp(drive['startTime'], zone).date()
    daily[day]['duration'] += drive['durationSeconds']
    daily[day]['engaged'] += drive['engagedSeconds']
    weekly[day - timedelta(days=day.weekday())] += drive['distanceMeters']
    if drive.get('distractedMoments') == 0:
      undistracted.append(drive)
    clean = drive.get('unresponsiveMoments') == 0
    if clean:
      clean_streak += 1
      best_clean_streak = max(best_clean_streak, clean_streak)
    else:
      clean_streak = 0
  engaged_days = [(100 * data['engaged'] / data['duration'], day) for day, data in daily.items() if data['duration'] > 0]
  best_day = max(engaged_days, default=(None, None))
  best_week = max(((distance, day) for day, distance in weekly.items()), default=(None, None))
  streak = best_streak = 0
  previous_day = None
  for day in sorted(daily):
    streak = streak + 1 if previous_day is not None and (day - previous_day).days == 1 else 1
    best_streak = max(best_streak, streak)
    previous_day = day
  best_undistracted = max(undistracted, key=lambda r: r['durationSeconds']) if undistracted else None
  return [
    {'id': 'longestDrive', 'label': 'Longest drive', 'value': longest['distanceMeters'], 'unit': 'm'},
    {'id': 'mostEngagedDay', 'label': 'Most engaged day', 'value': round(best_day[0], 1) if best_day[0] is not None else None, 'unit': '%'},
    {'id': 'bestWeek', 'label': 'Best week', 'value': best_week[0], 'unit': 'm'},
    {'id': 'highestStreak', 'label': 'Highest streak', 'value': best_streak, 'unit': 'days'},
    {'id': 'longestUndistractedDrive', 'label': 'Longest undistracted drive',
     'value': best_undistracted['durationSeconds'] if best_undistracted else None, 'unit': 's'},
    {'id': 'cleanDriveStreak', 'label': 'Clean drive streak', 'value': best_clean_streak, 'unit': 'drives'},
  ]


class DriveStatsOwner:
  def __init__(self, root: Path | None = None, store: Path | None = None, history: DriveHistory | None = None,
               analyzer: Callable | None = None, permitted: Callable[[], bool] | None = None,
               clock: Callable[[], float] | None = None, monotonic: Callable[[], float] | None = None,
               metric: Callable[[], bool] | None = None):
    self.root = Path(root) if root is not None else None
    self.store = Path(store) if store is not None else galaxy_storage_root() / 'drive-stats.json'
    self.history = history if history is not None else DriveHistory(self.root, max_segments_per_route=MAX_STATS_SEGMENTS_PER_ROUTE,
                                                                    max_segments=MAX_ROOT_ENTRIES)
    self.root = Path(self.history.root) if self.root is None else self.root
    if analyzer is None:
      from openpilot.starpilot.galaxy.drive_analysis import analyze_route
      analyzer = analyze_route
    self.analyzer = analyzer
    self.permitted = permitted if permitted is not None else lambda: False
    self.clock = clock if clock is not None else lambda: datetime.now(UTC).timestamp()
    self.monotonic = monotonic if monotonic is not None else time.monotonic
    self.metric = metric if metric is not None else lambda: True
    self._lock = threading.RLock()
    self._permit_lock = threading.Lock()
    self._permit_checked_at = float('-inf')
    self._permit_cached = False
    self._closed = threading.Event()
    self._worker: threading.Thread | None = None
    self._scheduler: threading.Thread | None = None
    self._next_scan = 0.0
    self._records: dict[str, dict] = {}
    self._failures: dict[str, dict] = {}
    self._scan_incomplete = False
    self._scanned = False
    self._truncated = False
    self._corrupt = False
    self._pending = 0
    self._cursor = 0
    self._load()

  def _load(self):
    try:
      with self.store.open('rb') as stream:
        raw = stream.read(MAX_CACHE_BYTES + 1)
      if len(raw) > MAX_CACHE_BYTES:
        raise ValueError('Oversized cache')
      data = json.loads(raw)
      records = data['routes']
      if data.get('schemaVersion') != SCHEMA_VERSION or not isinstance(records, dict) or len(records) > MAX_CACHED_ROUTES:
        raise ValueError('Invalid cache')
      if any(key != value.get('routeId') or not _valid_record(value) for key, value in records.items()):
        raise ValueError('Invalid route')
      failures = data.get('failures', {})
      if not isinstance(failures, dict) or len(failures) > MAX_CACHED_ROUTES:
        raise ValueError('Invalid retry cache')
      for key, failure in failures.items():
        if (type(key) is not str or len(key) > 180 or not isinstance(failure, dict) or
            type(failure.get('signature')) is not str or len(failure['signature']) > 128 or
            type(failure.get('attempts')) is not int or not 1 <= failure['attempts'] <= MAX_RETRY_ATTEMPTS or
            any(_finite(failure.get(field)) is None for field in ('firstAt', 'retryAt')) or
            type(failure.get('analysisVersion', 0)) is not int or not 0 <= failure.get('analysisVersion', 0) < 2**31 or
            type(failure.get('reason')) is not str or len(failure['reason']) > 200):
          raise ValueError('Invalid retry record')
      self._failures = failures
      self._records = records
      self._scan_incomplete = data.get('scanIncomplete') is True
      self._scanned = data.get('scanned') is True
      self._truncated = data.get('truncated') is True
    except FileNotFoundError:
      return
    except (OSError, ValueError, TypeError, KeyError, AttributeError):
      self._corrupt = True

  def _save(self):
    self.store.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
    payload = {'schemaVersion': SCHEMA_VERSION, 'routes': self._records,
               'scanned': self._scanned, 'scanIncomplete': self._scan_incomplete, 'truncated': self._truncated,
               'failures': self._failures}
    encoded = json.dumps(payload, separators=(',', ':'), allow_nan=False).encode()
    if len(encoded) > MAX_CACHE_BYTES:
      raise OSError('Drive statistics cache exceeds storage limit')
    temporary = self.store.with_name(f'.{self.store.name}.{os.getpid()}.{threading.get_ident()}.tmp')
    try:
      fd = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600)
      with os.fdopen(fd, 'wb') as stream:
        stream.write(encoded)
        stream.flush()
        os.fsync(stream.fileno())
      os.replace(temporary, self.store)
      directory_fd = os.open(self.store.parent, os.O_RDONLY | os.O_DIRECTORY)
      try:
        os.fsync(directory_fd)
      finally:
        os.close(directory_fd)
      self._corrupt = False
    finally:
      try:
        temporary.unlink()
      except FileNotFoundError:
        pass

  def _allowed(self) -> bool:
    if self._closed.is_set():
      return False
    with self._permit_lock:
      if self._closed.is_set():
        return False
      now = self.monotonic()
      if now - self._permit_checked_at >= .1 or now < self._permit_checked_at:
        try:
          self._permit_cached = bool(self.permitted())
        except Exception:
          self._permit_cached = False
        self._permit_checked_at = now
      return self._permit_cached

  def _run(self):
    if not self._allowed():
      return
    assert self.root is not None
    try:
      inventory = self.history.snapshot()
      routes = inventory.get('routes', [])
      if not isinstance(routes, list):
        raise ValueError('Invalid route inventory')
      with self._lock:
        self._scanned = True
        self._scan_incomplete = inventory.get('scanIncomplete') is True or len(routes) > MAX_SCAN_ROUTES
        self._pending = 0
        route_ids = {route.get('routeId') for route in routes if isinstance(route, dict) and type(route.get('routeId')) is str}
        self._failures = {key: value for key, value in self._failures.items() if key in route_ids}
      candidates = []
      for route in routes[:MAX_SCAN_ROUTES]:
        if not self._allowed():
          break
        route_id = route.get('routeId')
        if type(route_id) is not str or len(route_id) > 180:
          continue
        signature, manifest = _route_signature(self.root, route)
        with self._lock:
          existing = self._records.get(route_id)
          failed = self._failures.get(route_id)
          if (existing and existing.get('sourceSignature') == signature and existing.get('complete') and
              existing.get('analysisVersion', 0) >= ANALYSIS_VERSION):
            continue
          if (existing and existing.get('complete') and manifest != existing.get('sourceManifest') and
              _only_deleted_sources(existing, manifest)):
            prior_signature = existing['sourceSignature']
            existing['sourceSignature'] = signature
            try:
              self._save()
            except OSError:
              existing['sourceSignature'] = prior_signature
            continue
          if failed and failed['signature'] == signature and failed.get('analysisVersion', 0) == ANALYSIS_VERSION:
            now = self.clock()
            if (failed['attempts'] >= MAX_RETRY_ATTEMPTS or now - failed['firstAt'] >= MAX_RETRY_AGE_SECONDS or
                failed['firstAt'] <= now < failed['retryAt']):
              continue
          candidates.append((route, route_id, signature, manifest))
      with self._lock:
        self._pending = len(candidates)
      if candidates:
        start = self._cursor % len(candidates)
        candidates = candidates[start:] + candidates[:start]
      attempted = 0
      for offset, (route, route_id, signature, manifest) in enumerate(candidates):
        if not self._allowed() or attempted >= MAX_ANALYSES_PER_SCAN:
          break
        with self._lock:
          existing = self._records.get(route_id)
          failed = self._failures.get(route_id)
          if existing and existing.get('complete') and existing.get('sourceSignature') != signature:
            prior = existing.copy()
            existing['complete'] = False
            existing['reason'] = 'sourceChanged'
            try:
              self._save()
            except OSError:
              self._records[route_id] = prior
              raise
        attempted += 1
        self._cursor = (start + offset + 1) % len(candidates)
        try:
          result = self.analyzer(self.root, route, permitted=self._allowed)
          if _route_signature(self.root, route) != (signature, manifest):
            raise ValueError('Route source changed during analysis')
          if not isinstance(result, dict) or result.get('routeId') != route_id:
            raise ValueError('Invalid route analysis')
          if not self._allowed():
            break  # A drive starting or shutdown is not an unreadable recording.
          record = {key: result.get(key) for key in ('routeId', 'startTime', 'endTime', 'distanceMeters',
                    'durationSeconds', 'engagedSeconds', 'engagedPercent', 'aolSeconds', 'aolPercent', 'model',
                    'distractedMoments', 'unresponsiveMoments', 'complete', 'reason', 'segmentCount')}
          record['ignored'] = bool(existing and existing.get('ignored'))
          record['analysisVersion'] = result.get('analysisVersion', ANALYSIS_VERSION)
          record['sourceSignature'] = signature
          record['sourceManifest'] = manifest
          if not _valid_record(record):
            raise ValueError('Invalid route analysis')
          with self._lock:
            if self._closed.is_set():
              break
            current = self._records.get(route_id)
            prior_records = self._records.copy()
            prior_failure = self._failures.get(route_id)
            prior_truncated = self._truncated
            record['ignored'] = bool(current and current.get('ignored'))
            refreshing_complete = (current and current.get('complete') and current.get('sourceSignature') == signature)
            if record['complete'] or not refreshing_complete:
              self._records[route_id] = record
            if record['complete']:
              self._failures.pop(route_id, None)
            else:
              self._failures[route_id] = self._failure(signature, prior_failure, record.get('reason') or 'Incomplete recording')
            self._bound()
            try:
              self._save()
            except OSError:
              self._records = prior_records
              self._truncated = prior_truncated
              if prior_failure is None:
                self._failures.pop(route_id, None)
              else:
                self._failures[route_id] = prior_failure
              raise
        except Exception as error:
          with self._lock:
            self._failures[route_id] = self._failure(signature, failed, type(error).__name__)
            if len(self._failures) > MAX_CACHED_ROUTES:
              self._failures.pop(next(iter(self._failures)))
        finally:
          with self._lock:
            self._pending = max(0, self._pending - 1)
      with self._lock:
        if not self._closed.is_set():
          self._save()
    except Exception:
      with self._lock:
        self._scan_incomplete = True

  def _failure(self, signature, previous, reason):
    now = self.clock()
    same = (previous is not None and previous['signature'] == signature and
            previous.get('analysisVersion', 0) == ANALYSIS_VERSION)
    attempts = min(MAX_RETRY_ATTEMPTS, previous['attempts'] + 1 if same else 1)
    return {'signature': signature, 'analysisVersion': ANALYSIS_VERSION, 'attempts': attempts,
            'firstAt': previous['firstAt'] if same else now,
            'retryAt': now + min(MAX_RETRY_DELAY_SECONDS, RETRY_SECONDS * 2 ** (attempts - 1)),
            'reason': str(reason)[:200]}

  def _bound(self):
    estimated = 1000 + sum(len(key) + len(json.dumps(record, separators=(',', ':'))) + 8
                           for key, record in self._records.items())
    limit = max(1000, MAX_CACHE_BYTES - 1000)
    if len(self._records) <= MAX_CACHED_ROUTES and estimated <= limit:
      return
    ordered = sorted(self._records.values(), key=lambda r: (r.get('startTime') or 0, r['routeId']))
    for record in ordered:
      if len(self._records) <= MAX_CACHED_ROUTES and estimated <= limit:
        break
      del self._records[record['routeId']]
      estimated -= len(record['routeId']) + len(json.dumps(record, separators=(',', ':'))) + 8
      self._truncated = True

  def _start(self):
    with self._lock:
      if self._closed.is_set() or (self._worker is not None and self._worker.is_alive()):
        return
      now = self.monotonic()
      if now < self._next_scan or not self._allowed():
        return
      self._next_scan = now + SCAN_INTERVAL_SECONDS
      self._worker = threading.Thread(target=self._run, name='galaxy-drive-stats', daemon=True)
      self._worker.start()

  def start(self):
    """Start the device-owned analysis lifecycle independently of HTTP requests."""
    with self._lock:
      if self._closed.is_set() or self._scheduler is not None:
        return
      self._scheduler = threading.Thread(target=self._schedule, name='galaxy-drive-history', daemon=True)
      self._scheduler.start()

  def _schedule(self):
    was_allowed = False
    while not self._closed.is_set():
      allowed = self._allowed()
      if allowed:
        if not was_allowed:
          with self._lock:
            self._next_scan = 0.0
        self._start()
      was_allowed = allowed
      self._closed.wait(.1)

  def recording_dates(self, route_ids) -> dict:
    """Read already analyzed capture dates; listing recordings does not start analysis."""
    with self._lock:
      return {identifier: self._records[identifier]['startTime'] for identifier in route_ids
              if identifier in self._records and self._records[identifier].get('startTime') is not None}

  def snapshot(self, timezone: str = "UTC") -> dict:
    with self._lock:
      result = self._snapshot_locked(timezone)
    return result

  def _snapshot_locked(self, timezone: str = "UTC") -> dict:
    from zoneinfo import ZoneInfo
    if type(timezone) is not str or not 1 <= len(timezone) <= 128:
      raise ValueError("Invalid timezone")
    try:
      zone = ZoneInfo(timezone)
    except (KeyError, ValueError) as error:
      raise ValueError("Invalid timezone") from error
    try:
      is_metric = bool(self.metric())
    except Exception:
      is_metric = True
    now = datetime.fromtimestamp(self.clock(), zone)
    day = now.date()
    monday = day - timedelta(days=day.weekday())
    dates = [(monday + timedelta(days=n)).isoformat() for n in range(7)]
    complete = [r for r in self._records.values() if r['complete'] and not r['ignored']]
    complete.sort(key=lambda r: (r.get('startTime') or 0, r['routeId']), reverse=True)
    recent = sorted(self._records.values(), key=lambda r: (r.get('startTime') or 0, r['routeId']), reverse=True)
    week = [r for r in complete if datetime.fromtimestamp(r['startTime'], zone).date().isoformat() in dates]
    daily = defaultdict(float)
    for record in week:
      daily[datetime.fromtimestamp(record['startTime'], zone).date().isoformat()] += record['distanceMeters']
    def sums(drives):
      return {'distanceMeters': sum(r['distanceMeters'] for r in drives),
              'durationSeconds': sum(r['durationSeconds'] for r in drives),
              'engagedSeconds': sum(r['engagedSeconds'] for r in drives), 'drives': len(drives)}
    def aol_sums(drives):
      # Old or gapped recordings do not silently become zero-AOL drives.
      observed = [r for r in drives if r.get('aolSeconds') is not None]
      duration = sum(r['durationSeconds'] for r in observed)
      seconds = sum(r['aolSeconds'] for r in observed)
      return {'aolSeconds': seconds if observed else None,
              'aolPercent': round(100 * seconds / duration, 1) if duration else None,
              'aolDurationSeconds': duration, 'aolDrives': len(observed)}
    totals = sums(complete)
    totals.update(aol_sums(complete))
    week_totals = sums(week)
    week_result = {'days': [{'date': date, 'distanceMeters': daily[date]} for date in dates],
                   'distanceMeters': week_totals['distanceMeters'], 'durationSeconds': week_totals['durationSeconds'],
                   'engagedPercent': round(100 * week_totals['engagedSeconds'] / week_totals['durationSeconds'], 1)
                   if week_totals['durationSeconds'] else None, 'drives': len(week), 'timezone': timezone}
    week_result.update(aol_sums(week))
    model_totals = defaultdict(lambda: {'distanceMeters': 0.0, 'durationSeconds': 0.0, 'drives': 0})
    for record in complete:
      if type(record.get('model')) is str and record['model']:
        model = model_totals[record['model']]
        model['distanceMeters'] += record['distanceMeters']
        model['durationSeconds'] += record['durationSeconds']
        model['drives'] += 1
    models = [{'name': name, **amounts} for name, amounts in model_totals.items()]
    models.sort(key=lambda m: (-m['distanceMeters'], m['name']))
    records = _personal_records(complete, zone)
    running = self._worker is not None and self._worker.is_alive()
    pending = max(self._pending, int(not self._scanned))
    state = ('analyzing' if running else 'waitingParked' if pending and not self._allowed() else
             'queued' if pending else 'retrying' if any(
               f['attempts'] < MAX_RETRY_ATTEMPTS and self.clock() - f['firstAt'] < MAX_RETRY_AGE_SECONDS
               for f in self._failures.values()) else 'failed' if self._failures else 'ready')
    def public(record):
      return {key: value for key, value in record.items() if key not in ('sourceSignature', 'sourceManifest')}
    included_recent = [record for record in recent if not record['ignored']]
    return {'schemaVersion': SCHEMA_VERSION, 'isMetric': is_metric,
            'analysis': {'running': running, 'pending': pending, 'state': state,
                         'failed': len(self._failures),
                         'quarantined': sum(f['attempts'] >= MAX_RETRY_ATTEMPTS or
                                            self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS for f in self._failures.values()),
                         'failures': [{'routeId': key, 'reason': f['reason'], 'attempts': f['attempts'],
                                       'retryStopped': f['attempts'] >= MAX_RETRY_ATTEMPTS or
                                                       self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS}
                                      for key, f in self._failures.items()],
                         'scanIncomplete': self._scan_incomplete},
            'totals': totals, 'week': week_result, 'lastDrive': public(included_recent[0]) if included_recent else None,
            'recentDrives': [public(r) for r in recent[:RECENT_LIMIT]], 'records': records, 'models': models,
            'coverage': {'scope': 'analyzedLocalRoutes', 'timezone': timezone, 'partialHistory': True,
                         'scanIncomplete': self._scan_incomplete, 'scanned': self._scanned,
                         'cachedRoutes': len(self._records), 'completeRoutes': sum(r['complete'] for r in self._records.values()),
                         'incompleteRoutes': sum(not r['complete'] for r in self._records.values()),
                         'ignoredRoutes': sum(r['ignored'] for r in self._records.values()),
                         'truncated': self._truncated, 'cacheCorrupt': self._corrupt}}

  def ignore(self, route_id: str, ignored: bool, authorized: Callable[[], bool]) -> dict:
    if not authorized():
      raise PermissionError('Local Galaxy session expired')
    if type(route_id) is not str or type(ignored) is not bool:
      raise ValueError('Invalid route selection')
    with self._lock:
      record = self._records.get(route_id)
      if record is None:
        raise ValueError('Unknown route')
      if record['ignored'] != ignored:
        prior = record['ignored']
        record['ignored'] = ignored
        try:
          self._save()
        except OSError:
          record['ignored'] = prior
          raise
    return self.snapshot()

  def close(self):
    self._closed.set()
    scheduler = self._scheduler
    if scheduler is not None and scheduler is not threading.current_thread():
      scheduler.join(timeout=2)
    worker = self._worker
    if worker is not None and worker is not threading.current_thread():
      worker.join(timeout=31)