#!/usr/bin/env python3 """Bounded, read-only local runtime sampling and offline capture comparison.""" from __future__ import annotations import argparse from collections import Counter from datetime import UTC, datetime import hashlib import importlib.util import json from itertools import islice import math import os from pathlib import Path import platform import subprocess import time ROOT = Path(__file__).resolve().parents[2] SERVICES = ('uiDebug', 'managerState', 'deviceState', 'modelV2', 'controlsState', 'selfdriveState') MAX_PROCESSES = 64 MAX_INSTANCES = 256 FIELDS = { 'uiDebug': {'reportedFrameIntervalMs': ('frameTimeMillis', 1), 'reportedPrePresentWallMs': ('cpuTimeMillis', 1)}, 'modelV2': {'reportedModelExecutionMs': ('modelExecutionTime', 1000), 'reportedFrameDropPercent': ('frameDropPerc', 1)}, 'deviceState': {name: (field, 1) for name, field in ( ('reportedMemoryUsagePercent', 'memoryUsagePercent'), ('reportedGpuUsagePercent', 'gpuUsagePercent'), ('reportedMaxTemperatureC', 'maxTempC'), ('reportedPowerDrawW', 'powerDrawW'))}, } CATEGORIES = {'deviceState': ('started', 'thermalStatus', 'deviceType', 'chestnutPresent'), 'selfdriveState': ('enabled', 'active'), 'modelV2': ('big',)} METRICS = { 'reportedFrameIntervalMs': 'uiDebug.frameTimeMillis; Raylib reported frame interval, not measured present duration', 'reportedPrePresentWallMs': 'uiDebug.cpuTimeMillis; publisher monotonic wall time for drawing/update before present, not thread CPU', 'reportedModelExecutionMs': 'modelV2.modelExecutionTime * 1000; publisher execution interval, not sensor-to-actuator latency', 'observedReceiveRateHz': 'fresh conflated messages observed / actual capture seconds; capped by collector polling, not publisher rate', 'receiveAgeMs': 'time since this collector last received a message, sampled each poll after first receipt', 'publicationAgeAtReceiveMs': 'collector monotonic time minus Event.logMonoTime at receipt; includes publication/transport/sampling delay', 'cpuPercentOneCore': '100 * delta process(user+system ticks) / clock ticks per second / elapsed sample seconds; can exceed 100', 'rssMiB': 'Linux /proc/PID/stat resident pages * page size / 1048576; excludes swapped/nonresident memory', } class Distribution: def __init__(self): self.values: list[float] = [] self.missing: Counter[str] = Counter() def add(self, value, reason='missing_field'): if value is None: self.missing[reason] += 1 elif not isinstance(value, (int, float)) or isinstance(value, bool) or not math.isfinite(value): self.missing['nonfinite_or_nonnumeric'] += 1 else: self.values.append(float(value)) def summary(self): values = sorted(self.values) result = {'status': 'observed' if values else 'no_samples', 'sampleCount': len(values), 'missingCounts': dict(self.missing)} if not values: return {**result, **dict.fromkeys(('min', 'mean', 'p50', 'p90', 'p95', 'p99', 'max'))} def percentile(q): index = (len(values) - 1) * q lo, hi = math.floor(index), math.ceil(index) return values[lo] + (values[hi] - values[lo]) * (index - lo) return {**result, 'min': values[0], 'mean': math.fsum(values) / len(values), 'max': values[-1], **{f'p{q}': percentile(q / 100) for q in (50, 90, 95, 99)}} def field(body, name): try: return body.get(name) if isinstance(body, dict) else getattr(body, name) except (AttributeError, RuntimeError): return None class MessageSamples: def __init__(self): self.count = 0 self.invalid = 0 self.last_received = None self.last_published = None self.timestamps = Counter() self.receive_age = Distribution() self.publication_age = Distribution() self.intervals = Distribution() self.metrics: dict[str, Distribution] = {} self.states: dict[str, Counter] = {} def observe(self, now, event): self.count += 1 self.invalid += not event['valid'] if self.last_received is not None: self.intervals.add((now - self.last_received) * 1000) self.last_received = now published = event['logMonoTime'] if not isinstance(published, int) or published <= 0: self.publication_age.add(None, 'missing_or_zero_timestamp') elif published > now * 1e9: self.publication_age.add(None, 'future_timestamp_clock_mismatch') else: self.publication_age.add((now - published / 1e9) * 1000) if isinstance(published, int) and published > 0: if self.last_published is not None: if published == self.last_published: self.timestamps['repeated'] += 1 elif published < self.last_published: self.timestamps['regressed'] += 1 self.last_published = published def summary(self, duration, source_status): return {'status': 'observed' if self.count else ('no_messages' if source_status == 'available' else source_status), 'observedMessages': self.count, 'invalidMessages': self.invalid, 'observedReceiveRateHz': self.count / duration if duration > 0 and source_status == 'available' else None, 'timestampDiscontinuities': dict(self.timestamps), 'receiveAgeMs': self.receive_age.summary(), 'publicationAgeAtReceiveMs': self.publication_age.summary(), 'observedReceiveIntervalMs': self.intervals.summary(), 'reportedStateCounts': {name: dict(value) for name, value in self.states.items()}, 'metrics': {name: value.summary() for name, value in self.metrics.items()}} class LocalMessages: def __init__(self): self.status = 'available' self.error = None self.master = None self.expected_rates = {} self.module_path = None try: from openpilot.cereal import messaging from openpilot.cereal.services import SERVICE_LIST self.module_path = messaging.__file__ self.master = messaging.SubMaster(list(SERVICES)) self.expected_rates = {name: SERVICE_LIST[name].frequency for name in SERVICES} except Exception as error: self.status, self.error = 'unavailable', type(error).__name__ def poll(self): if self.master is None: return {} try: self.master.update(0) return {name: {'body': self.master[name], 'valid': self.master.valid[name], 'logMonoTime': self.master.logMonoTime[name]} for name in SERVICES if self.master.updated[name]} except Exception as error: self.status, self.error, self.master = 'error', type(error).__name__, None return {} class ProcReader: def __init__(self, root=Path('/proc'), ticks=None, page_size=None): self.root = root self.available = (root / 'self/stat').exists() self.ticks = ticks or (os.sysconf('SC_CLK_TCK') if self.available else None) self.page_size = page_size or (os.sysconf('SC_PAGE_SIZE') if self.available else None) def read(self, pid): if not self.available: return None, 'proc_unavailable' try: raw = (self.root / str(pid) / 'stat').read_text() begin, end = raw.index('('), raw.rindex(')') parts = raw[end + 2:].split() if int(raw[:begin].strip()) != pid: raise ValueError('PID mismatch') result = {'comm': raw[begin + 1:end], 'cpuTicks': int(parts[11]) + int(parts[12]), 'startTicks': int(parts[19]), 'rssPages': int(parts[21])} if min(result['cpuTicks'], result['startTicks'], result['rssPages']) < 0: raise ValueError('negative stat') return result, None except FileNotFoundError: return None, 'process_gone' except PermissionError: return None, 'permission_denied' except (OSError, ValueError, IndexError): return None, 'unreadable_or_malformed_stat' class ProcessSamples: def __init__(self, reader): self.reader = reader self.instances = {} self.previous = {} self.read_errors = Counter() self.pid_reuses = 0 self.skipped_instances = 0 self.skipped_targets = 0 self.target_samples = 0 def poll(self, now, targets): self.skipped_targets += max(0, len(targets) - MAX_PROCESSES) self.target_samples += min(MAX_PROCESSES, len(targets)) for pid, name in sorted(targets.items())[:MAX_PROCESSES]: record, error = self.reader.read(pid) if error: self.read_errors[error] += 1 previous = self.previous.pop(pid, None) if previous is not None: for metric in ('cpuPercentOneCore', 'rssMiB'): self.instances[previous[0]][metric].add(None, error) continue identity = (pid, record['startTicks']) if identity not in self.instances: if len(self.instances) >= MAX_INSTANCES: self.skipped_instances += 1 continue self.instances[identity] = {'pid': pid, 'startTicks': record['startTicks'], 'requestedName': name, 'comm': record['comm'], 'firstObservedMonotonic': now, 'lastObservedMonotonic': now, 'cpuPercentOneCore': Distribution(), 'rssMiB': Distribution()} item = self.instances[identity] item['lastObservedMonotonic'] = now item['rssMiB'].add(record['rssPages'] * self.reader.page_size / 1048576) before = self.previous.get(pid) reason = None if before is None: reason = 'baseline_required' elif before[0] != identity: reason = 'pid_reused_or_restarted' self.pid_reuses += 1 elif now <= before[1]: reason = 'nonpositive_sample_interval' elif record['cpuTicks'] < before[2]: reason = 'cpu_counter_regressed' if reason: item['cpuPercentOneCore'].add(None, reason) else: assert before is not None item['cpuPercentOneCore'].add(100 * (record['cpuTicks'] - before[2]) / self.reader.ticks / (now - before[1])) self.previous[pid] = (identity, now, record['cpuTicks']) self.previous = {pid: value for pid, value in self.previous.items() if pid in targets} def summary(self): names = Counter(item['requestedName'] for item in self.instances.values()) return {'status': 'available' if self.reader.available else 'unavailable', 'readErrors': dict(self.read_errors), 'samplingStatus': 'observed' if self.instances else ('no_targets' if not self.target_samples else 'no_readable_processes'), 'requestedTargetSamples': self.target_samples, 'clockTicksPerSecond': self.reader.ticks, 'pageSizeBytes': self.reader.page_size, 'pidReuseOrRestartEvents': self.pid_reuses, 'skippedInstanceSamples': self.skipped_instances, 'observedInstanceCountsByName': dict(names), 'skippedTargetSamples': self.skipped_targets, 'instances': [{name: value.summary() if isinstance(value, Distribution) else value for name, value in item.items()} for item in self.instances.values()]} def capture(duration=30, rate=20, *, messages=None, proc=None, pids=(), clock=time.monotonic, sleep=time.sleep): if not math.isfinite(duration) or not 0.1 <= duration <= 120 or not math.isfinite(rate) or not 1 <= rate <= 50: raise ValueError('duration must be 0.1..120 seconds and sample rate 1..50 Hz') if len(pids) > MAX_PROCESSES or any(type(pid) is not int or pid <= 0 for pid in pids): raise ValueError('at most 64 positive PIDs are supported') messages = messages if messages is not None else LocalMessages() processes = ProcessSamples(proc if proc is not None else ProcReader()) samples = {name: MessageSamples() for name in SERVICES} for service, metrics in FIELDS.items(): samples[service].metrics = {name: Distribution() for name in metrics} for name in ('reportedCpuUsageMeanPercent', 'reportedCpuUsageMaxPercent'): samples['deviceState'].metrics[name] = Distribution() explicit = {pid: f'explicit_pid_{pid}' for pid in pids} targets = {} manager_counts = Counter() polls, missed_slots = 0, 0 poll_intervals = Distribution() started = clock() previous_poll = None scheduled = started while polls < math.ceil(duration * rate) and clock() < started + duration: if clock() < scheduled: sleep(min(scheduled, started + duration) - clock()) if clock() >= started + duration: break events = messages.poll() now = clock() if previous_poll is not None: poll_intervals.add((now - previous_poll) * 1000) previous_poll = now for service, event in events.items(): sample = samples[service] sample.observe(now, event) body = event['body'] for name in CATEGORIES.get(service, ()): value = field(body, name) sample.states.setdefault(name, Counter())['missing' if value is None else str(value)[:128]] += 1 for name, (source, scale) in FIELDS.get(service, {}).items(): value = field(body, source) if isinstance(value, (int, float)) and value < 0: sample.metrics[name].add(None, 'negative_reported_value') else: sample.metrics[name].add(value * scale if isinstance(value, (int, float)) else value) if service == 'deviceState': values = field(body, 'cpuUsagePercent') values = list(islice(values, 256)) if values is not None else [] valid = [value for value in values if isinstance(value, (int, float)) and math.isfinite(value) and value >= 0] reason = 'missing_or_empty_cpu_list' if not values else 'invalid_cpu_list' for name, value in [('reportedCpuUsageMeanPercent', sum(valid) / len(valid) if valid else None), ('reportedCpuUsageMaxPercent', max(valid) if valid else None)]: sample.metrics[name].add(value if len(valid) == len(values) else None, reason) if service == 'managerState': targets = {} entries = field(body, 'processes') if entries is None: manager_counts['missing_process_list'] += 1 else: for index, process in enumerate(entries): if index >= MAX_PROCESSES: manager_counts['truncated_process_lists'] += 1 break running, expected = field(process, 'running'), field(process, 'shouldBeRunning') manager_counts['process_observations'] += 1 manager_counts['reported_running'] += running is True manager_counts['expected_but_stopped'] += expected is True and running is False pid = field(process, 'pid') if running is True and isinstance(pid, int) and pid > 0: targets[pid] = str(field(process, 'name'))[:128] for sample in samples.values(): if sample.last_received is not None: sample.receive_age.add((now - sample.last_received) * 1000) processes.poll(now, {**targets, **explicit}) polls += 1 next_slot = max(polls + missed_slots, math.floor((clock() - started) * rate) + 1) missed_slots = next_slot - polls scheduled = started + next_slot / rate if clock() < started + duration: sleep(started + duration - clock()) elapsed = clock() - started return {'schemaVersion': 1, 'kind': 'runtime-performance-capture', 'capture': {'requestedDurationSeconds': duration, 'actualDurationSeconds': elapsed, 'requestedSampleRateHz': rate, 'pollCount': polls, 'missedScheduledPolls': missed_slots, 'pollIntervalMs': poll_intervals.summary()}, 'messageSource': {'status': messages.status, 'errorType': messages.error, 'transport': 'local conflated SubMaster', 'modulePath': getattr(messages, 'module_path', None), 'configuredPublisherRatesHz': messages.expected_rates}, 'services': {name: value.summary(elapsed, messages.status) for name, value in samples.items()}, 'managerObservations': dict(manager_counts), 'processes': processes.summary(), 'metricDefinitions': METRICS, 'bounds': {'maxSeconds': 120, 'maxPollHz': 50, 'maxProcessesPerPoll': MAX_PROCESSES, 'maxProcessInstances': MAX_INSTANCES}} def metadata(root=None): try: spec = importlib.util.find_spec('openpilot') imported_root = Path(spec.origin).resolve().parent.parent if spec is not None and spec.origin else None except (ImportError, ValueError): imported_root = None binding = 'explicit' if root is not None else 'imported_openpilot' if root is None: root = imported_root if root is None and (ROOT / 'openpilot/cereal/log.capnp').is_file(): root, binding = ROOT, 'tool_repository_fallback' if root is not None: root = Path(root).resolve() if not (root / 'openpilot/cereal/log.capnp').is_file(): raise ValueError('source root must contain openpilot/cereal/log.capnp') def read(path): try: return path.read_text().strip()[:256] except OSError: return None def git(*args): if root is None: return None try: return subprocess.run(['git', '-C', str(root), *args], capture_output=True, text=True, timeout=3, check=True).stdout.strip() except (OSError, subprocess.SubprocessError): return None paths = ('openpilot/cereal/log.capnp', 'openpilot/cereal/services.py', 'openpilot/selfdrive/ui/ui.py', 'openpilot/system/ui/lib/application.py', 'openpilot/selfdrive/modeld/modeld.py') return {'recordedAtUtc': datetime.now(UTC).isoformat(), 'system': platform.system(), 'release': platform.release(), 'machine': platform.machine(), 'python': platform.python_version(), 'logicalCpuCount': os.cpu_count(), 'agnosVersion': read(Path('/VERSION')), 'bootId': read(Path('/proc/sys/kernel/random/boot_id')), 'sourceRoot': str(root) if root is not None else None, 'sourceBinding': binding if root is not None else 'unavailable', 'importedOpenpilotRoot': str(imported_root) if imported_root is not None else None, 'declaredRootMatchesImported': root == imported_root if root is not None and imported_root is not None else None, 'tool': {'path': str(Path(__file__).resolve()), 'sha256': hashlib.sha256(Path(__file__).read_bytes()).hexdigest()}, 'gitHead': git('rev-parse', 'HEAD'), 'gitDirty': bool(status) if (status := git('status', '--porcelain')) is not None else None, 'sourceSha256': {name: hashlib.sha256((root / name).read_bytes()).hexdigest() for name in paths if root is not None and (root / name).is_file()}, 'collectorEnvironment': {name: os.environ.get(name) for name in ('UI_FRAME_TIMING', 'SIMULATION', 'SCALE', 'ZMQ', 'MSGQ_PREFIX')}} def compare(before, after): for report in (before, after): if report.get('schemaVersion') != 1 or report.get('kind') != 'runtime-performance-capture': raise ValueError('unsupported capture schema') differences = [] for field_name in ('system', 'machine', 'agnosVersion', 'logicalCpuCount'): left, right = before.get('environment', {}).get(field_name), after.get('environment', {}).get(field_name) if left != right or left is None: differences.append({'field': field_name, 'before': left, 'after': right}) for name in ('requestedDurationSeconds', 'requestedSampleRateHz'): if before['capture'][name] != after['capture'][name]: differences.append({'field': name, 'before': before['capture'][name], 'after': after['capture'][name]}) def flatten(report): result = {} for service, value in report['services'].items(): result[f'{service}.observedReceiveRateHz'] = value['observedReceiveRateHz'] for metric, data in {**value['metrics'], **{name: value[name] for name in ('receiveAgeMs', 'publicationAgeAtReceiveMs', 'observedReceiveIntervalMs')}}.items(): for statistic in ('sampleCount', 'p50', 'p95', 'max'): result[f'{service}.{metric}.{statistic}'] = data[statistic] groups = {} for item in report['processes']['instances']: group = groups.setdefault(item['requestedName'], []) group.append(item) for name, items in groups.items(): result[f'process.{name}.observedInstances'] = len(items) # Do not average per-incarnation percentiles into a false pooled percentile. for metric in ('cpuPercentOneCore', 'rssMiB'): available = [item[metric]['max'] for item in items if item[metric]['sampleCount']] result[f'process.{name}.{metric}.max'] = max(available) if available else None return result left, right = flatten(before), flatten(after) rows = [] for key in sorted(left.keys() | right.keys()): a, b = left.get(key), right.get(key) rows.append({'metric': key, 'before': a, 'after': b, 'status': 'comparable' if a is not None and b is not None else 'missing_data', 'delta': b - a if a is not None and b is not None else None, 'changePercent': (b - a) / abs(a) * 100 if a is not None and b is not None and a != 0 else None}) return {'schemaVersion': 1, 'kind': 'runtime-performance-comparison', 'environmentOrSamplingDifferences': differences, 'interpretation': 'Descriptive differences only; workload, thermal state, conflation and instrumentation affect results.', 'captureContext': {name: {'gitHead': report.get('environment', {}).get('gitHead'), 'messageSource': report['messageSource'], 'processStatus': report['processes']['status'], 'reportedStateCounts': {service: value['reportedStateCounts'] for service, value in report['services'].items()}} for name, report in (('before', before), ('after', after))}, 'summary': [row for row in rows if row['metric'] in ( 'uiDebug.reportedFrameIntervalMs.p95', 'uiDebug.reportedPrePresentWallMs.p95', 'modelV2.reportedModelExecutionMs.p95', 'modelV2.reportedFrameDropPercent.max', 'deviceState.reportedMaxTemperatureC.max', 'deviceState.reportedMemoryUsagePercent.max') or row['metric'].startswith('process.')], 'metrics': rows} def main(): parser = argparse.ArgumentParser(description=__doc__) commands = parser.add_subparsers(dest='command', required=True) collect = commands.add_parser('capture', help='Read local messages and Linux /proc; write one final JSON file') collect.add_argument('--duration', type=float, default=30) collect.add_argument('--rate', type=float, default=20) collect.add_argument('--pid', type=int, action='append', default=[], help='Also sample this local PID (repeatable, maximum 64)') collect.add_argument('--source-root', type=Path, help='Source checkout to identify; default is the imported openpilot checkout') collect.add_argument('--output', type=Path, required=True) diff = commands.add_parser('compare', help='Compare two saved captures without connecting to anything') diff.add_argument('before', type=Path) diff.add_argument('after', type=Path) diff.add_argument('--output', type=Path, required=True) args = parser.parse_args() try: if args.output.exists(): raise FileExistsError(f'output already exists: {args.output}') if args.command == 'capture': environment = metadata(args.source_root) report = capture(args.duration, args.rate, pids=args.pid) report['environment'] = environment else: report = compare(json.loads(args.before.read_text()), json.loads(args.after.read_text())) report['inputs'] = {name: {'path': str(path), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} for name, path in (('before', args.before), ('after', args.after))} args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open('x') as output: json.dump(report, output, indent=2, allow_nan=False) output.write('\n') except (ValueError, OSError) as error: parser.exit(1, f'{type(error).__name__}: {error}\n') print(f'{report["kind"]}: {args.output}') return 0 if __name__ == '__main__': raise SystemExit(main())