#!/usr/bin/env python3 """Report exact source-platform obligations without qualifying any vehicle.""" import argparse from collections import Counter import hashlib import importlib.machinery import json from pathlib import Path import re import subprocess from openpilot.starpilot.validation.control_modes import SCENARIOS, configuration_id, validate_matrix ROOT = Path(__file__).resolve().parents[2] DEFAULT_MANIFEST = ROOT / "openpilot/starpilot/validation/required-platforms.json" SHA256 = re.compile(r"[0-9a-f]{64}\Z") REVISION = re.compile(r"[0-9a-f]{40}\Z") NATIVE_CAN_SOURCES = {"opendbc_repo/opendbc/can/packer.py", "opendbc_repo/opendbc/can/parser.py"} RELEVANT_SOURCE_PATHS = ("opendbc_repo", "openpilot/common", "openpilot/cereal", "msgq_repo", "tools/ci/run_vehicle_tests.py", "tools/ci/vehicle_pytest.py", "tools/test_runner.py", "upstream-sync.json") def sha256(path): return hashlib.sha256(Path(path).read_bytes()).hexdigest() def canonical_sha256(value): """Hash compact, key-sorted UTF-8 JSON without a trailing newline.""" return hashlib.sha256(json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False).encode("utf-8")).hexdigest() def relevant_dirty_paths(): output = subprocess.check_output(["git", "status", "--porcelain", "--untracked-files=all", "--", *RELEVANT_SOURCE_PATHS], cwd=ROOT, text=True) return output.splitlines() def _relevant_status_lines(status): if not isinstance(status, str): return [] return [line for line in status.splitlines() if len(line) >= 4 and any( line[3:] == path or line[3:].startswith(path + "/") for path in RELEVANT_SOURCE_PATHS)] def source_snapshot_matches(source, source_after, current): from tools.ci.run_vehicle_tests import SOURCE_INPUT_PATHS if not all(isinstance(value, dict) for value in (source, source_after, current)): return False before = source.get("inputs") after = source_after.get("inputs") if not isinstance(before, dict) or before != after or before != current: return False if set(before) != {"revision", "paths", "tracked_patch_sha256", "untracked_sha256"}: return False if (not REVISION.fullmatch(str(before["revision"])) or before["paths"] != list(SOURCE_INPUT_PATHS) or not SHA256.fullmatch(str(before["tracked_patch_sha256"])) or not isinstance(before["untracked_sha256"], dict)): return False if any(not isinstance(path, str) or not path or not SHA256.fullmatch(str(digest)) for path, digest in before["untracked_sha256"].items()): return False return source.get("revision") == source_after.get("revision") == before["revision"] def _ids(values, label, errors): if not isinstance(values, list) or any(not isinstance(v, str) or not v for v in values): errors.append(f"{label}: expected nonempty string IDs") return [] duplicates = sorted(k for k, count in Counter(values).items() if count > 1) if duplicates: errors.append(f"{label}: duplicate exact IDs {duplicates}") return values def evaluate(manifest, coverage, results, mode_evidence=None, *, expected_revision=None, hashes=None, dirty_relevant=None, current_snapshot=None, expected_registry=None): """Check independent registration, test execution and per-config mode evidence. All input documents are caller supplied; their physical origin is not asserted. """ errors, failures, uncovered = [], [], [] if not isinstance(manifest, dict): errors.append("required-platform manifest: expected object") manifest = {} if not isinstance(coverage, dict): errors.append("interface coverage: expected object") coverage = {} if not isinstance(results, dict): errors.append("interface results: expected object") results = {} if mode_evidence is not None and not isinstance(mode_evidence, dict): errors.append("mode evidence: expected object") mode_evidence = {} if dirty_relevant is None: dirty_relevant = relevant_dirty_paths() platforms = manifest.get("platforms") if manifest.get("schema_version") != 1 or not isinstance(platforms, list): errors.append("required-platform manifest: expected schema v1 and platforms list") platforms = [] required = _ids([r.get("id") if isinstance(r, dict) else None for r in platforms], "required platforms", errors) required_set = set(required) if not required_set: errors.append("required-platform manifest: at least one exact source ID required") additions = _ids(manifest.get("upstream_additions"), "upstream additions", errors) addition_set = set(additions) if required_set & addition_set: errors.append("required source platforms and upstream additions overlap") all_obligations = required_set | addition_set for row in platforms: if isinstance(row, dict) and (not row.get("family") or row.get("source_declaration") not in ("source_only", "same_declaration", "declaration_changed")): errors.append(f"required platform metadata invalid: {row.get('id')}") if not REVISION.fullmatch(str(manifest.get("source_revision", ""))) or not SHA256.fullmatch(str(manifest.get("source_inventory_sha256", ""))): errors.append("required-platform manifest source provenance missing") registered = _ids(coverage.get("platform_ids"), "reported registry", errors) registered_set = set(registered) mapped = coverage.get("interface_test_ids") if not isinstance(mapped, dict): errors.append("interface_test_ids: expected exact ID to test ID mapping") mapped = {} elif any(not isinstance(k, str) or not k for k in mapped): errors.append("interface_test_ids: mapping keys must be exact string IDs") mapped = {k: v for k, v in mapped.items() if isinstance(k, str) and k} selected = _ids(coverage.get("selected_test_ids"), "selected tests", errors) raw_records = results.get("results") if not isinstance(raw_records, list): errors.append("results: expected per-test records") raw_records = [] record_ids = [r.get("id") if isinstance(r, dict) else None for r in raw_records] if any(not isinstance(v, str) for v in record_ids) or len(record_ids) != len({v for v in record_ids if isinstance(v, str)}): errors.append("results: invalid or duplicate test IDs") by_test = {r["id"]: r for r in raw_records if isinstance(r, dict) and isinstance(r.get("id"), str)} nonpassing_records = [r for r in raw_records if isinstance(r, dict) and r.get("status") != "passed"] for record in nonpassing_records: issue = f"execution record {record.get('id')}: {record.get('status', 'missing status')}" if record.get("status") == "skipped": uncovered.append(issue) else: failures.append(issue) if coverage.get("suite") != "interfaces" or coverage.get("schema_version") != 1: errors.append("coverage: expected interface suite schema v1") if results.get("schema_version") != 1: errors.append("results: expected schema v1") result_selected = results.get("selected_test_ids") if not isinstance(result_selected, list) or any(not isinstance(v, str) for v in result_selected): errors.append("results selected_test_ids: expected test IDs") result_selected = [] if set(selected) != set(by_test) or set(selected) != set(result_selected): errors.append("selected tests and execution records differ") mapped_values = list(mapped.values()) unique_mapped = {v for v in mapped_values if isinstance(v, str)} if set(mapped) != registered_set or len(unique_mapped) != len(mapped): errors.append("registered platforms and unique interface test mapping differ") if coverage.get("platform_count") != len(registered) or coverage.get("result_sha256") != (hashes or {}).get("results"): errors.append("interface coverage count or companion results hash invalid") if coverage.get("interface_results") != {p: by_test.get(t) if isinstance(t, str) else None for p, t in mapped.items()}: errors.append("embedded interface results differ from execution records") for p, test_id in mapped.items(): if not isinstance(test_id, str) or not test_id.endswith(".test_car_interfaces_" + p) or test_id not in selected: errors.append(f"{p}: concrete interface test mapping invalid") if coverage.get("status") != "completed" or results.get("status") != "completed" or coverage.get("exit_code") != 0 or results.get("exit_code") != 0: failures.append("interface suite did not complete successfully") if coverage.get("collection_errors") or coverage.get("gate_errors") or results.get("collection_errors") or results.get("gate_errors"): failures.append("interface suite reported collection or gate failures") source = coverage.get("source") source = source if isinstance(source, dict) else {} if not isinstance(source.get("status"), str): uncovered.append("interface report working-tree status missing or malformed") reported_dirty = _relevant_status_lines(source.get("status")) if current_snapshot is None: from tools.ci.run_vehicle_tests import source_input_snapshot current_snapshot = source_input_snapshot() stable_snapshot = source_snapshot_matches(source, coverage.get("source_after"), current_snapshot) if not stable_snapshot: uncovered.append("interface report source inputs missing, changed during execution, or differ from current checkout") if dirty_relevant: uncovered.append("relevant vehicle/runner source differs from committed checkout") if reported_dirty: uncovered.append("interface report recorded relevant dirty vehicle/runner source") revision = source.get("revision") dependencies = source.get("dependencies") opendbc = [d for d in dependencies if isinstance(d, dict) and d.get("path") == "opendbc_repo"] if isinstance(dependencies, list) else [] if not REVISION.fullmatch(str(revision or "")) or len(opendbc) != 1 or not REVISION.fullmatch(str(opendbc[0].get("commit", ""))): errors.append("interface source/dependency provenance missing") if expected_revision is not None and revision != expected_revision: uncovered.append("interface report source revision differs from requested checkout") current_manifest = ROOT / "upstream-sync.json" current_runner = ROOT / "tools/ci/run_vehicle_tests.py" current_test_runner = ROOT / "tools/test_runner.py" for field, path in (("manifest_sha256", current_manifest), ("suite_runner_sha256", current_runner), ("runner_sha256", current_test_runner), ("pytest_runner_sha256", ROOT / "tools/ci/vehicle_pytest.py")): if source.get(field) != sha256(path): uncovered.append(f"interface report {field} differs from current source") pinned = json.loads(current_manifest.read_text()).get("dependencies", []) pinned_opendbc = [d for d in pinned if d.get("path") == "opendbc_repo"] if len(pinned_opendbc) != 1 or len(opendbc) != 1 or opendbc[0].get("commit") != pinned_opendbc[0].get("commit"): uncovered.append("interface report opendbc revision differs from current pin") native = coverage.get("native") if not isinstance(native, dict) or not isinstance(native.get("can_sources"), dict) or not isinstance(native.get("pycapnp"), dict): uncovered.append("interface report native parser/packer provenance missing") else: if set(native["can_sources"]) != NATIVE_CAN_SOURCES: uncovered.append("interface report native CAN source set incomplete or unexpected") for source_path, digest in native["can_sources"].items(): if not isinstance(source_path, str) or not (ROOT / source_path).is_file() or digest != sha256(ROOT / source_path): uncovered.append(f"interface report native CAN source mismatch: {source_path}") pycapnp = native["pycapnp"] binary_name = pycapnp.get("path") binary_path = Path(binary_name) if isinstance(binary_name, str) else None if (binary_path is None or not any(str(binary_path).endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES) or not binary_path.is_file() or pycapnp.get("sha256") != sha256(binary_path)): uncovered.append("interface report pycapnp binary unavailable or changed") if results.get("selected_test_ids") != selected: errors.append("selected test order differs between coverage and results") current_execution_compatible = not errors and not failures and stable_snapshot current_execution_compatible &= not any(issue.startswith("interface report ") for issue in uncovered) platform_rows = [] for p in sorted(required_set): if p not in registered_set: interface = "missing_port" elif p not in mapped: interface = "missing_test" else: test_id = mapped[p] status = by_test.get(test_id, {}).get("status") if isinstance(test_id, str) else None interface = "passed" if status == "passed" else "missing_test" if status is None else status current_interface = interface if interface != "passed" else "passed" if current_execution_compatible else "historical_pass_only" platform_rows.append({"id": p, "obligation": "source", "interface": interface, "current_interface": current_interface, "mode": "uncovered"}) if interface != "passed": uncovered.append(f"{p}: interface {interface}") if interface not in ("missing_port", "missing_test", "skipped"): failures.append(f"{p}: concrete interface test {interface}") upstream_only = sorted(registered_set - required_set) if set(upstream_only) != addition_set: uncovered.append("upstream additions differ from frozen manifest") # Additions remain independent obligations even if a future registry drops one. for p in sorted(addition_set): status = by_test.get(mapped.get(p), {}).get("status") if isinstance(mapped.get(p), str) else None reported_interface = "passed" if status == "passed" else "missing_port" if p not in registered_set else status or "missing_test" current_interface = "historical_pass_only" if reported_interface == "passed" and not current_execution_compatible else reported_interface platform_rows.append({"id": p, "obligation": "upstream_addition", "interface": reported_interface, "current_interface": current_interface, "mode": "uncovered"}) if status != "passed": uncovered.append(f"upstream addition {p}: concrete interface test {status or 'missing'}") if status not in (None, "skipped"): failures.append(f"upstream addition {p}: concrete interface test {status}") if expected_registry is None: from opendbc.car.values import PLATFORMS expected_registry = set(PLATFORMS) registry_gaps = sorted(set(expected_registry) ^ registered_set) interface_gaps = [issue for issue in uncovered if issue.startswith("interface report ")] if registry_gaps: interface_gaps.append(f"current production registry differs from reported registry: {registry_gaps}") for platform in sorted(registered_set): test_id = mapped.get(platform) if not isinstance(test_id, str) or by_test.get(test_id, {}).get("status") != "passed": interface_gaps.append(f"{platform}: registered concrete interface test not passed") interface_gate = {"scope": "current_registered_interface_regression", "status": "error" if errors else "failed" if failures else "uncovered" if interface_gaps else "pass", "errors": list(errors), "failures": list(failures), "uncovered": interface_gaps} mode_summary = {"input": "absent", "required_scenarios": list(SCENARIOS), "configuration_count": 0, "recorded_trace_count": 0} if mode_evidence is None: uncovered.append("independent per-configuration mode evidence absent") else: mode_summary["input"] = "supplied" if mode_evidence.get("schema_version") != 1: errors.append("mode evidence: expected schema v1") provenance = mode_evidence.get("provenance") provenance_keys = ("configuration_inventory_sha256", "trace_catalog_sha256") if not isinstance(provenance, dict) or any(not SHA256.fullmatch(str(provenance.get(k, ""))) for k in provenance_keys): uncovered.append("mode evidence: configuration inventory or trace catalog provenance missing") configurations = mode_evidence.get("configurations") traces = mode_evidence.get("traces") if not isinstance(configurations, list) or not isinstance(traces, list): errors.append("mode evidence: configurations and traces must be lists") configurations, traces = [], [] if isinstance(provenance, dict): if provenance.get("configuration_inventory_sha256") != canonical_sha256(configurations): errors.append("mode evidence: configuration inventory digest mismatch") if provenance.get("trace_catalog_sha256") != canonical_sha256(traces): errors.append("mode evidence: trace catalog digest mismatch") mode_summary["configuration_count"] = len(configurations) mode_summary["recorded_trace_count"] = len(traces) config_ids = [configuration_id(c) for c in configurations if isinstance(c, dict)] if len(config_ids) != len(configurations) or len(config_ids) != len(set(config_ids)): errors.append("mode evidence: invalid or duplicate exact configurations") if any(not isinstance(c, dict) or not isinstance(c.get("platform"), str) or c["platform"] not in all_obligations for c in configurations): errors.append("mode evidence: configuration platform is not an exact obligated ID") if any(not isinstance(t, dict) or not isinstance(t.get("configuration"), dict) or configuration_id(t["configuration"]) not in config_ids for t in traces): errors.append("mode evidence: trace has no exact declared configuration") complete = mode_evidence.get("enumeration_complete") if complete is not True: uncovered.append("mode evidence: exact configuration enumeration not asserted complete") for platform in sorted(all_obligations): row = next((r for r in platform_rows if r["id"] == platform), None) configs = [c for c in configurations if isinstance(c, dict) and c.get("platform") == platform] if not configs: uncovered.append(f"{platform}: no exact configurations enumerated") continue platform_traces = [t for t in traces if isinstance(t, dict) and isinstance(t.get("configuration"), dict) and t["configuration"].get("platform") == platform] try: matrix = validate_matrix(platform_traces, configs) except (KeyError, TypeError, ValueError) as exc: errors.append(f"{platform}: malformed mode matrix: {type(exc).__name__}") continue if row is not None: row["mode"] = matrix["status"] if matrix["status"] == "error": errors.append(f"{platform}: malformed mode trace") elif matrix["status"] == "failed": failures.append(f"{platform}: mode invariant or expectation failed") elif matrix["status"] == "uncovered": uncovered.append(f"{platform}: {len(matrix['uncovered'])} mode scenario/configuration gaps") status = "error" if errors else "failed" if failures else "uncovered" if uncovered else "pass" if errors or failures: current_execution_compatible = False for row in platform_rows: if row["current_interface"] == "passed": row["current_interface"] = "historical_pass_only" return {"schema_version": 1, "scope": "exact_source_platform_and_recorded_configuration_coverage", "vehicle_qualification": "not_established", "status": status, "interface_gate": interface_gate, "source_revision": manifest.get("source_revision"), "interface_report_revision": revision, "hashes": hashes or {}, "required_platform_count": len(required_set), "upstream_addition_count": len(addition_set), "registered_platform_count": len(registered_set), "upstream_only": upstream_only, "required_upstream_additions": sorted(addition_set), "evidence_limit": "Supplied recording labels and hashes do not authenticate trace adapters or physical origin.", "relevant_dirty_paths": dirty_relevant, "reported_relevant_dirty_paths": reported_dirty, "interface_execution_context": "current_compatible" if current_execution_compatible else "historical_or_incompatible", "reported_interface_counts": dict(Counter(r["interface"] for r in platform_rows)), "current_interface_counts": dict(Counter(r["current_interface"] for r in platform_rows)), "platforms": platform_rows, "mode_evidence": mode_summary, "errors": errors, "failures": failures, "uncovered": uncovered} def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--manifest", type=Path, default=DEFAULT_MANIFEST) parser.add_argument("--interface-coverage", type=Path, required=True) parser.add_argument("--interface-results", type=Path, required=True) parser.add_argument("--require", choices=("complete", "interfaces"), default="complete", help="Exit gate; interfaces checks the current registry, not full Dom parity or mode readiness") parser.add_argument("--mode-evidence", type=Path) parser.add_argument("--expected-revision", help="40-character checkout revision; defaults to current HEAD") parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.output.exists(): parser.error(f"output already exists; refusing to replace evidence: {args.output}") paths = {"manifest": args.manifest, "coverage": args.interface_coverage, "results": args.interface_results} if args.mode_evidence: paths["mode_evidence"] = args.mode_evidence hashes = {k: sha256(v) for k, v in paths.items()} checkout_revision = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip() if args.expected_revision is not None and args.expected_revision != checkout_revision: parser.error("expected revision differs from current checkout HEAD") expected = checkout_revision report = evaluate(json.loads(args.manifest.read_text()), json.loads(args.interface_coverage.read_text()), json.loads(args.interface_results.read_text()), json.loads(args.mode_evidence.read_text()) if args.mode_evidence else None, expected_revision=expected, hashes=hashes, dirty_relevant=relevant_dirty_paths()) report["required_gate"] = args.require report["required_gate_status"] = report["interface_gate"]["status"] if args.require == "interfaces" else report["status"] args.output.parent.mkdir(parents=True, exist_ok=True) with args.output.open("x") as stream: stream.write(json.dumps(report, indent=2, sort_keys=True) + "\n") summary = (f"fleet coverage: {report['status']}; {args.require} gate={report['required_gate_status']}; required={report['required_platform_count']}; " + f"registered={report['registered_platform_count']}; missing={report['reported_interface_counts'].get('missing_port', 0)}; " + f"current passes={report['current_interface_counts'].get('passed', 0)}; " + f"mode configurations={report['mode_evidence']['configuration_count']}; report={args.output}") print(summary) return 0 if report["required_gate_status"] == "pass" else 1 if __name__ == "__main__": raise SystemExit(main())