diff --git a/models b/models new file mode 100755 index 0000000000..73adfa3827 --- /dev/null +++ b/models @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -eo pipefail +DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/launch_env.sh" +exec python3 "$DIR/scripts/model_compiler.py" "$@" diff --git a/openpilot/starpilot/model_geometry.py b/openpilot/starpilot/model_geometry.py new file mode 100644 index 0000000000..69503f587d --- /dev/null +++ b/openpilot/starpilot/model_geometry.py @@ -0,0 +1,51 @@ +"""Validated shared model geometry; no control authority or mutable state.""" + +import math + +ORIGIN_ROUNDOFF_M = 1e-6 + + +def normalized_origin(distances: tuple) -> tuple: + """Only remove sub-micrometre model-origin roundoff; callers validate all geometry.""" + if distances and _finite(distances[0], low=-ORIGIN_ROUNDOFF_M, high=0.0) is not None and distances[0] < 0: + return (0.0, *distances[1:]) + return distances + + +def _finite(value: object, *, low: float, high: float) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + result = float(value) + except (OverflowError, ValueError): + return None + return result if math.isfinite(result) and low <= result <= high else None + + +def road_curvature(model: object, speed_mps: float) -> tuple[float, float] | None: + """Exact frozen max-|orientationRate.z * velocity.x| selection. + + The original helper divides by max(vEgo, 1)^2 and floors time at 1 s. + Invalid or incomplete current model arrays remain unknown. + """ + speed = _finite(speed_mps, low=0.0, high=80.0) + if speed is None: + return None + try: + rates = tuple(model.orientationRate.z) + times = tuple(model.orientationRate.t) + velocities = tuple(model.velocity.x) + except (AttributeError, TypeError, ValueError, OverflowError): + return None + if not len(rates) == len(times) == len(velocities) == 33: + return None + triples = [] + for rate, timestamp, velocity in zip(rates, times, velocities, strict=True): + r = _finite(rate, low=-10.0, high=10.0) + t = _finite(timestamp, low=0.0, high=20.0) + v = _finite(velocity, low=0.0, high=100.0) + if r is None or t is None or v is None: + return None + triples.append((r * v, t)) + acceleration, timestamp = max(triples, key=lambda pair: abs(pair[0])) + return acceleration / max(speed, 1.0) ** 2, max(timestamp, 1.0) diff --git a/openpilot/starpilot/models/README.md b/openpilot/starpilot/models/README.md new file mode 100644 index 0000000000..b449d11057 --- /dev/null +++ b/openpilot/starpilot/models/README.md @@ -0,0 +1,15 @@ +# Driving models + +Model Manager installs the versioned catalog into `/data/models/v26`. Downloads use the catalog's declared size and SHA-256, publish atomically, and share progress and cancellation between Galaxy and the native menus. Small and Chestnut selections take effect when modeld next starts. Randomization chooses only installed, verified models for the available hardware and respects exclusions; favorites are a separate browsing preference. + +Catalog behavior versions describe model inputs and outputs. The artifact ABI and compiler revision separately identify runtime compatibility. Existing compiled artifacts can be migrated when their kernels and state can be preserved and verified; changing the version label alone does not establish compatibility. Unavailable catalog entries cannot be downloaded or selected. + +`modeld` records the loaded artifact digest, process identity, variant, and fallback reason in a bounded volatile receipt. Status requires a matching running process and fresh valid model outputs before reporting an active model. A failed Chestnut load falls back to the initialized small runner. Artifact integrity and bench inference do not establish on-road driving quality. + +## Model Laboratory + +Laboratory artifact management uses separate AMD variants of small models. A standard on-device artifact or a Chestnut-class large model cannot substitute for one. The manifest declares each variant under `accelerator_artifacts.amd`, with `execution_device: "AMD"`, the current `compiler_revision`, and the same artifact format, digest, size, and chunk-count fields as ordinary catalog artifacts. Files are named `_driving_amd_tinygrad.pkl` under the model's existing catalog directory. Deleting an AMD variant preserves the ordinary on-device model. + +The Galaxy page and parked configuration owner are available. The standalone pair runner verifies both AMD artifacts, shares compatible camera preprocessing, keeps each model's state separate, and rejects the whole frame if either role fails. It decodes each model's own action units before choosing lateral curvature and longitudinal acceleration/stopping. The compositor assigns lateral geometry and the corresponding plan columns to the lateral model, and leads, scene metadata, and current-frame pose to the longitudinal model. + +Paired inference is not integrated into modeld yet, so production reports `runtimeSupported: false` and rejects enabling a pair. Completing it requires two verified AMD small-model variants, modeld loading/fallback and fresh pair status, then a Chestnut bench run covering real shared-warp execution, memory, timing and driver monitoring coexistence. CPU state-isolation and mocked frame tests do not qualify the AMD pair. diff --git a/openpilot/starpilot/models/__init__.py b/openpilot/starpilot/models/__init__.py new file mode 100644 index 0000000000..798dbd872c --- /dev/null +++ b/openpilot/starpilot/models/__init__.py @@ -0,0 +1 @@ +"""Driving-model identity and runtime status for StarPilot surfaces.""" diff --git a/openpilot/starpilot/models/catalog-v26.json b/openpilot/starpilot/models/catalog-v26.json new file mode 100644 index 0000000000..601b075935 --- /dev/null +++ b/openpilot/starpilot/models/catalog-v26.json @@ -0,0 +1,1210 @@ +{ + "generation": "v26", + "compiler_revision": "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf", + "artifact_abi": "tinygrad_single_v1_arena", + "models": [ + { + "id": "tr14223", + "name": "TR14 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-06-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "288980456db5ba083d333c0d08924f896e4141ba41642a9760eeefb497ff9797", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "tr15223", + "name": "TR15 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "551e01170bc6e7f40a32329122a3b07d4e202b45433c7d279243fe5003b00f27", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "tr16223", + "name": "TR16 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-21", + "community_favorite": false, + "uses_external_gpu": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5503a19a84fea7de25887fc3a2ec7148be8f84c5d9ea0f9e118f0df39743da25", + "artifact_size": 89751287, + "artifact_chunk_count": 2 + }, + { + "id": "letr223", + "name": "LeTR 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e55f53c970b011f4dbc5d90b119ba097dec6d390f3eed7cfb55de5233be72ea7", + "artifact_size": 89751287, + "artifact_chunk_count": 2 + }, + { + "id": "spacelab22223", + "name": "Space Lab 2 👀📡", + "version": "v8", + "series": "SL Series", + "released": "2025-07-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "1dae96a137e786bd75b80a8a1e89d215cb7c5b4fbe588c0ac270d4f5f33aa10e", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "vikander223", + "name": "Vikander 👀📡", + "version": "v8", + "series": "Custom Series", + "released": "2025-05-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "3cae48b71fc6db7339e15d0b329a1ee5ebff1e0ea469e1b7d3aac1ffa4db2aa4", + "artifact_size": 89737007, + "artifact_chunk_count": 2 + }, + { + "id": "fof223", + "name": "Filet-O-Fish 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-03-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "06e0967f88f022b3f9fca0164a1aac9ee95b6256a4a457388961d55de0e33ba0", + "artifact_size": 77785913, + "artifact_chunk_count": 2 + }, + { + "id": "vfof223", + "name": "Vegetarian-Filet-O-Fish 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-06-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7a001de39fef29d0ddde3f59e9ba7673d7dce7fbd3145ca2c1e9b715adce0f0c", + "artifact_size": 77800671, + "artifact_chunk_count": 2 + }, + { + "id": "dtr223", + "name": "Down To Ride 👀📡", + "version": "v8", + "series": "DTR Series", + "released": "2025-07-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "747792c151e9c119e138836638731cc7c69782d414805f7e02125ea7c59f4fe5", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "dtr6223", + "name": "Down To Ride v6 👀📡", + "version": "v10", + "series": "DTR Series", + "released": "2025-08-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "af8417340b6f3b70cd0117709af09e85a75b33aee9a66decd69e05936ca48872", + "artifact_size": 89750663, + "artifact_chunk_count": 2 + }, + { + "id": "uvdtr6223", + "name": "UV DTR v6 👀📡", + "version": "v10", + "series": "DTR Series", + "released": "2025-08-13", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "44184497255d0689861caa7719509eb680ce94ea7ac0da6e02afc1b9dece8103", + "artifact_size": 88543399, + "artifact_chunk_count": 2 + }, + { + "id": "sp2223", + "name": "Steam Powered 👀📡", + "version": "v10", + "series": "SP Series", + "released": "2025-08-16", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "93b34adeb2db586072ff179fd11e744fd3a9a39889795538a7933443cc1a4880", + "artifact_size": 83632118, + "artifact_chunk_count": 2 + }, + { + "id": "fp223", + "name": "Falling Phoenix 👀📡", + "version": "v10", + "series": "Custom Series", + "released": "2025-08-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "19b861ae053fd9e5182f1d28ce8875594825c53f3819e226ff57407d861a4a5b", + "artifact_size": 89750883, + "artifact_chunk_count": 2 + }, + { + "id": "kv223", + "name": "Kumar's Vibe 👀📡", + "version": "v10", + "series": "Custom Series", + "released": "2025-08-16", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "f91ce19bbcea90cf5416f01329411fbc8be0a9a467be2be7e62cbd4bfc2d7d6c", + "artifact_size": 89750882, + "artifact_chunk_count": 2 + }, + { + "id": "gwm3223", + "name": "Green Watermelon v3 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-08-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c74cc5e56972469328fb9710794d199aa14d6653412baf19bb6e81bb0d0db917", + "artifact_size": 83632118, + "artifact_chunk_count": 2 + }, + { + "id": "gwm5223", + "name": "Green Watermelon v5 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "2671cfa5daa6d0768ccb81d6eba5d756625ee06cb7c7bfe51f335e831f00ec6f", + "artifact_size": 83632124, + "artifact_chunk_count": 2 + }, + { + "id": "gwm6223", + "name": "Green Watermelon v6 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "gwm8223", + "name": "Green Watermelon v8 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-28", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e0263bb34dba65d4d55d679675dcc3198728f817f2cd98bb872cf3774fa4eabe", + "artifact_size": 83631220, + "artifact_chunk_count": 2 + }, + { + "id": "gwm9223", + "name": "Green Watermelon v9 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-10-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "afc80e282c27c2454da85b1f793fc52dad9d9dd07d88d88630e888317439bd1c", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "cgwm223", + "name": "Cool Green Watermelon 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-10-24", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5e1ecd3c91b7847e855885ab73f1399eda80179667f3c7b310b633479dadc501", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "bd223", + "name": "Firehose 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-08-31", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b1dfa01aa7b2031c4add906445c446844e36c7c11fb47696600fa2ea04ec7cd0", + "artifact_size": 83632124, + "artifact_chunk_count": 2 + }, + { + "id": "nr223", + "name": "Neurips 👀📡", + "version": "v11", + "series": "NR Series", + "released": "2025-11-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "df616dfd649dec7141e4190a3e66c61afd684561d939030c4d6eaa2f709e3143", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "tcp2223", + "name": "The Cool People v2 👀📡", + "version": "v11", + "series": "TCP Series", + "released": "2025-10-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "795b593cb375f299a34c561ad18232690369b251adbde9fd10f26dc3cd6e9ade", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "tcp3223", + "name": "The Cool People v3 👀📡", + "version": "v11", + "series": "TCP Series", + "released": "2025-10-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "85f9ce6a0e88db51ce02d61f594a9ada7de6a8aa2ee206f5f014a7fe9dce59bc", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "fbw223", + "name": "Fly By Wire 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-04", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5a58a2bbf05fef21e470efc837d29301ddda5d23b959c9c46d5736c659aaed47", + "artifact_size": 71032111, + "artifact_chunk_count": 2 + }, + { + "id": "nevada223", + "name": "Nevada 👀📡", + "version": "v11", + "series": "NV Series", + "released": "2025-09-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "6f497406e2e914bd29d7140710e3214ffde7ac36cb60a7333e5770517864a1c2", + "artifact_size": 83632120, + "artifact_chunk_count": 2 + }, + { + "id": "wmi4223", + "name": "Watermelon Indigo v4 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7f0983957c87cc280e6b51f9e8ed2f40817da449669abd7410ff3319e287c050", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi523", + "name": "Watermelon Indigo v5 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-29", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c98c2dbe228aad773a2fc85949eb7dd07cb4cb9621a81074276f4aaf06f926da", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi623", + "name": "Watermelon Indigo v6 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c94059abcb0b95cf4a11d912e0aa36eadcb423832dfae1af7c0cbe2f406428b6", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi723", + "name": "Watermelon Indigo v7 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c91bbdb296f56acc084097aafb30d1050abb5213ad2e6fce73e9ddc5a3ab98fe", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi7223", + "name": "Watermelon Indigo v7 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "673ac43ce0f52b0324322ee26b5baad165f71ea08c599385328035620a6cb560", + "artifact_size": 83635845, + "artifact_chunk_count": 2 + }, + { + "id": "wmi823", + "name": "Watermelon Indigo v8 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "eb3f263d370ebd74d5feea288686844901c2e10ab5e941105e7eeed6d6fb06d3", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi923", + "name": "Watermelon Indigo v9 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "04528d7005d5913f35313c346235c5e94111bf8d4baf9765e79a50d093e01c33", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "wmi1023", + "name": "Watermelon Indigo v10 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "8dd02dc16a0ed51978ec55f34875f779e8089757b5153a9216de9010933cf25d", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi1123", + "name": "Watermelon Indigo v11 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-13", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "917894794ddc28ceac39a1f098e875fb3449dbc9b7d4e0f78f0035a7a2651580", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "cd21023", + "name": "CD210 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-31", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "cb4b9ee7a1a7ffec8883c9e566a051ccf06736195645fa1392cf3fd6af8d5bb8", + "artifact_size": 83728468, + "artifact_chunk_count": 2 + }, + { + "id": "op223", + "name": "Off Policy v2 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "518a4c91cc916eec2d533f68546c2df07222b514bdbcc94bda08f8a581f96c5d", + "artifact_size": 72331687, + "artifact_chunk_count": 2 + }, + { + "id": "op323", + "name": "Off Policy v3 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "791ba9270e6329da9d64c6175e0c0bdbc1a42d84f821bf1c8274e138fbaa074e", + "artifact_size": 60720006, + "artifact_chunk_count": 2 + }, + { + "id": "op423", + "name": "Off Policy v4 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-27", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "3334290c8ca613a487a3a267c83a6c4742b8e145e96796a9ff04f8d31c5a3802", + "artifact_size": 67107030, + "artifact_chunk_count": 2 + }, + { + "id": "op523", + "name": "Off Policy v5 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-03-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7cf2fb630b3fdab3c68f809f2cb409091bcec90348351fffa126b8378a1aaf6f", + "artifact_size": 69394003, + "artifact_chunk_count": 2 + }, + { + "id": "op623", + "name": "Off Policy v6 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-03-28", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "4dab4296e6e07470ccecfe17555667b7b5894c69e5d01bd70b770fef21208c47", + "artifact_size": 73634626, + "artifact_chunk_count": 2 + }, + { + "id": "opv73", + "name": "Off Policy v7 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e7055bf07e9bdcd223d174545d073e331a6f9a39564b80ffbcb22fafc1a42448", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv83", + "name": "Off Policy v8 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "ab6847f204d935b9cbf51a892d7dd0b7fadf3395c78766da74d06f79e1265c8d", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv93", + "name": "Off Policy v9 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "d6771747aae786f9bf158ed067ea2c77d1eb1b31cb5cbcb5c754c6ba0df1190d", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv103", + "name": "Off Policy v10 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv113", + "name": "Off Policy v11 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-18", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv123", + "name": "Off Policy v12 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv133", + "name": "Off Policy v13 👀📡", + "version": "v14", + "series": "OP Series", + "released": "2026-05-05", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op163", + "name": "Off Policy v16 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-19", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op16d3", + "name": "Off Policy v16 Deep 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op16dv23", + "name": "Off Policy v16 Deep V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rlv1dl3", + "name": "RL V1 Lateral Only 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "deeprl33", + "name": "Deep RL 3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "deeprl333", + "name": "Deep RL 3 V3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rl343", + "name": "Rebel Legion 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "drl3", + "name": "Divided RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "nopp3", + "name": "No PP 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tobyrl3", + "name": "Toby RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-28", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "michael-rl3", + "name": "Michael RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "michael-rl23", + "name": "Michael RL V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "karnbir3", + "name": "Karnbir 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "karnbir23", + "name": "Karnbir V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "ms23", + "name": "Macrostiff 👀📡", + "version": "v11", + "series": "MS Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "874e4c8a00c483874fc704a449b003ab59bcf0e5d99c8dd591bbc20abf386029", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "pp2223", + "name": "Plan Plus v2 👀📡", + "version": "v11", + "series": "PL Series", + "released": "2025-12-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "ds2223", + "name": "Dark Souls v2 👀📡", + "version": "v11", + "series": "DS Series", + "released": "2025-12-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7f4a4e9cad0c357f4e66d8e772c7b6748304fc55c440db768671242fccc28431", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "nn2223", + "name": "North Nevada v2 👀📡", + "version": "v11", + "series": "NV Series", + "released": "2025-10-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b86bd737f158a926846f63ce8e2a1e28e0d2c18ad0c3871d43160a566797128f", + "artifact_size": 83632126, + "artifact_chunk_count": 2 + }, + { + "id": "sc23", + "name": "South Carolina 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-01-08", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "66e4837bde0e12bdd90e7eb3e180ea2eab62d3a6af279df366cbf0cc69c469fd", + "artifact_size": 83737646, + "artifact_chunk_count": 2 + }, + { + "id": "pop23", + "name": "Pop Model 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-03-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "d0f287497057b47fd91bbfa9eeb6d86f91e6bad84f1c846e4afec1b8b6deb4bb", + "artifact_size": 83728468, + "artifact_chunk_count": 2 + }, + { + "id": "pop223", + "name": "Pop Model V2 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-03-24", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e147dd4f0d7be4e7931ebc2b07eb28a0eda08a7b26e562188d53e22efc340e58", + "artifact_size": 83737200, + "artifact_chunk_count": 2 + }, + { + "id": "nid223", + "name": "Nuggets in Dijon 👀📡", + "version": "v11", + "series": "NUG Series", + "released": "2025-10-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "32cea751d38aeaed25e01882c4dd24e4467d9139ddba4c74ff42c7a2084a9045", + "artifact_size": 83632126, + "artifact_chunk_count": 2 + }, + { + "id": "kerrygold223", + "name": "Kerrygold 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-06-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "9379c9a1962d743ea80f7708ae7ab42235f3787dc7f65e1c7b320f73a6e08935", + "artifact_size": 77797177, + "artifact_chunk_count": 2 + }, + { + "id": "deeprl3v23", + "name": "Deep RL 3 V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rh3", + "name": "Rebellious Hope 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-27", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "gyhu3", + "name": "GYHU 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-25", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdfv23", + "name": "Regret Driven Framework 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-05", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf23", + "name": "Regret Driven Framework V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf43", + "name": "Regret Driven Framework V4 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf33", + "name": "Regret Driven Framework V3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf53", + "name": "Regret Driven Framework V5 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf63", + "name": "Regret Driven Framework V6 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tsf", + "name": "Terrific Super Fantastic 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tsfdo", + "name": "Terrific Super Fantastic Do Over 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-15", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "lebowski3", + "name": "Lebowski 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-15", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "9faae0161a46cce693927da12be5cdac32df5e9855e2e521c213c2de2279e4fe", + "artifact_size": 1773071525, + "artifact_chunk_count": 38 + }, + { + "id": "tt-driving", + "name": "TT Driving 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-17", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b5ee71cff876bcc5e910c237abac783c8c4d8e51121af78a2c4d94ee13764ba2", + "artifact_size": 1773202548, + "artifact_chunk_count": 38 + }, + { + "id": "tee-time", + "name": "Tee Time 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-23", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "608728fb6c53cf39a0d417a2245557c80cdc740ef651ac148cd805ccd2f5a9de", + "artifact_size": 1773210625, + "artifact_chunk_count": 38 + }, + { + "id": "bmrlnapv2", + "name": "BMRLNAP V2 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-26", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c828394a69d59c101427ffdc55cde6f61bd432abdf15a7e10e03eaa610a03bc0", + "artifact_size": 1776069700, + "artifact_chunk_count": 38 + }, + { + "id": "happybirthday", + "name": "Happy Birthday 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-24", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "f729e4b4711212f314a1fff14058657a86c36405deca5ae92e3ed5b213f3cbdc", + "artifact_size": 1776069700, + "artifact_chunk_count": 38 + }, + { + "id": "berightthere", + "name": "Be Right Here 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-01", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv4", + "name": "Remove AvgPool V4 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-26", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv3", + "name": "Remove AvgPool V3 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-24", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv2", + "name": "Remove AvgPool V2 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-20", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv1", + "name": "Remove AvgPool V1 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-20", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "rdf-driving", + "name": "RDF Driving 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-09", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "sad", + "name": "Sad 😢", + "version": "v16", + "series": "OP Series", + "released": "2026-08-28", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "bmrlnapv6", + "name": "BMRLNAP V6 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-01", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "deeprl3v2", + "name": "Deep RL 3 V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "cinque-driving", + "name": "Cinque-Driving 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-04", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "cdm", + "name": "Cinque Divergence Mask 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-08", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "cinquev3", + "name": "Cinque v3 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-18", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "261454e0d170f5282d989d67ea7db08c90d7297369260085fbdf44af4e327a08", + "artifact_size": 779382267, + "artifact_chunk_count": 17 + } + ] +} diff --git a/openpilot/starpilot/models/catalog.py b/openpilot/starpilot/models/catalog.py new file mode 100644 index 0000000000..252dd84c8c --- /dev/null +++ b/openpilot/starpilot/models/catalog.py @@ -0,0 +1,54 @@ +"""Reviewed local driving-model catalog. This module never loads model artifacts.""" + +from dataclasses import dataclass +import json +from pathlib import Path + + +BUNDLED_CURRENT = "bundled-current" +GENERATION = "v26" +COMPILER_REVISION = "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf" +ARTIFACT_ABI = "tinygrad_single_v1_arena" +CATALOG_PATH = Path(__file__).with_name("catalog-v26.json") + + +@dataclass(frozen=True) +class ModelEntry: + model_id: str + name: str + source_sha256: str + source_path: str + compiler_revision: str + artifact_abi: str + selectable: bool + version: str = "current" + uses_external_gpu: bool = False + + +BUNDLED = ( + ModelEntry( + model_id=BUNDLED_CURRENT, + name="Bundled driving model", + source_sha256="65a08adc31d5c456219687d99b7bf5e44d61dae2d49ea67850e76105c7248cce", + source_path="openpilot/selfdrive/modeld/models/driving_supercombo.onnx", + compiler_revision="9d0446a4ba8a532c8b674fb6ad795af015cd9dcf", + artifact_abi="current-onnx-tinygrad-oob", + selectable=True, + ), +) + +CATALOG = BUNDLED + tuple(ModelEntry( + model_id=m["id"], name=m["name"], source_sha256="", source_path="", + compiler_revision=COMPILER_REVISION, artifact_abi=ARTIFACT_ABI, selectable=True, + version=m["version"], uses_external_gpu=bool(m.get("uses_external_gpu", False)), +) for m in json.loads(CATALOG_PATH.read_text())["models"]) + +BY_ID = {entry.model_id: entry for entry in CATALOG} + + +def resolve_selection(requested_id: str | None) -> ModelEntry: + """Resolve a saved request; absence means the bundled model, never an unknown ID.""" + model_id = BUNDLED_CURRENT if requested_id is None else requested_id + if not isinstance(model_id, str) or model_id not in BY_ID or not BY_ID[model_id].selectable: + raise ValueError("unqualified driving-model selection") + return BY_ID[model_id] diff --git a/openpilot/starpilot/models/compile.py b/openpilot/starpilot/models/compile.py new file mode 100644 index 0000000000..b121a23590 --- /dev/null +++ b/openpilot/starpilot/models/compile.py @@ -0,0 +1,697 @@ +"""Compile StarPilot unified driving artifacts against the vendored tinygrad runtime.""" +import argparse +import base64 +import math +import os +from pathlib import Path +import pickle +import tempfile +import time +from collections import namedtuple +from functools import partial + +import numpy as np +from tinygrad import Device, TinyJit, Context, Tensor +from tinygrad.dtype import _to_np_dtype +from tinygrad_repo.examples.openpilot.helpers import dump_pickle, load_pickle + +COMPILER_REVISION = "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf" +ARTIFACT_ABI = "tinygrad_single_v1_arena" + + +def make_metadata_dict(path, runner): + from tinygrad.nn.onnx import OnnxPBParser + + parser = OnnxPBParser(path, load_external_data=False) + properties, output_shapes = {}, {} + for field, wire_type in parser._parse_message(parser.reader.len): + if field == 14: + entry = parser._parse_StringStringEntryProto() + properties[entry['key']] = entry['value'] + elif field == 7: + for field, wire_type in parser._parse_message(parser._decode_end_pos()): + if field == 12: + value = parser._parse_ValueInfoProto() + output_shapes[value['name']] = value['parsed_type'].shape if value['parsed_type'] is not None else () + else: + parser.reader.skip_field(wire_type) + else: + parser.reader.skip_field(wire_type) + if "output_slices" not in properties: + raise ValueError(f"Missing output_slices metadata: {path}") + return { + "model_checkpoint": properties.get("model_checkpoint"), + "output_slices": pickle.loads(base64.b64decode(properties["output_slices"])), + "input_shapes": {name: tuple(spec.shape) for name, spec in runner.graph_inputs.items()}, + "output_shapes": output_shapes, + } + + +ARTIFACT_FORMAT_VERSION = 1 +MODEL_TYPES = ("vision_policy", "vision_multi_policy", "supercombo") +NV12Frame = namedtuple("NV12Frame", ["width", "height", "stride", "y_height", "uv_height", "size"]) +IMAGE_HISTORY_IN_WARP = "warp" +IMAGE_HISTORY_IN_POLICY = "policy" +IMAGE_HISTORY_PIPELINES = (IMAGE_HISTORY_IN_WARP, IMAGE_HISTORY_IN_POLICY) +LEGACY_WARP_INPUTS = ("img_q", "big_img_q", "tfm", "big_tfm") +FAST_WARP_INPUTS = ("tfm", "big_tfm") +BASE_POLICY_INPUTS = ("feat_q", "desire_q", "packed_npy_inputs") +FAST_POLICY_INPUTS = ("img_q", "big_img_q", *BASE_POLICY_INPUTS) +WARP_INPUTS = LEGACY_WARP_INPUTS +SPLIT_POLICY_INPUTS = BASE_POLICY_INPUTS +SUPERCOMBO_POLICY_INPUTS = BASE_POLICY_INPUTS +WARP_DEV = os.getenv("WARP_DEV") + + + +def _detect_desire_key(input_shapes): + return next((key for key in input_shapes if key.startswith("desire")), None) + + +def _detect_vision_keys(input_shapes): + image_keys = sorted(key for key in input_shapes if "img" in key) + road_key = next((key for key in image_keys if "big" not in key), None) + wide_key = next((key for key in image_keys if "big" in key), None) + if road_key is None or wide_key is None: + raise ValueError(f"Cannot determine road/wide image keys from {list(input_shapes)}") + return road_key, wide_key + + +def derive_frame_skip(input_shapes): + features_shape = input_shapes.get("features_buffer") + if features_shape is None: + return 1 + return 1 if features_shape[1] >= 99 else 4 + + +def make_random_images(keys, shape, device=None, *, rng=None): + return {key: Tensor.randint(shape, low=0, high=256, dtype="uint8", device=device).realize() for key in keys} + + +def make_random_blob_images(keys, size, device=None): + keepalive: list[np.ndarray] = [] + + def make_inputs(*, rng): + nonlocal keepalive + keepalive = [] + tensors = {} + for key in keys: + frame = (32 * rng.standard_normal(size).astype(np.float32) + 128).clip(0, 255).astype(np.uint8) + keepalive.append(frame) + tensors[key] = Tensor.from_blob(frame.ctypes.data, (size,), dtype="uint8", device=device).realize() + return tensors + + return make_inputs + + +def warp_perspective_tinygrad(src_flat, matrix_inverse, dst_shape, src_shape, stride_pad, border_fill_val=None): + width_dst, height_dst = dst_shape + height_src, width_src = src_shape + + x = Tensor.arange(width_dst).reshape(1, width_dst).expand(height_dst, width_dst).reshape(-1) + y = Tensor.arange(height_dst).reshape(height_dst, 1).expand(height_dst, width_dst).reshape(-1) + + src_x = matrix_inverse[0, 0] * x + matrix_inverse[0, 1] * y + matrix_inverse[0, 2] + src_y = matrix_inverse[1, 0] * x + matrix_inverse[1, 1] * y + matrix_inverse[1, 2] + src_w = matrix_inverse[2, 0] * x + matrix_inverse[2, 1] * y + matrix_inverse[2, 2] + src_x = src_x / src_w + src_y = src_y / src_w + + x_round = Tensor.round(src_x) + y_round = Tensor.round(src_y) + x_nn_clipped = x_round.clip(0, width_src - 1).cast("int") + y_nn_clipped = y_round.clip(0, height_src - 1).cast("int") + sampled = src_flat[y_nn_clipped * (width_src + stride_pad) + x_nn_clipped] + + if border_fill_val is None: + return sampled + + in_bounds = ((x_round >= 0) & (x_round <= width_src - 1) & + (y_round >= 0) & (y_round <= height_src - 1)).cast(sampled.dtype) + return sampled * in_bounds + Tensor(border_fill_val, dtype=sampled.dtype) * (1 - in_bounds) + + +def frames_to_tensor(frames): + height = (frames.shape[0] * 2) // 3 + width = frames.shape[1] + return Tensor.cat( + frames[0:height:2, 0::2], + frames[1:height:2, 0::2], + frames[0:height:2, 1::2], + frames[1:height:2, 1::2], + frames[height:height + height // 4].reshape((height // 2, width // 2)), + frames[height + height // 4:height + height // 2].reshape((height // 2, width // 2)), + dim=0, + ).reshape((6, height // 2, width // 2)) + + +def make_frame_prepare(nv12: NV12Frame, model_w, model_h): + cam_w, cam_h, stride, y_height, uv_height, _ = nv12 + uv_offset = stride * y_height + stride_pad = stride - cam_w + + def frame_prepare(input_frame, matrix_inverse): + matrix_inverse_uv = matrix_inverse * Tensor( + [[1.0, 1.0, 0.5], [1.0, 1.0, 0.5], [2.0, 2.0, 1.0]], + device=WARP_DEV, + ) + uv = input_frame[uv_offset:uv_offset + uv_height * stride].reshape(uv_height, stride) + with Context(SPLIT_REDUCEOP=0): + y = warp_perspective_tinygrad( + input_frame[:cam_h * stride], matrix_inverse, (model_w, model_h), (cam_h, cam_w), stride_pad, + ).realize() + u = warp_perspective_tinygrad( + uv[:cam_h // 2, :cam_w:2].flatten(), matrix_inverse_uv, + (model_w // 2, model_h // 2), (cam_h // 2, cam_w // 2), 0, + ).realize() + v = warp_perspective_tinygrad( + uv[:cam_h // 2, 1:cam_w:2].flatten(), matrix_inverse_uv, + (model_w // 2, model_h // 2), (cam_h // 2, cam_w // 2), 0, + ).realize() + return frames_to_tensor(y.cat(u).cat(v).reshape((model_h * 3 // 2, model_w))) + + return frame_prepare + + +def make_warp_input_queues(vision_input_shapes, frame_skip, device): + road_key, _ = _detect_vision_keys(vision_input_shapes) + image_shape = vision_input_shapes[road_key] + frame_count = image_shape[1] // 6 + image_buffer_shape = (frame_skip * (frame_count - 1) + 1, 6, image_shape[2], image_shape[3]) + + npy = { + "tfm": np.zeros((3, 3), dtype=np.float32), + "big_tfm": np.zeros((3, 3), dtype=np.float32), + } + queues = { + "img_q": Tensor(np.zeros(image_buffer_shape, dtype=np.uint8), device=device).contiguous().realize(), + "big_img_q": Tensor(np.zeros(image_buffer_shape, dtype=np.uint8), device=device).contiguous().realize(), + **{key: Tensor(value, device="NPY").realize() for key, value in npy.items()}, + } + return queues, npy + + +def _packed_policy_shapes(input_shapes, include_prev_feature=False): + desire_key = _detect_desire_key(input_shapes) + if desire_key is None: + raise ValueError(f"No desire input found in {list(input_shapes)}") + + shapes = {"desire": (input_shapes[desire_key][2],)} + for key, shape in input_shapes.items(): + if key in ("features_buffer", desire_key) or "img" in key: + continue + shapes[key] = tuple(shape) + if include_prev_feature: + features_shape = input_shapes["features_buffer"] + shapes["prev_feat"] = (features_shape[0], math.prod(features_shape[2:])) + return shapes, [math.prod(shape) for shape in shapes.values()] + + +def make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device): + queues, npy = make_warp_input_queues(vision_input_shapes, frame_skip, device) + features_shape = policy_input_shapes["features_buffer"] + feature_dim = math.prod(features_shape[2:]) + desire_key = _detect_desire_key(policy_input_shapes) + desire_shape = policy_input_shapes[desire_key] + packed_shapes, packed_sizes = _packed_policy_shapes(policy_input_shapes) + + packed_inputs = np.zeros(sum(packed_sizes), dtype=np.float32) + npy.update({ + key: value.reshape(shape) + for (key, shape), value in zip( + packed_shapes.items(), np.split(packed_inputs, np.cumsum(packed_sizes[:-1])), strict=True, + ) + }) + queues.update({ + "feat_q": Tensor( + np.zeros((frame_skip * (features_shape[1] - 1) + 1, features_shape[0], feature_dim), dtype=np.float32), + device=device, + ).contiguous().realize(), + "desire_q": Tensor( + np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), dtype=np.float32), + device=device, + ).contiguous().realize(), + "packed_npy_inputs": Tensor(packed_inputs, device="NPY").realize(), + }) + return queues, npy + + +def make_supercombo_input_queues(input_shapes, frame_skip, device): + queues, npy = make_warp_input_queues(input_shapes, frame_skip, device) + features_shape = input_shapes["features_buffer"] + feature_dim = math.prod(features_shape[2:]) + desire_key = _detect_desire_key(input_shapes) + desire_shape = input_shapes[desire_key] + packed_shapes, packed_sizes = _packed_policy_shapes(input_shapes, include_prev_feature=True) + + packed_inputs = np.zeros(sum(packed_sizes), dtype=np.float32) + npy.update({ + key: value.reshape(shape) + for (key, shape), value in zip( + packed_shapes.items(), np.split(packed_inputs, np.cumsum(packed_sizes[:-1])), strict=True, + ) + }) + queues.update({ + "feat_q": Tensor( + np.zeros((frame_skip * features_shape[1], features_shape[0], feature_dim), dtype=np.float32), + device=device, + ).contiguous().realize(), + "desire_q": Tensor( + np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), dtype=np.float32), + device=device, + ).contiguous().realize(), + "packed_npy_inputs": Tensor(packed_inputs, device="NPY").realize(), + }) + return queues, npy + + +def shift_and_sample(buffer, new_value, sample_fn): + buffer.assign(buffer[1:].cat(new_value, dim=0).contiguous()) + return sample_fn(buffer) + + +def sample_skip(buffer, frame_skip): + return buffer[::frame_skip].contiguous().flatten(0, 1).unsqueeze(0) + + +def sample_desire(buffer, frame_skip): + return buffer.reshape(-1, frame_skip, *buffer.shape[1:]).max(1).flatten(0, 1).unsqueeze(0) + + +def make_warp(nv12, model_w, model_h, frame_skip, image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + frame_prepare = make_frame_prepare(nv12, model_w, model_h) + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def warp(tfm, big_tfm, frame, big_frame): + tfm = tfm.to(WARP_DEV) + big_tfm = big_tfm.to(WARP_DEV) + Tensor.realize(tfm, big_tfm) + + return Tensor.cat( + frame_prepare(frame, tfm).unsqueeze(0), + frame_prepare(big_frame, big_tfm).unsqueeze(0), + ) + + return warp + + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + + def warp_enqueue(img_q, big_img_q, tfm, big_tfm, frame, big_frame): + tfm = tfm.to(WARP_DEV) + big_tfm = big_tfm.to(WARP_DEV) + Tensor.realize(tfm, big_tfm) + + warped = Tensor.cat( + frame_prepare(frame, tfm).unsqueeze(0), + frame_prepare(big_frame, big_tfm).unsqueeze(0), + ).to(Device.DEFAULT) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return img, big_img + + return warp_enqueue + + +def make_run_split_policy(vision_runner, policy_runners, metadata, policy_order, frame_skip, + image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + vision_metadata = metadata["vision"] + policy_metadata = metadata[policy_order[0]] + vision_features_slice = vision_metadata["output_slices"]["hidden_state"] + desire_key = _detect_desire_key(policy_metadata["input_shapes"]) + packed_shapes, packed_sizes = _packed_policy_shapes(policy_metadata["input_shapes"]) + road_key, wide_key = _detect_vision_keys(vision_metadata["input_shapes"]) + + def run_model(img, big_img, feat_q, desire_q, packed_npy_inputs): + unpacked = { + key: tensor.reshape(shape) + for (key, shape), tensor in zip( + packed_shapes.items(), packed_npy_inputs.split(packed_sizes), strict=True, + ) + } + desire_buffer = shift_and_sample( + desire_q, unpacked.pop("desire").reshape(1, 1, -1), sample_desire_fn, + ) + vision_output = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast("float32") + new_feature = vision_output[:, vision_features_slice].reshape(1, -1).unsqueeze(0) + features_buffer = shift_and_sample(feat_q, new_feature, sample_skip_fn) + features_buffer = features_buffer.reshape(policy_metadata["input_shapes"]["features_buffer"]) + + policy_inputs = { + "features_buffer": features_buffer, + desire_key: desire_buffer, + **unpacked, + } + policy_outputs = [ + next(iter(policy_runners[key](policy_inputs).values())).cast("float32") + for key in policy_order + ] + return (vision_output, *policy_outputs) + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT) + warped = warped.to(Device.DEFAULT) + Tensor.realize(packed_npy_inputs, warped) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + def run_policy(img, big_img, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT).realize() + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + +def make_run_supercombo(model_runner, metadata, frame_skip, image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + input_shapes = metadata["model"]["input_shapes"] + sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + desire_key = _detect_desire_key(input_shapes) + packed_shapes, packed_sizes = _packed_policy_shapes(input_shapes, include_prev_feature=True) + road_key, wide_key = _detect_vision_keys(input_shapes) + + def run_model(img, big_img, feat_q, desire_q, packed_npy_inputs): + unpacked = { + key: tensor.reshape(shape) + for (key, shape), tensor in zip( + packed_shapes.items(), packed_npy_inputs.split(packed_sizes), strict=True, + ) + } + desire_buffer = shift_and_sample( + desire_q, unpacked.pop("desire").reshape(1, 1, -1), sample_desire_fn, + ) + previous_feature = unpacked.pop("prev_feat") + features_buffer = shift_and_sample( + feat_q, previous_feature.reshape(1, 1, -1), sample_skip_fn, + ) + features_buffer = features_buffer.reshape(input_shapes["features_buffer"]) + model_inputs = { + road_key: img, + wide_key: big_img, + "features_buffer": features_buffer, + desire_key: desire_buffer, + **unpacked, + } + model_output = next(iter(model_runner(model_inputs).values())).cast("float32") + return model_output, + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT) + warped = warped.to(Device.DEFAULT) + Tensor.realize(packed_npy_inputs, warped) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + def run_policy(img, big_img, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT).realize() + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + +def stateful_image_shapes(metadata): + shape = tuple(metadata['input_shapes']['new_img']) + if len(shape) != 4 or shape[:2] != (2, 6): + raise ValueError(f"Unsupported stateful image shape: {shape}") + return dict.fromkeys(('img', 'big_img'), (1, *shape[1:])) + + +def stateful_host_shapes(metadata): + return {name: shape for name, shape in metadata['input_shapes'].items() + if name != 'new_img' and name not in metadata['state_pairs']} + + +def make_stateful_input_queues(metadata, device): + queues, npy = make_warp_input_queues(stateful_image_shapes(metadata), 1, device) + shapes = stateful_host_shapes(metadata) + sizes = [math.prod(shape) for shape in shapes.values()] + packed = np.zeros(sum(sizes), dtype=np.float32) + npy.update({name: value.reshape(shape) for (name, shape), value in + zip(shapes.items(), np.split(packed, np.cumsum(sizes[:-1])), strict=True)}) + queues['packed_npy_inputs'] = Tensor(packed, device='NPY').realize() + for name in metadata['state_pairs']: + queues[name] = Tensor(np.zeros(metadata['input_shapes'][name], dtype=metadata['input_dtypes'][name]), + device=device).contiguous().realize() + return queues, npy + + +def make_run_stateful_supercombo(model_runner, metadata): + shapes = stateful_host_shapes(metadata) + sizes = [math.prod(shape) for shape in shapes.values()] + + def run_policy(warped, packed_npy_inputs, **state): + packed = packed_npy_inputs.to(Device.DEFAULT).realize() + inputs = {name: value.reshape(shape).cast(model_runner.graph_inputs[name].dtype) + for (name, shape), value in zip(shapes.items(), packed.split(sizes), strict=True)} + inputs['new_img'] = warped.to(Device.DEFAULT).cast(model_runner.graph_inputs['new_img'].dtype) + outputs = {name: value.contiguous() for name, value in model_runner(inputs | state).items()} + for name, next_name in metadata['state_pairs'].items(): + if outputs[next_name].dtype != state[name].dtype: + raise ValueError(f'State dtype mismatch: {name} -> {next_name}') + Tensor.realize(*outputs.values()) + Tensor.realize(*(state[name].assign(outputs[next_name]) for name, next_name in metadata['state_pairs'].items())) + return outputs['outputs'].cast('float32'), + + return run_policy + + +def compile_jit(jit, make_random_inputs, input_keys, make_queues, validation_runs=1): + seed = 42 + + def random_inputs_run(fn, current_seed, test_values=None, test_buffers=None, expect_match=True): + input_queues, npy = make_queues(Device.DEFAULT) + rng = np.random.default_rng(current_seed) + Tensor.manual_seed(current_seed) + testing = test_values is not None or test_buffers is not None + run_count = validation_runs if testing else max(3, validation_runs) + values, buffers = [], [] + + for index in range(run_count): + for value in npy.values(): + value[:] = rng.standard_normal(value.shape).astype(value.dtype) + Device.default.synchronize() + random_inputs = make_random_inputs(rng=rng) + start = time.perf_counter() + outputs = fn(**{key: input_queues[key] for key in input_keys}, **random_inputs) + mid = time.perf_counter() + Device.default.synchronize() + end = time.perf_counter() + print(f" [{index + 1}/{run_count}] enqueue {(mid - start) * 1e3:6.2f} ms -- total {(end - start) * 1e3:6.2f} ms") + + if index < validation_runs: + values.extend(np.copy(value.numpy()) for value in outputs) + buffers.extend(np.copy(value.numpy()) for value in input_queues.values()) + if not all(np.isfinite(value).all() for value in values): + raise ValueError("Compiled JIT produced non-finite outputs") + + if test_values is not None: + match = all(np.array_equal(lhs, rhs) for lhs, rhs in zip(values, test_values, strict=True)) + assert match == expect_match, f"outputs {'differ from' if expect_match else 'match'} baseline (seed={current_seed})" + if test_buffers is not None: + match = all(np.array_equal(lhs, rhs) for lhs, rhs in zip(buffers, test_buffers, strict=True)) + assert match == expect_match, f"buffers {'differ from' if expect_match else 'match'} baseline (seed={current_seed})" + return values, buffers + + print("capture + replay") + test_values, test_buffers = random_inputs_run(jit, seed) + print("pickle round trip") + with tempfile.TemporaryDirectory(prefix="starpilot-jit-") as temporary: + path = Path(temporary) / "captured.pkl" + dump_pickle(jit, path) + loaded = load_pickle(path, out_of_band=True) + random_inputs_run(loaded, seed, test_values, test_buffers, expect_match=True) + random_inputs_run(loaded, seed + 1, test_values, test_buffers, expect_match=False) + return jit + + +def _parse_size(value): + width, height = value.lower().split("x") + return int(width), int(height) + + +def validate_metadata(metadata): + output_shapes = metadata.get("output_shapes", {}) + output_shape = output_shapes.get("outputs") + if not output_shape or len(output_shape) < 2: + raise ValueError(f"Invalid model output shape metadata: {output_shapes}") + output_size = output_shape[-1] + for name, output_slice in metadata.get("output_slices", {}).items(): + start, stop, step = output_slice.indices(output_size) + if step != 1 or start < 0 or stop < start or stop > output_size: + raise ValueError(f"Invalid output slice {name}={output_slice} for output size {output_size}") + + +def main(): + from tinygrad.nn.onnx import OnnxRunner + + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + parser = argparse.ArgumentParser() + parser.add_argument("--model-type", choices=MODEL_TYPES, required=True) + parser.add_argument("--model-size", type=_parse_size, required=True) + parser.add_argument("--camera-resolutions", type=_parse_size, nargs="+", required=True) + parser.add_argument("--frame-skip", type=int) + parser.add_argument("--behavior-version") + parser.add_argument("--output", required=True) + parser.add_argument("--out-of-band", action="store_true", help="Compatibility flag; all artifacts use the current buffer arena format.") + parser.add_argument("--vision-onnx") + parser.add_argument("--policy-onnx") + parser.add_argument("--off-policy-onnx") + parser.add_argument("--on-policy-onnx") + parser.add_argument("--supercombo-onnx") + parser.add_argument( + "--image-history-pipeline", + choices=IMAGE_HISTORY_PIPELINES, + default=IMAGE_HISTORY_IN_POLICY, + help="Where img/big_img history queues are updated. 'policy' is the newer faster ABI; 'warp' reproduces legacy v22 artifacts.", + ) + args = parser.parse_args() + if args.frame_skip is not None and args.frame_skip < 1: + parser.error("--frame-skip must be positive") + + output = { + "format_version": ARTIFACT_FORMAT_VERSION, + "artifact_abi": ARTIFACT_ABI, + "compiler_revision": COMPILER_REVISION, + "execution_device": Device.DEFAULT, + "warp_device": WARP_DEV or Device.DEFAULT, + "model_type": args.model_type, + "metadata": {}, + "image_history_pipeline": args.image_history_pipeline, + } + if args.behavior_version: + output["behavior_version"] = args.behavior_version + + if args.model_type == "supercombo": + if not args.supercombo_onnx: + parser.error("--supercombo-onnx is required for supercombo") + model_path = Path(args.supercombo_onnx) + model_runner = OnnxRunner(model_path) + output["metadata"]["model"] = make_metadata_dict(model_path, model_runner) + validate_metadata(output["metadata"]["model"]) + policy_shapes = output["metadata"]["model"]["input_shapes"] + if 'new_img' in policy_shapes: + if args.image_history_pipeline != IMAGE_HISTORY_IN_POLICY: + parser.error('ONNX-managed history requires --image-history-pipeline policy') + metadata = output['metadata']['model'] + metadata['state_pairs'] = {name: f'next_{name}' for name in policy_shapes + if f'next_{name}' in metadata['output_shapes']} + if not metadata['state_pairs']: + raise ValueError('Stateful supercombo is missing next-state outputs') + metadata['input_dtypes'] = {name: np.dtype(_to_np_dtype(spec.dtype)).name for name, spec in model_runner.graph_inputs.items()} + for name, next_name in metadata['state_pairs'].items(): + if policy_shapes[name] != metadata['output_shapes'][next_name]: + raise ValueError(f'State shape mismatch: {name} -> {next_name}') + frame_skip = 1 + make_policy_queues = partial(make_stateful_input_queues, metadata) + run_policy = make_run_stateful_supercombo(model_runner, metadata) + image_shapes = stateful_image_shapes(metadata) + policy_input_keys = ('packed_npy_inputs', *metadata['state_pairs']) + else: + frame_skip = args.frame_skip or derive_frame_skip(policy_shapes) + make_policy_queues = partial(make_supercombo_input_queues, policy_shapes, frame_skip) + run_policy = make_run_supercombo( + model_runner, output["metadata"], frame_skip, args.image_history_pipeline, + ) + image_shapes = policy_shapes + policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SUPERCOMBO_POLICY_INPUTS + else: + if not args.vision_onnx: + parser.error("--vision-onnx is required for split models") + policy_paths = {} + if args.policy_onnx: + policy_paths["policy"] = args.policy_onnx + if args.off_policy_onnx: + policy_paths["off_policy"] = args.off_policy_onnx + if args.on_policy_onnx: + policy_paths["on_policy"] = args.on_policy_onnx + if args.model_type == "vision_policy" and set(policy_paths) != {"policy"}: + parser.error("vision_policy requires --policy-onnx") + if args.model_type == "vision_multi_policy" and not policy_paths: + parser.error("vision_multi_policy requires at least one policy ONNX") + + vision_path = Path(args.vision_onnx) + resolved_policy_paths = {key: Path(path) for key, path in policy_paths.items()} + vision_runner = OnnxRunner(vision_path) + policy_runners = {key: OnnxRunner(path) for key, path in resolved_policy_paths.items()} + output["metadata"]["vision"] = make_metadata_dict(vision_path, vision_runner) + validate_metadata(output["metadata"]["vision"]) + for key, path in resolved_policy_paths.items(): + output["metadata"][key] = make_metadata_dict(path, policy_runners[key]) + validate_metadata(output["metadata"][key]) + + policy_order = [key for key in ("on_policy", "off_policy", "policy") if key in policy_runners] + output["policy_order"] = policy_order + first_policy_shapes = output["metadata"][policy_order[0]]["input_shapes"] + for key in policy_order[1:]: + if output["metadata"][key]["input_shapes"] != first_policy_shapes: + raise ValueError(f"Policy input shapes differ between {policy_order[0]} and {key}") + frame_skip = args.frame_skip or derive_frame_skip(first_policy_shapes) + make_policy_queues = partial( + make_split_input_queues, + output["metadata"]["vision"]["input_shapes"], + first_policy_shapes, + frame_skip, + ) + run_policy = make_run_split_policy( + vision_runner, policy_runners, output["metadata"], policy_order, frame_skip, + args.image_history_pipeline, + ) + image_shapes = output["metadata"]["vision"]["input_shapes"] + policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SPLIT_POLICY_INPUTS + + output["frame_skip"] = frame_skip + output["policy_input_keys"] = policy_input_keys + warp_input_keys = FAST_WARP_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else LEGACY_WARP_INPUTS + output["warp_input_keys"] = warp_input_keys + run_policy_jit = TinyJit(run_policy, prune=True) + road_key, wide_key = _detect_vision_keys(image_shapes) + if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + make_random_model_inputs = partial( + make_random_images, + keys=["warped"], + shape=(2, 6, *image_shapes[road_key][2:]), + device=WARP_DEV, + ) + else: + make_random_model_inputs = partial( + make_random_images, + keys=[road_key, wide_key], + shape=image_shapes[road_key], + ) + output["run_policy"] = compile_jit( + run_policy_jit, make_random_model_inputs, policy_input_keys, make_policy_queues, + validation_runs=5 if output['metadata'].get('model', {}).get('state_pairs') else 1, + ) + + model_w, model_h = args.model_size + for cam_w, cam_h in args.camera_resolutions: + nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) + warp_enqueue = TinyJit( + make_warp(nv12, model_w, model_h, frame_skip, args.image_history_pipeline), + prune=True, + ) + make_random_warp_inputs = make_random_blob_images( + keys=["frame", "big_frame"], size=nv12.size, device=WARP_DEV, + ) + make_warp_queues = partial(make_warp_input_queues, image_shapes, frame_skip) + output[(cam_w, cam_h)] = compile_jit( + warp_enqueue, make_random_warp_inputs, warp_input_keys, make_warp_queues, + ) + + dump_pickle(output, args.output) + print(f"Saved JITs to {args.output} ({os.path.getsize(args.output) / 1e6:.2f} MB)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/openpilot/starpilot/models/laboratory.py b/openpilot/starpilot/models/laboratory.py new file mode 100644 index 0000000000..e83fff584d --- /dev/null +++ b/openpilot/starpilot/models/laboratory.py @@ -0,0 +1,132 @@ +from pathlib import Path + +from openpilot.starpilot.models.catalog import BY_ID, BUNDLED_CURRENT, GENERATION +from openpilot.starpilot.models.manager import ( + ModelError, ModelManager, artifact_entry, artifact_path, atomic_json, catalog, read_json, state_lock, verified_artifact, +) + + +def configuration(root: Path) -> dict: + default = {"enabled": False, "lateralModel": "", "longitudinalModel": ""} + try: + saved = read_json(root / "laboratory.json", 4096) + validate_configuration(saved) + return saved + except (OSError, ValueError, TypeError): + return default + + +def validate_configuration(value: dict) -> None: + if (not isinstance(value, dict) or set(value) != {"enabled", "lateralModel", "longitudinalModel"} or + type(value["enabled"]) is not bool): + raise ModelError("Invalid laboratory configuration") + for key in ("lateralModel", "longitudinalModel"): + mid = value[key] + if not isinstance(mid, str) or (mid and (mid not in BY_ID or mid == BUNDLED_CURRENT or BY_ID[mid].uses_external_gpu)): + raise ModelError("Choose small models for the laboratory pair") + if value["enabled"] and (not value["lateralModel"] or not value["longitudinalModel"] or + value["lateralModel"] == value["longitudinalModel"]): + raise ModelError("Choose two different small models") + + +class ModelLaboratory: + def __init__(self, manager: ModelManager, *, runtime_supported: bool = False): + self.manager = manager + self.runtime_supported = runtime_supported + + @property + def runtime_unavailable_reason(self) -> str: + return "" if self.runtime_supported else "Pair inference is not available in this build yet." + + def snapshot(self) -> dict: + owner = self.manager + with owner.lock: + entries = catalog(owner.root) + config = configuration(owner.root) + parked, gpu = owner.parked(), owner.gpu_present() + job = owner._job_status() + models = [] + for mid, row in entries.items(): + small = not BY_ID[mid].uses_external_gpu + eligible = small and row.get("model_lab_eligible") is True + artifact = artifact_entry(mid, entries, "amd") if eligible else {} + published = "artifact_sha256" in artifact + installed, checking = owner._snapshot_installed(mid, row, "amd") if published else (False, False) + if not small: + state, reason = "unsupported", "Big models cannot be combined in this build." + elif not eligible: + state, reason = "unsupported", "This model is not available for combining." + elif not published: + state, reason = "unpublished", "A Chestnut version for combining has not been published for this model." + elif checking: + state, reason = "checking", "Checking the downloaded Chestnut version." + elif not installed: + state, reason = "missing", "Download the Chestnut version to prepare this model for combining." + elif not self.runtime_supported: + state, reason = "runtime-unavailable", "Download verified. Running two models together is unavailable in this build." + elif not gpu: + state, reason = "gpu-unavailable", "Download verified. Connect a ready Chestnut to use it." + else: + state, reason = "installed", "Download verified. Choose another compatible model to form a pair." + models.append({"value": mid, "label": row["name"], "version": row["version"], "series": row.get("series", ""), + "small": small, "modelSize": "small" if small else "big", "modelLabEligible": eligible, + "modelLabArtifactAvailable": published, "modelLabArtifactInstalled": installed, "checking": checking, + "modelLabStatus": state, "modelLabReason": reason, + "artifactSize": artifact.get("artifact_size", 0)}) + return {"schemaVersion": 1, "chestnutReady": gpu, "isOnroad": not parked, "configuration": config, + "configurationError": self.runtime_unavailable_reason if config["enabled"] else "", + "runtimeSupported": self.runtime_supported, "runtimeUnavailableReason": self.runtime_unavailable_reason, + "runtime": {"requested": config["enabled"], "active": False, "lateralModel": "", "longitudinalModel": "", + "error": "", "health": "unavailable"}, + "download": {"model": job.get("model", ""), "progress": job.get("progress", ""), + "downloading": job["downloading"], "jobId": job.get("jobId", ""), + "variant": job.get("variant", "standard")}, + "summary": {"catalog": len(models), "eligible": sum(row["modelLabEligible"] for row in models), "ready": sum(row["modelLabArtifactInstalled"] for row in models), + "published": sum(row["modelLabArtifactAvailable"] for row in models), + "declaredSize": sum(row["artifactSize"] for row in models)}, + "models": models, "manifest": {"version": GENERATION}, + "capabilities": {"configure": parked, "download": parked, + "delete": parked, "cancel": True, "refresh": parked}} + + def action(self, action: str, payload: dict) -> dict: + owner = self.manager + if not isinstance(payload, dict): + raise ModelError("Invalid laboratory request") + with owner.lock: + owner._require_parked() + if action == "download": + if set(payload) != {"model"} or not isinstance(payload["model"], str): + raise ModelError("Unknown laboratory model") + return owner.action("download", {"model": payload["model"], "variant": "amd", "allowGpuWithoutGpu": True}) + if action not in ("configure", "delete"): + raise ModelError("Unknown laboratory action") + with state_lock(owner.root): + owner._require_parked() + if owner._job_busy(): + raise ModelError("Wait for the model download to finish") + if action == "configure": + validate_configuration(payload) + if payload["enabled"]: + if not self.runtime_supported: + raise ModelError(self.runtime_unavailable_reason) + if not owner.gpu_present(): + raise ModelError("Connect a firmware-ready Chestnut to enable this pair") + entries = catalog(owner.root) + for role in ("lateralModel", "longitudinalModel"): + if verified_artifact(payload[role], entries, owner.root, "amd") is None: + raise ModelError("Download and verify both eGPU variants first") + owner._require_parked() + atomic_json(owner.root / "laboratory.json", payload) + message = "Model Laboratory configuration saved" + else: + if set(payload) != {"model"} or not isinstance(payload["model"], str): + raise ModelError("Unknown laboratory model") + mid = payload["model"] + destination = artifact_path(mid, owner.root, "amd") + config = configuration(owner.root) + if config["enabled"] and mid in (config["lateralModel"], config["longitudinalModel"]): + raise ModelError("Disable the laboratory pair before deleting either eGPU variant") + destination.unlink(missing_ok=True) + owner.checked.pop(f"{mid}:amd", None) + message = "eGPU variant deleted" + return {**self.snapshot(), "message": message} diff --git a/openpilot/starpilot/models/laboratory_outputs.py b/openpilot/starpilot/models/laboratory_outputs.py new file mode 100644 index 0000000000..02e5c0badb --- /dev/null +++ b/openpilot/starpilot/models/laboratory_outputs.py @@ -0,0 +1,60 @@ +from typing import Any + +import numpy as np + + +LATERAL_PLAN_COLUMNS = (1, 4, 7, 11, 14) +LATERAL_OUTPUT_KEYS = ( + "desired_curvature", "desired_curvature_stds", "lat_planner_solution", "lat_planner_solution_stds", + "lane_lines", "lane_lines_stds", "lane_lines_prob", "road_edges", "road_edges_stds", "desire_state", "desire_pred", +) +CURRENT_FRAME_OUTPUT_KEYS = ( + "pose", "pose_stds", "wide_from_device_euler", "wide_from_device_euler_stds", "road_transform", "road_transform_stds", +) + + +def _validate_outputs(outputs: dict[str, np.ndarray]) -> None: + for key, value in outputs.items(): + if not isinstance(value, np.ndarray) or not np.isfinite(value).all(): + raise ValueError(f"Model Laboratory requires finite normalized arrays: {key}") + + +def _merge_plan(lateral: np.ndarray, longitudinal: np.ndarray) -> np.ndarray: + if lateral.shape != longitudinal.shape or lateral.ndim < 2 or lateral.shape[-1] < 15: + raise ValueError(f"Model Laboratory incompatible plan shapes: {lateral.shape}, {longitudinal.shape}") + merged = longitudinal.copy() + merged[..., LATERAL_PLAN_COLUMNS] = lateral[..., LATERAL_PLAN_COLUMNS] + return merged + + +def compose_model_outputs(lateral_output: dict[str, np.ndarray], longitudinal_output: dict[str, np.ndarray], + current_frame_output: dict[str, np.ndarray] | None = None) -> dict[str, np.ndarray]: + """Own lateral plan/geometry and longitudinal extras; decode actions separately per generation.""" + frame_output = longitudinal_output if current_frame_output is None else current_frame_output + for outputs in (lateral_output, longitudinal_output, frame_output): + _validate_outputs(outputs) + if "plan" not in lateral_output or "plan" not in longitudinal_output: + raise ValueError("Model Laboratory requires a plan from both models") + if ("plan_stds" in lateral_output) != ("plan_stds" in longitudinal_output): + raise ValueError("Model Laboratory requires matching plan uncertainty outputs") + + composed = {key: value.copy() for key, value in longitudinal_output.items() if key not in ("action", "action_stds")} + composed["plan"] = _merge_plan(lateral_output["plan"], longitudinal_output["plan"]) + if "plan_stds" in lateral_output: + composed["plan_stds"] = _merge_plan(lateral_output["plan_stds"], longitudinal_output["plan_stds"]) + for keys, outputs in ((LATERAL_OUTPUT_KEYS, lateral_output), (CURRENT_FRAME_OUTPUT_KEYS, frame_output)): + for key in keys: + if key in outputs: + composed[key] = outputs[key].copy() + else: + composed.pop(key, None) + return composed + + +def hybrid_action_values(lateral_action: Any, longitudinal_action: Any) -> dict[str, Any]: + """Accept already decoded actions, never raw generation-dependent action tensors.""" + curvature = float(lateral_action.desiredCurvature) + acceleration = float(longitudinal_action.desiredAcceleration) + if not np.isfinite((curvature, acceleration)).all(): + raise ValueError("Model Laboratory requires finite decoded actions") + return {"desiredCurvature": curvature, "desiredAcceleration": acceleration, "shouldStop": bool(longitudinal_action.shouldStop)} diff --git a/openpilot/starpilot/models/laboratory_runner.py b/openpilot/starpilot/models/laboratory_runner.py new file mode 100644 index 0000000000..fb407bfbb8 --- /dev/null +++ b/openpilot/starpilot/models/laboratory_runner.py @@ -0,0 +1,81 @@ +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from openpilot.cereal import log +from openpilot.starpilot.models.laboratory import validate_configuration +from openpilot.starpilot.models.laboratory_outputs import compose_model_outputs, hybrid_action_values +from openpilot.starpilot.models.manager import ROOT, artifact_entry, artifact_path, catalog +from openpilot.starpilot.models.receipt import PreparedArtifact +from openpilot.starpilot.models.runner import CatalogModelState, action_from_outputs, load_verified_model + + +@dataclass(frozen=True) +class LaboratoryRole: + model_id: str + behavior_version: str + artifact: PreparedArtifact + + +@dataclass(frozen=True) +class LaboratoryFrame: + outputs: dict[str, np.ndarray] + action: log.ModelDataV2.Action + + +class LaboratoryPair: + def __init__(self, lateral: CatalogModelState, longitudinal: CatalogModelState, + lateral_artifact: PreparedArtifact, longitudinal_artifact: PreparedArtifact): + if lateral is longitudinal or lateral.model_id == longitudinal.model_id: + raise ValueError("Model Laboratory requires two distinct models") + if not lateral.chestnut or not longitudinal.chestnut: + raise ValueError("Model Laboratory requires two AMD variants") + if lateral.camera_warp_descriptor is None or lateral.camera_warp_descriptor != longitudinal.camera_warp_descriptor: + raise ValueError("Model Laboratory requires compatible policy-history camera warps") + self.lateral = lateral + self.longitudinal = longitudinal + self.lateral_role = LaboratoryRole(lateral.model_id, lateral.behavior_version, lateral_artifact) + self.longitudinal_role = LaboratoryRole(longitudinal.model_id, longitudinal.behavior_version, longitudinal_artifact) + self.failed = False + + def run_frame(self, bufs: dict, transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], + previous: log.ModelDataV2.Action, lat_action_t: float, long_action_t: float, v_ego: float, + lat_smooth_seconds: float = 0.0, long_smooth_seconds: float = 0.3, + after_enqueue: Callable[[], None] | None = None) -> LaboratoryFrame: + """Run both roles before releasing the camera frame; any failure requires fallback.""" + if self.failed: + raise RuntimeError("Model Laboratory pair failed; load a fallback before continuing") + try: + lateral_output = self.lateral.run(bufs, transforms, inputs) + shared_warp = self.lateral.last_warp + if shared_warp is None: + raise ValueError("Model Laboratory camera warp unavailable") + longitudinal_output = self.longitudinal.run(bufs, transforms, inputs, after_enqueue, shared_warp=shared_warp) + actions = [action_from_outputs(output, model.behavior_version, previous, lat_action_t, long_action_t, v_ego, + lat_smooth_seconds, long_smooth_seconds) + for model, output in ((self.lateral, lateral_output), (self.longitudinal, longitudinal_output))] + action = log.ModelDataV2.Action(**hybrid_action_values(*actions)) + return LaboratoryFrame(compose_model_outputs(lateral_output, longitudinal_output), action) + except Exception: + self.failed = True + self.lateral.last_warp = self.longitudinal.last_warp = None + raise + + +def load_laboratory_pair(cam_w: int, cam_h: int, config: dict, *, root: Path = ROOT) -> LaboratoryPair: + validate_configuration(config) + if not config["enabled"]: + raise ValueError("Model Laboratory is disabled") + entries = catalog(root) + requests = [] + for role in ("lateralModel", "longitudinalModel"): + mid = config[role] + artifact = artifact_entry(mid, entries, "amd") + if "artifact_sha256" not in artifact: + raise ValueError("Model Laboratory requires published AMD variants for both models") + requests.append((mid, entries[mid]["version"], artifact["artifact_sha256"])) + loaded = [load_verified_model(cam_w, cam_h, artifact_path(mid, root, "amd"), version, True, mid, digest, root=root) + for mid, version, digest in requests] + return LaboratoryPair(loaded[0][0], loaded[1][0], loaded[0][1], loaded[1][1]) diff --git a/openpilot/starpilot/models/manager.py b/openpilot/starpilot/models/manager.py new file mode 100644 index 0000000000..cb4c92ec92 --- /dev/null +++ b/openpilot/starpilot/models/manager.py @@ -0,0 +1,642 @@ +"""Versioned model downloads and next-start Small/Chestnut selections.""" + +from __future__ import annotations + +from contextlib import contextmanager +from collections import deque +from dataclasses import dataclass +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import random +import shutil +import tempfile +import threading +import uuid +from collections.abc import Callable +from urllib.request import urlopen + +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, BUNDLED_CURRENT, BY_ID, CATALOG_PATH, COMPILER_REVISION, GENERATION + +RESOURCE_URL = "https://huggingface.co/buckets/StarPilot-Driving/StarPilot-Resources/resolve" +ROOT = Path("/data/models") / GENERATION +MAX_MANIFEST = 2 * 1024 * 1024 +MAX_ARTIFACT = 2 * 1024 * 1024 * 1024 +SHA = re.compile(r"[0-9a-f]{64}\Z") +MODEL_RUNNER_REVISION = 2 +RUNTIME_ARTIFACT_KEYS = {"min_runner_revision", "artifact_format", "artifact_sha256", "artifact_size", + "artifact_chunk_count", "conversion_provenance", "validation"} + + +class ModelError(ValueError): + pass + + +def read_json(path: Path, limit: int = MAX_MANIFEST) -> dict: + with path.open("rb") as stream: + raw = stream.read(limit + 1) + if len(raw) > limit: + raise ModelError("Model document is too large") + data = json.loads(raw) + if not isinstance(data, dict): + raise ModelError("Invalid model document") + return data + + +def atomic_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=".model-", dir=path.parent) + try: + with os.fdopen(fd, "w") as out: + json.dump(value, out, ensure_ascii=False, allow_nan=False, sort_keys=True) + out.flush() + os.fsync(out.fileno()) + os.replace(name, path) + finally: + Path(name).unlink(missing_ok=True) + + +@contextmanager +def state_lock(root: Path): + root.mkdir(parents=True, exist_ok=True) + with (root / ".state.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +def validate_manifest(data: dict) -> dict[str, dict]: + if (data.get("generation") != GENERATION or data.get("compiler_revision") != COMPILER_REVISION or + data.get("artifact_abi") != ARTIFACT_ABI or not isinstance(data.get("models"), list)): + raise ModelError("The model catalog targets a different runtime") + rows = {} + for entry in data["models"]: + if not isinstance(entry, dict): + raise ModelError("Invalid model entry") + model_id = entry.get("id") + if not isinstance(model_id, str) or model_id not in BY_ID or model_id == BUNDLED_CURRENT or model_id in rows: + raise ModelError("Unknown or duplicate model") + known = BY_ID[model_id] + if entry.get("version") != known.version or bool(entry.get("uses_external_gpu", False)) != known.uses_external_gpu: + raise ModelError("Model behavior or hardware changed") + variants = entry.get("accelerator_artifacts", {}) + if not isinstance(variants, dict) or set(variants) - {"amd"}: + raise ModelError("Unknown model accelerator") + amd = variants.get("amd") + if amd is not None and (not isinstance(amd, dict) or known.uses_external_gpu or + entry.get("model_lab_eligible") is not True or amd.get("execution_device") != "AMD" or + amd.get("compiler_revision") != COMPILER_REVISION): + raise ModelError("Invalid laboratory artifact hardware or runtime") + runtime_artifacts = entry.get("runtime_artifacts", []) + if not isinstance(runtime_artifacts, list): + raise ModelError("Invalid runtime artifacts") + revisions = set() + for artifact in runtime_artifacts: + if (not isinstance(artifact, dict) or set(artifact) - RUNTIME_ARTIFACT_KEYS or + set(artifact) & {"artifact_format", "artifact_sha256", "artifact_size", "artifact_chunk_count"} != + {"artifact_format", "artifact_sha256", "artifact_size", "artifact_chunk_count"} or + type(artifact.get("min_runner_revision")) is not int or artifact["min_runner_revision"] < 2 or + artifact["min_runner_revision"] in revisions): + raise ModelError("Invalid runtime artifact variant") + revisions.add(artifact["min_runner_revision"]) + for artifact in (entry, *variants.values(), *runtime_artifacts): + if "artifact_sha256" not in artifact: + continue + size, chunks = artifact.get("artifact_size"), artifact.get("artifact_chunk_count") + if (not isinstance(artifact["artifact_sha256"], str) or not SHA.fullmatch(artifact["artifact_sha256"]) or + type(size) is not int or not 0 < size <= MAX_ARTIFACT or type(chunks) is not int or not 0 <= chunks <= 99 or + artifact.get("artifact_format") != ARTIFACT_ABI): + raise ModelError("Invalid compiled artifact metadata") + row = dict(entry) + compatible = [artifact for artifact in runtime_artifacts if artifact["min_runner_revision"] <= MODEL_RUNNER_REVISION] + if compatible: + selected = max(compatible, key=lambda artifact: artifact["min_runner_revision"]) + row.update({key: value for key, value in selected.items() if key != "min_runner_revision"}) + rows[model_id] = row + return rows + + +def catalog(root: Path = ROOT) -> dict[str, dict]: + base = validate_manifest(read_json(CATALOG_PATH)) + try: + base.update(validate_manifest(read_json(root / "catalog.json"))) + except (OSError, ValueError, TypeError): + pass + return base + + +def preferences(root: Path = ROOT) -> dict: + default = {"small": BUNDLED_CURRENT, "big": BUNDLED_CURRENT, "userFavorites": [], "sortMode": "name", + "randomizer": False, "blacklistedModels": []} + try: + saved = read_json(root / "preferences.json", 16384) + for profile in ("small", "big"): + mid = saved.get(profile) + if isinstance(mid, str) and (mid == BUNDLED_CURRENT or (profile == "big" and mid == "") or ( + mid in BY_ID and BY_ID[mid].uses_external_gpu == (profile == "big"))): + default[profile] = mid + for key in ("userFavorites", "blacklistedModels"): + if isinstance(saved.get(key), list): + default[key] = list(dict.fromkeys(x for x in saved[key] if isinstance(x, str) and x in BY_ID)) + if type(saved.get("randomizer")) is bool: + default["randomizer"] = saved["randomizer"] + if saved.get("sortMode") in ("name", "date", "date_oldest", "community", "series", "favorites", "released"): + default["sortMode"] = saved["sortMode"] + except (OSError, ValueError, TypeError): + pass + return default + + +def artifact_path(model_id: str, root: Path = ROOT, variant: str = "standard") -> Path: + if not isinstance(model_id, str) or model_id not in BY_ID or model_id == BUNDLED_CURRENT: + raise ModelError("Unknown downloadable model") + if variant not in ("standard", "amd") or (variant == "amd" and BY_ID[model_id].uses_external_gpu): + raise ModelError("Unknown model artifact variant") + suffix = "amd_" if variant == "amd" else "" + return root / model_id / f"{model_id}_driving_{suffix}tinygrad.pkl" + + +def artifact_entry(model_id: str, entries: dict[str, dict], variant: str = "standard") -> dict: + row = entries.get(model_id, {}) + if variant == "standard": + return row + if variant == "amd" and not row.get("uses_external_gpu") and row.get("model_lab_eligible") is True: + return row.get("accelerator_artifacts", {}).get("amd", {}) + return {} + + +def verified_artifact(model_id: str, entries: dict[str, dict], root: Path = ROOT, variant: str = "standard") -> Path | None: + row = artifact_entry(model_id, entries, variant) + if "artifact_sha256" not in row: + return None + path = artifact_path(model_id, root, variant) + try: + if path.is_symlink() or not path.is_file() or path.stat().st_size != row["artifact_size"]: + return None + with path.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + return path if digest == row["artifact_sha256"] else None + except OSError: + return None + + +@dataclass(frozen=True) +class RuntimeSelection: + small_id: str = BUNDLED_CURRENT + small_path: Path | None = None + small_version: str = "current" + big_id: str = BUNDLED_CURRENT + big_path: Path | None = None + big_version: str = "current" + allow_big: bool = True + small_sha256: str | None = None + big_sha256: str | None = None + + +def requested_runtime_id(chestnut_available: bool, *, root: Path = ROOT) -> str: + prefs = preferences(root) + return prefs["big"] if chestnut_available and prefs["big"] else prefs["small"] + + +def randomize_next_start(chestnut_available: bool, *, root: Path = ROOT, chooser=random.choice) -> dict: + with state_lock(root): + prefs = preferences(root) + if not prefs["randomizer"]: + return prefs + profile = "big" if chestnut_available and prefs["big"] else "small" + entries = catalog(root) + blocked = set(prefs["blacklistedModels"]) + choices = [mid for mid in entries if mid not in blocked and + BY_ID[mid].uses_external_gpu == (profile == "big") and verified_artifact(mid, entries, root) is not None] + if profile == "small" and BUNDLED_CURRENT not in blocked: + choices.append(BUNDLED_CURRENT) + alternatives = [mid for mid in choices if mid != prefs[profile]] + prefs[profile] = chooser(alternatives or choices) if choices else ("" if profile == "big" else BUNDLED_CURRENT) + atomic_json(root / "preferences.json", prefs) + return prefs + + +def resolve_runtime(chestnut_available: bool, *, root: Path = ROOT, randomize: bool = False) -> RuntimeSelection: + prefs = randomize_next_start(chestnut_available, root=root) if randomize else preferences(root) + entries = catalog(root) + small, big = prefs["small"], prefs["big"] + small_path = verified_artifact(small, entries, root) if small != BUNDLED_CURRENT else None + if small_path is None: + small = BUNDLED_CURRENT + big_path = verified_artifact(big, entries, root) if big not in ("", BUNDLED_CURRENT) and chestnut_available else None + allow_big = chestnut_available and (big == BUNDLED_CURRENT or big_path is not None) + return RuntimeSelection(small, small_path, BY_ID[small].version, big or BUNDLED_CURRENT, big_path, + BY_ID[big].version if big in BY_ID else "current", allow_big, + entries[small]["artifact_sha256"] if small_path is not None else None, + entries[big]["artifact_sha256"] if big_path is not None else None) + + +class ModelManager: + def __init__(self, *, root: Path = ROOT, parked: Callable[[], bool] | None = None, + gpu_present: Callable[[], bool] | None = None, opener=urlopen): + self.root, self.opener = root, opener + self.parked = parked or (lambda: False) + if gpu_present is None: + from openpilot.selfdrive.modeld.helpers import chestnut_present + gpu_present = chestnut_present + self.gpu_present = gpu_present + self.lock = threading.RLock() + self.cancelled = threading.Event() + self.worker: threading.Thread | None = None + self.progress = "" + self.downloading = "" + self.download_all = False + self.job_id = "" + self.checked: dict[str, tuple[tuple, bool]] = {} + self.verify_queue: deque[tuple[str, dict, tuple, str]] = deque() + self.verifying: dict[str, tuple] = {} + self.verify_worker: threading.Thread | None = None + self.verify_closed = False + + def _read_job(self) -> dict: + try: + job = read_json(self.root / ".download-job.json", 16384) + if (job.get("schemaVersion") == 1 and isinstance(job.get("jobId"), str) and + re.fullmatch(r"[0-9a-f]{32}", job["jobId"]) and job.get("state") in ("running", "completed", "failed", "interrupted") and + job.get("model") in ("", *BY_ID) and isinstance(job.get("progress"), str) and + type(job.get("downloadAll")) is bool and type(job.get("cancelRequested")) is bool and + job.get("variant", "standard") in ("standard", "amd")): + return job + except (OSError, ValueError, TypeError): + pass + return {} + + def _job_busy(self) -> bool: + with (self.root / ".download.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return True + return False + + def _job_status(self) -> dict: + with state_lock(self.root): + job, busy = self._read_job(), self._job_busy() + if not busy and job.get("state") == "running": + job.update(state="interrupted", progress="Previous model download was interrupted", model="", downloadAll=False, cancelRequested=False) + atomic_json(self.root / ".download-job.json", job) + return {**job, "downloading": busy} + + def _publish_job(self, *, progress: str | None = None, model: str | None = None) -> None: + with state_lock(self.root): + job = self._read_job() + if job.get("jobId") != self.job_id or job.get("state") != "running": + raise ModelError("Download job state is unavailable") + if progress is not None: + self.progress = progress[:1000] + job["progress"] = self.progress + if model is not None: + self.downloading = model + job["model"] = model + atomic_json(self.root / ".download-job.json", job) + + def _check_cancelled(self) -> None: + if self.cancelled.is_set(): + raise ModelError("Download cancelled") + job = self._read_job() + if job.get("jobId") != self.job_id or job.get("state") != "running": + raise ModelError("Download job state is unavailable") + if job["cancelRequested"]: + raise ModelError("Download cancelled") + + def _finish_job(self, lock, failed: bool) -> None: + self.downloading, self.download_all = "", False + try: + with state_lock(self.root): + try: + job = self._read_job() + if job.get("jobId") == self.job_id: + job.update(state="failed" if failed else "completed", progress=self.progress[:1000], model="", downloadAll=False) + atomic_json(self.root / ".download-job.json", job) + finally: + lock.close() + finally: + if not lock.closed: + lock.close() + + def _signature(self, mid: str, row: dict, variant: str = "standard") -> tuple | None: + path = artifact_path(mid, self.root, variant) + try: + st = path.stat() + return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns, + row["artifact_sha256"], row["artifact_size"], path.is_symlink()) + except OSError: + return None + + def _installed(self, mid: str, entries: dict, variant: str = "standard") -> bool: + if mid == BUNDLED_CURRENT: + return True + row = artifact_entry(mid, entries, variant) + if "artifact_sha256" not in row: + return False + signature = self._signature(mid, row, variant) + if signature is None: + return False + key = f"{mid}:{variant}" + if self.checked.get(key, (None,))[0] != signature: + self.checked[key] = (signature, verified_artifact(mid, entries, self.root, variant) is not None) + return self.checked[key][1] + + def _snapshot_installed(self, mid: str, row: dict, variant: str = "standard") -> tuple[bool, bool]: + artifact = artifact_entry(mid, {mid: row}, variant) + if "artifact_sha256" not in artifact: + return False, False + signature = self._signature(mid, artifact, variant) + if signature is None or signature[-1] or signature[2] != artifact["artifact_size"]: + return False, False + key = f"{mid}:{variant}" + checked = self.checked.get(key) + if checked is not None and checked[0] == signature: + return checked[1], False + if not self.verify_closed and self.verifying.get(key) != signature: + self.verifying[key] = signature + self.verify_queue.append((mid, row, signature, variant)) + if self.verify_worker is None: + self.verify_worker = threading.Thread(target=self._verify_pending, daemon=True) + self.verify_worker.start() + return False, True + + def _verify_pending(self) -> None: + while True: + with self.lock: + if self.verify_closed or not self.verify_queue: + self.verify_worker = None + return + mid, row, signature, variant = self.verify_queue.popleft() + key = f"{mid}:{variant}" + if self.verifying.get(key) != signature: + continue + valid = verified_artifact(mid, {mid: row}, self.root, variant) is not None + with self.lock: + if self.verifying.get(key) == signature: + self.verifying.pop(key) + if self._signature(mid, artifact_entry(mid, {mid: row}, variant), variant) == signature: + self.checked[key] = (signature, valid) + + def snapshot(self) -> dict: + with self.lock: + entries, prefs = catalog(self.root), preferences(self.root) + parked, gpu = self.parked(), self.gpu_present() + job = self._job_status() + bundled_dir = Path(__file__).resolve().parents[2] / "selfdrive/modeld/models" + bundled_big = all((bundled_dir / name).is_file() for name in + ("big_driving_tinygrad.pkl", "big_driving_warp_1344x760_tinygrad.pkl", "big_driving_warp_1928x1208_tinygrad.pkl")) + models = [{"value": BUNDLED_CURRENT, "label": "Bundled driving model", "series": "openpilot", "version": "current", + "installed": True, "builtin": True, "selectable": True, "requiresGpu": False, "gpuAvailable": True, + "profiles": ["small", "big"] if bundled_big else ["small"], "bundledBigAvailable": bundled_big, + "communityFavorite": False, "userFavorite": BUNDLED_CURRENT in prefs["userFavorites"], + "blacklisted": BUNDLED_CURRENT in prefs["blacklistedModels"], "unavailableReason": ""}] + for mid, row in entries.items(): + installed, checking = self._snapshot_installed(mid, row) + requires_gpu = bool(row.get("uses_external_gpu")) + published = "artifact_sha256" in row + models.append({"value": mid, "label": row["name"], "version": row["version"], "series": row.get("series", ""), + "released": row.get("released", ""), "communityFavorite": bool(row.get("community_favorite")), + "userFavorite": mid in prefs["userFavorites"], "blacklisted": mid in prefs["blacklistedModels"], + "installed": installed, "checking": checking, + "builtin": False, + "requiresGpu": requires_gpu, "gpuAvailable": gpu or not requires_gpu, "small": not requires_gpu, + "selectable": installed, "downloadAvailable": published, "artifactSize": row.get("artifact_size", 0), + "unavailableReason": "" if installed else "Checking artifact" if checking else + "Download required" if published else f"{GENERATION} download not published"}) + count = sum(bool(row["installed"]) for row in models) + return {"schemaVersion": 1, "models": models, "activeSmallModel": prefs["small"], "activeBigModel": prefs["big"], + "currentModel": prefs["big"] if gpu and prefs["big"] else prefs["small"], "userFavorites": prefs["userFavorites"], + "sortMode": prefs["sortMode"], "randomizer": prefs["randomizer"], "blacklistedModels": prefs["blacklistedModels"], + "summary": {"total": len(models), "installed": count, "missing": len(models)-count}, + "jobId": job.get("jobId", ""), "cancelRequested": job.get("cancelRequested", False), + "downloadVariant": job.get("variant", "standard"), + "modelToDownload": job.get("model", ""), "downloadAll": job.get("downloadAll", False), + "downloading": job["downloading"], "progress": job.get("progress", ""), + "isOnroad": not parked, "gpuAvailable": gpu, + "capabilities": {"select": parked, "download": parked, "downloadAll": parked, "cancel": True, + "refresh": parked, "favorites": True, "delete": parked, "randomizer": parked, "exclusions": parked}} + + def _require_parked(self) -> None: + if not self.parked(): + raise ModelError("Turn the vehicle off before changing driving models") + + def action(self, action: str, payload: dict) -> dict: + if not isinstance(payload, dict): + raise ModelError("Invalid model request") + with self.lock: + if action == "cancel": + if set(payload) - {"jobId"} or ("jobId" in payload and not isinstance(payload["jobId"], str)): + raise ModelError("Invalid cancel request") + with state_lock(self.root): + job, busy = self._read_job(), self._job_busy() + if "jobId" in payload and payload["jobId"] != job.get("jobId"): + raise ModelError("This download job is no longer running") + if not busy: + return {"message": "No model download is running"} + if job.get("state") != "running": + raise ModelError("Download job state is unavailable") + job["cancelRequested"] = True + atomic_json(self.root / ".download-job.json", job) + if job["jobId"] == self.job_id: + self.cancelled.set() + return {"message": "Download cancellation requested"} + if action == "preferences": + if not payload or set(payload) - {"userFavorites", "sortMode", "randomizer", "blacklistedModels"}: + raise ModelError("Invalid model preferences") + for key in ("userFavorites", "blacklistedModels"): + values = payload.get(key, []) + if not isinstance(values, list) or len(values) > len(BY_ID) or any(not isinstance(x, str) or x not in BY_ID for x in values): + raise ModelError("Unknown model preference") + changes_selection = bool(set(payload) & {"randomizer", "blacklistedModels"}) + if "randomizer" in payload and type(payload["randomizer"]) is not bool: + raise ModelError("Invalid randomizer preference") + if changes_selection: + self._require_parked() + if "sortMode" in payload and payload["sortMode"] not in ("name", "date", "date_oldest", "community", "series", "favorites", "released"): + raise ModelError("Invalid sort mode") + with state_lock(self.root): + if changes_selection: + self._require_parked() + saved = preferences(self.root) + saved.update(payload) + atomic_json(self.root / "preferences.json", saved) + return {"message": "Model preferences saved"} + self._require_parked() + if action in ("download", "download_all", "refresh_manifest"): + if self.worker is not None and self.worker.is_alive(): + raise ModelError("A model download is already running") + if action == "download" and (set(payload) - {"model", "allowGpuWithoutGpu", "variant"} or + not isinstance(payload.get("model"), str) or payload["model"] not in BY_ID): + raise ModelError("Unknown model download") + if action != "download" and set(payload) - {"allowGpuWithoutGpu"}: + raise ModelError("Invalid download request") + if "allowGpuWithoutGpu" in payload and type(payload["allowGpuWithoutGpu"]) is not bool: + raise ModelError("Invalid GPU download preference") + variant = payload.get("variant", "standard") + if variant not in ("standard", "amd"): + raise ModelError("Unknown model artifact variant") + if variant == "amd" and not artifact_entry(payload["model"], catalog(self.root), "amd").get("artifact_sha256"): + raise ModelError("This model has no eGPU variant for this software version yet") + if (action == "download" and BY_ID[payload["model"]].uses_external_gpu and + not self.gpu_present() and not payload.get("allowGpuWithoutGpu")): + raise ModelError("Connect Chestnut or choose to download for later") + with state_lock(self.root): + self._require_parked() + lock = (self.root / ".download.lock").open("a") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + lock.close() + raise ModelError("Another model manager is downloading") from error + except OSError: + lock.close() + raise + self.cancelled.clear() + self.job_id = uuid.uuid4().hex + self.progress = "Refreshing catalog" if action == "refresh_manifest" else "Starting download" + self.downloading = payload.get("model", "") + self.download_all = action == "download_all" + try: + atomic_json(self.root / ".download-job.json", { + "schemaVersion": 1, "jobId": self.job_id, "state": "running", "model": self.downloading, + "progress": self.progress, "downloadAll": self.download_all, "cancelRequested": False, "variant": variant, + }) + except BaseException: + lock.close() + raise + message = self.progress + self.worker = threading.Thread(target=self._work, args=(action, dict(payload), lock), daemon=True) + try: + self.worker.start() + except BaseException: + self.progress = "Model download could not start" + self._finish_job(lock, True) + raise + return {"message": message} + if action == "active": + if self._job_status()["downloading"]: + raise ModelError("Wait for the model download to finish before selecting a model") + if (set(payload) != {"profile", "model"} or payload["profile"] not in ("small", "big") or + not isinstance(payload["model"], str)): + raise ModelError("Invalid model selection") + if preferences(self.root)["randomizer"]: + raise ModelError("Turn Model Randomizer off before choosing a model") + mid, profile = payload["model"], payload["profile"] + if mid != BUNDLED_CURRENT and not (profile == "big" and mid == ""): + if mid not in BY_ID or BY_ID[mid].uses_external_gpu != (profile == "big"): + raise ModelError("Model does not match the selected hardware profile") + if verified_artifact(mid, catalog(self.root), self.root) is None: + raise ModelError("Download and verify this model first") + with state_lock(self.root): + self._require_parked() + saved = preferences(self.root) + if saved["randomizer"]: + raise ModelError("Turn Model Randomizer off before choosing a model") + if self._job_busy(): + raise ModelError("Wait for the model download to finish before selecting a model") + saved[profile] = mid + atomic_json(self.root / "preferences.json", saved) + return {"message": "Model selected for the next drive"} + if action == "delete": + if set(payload) != {"model"}: + raise ModelError("Invalid delete request") + mid = payload["model"] + path = artifact_path(mid, self.root) + with state_lock(self.root): + saved = preferences(self.root) + if self._job_busy(): + raise ModelError("Wait for the model download to finish before deleting a model") + if mid in (saved["small"], saved["big"]): + raise ModelError("Select another model before deleting this one") + self._require_parked() + path.unlink(missing_ok=True) + self.checked.pop(f"{mid}:standard", None) + return {"message": "Downloaded model deleted"} + raise ModelError("Unknown model action") + + def _refresh(self) -> None: + url = f"{RESOURCE_URL}/manifests/model_names_{GENERATION}.json" + with self.opener(url, timeout=30) as response: + raw = response.read(MAX_MANIFEST + 1) + if len(raw) > MAX_MANIFEST: + raise ModelError("Model catalog is too large") + data = json.loads(raw) + validate_manifest(data) + atomic_json(self.root / "catalog.json", data) + + def _work(self, action: str, payload: dict, lock) -> None: + failed = True + try: + self._check_cancelled() + self._refresh() + self._check_cancelled() + entries = catalog(self.root) + variant = payload.get("variant", "standard") + ids = [payload["model"]] if action == "download" else [mid for mid, row in entries.items() + if "artifact_sha256" in row and (not row.get("uses_external_gpu") or self.gpu_present() or payload.get("allowGpuWithoutGpu"))] + if action != "refresh_manifest": + for mid in ids: + self._check_cancelled() + if not self._installed(mid, entries, variant): + self._publish_job(model=mid) + self._download(mid, artifact_entry(mid, entries, variant), variant) + self._check_cancelled() + self.progress = "Catalog updated" if action == "refresh_manifest" else "Downloaded!" + failed = False + except (OSError, ValueError, KeyError, TimeoutError) as error: + failed = True + self.progress = str(error) or "Model download failed" + finally: + self._finish_job(lock, failed) + + def _download(self, mid: str, entry: dict, variant: str = "standard") -> None: + if "artifact_sha256" not in entry: + raise ModelError("This model is not available for this software version yet") + destination = artifact_path(mid, self.root, variant) + destination.parent.mkdir(parents=True, exist_ok=True) + size, chunks = entry["artifact_size"], entry["artifact_chunk_count"] + if shutil.disk_usage(self.root).free < size + 256 * 1024 * 1024: + raise ModelError("Not enough storage for this model") + filename = destination.name + pieces = [filename] if chunks == 0 else [f"{filename}.chunk{i:02d}of{chunks:02d}" for i in range(1, chunks+1)] + fd, temp = tempfile.mkstemp(prefix=".download-", dir=destination.parent) + try: + digest, received = hashlib.sha256(), 0 + with os.fdopen(fd, "wb") as out: + for piece in pieces: + with self.opener(f"{RESOURCE_URL}/models/{GENERATION}/{mid}/{piece}", timeout=30) as response: + while data := response.read(1024 * 1024): + self._check_cancelled() + received += len(data) + if received > size: + raise ModelError("Model exceeds its declared size") + out.write(data) + digest.update(data) + progress = f"{BY_ID[mid].name}: {received * 100 // size}%" + if progress != self.progress: + self._publish_job(progress=progress) + out.flush() + os.fsync(out.fileno()) + if received != size or digest.hexdigest() != entry["artifact_sha256"]: + raise ModelError("Model checksum did not match; download discarded") + with state_lock(self.root): + self._require_parked() + self._check_cancelled() + os.replace(temp, destination) + self.checked.pop(f"{mid}:{variant}", None) + finally: + Path(temp).unlink(missing_ok=True) + + def close(self) -> None: + self.cancelled.set() + with self.lock: + self.verify_closed = True + self.verify_queue.clear() + verify_worker = self.verify_worker + if verify_worker is not None: + verify_worker.join(timeout=2) + if self.worker is not None: + self.worker.join(timeout=2) diff --git a/openpilot/starpilot/models/parser.py b/openpilot/starpilot/models/parser.py new file mode 100644 index 0000000000..b2131b8b45 --- /dev/null +++ b/openpilot/starpilot/models/parser.py @@ -0,0 +1,87 @@ +"""Output parsing for the v8–v16 model catalog; bundled parsing stays upstream.""" + +import numpy as np +from openpilot.selfdrive.modeld.constants import ModelConstants +from openpilot.selfdrive.modeld.parse_model_outputs import Parser as CurrentParser + + +class Parser(CurrentParser): + def split_outputs(self, outs: dict[str, np.ndarray]) -> None: + if 'lead' in outs: + if outs['lead'].shape[1] == 2 * ModelConstants.LEAD_MHP_SELECTION * ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH: + self.parse_mdn('lead', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH)) + else: + self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION, + out_shape=(ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH)) + if 'plan' in outs: + plan_mhp = outs['plan'].shape[1] != 2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + plan_in_N, plan_out_N = (ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) + self.parse_mdn('plan', outs, in_N=plan_in_N, out_N=plan_out_N, + out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + if 'planplus' in outs: + self.parse_mdn('planplus', outs, in_N=plan_in_N, out_N=plan_out_N, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + elif 'planplus' in outs: + planplus_mhp = outs['planplus'].shape[1] != 2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + planplus_in_N, planplus_out_N = (ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) if planplus_mhp else (0, 0) + self.parse_mdn('planplus', outs, in_N=planplus_in_N, out_N=planplus_out_N, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + if 'lane_lines' in outs: + self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + if 'road_edges' in outs: + self.parse_mdn('road_edges', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.NUM_ROAD_EDGES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + if 'sim_pose' in outs: + self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + if 'lane_lines_prob' in outs: + self.parse_binary_crossentropy('lane_lines_prob', outs) + if 'lead_prob' in outs: + self.parse_binary_crossentropy('lead_prob', outs) + + def parse_off_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.split_outputs(outs) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs + + def parse_vision_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) + self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.split_outputs(outs) + self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN, ModelConstants.DESIRE_PRED_WIDTH)) + self.parse_binary_crossentropy('meta', outs) + return outs + + def parse_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.split_outputs(outs) + if 'action' in outs: + self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(ModelConstants.ACTION_WIDTH,)) + if 'lat_planner_solution' in outs: + self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N, ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) + if 'desired_curvature' in outs: + self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs + + def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + # Combined supercombo outputs contain both vision and policy slices in the + # same dictionary. Parse shared MDN outputs once; the split runtime keeps + # using parse_vision_outputs/parse_policy_outputs independently. + self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) + self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.split_outputs(outs) + self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN, ModelConstants.DESIRE_PRED_WIDTH)) + self.parse_binary_crossentropy('meta', outs) + if 'action' in outs: + self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(ModelConstants.ACTION_WIDTH,)) + if 'lat_planner_solution' in outs: + self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.IDX_N, ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) + if 'desired_curvature' in outs: + self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs diff --git a/openpilot/starpilot/models/receipt.py b/openpilot/starpilot/models/receipt.py new file mode 100644 index 0000000000..dcee19bdd3 --- /dev/null +++ b/openpilot/starpilot/models/receipt.py @@ -0,0 +1,363 @@ +"""Small volatile modeld load receipt; no model artifact is deserialized here.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import stat +import time + +from openpilot.common.hardware.hw import Paths +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, BY_ID +from openpilot.starpilot.models.status import ModelLoad, ModelVariant + + +MAX_RECEIPT_BYTES = 1024 +MAX_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024 +_PREFIX = re.compile(r"[A-Za-z0-9_-]{1,32}\Z") +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") +_REASONS = {None, "chestnut-load-failed", "chestnut-run-stalled", "selected-load-failed"} + + +class ReceiptUnavailable(ValueError): + pass + + +def receipt_path() -> Path: + prefix = os.environ.get("OPENPILOT_PREFIX", "d") + if _PREFIX.fullmatch(prefix) is None: + raise ReceiptUnavailable("invalid IPC namespace") + return Path(Paths.shm_path()) / f"starpilot_modeld_{prefix}" / "load-v1.json" + + +def _root_fd(path: Path, *, create: bool) -> int: + if create: + try: + os.mkdir(path.parent, 0o700) + except FileExistsError: + pass + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(fd) + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ReceiptUnavailable("model receipt directory unavailable") + return fd + except BaseException: + os.close(fd) + raise + + +def process_start_ticks(pid: int, *, proc_root: Path = Path("/proc")) -> int: + """Return Linux's immutable per-process start ticks; zero means unavailable.""" + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + return 0 + try: + raw = (proc_root / str(pid) / "stat").read_bytes() + tail = raw[raw.rfind(b")") + 2:].split() + ticks = int(tail[19]) + return ticks if ticks > 0 else 0 + except (OSError, ValueError, IndexError): + return 0 + + +def _identity(info: os.stat_result) -> tuple[int, int, int, int, int]: + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns + + +@dataclass(frozen=True) +class PreparedArtifact: + identity: tuple[int, int, int, int, int] + sha256: str + + +def artifact_identity(path: Path) -> tuple[int, int, int, int, int]: + """Capture the installed file identity immediately before ModelState opens it.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_size <= 0 or info.st_size > MAX_ARTIFACT_BYTES: + raise ReceiptUnavailable("compiled artifact is not a bounded regular file") + return _identity(info) + finally: + os.close(fd) + + +def artifact_digest(path: Path, expected_identity: tuple[int, int, int, int, int]) -> str: + """Hash a bounded installed compiled artifact once, without following a link or FIFO.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_size <= 0 or before.st_size > MAX_ARTIFACT_BYTES or + _identity(before) != expected_identity): + raise ReceiptUnavailable("compiled artifact is not a bounded regular file") + digest = hashlib.sha256() + remaining = before.st_size + while remaining: + chunk = os.read(fd, min(1024 * 1024, remaining)) + if not chunk: + raise ReceiptUnavailable("compiled artifact changed during hash") + digest.update(chunk) + remaining -= len(chunk) + after = os.fstat(fd) + named = os.stat(path, follow_symlinks=False) + if _identity(before) != _identity(after) or _identity(before) != _identity(named): + raise ReceiptUnavailable("compiled artifact changed during hash") + return digest.hexdigest() + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + finally: + os.close(fd) + + +def prepare_artifact(path: Path, expected_identity: tuple[int, int, int, int, int]) -> PreparedArtifact: + """Complete bounded installed-file hashing before modeld's live loop starts.""" + return PreparedArtifact(expected_identity, artifact_digest(path, expected_identity)) + + +def _pairs(items: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in items: + if key in result: + raise ReceiptUnavailable("duplicate model receipt field") + result[key] = value + return result + + +def _positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 0 < value <= (1 << 64) - 1 + + +def _decode(raw: bytes) -> ModelLoad: + try: + record = json.loads(raw.decode("utf-8"), object_pairs_hook=_pairs) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc: + raise ReceiptUnavailable("malformed model receipt") from exc + if not isinstance(record, dict) or set(record) != {"version", "pid", "processStartTicks", "loadedMonoNs", + "modelId", "variant", "artifactSha256", "fallbackReason"}: + raise ReceiptUnavailable("invalid model receipt fields") + if record["version"] != 1 or type(record["version"]) is not int: + raise ReceiptUnavailable("unsupported model receipt") + if not all(_positive_int(record[key]) for key in ("pid", "processStartTicks", "loadedMonoNs")): + raise ReceiptUnavailable("invalid model process identity") + if not isinstance(record["modelId"], str) or record["modelId"] not in BY_ID: + raise ReceiptUnavailable("unknown loaded model") + try: + variant = ModelVariant(record["variant"]) + except (ValueError, TypeError) as exc: + raise ReceiptUnavailable("unknown model variant") from exc + if record["modelId"] != BUNDLED_CURRENT and BY_ID[record["modelId"]].uses_external_gpu != (variant is ModelVariant.CHESTNUT): + raise ReceiptUnavailable("model receipt hardware mismatch") + digest = record["artifactSha256"] + if not isinstance(digest, str) or _DIGEST.fullmatch(digest) is None or record["fallbackReason"] not in _REASONS: + raise ReceiptUnavailable("invalid model artifact identity") + return ModelLoad(record["pid"], record["processStartTicks"], record["loadedMonoNs"], + record["modelId"], variant, digest, record["fallbackReason"]) + + +def read_receipt(path: Path | None = None) -> ModelLoad | None: + """Missing or malformed volatile state is unavailable, never a default identity.""" + try: + path = receipt_path() if path is None else path + root_fd = _root_fd(path, create=False) + except (OSError, ReceiptUnavailable): + return None + try: + fd = os.open(path.name, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=root_fd) + except (OSError, ReceiptUnavailable): + os.close(root_fd) + return None + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_RECEIPT_BYTES: + return None + raw = os.read(fd, MAX_RECEIPT_BYTES + 1) + after = os.fstat(fd) + named = os.stat(path.name, dir_fd=root_fd, follow_symlinks=False) + if len(raw) > MAX_RECEIPT_BYTES or _identity(before) != _identity(after) or _identity(before) != _identity(named): + return None + return _decode(raw) + except (OSError, ReceiptUnavailable, TypeError): + return None + finally: + os.close(fd) + os.close(root_fd) + + +def write_receipt(load: ModelLoad, path: Path | None = None) -> None: + """Replace one volatile record after the selected model has warmed successfully.""" + path = receipt_path() if path is None else path + raw = json.dumps({"version": 1, "pid": load.pid, "processStartTicks": load.process_start_ticks, + "loadedMonoNs": load.loaded_mono_ns, "modelId": load.model_id, + "variant": load.variant.value, "artifactSha256": load.artifact_sha256, + "fallbackReason": load.fallback_reason}, separators=(",", ":"), sort_keys=True).encode() + if len(raw) > MAX_RECEIPT_BYTES or _decode(raw) != load: + raise ReceiptUnavailable("invalid model receipt") + root_fd = _root_fd(path, create=True) + temporary = f".load-{secrets.token_hex(8)}" + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, dir_fd=root_fd) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + except BaseException: + os.unlink(temporary, dir_fd=root_fd) + raise + os.replace(temporary, path.name, src_dir_fd=root_fd, dst_dir_fd=root_fd) + finally: + try: + os.unlink(temporary, dir_fd=root_fd) + except FileNotFoundError: + pass + os.close(root_fd) + + +def recorded_model_message(load: ModelLoad) -> str: + return json.dumps({'process': load.pid, 'msg': {'event': 'modeld.loaded', 'version': 1, + 'pid': load.pid, 'processStartTicks': load.process_start_ticks, + 'loadedMonoNs': load.loaded_mono_ns, 'modelId': load.model_id, + 'variant': load.variant.value, 'artifactSha256': load.artifact_sha256, + 'fallbackReason': load.fallback_reason}}, separators=(',', ':')) + + +def log_loaded_model(load: ModelLoad) -> None: + """Record the actual initialized runner once per load or fallback in the rlog.""" + from openpilot.common.swaglog import cloudlog + cloudlog.event("modeld.loaded", version=1, pid=load.pid, processStartTicks=load.process_start_ticks, + loadedMonoNs=load.loaded_mono_ns, modelId=load.model_id, variant=load.variant.value, + artifactSha256=load.artifact_sha256, fallbackReason=load.fallback_reason) + + +def logged_model_load(raw: str) -> ModelLoad | None: + """Read our load event, never a requested selection or a free-form load message.""" + if len(raw) > 8192: + return None + try: + record = json.loads(raw, object_pairs_hook=_pairs) + payload = record.get("msg") if isinstance(record, dict) else None + if not isinstance(payload, dict) or payload.get("event") != "modeld.loaded": + return None + payload = {key: value for key, value in payload.items() if key != "event"} + load = _decode(json.dumps(payload).encode()) + return load if type(record.get("process")) is int and record["process"] == load.pid else None + except (ReceiptUnavailable, ValueError, TypeError, RecursionError): + return None + + +def clear_receipt(path: Path | None = None) -> None: + """Invalidate a prior run without following the previous file or directory.""" + path = receipt_path() if path is None else path + try: + root_fd = _root_fd(path, create=False) + except (OSError, ReceiptUnavailable): + return + try: + os.unlink(path.name, dir_fd=root_fd) + except FileNotFoundError: + pass + finally: + os.close(root_fd) + + +def record_prepared(prepared: PreparedArtifact, variant: ModelVariant, fallback_reason: str | None = None, + *, model_id: str = BUNDLED_CURRENT, receipt_file: Path | None = None, pid: int | None = None, + start_ticks: int | None = None, now_mono_ns: int | None = None) -> ModelLoad: + """Write a tiny receipt from a startup-verified artifact; never hash on fallback.""" + pid = os.getpid() if pid is None else pid + start_ticks = process_start_ticks(pid) if start_ticks is None else start_ticks + now_mono_ns = time.monotonic_ns() if now_mono_ns is None else now_mono_ns + if not _positive_int(start_ticks) or not _positive_int(now_mono_ns): + raise ReceiptUnavailable("process identity unavailable") + load = ModelLoad(pid, start_ticks, now_mono_ns, model_id, variant, + prepared.sha256, fallback_reason) + write_receipt(load, receipt_file) + return load + + +class ModelReceiptOwner: + """Best-effort modeld hook: diagnostic failures never stop inference.""" + + def __init__(self, receipt_file: Path | None = None, report: Callable[[str], None] | None = None, + record: Callable[[ModelLoad], None] = log_loaded_model): + self.receipt_file = receipt_file + self.report = report if report is not None else lambda _reason: None + self.record = record + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + + def start(self) -> None: + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + try: + clear_receipt(self.receipt_file) + except Exception: + self.report("clear-failed") + + def capture(self, path: Path) -> tuple[int, int, int, int, int] | None: + try: + return artifact_identity(path) + except Exception: + self.report("artifact-unavailable") + return None + + def prepare(self, path: Path, expected_identity: tuple[int, int, int, int, int] | None) -> PreparedArtifact | None: + if expected_identity is None: + self.report("artifact-identity-unavailable") + return None + try: + return prepare_artifact(path, expected_identity) + except Exception: + self.report("artifact-digest-unavailable") + return None + + def publish(self, publisher, now_ns: int) -> bool: + """Repeat actual load identity so routes started after warm-up retain it.""" + if self._last_publication_ns and now_ns - self._last_publication_ns < 5_000_000_000: + return False + try: + from openpilot.cereal import messaging + message = messaging.new_message(None, valid=True, logMonoTime=now_ns) + message.logMessage = (recorded_model_message(self.load) if self.load is not None else + json.dumps({'msg': {'event': 'modeld.identity-unavailable', + 'sinceMonoNs': self._identity_since_ns}})) + publisher.send('modelIdentity', message) + self._last_publication_ns = now_ns + return True + except Exception: + self.report('publication-failed') + return False + + def loaded(self, prepared: PreparedArtifact | None, variant: ModelVariant, + fallback_reason: str | None = None, *, model_id: str = BUNDLED_CURRENT) -> bool: + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + if prepared is None: + self.report("prepared-artifact-unavailable") + return False + try: + load = record_prepared(prepared, variant, fallback_reason, model_id=model_id, receipt_file=self.receipt_file) + except Exception: + self.report("record-failed") + return False + self.load = load + self._last_publication_ns = 0 + try: + self.record(load) + except Exception: + self.report("log-failed") + return True diff --git a/openpilot/starpilot/models/recovery.py b/openpilot/starpilot/models/recovery.py new file mode 100644 index 0000000000..17a452e1be --- /dev/null +++ b/openpilot/starpilot/models/recovery.py @@ -0,0 +1,122 @@ +"""Drive-scoped process recovery after an established Big publisher stops. + +This owner never touches GPU buffers or retained model selection. Small may start +only after the exact failed modeld process has exited. +""" +from dataclasses import dataclass + +ARM_SPAN_NS = 1_000_000_000 +ARM_MAX_GAP_NS = 1_000_000_000 +STALL_NS = 2_000_000_000 +CAMERA_MAX_AGE_NS = 200_000_000 +INT_GRACE_NS = 500_000_000 +KILL_GRACE_NS = 1_000_000_000 + + +@dataclass(frozen=True) +class RecoveryEvidence: + now_ns: int + drive_id: int + identity: tuple[int, int] | None + alive: bool + big_receipt: bool + loaded_ns: int + model_ns: int + driving_ns: int + output_valid: bool + output_big: bool + cameras: tuple[tuple[int, int], ...] # producer timestamp, frameId + + +class ModelRecoveryOwner: + def __init__(self): + self.drive_id = 0 + self.small_only = False + self.phase = 'monitor' + self.identity = None + self.first_output_ns = 0 + self.last_output = (0, 0) + self.last_progress_ns = 0 + self.last_cameras = () + self.camera_progress_since = 0 + self.deadline_ns = 0 + self.armed = False + + def _withdraw(self): + self.identity = None + self.first_output_ns = 0 + self.last_output = (0, 0) + self.last_progress_ns = 0 + self.last_cameras = () + self.camera_progress_since = 0 + self.armed = False + + def step(self, e: RecoveryEvidence) -> str | None: + if self.phase in ('sigint', 'sigkill', 'failed'): + if not e.alive: + if self.phase == 'failed' and e.drive_id == self.drive_id: + return None + if e.drive_id == self.drive_id and e.drive_id > 0: + self.phase = 'small' + return 'small' + self.__init__() + self.drive_id = e.drive_id + return 'exit' + if e.identity != self.identity: + self.phase = 'failed' + return None + if self.phase != 'failed' and e.now_ns >= self.deadline_ns: + if self.phase == 'sigint': + self.phase = 'sigkill' + self.deadline_ns = e.now_ns + KILL_GRACE_NS + return 'kill' + self.phase = 'failed' + return 'failed' + return None + if e.drive_id <= 0: + self.__init__() + return None + if e.drive_id != self.drive_id: + self.__init__() + self.drive_id = e.drive_id + if self.phase != 'monitor' or self.small_only: + return None + if (not e.alive or e.identity is None or min(e.identity) <= 0 or not e.big_receipt or + not self.drive_id <= e.loaded_ns <= e.now_ns): + self._withdraw() + return None + if e.identity != self.identity: + self._withdraw() + self.identity = e.identity + cameras_fresh = (len(e.cameras) == 2 and all(self.drive_id <= t <= e.now_ns and + e.now_ns - t <= CAMERA_MAX_AGE_NS and f >= 0 for t, f in e.cameras)) + if not cameras_fresh: + self.camera_progress_since = 0 + self.last_cameras = () + return None + cameras_advance = bool(self.last_cameras and all(t > pt and f > pf for (t, f), (pt, pf) in zip(e.cameras, self.last_cameras))) + if cameras_advance: + if not self.camera_progress_since: + self.camera_progress_since = e.now_ns + else: + self.camera_progress_since = 0 + self.last_cameras = e.cameras + output = (e.model_ns, e.driving_ns) + valid_output = (e.output_valid and e.output_big and all(e.loaded_ns <= t <= e.now_ns for t in output) and + e.now_ns - min(output) <= CAMERA_MAX_AGE_NS) + if valid_output and all(t > old for t, old in zip(output, self.last_output)): + if not self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns > ARM_MAX_GAP_NS: + self.first_output_ns = 0 + if not self.first_output_ns: + self.first_output_ns = min(output) + self.last_output = output + self.last_progress_ns = e.now_ns + self.armed = min(output) - self.first_output_ns >= ARM_SPAN_NS + return None + if (self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns >= STALL_NS and + self.camera_progress_since and e.now_ns - self.camera_progress_since >= STALL_NS): + self.small_only = True + self.phase = 'sigint' + self.deadline_ns = e.now_ns + INT_GRACE_NS + return 'interrupt' + return None diff --git a/openpilot/starpilot/models/recovery_process.py b/openpilot/starpilot/models/recovery_process.py new file mode 100644 index 0000000000..7514a71d9e --- /dev/null +++ b/openpilot/starpilot/models/recovery_process.py @@ -0,0 +1,110 @@ +"""Manager adapter for process-isolated modeld recovery; no GPU context here.""" +import os +import signal +import time +from multiprocessing import Process + +from openpilot.common.swaglog import cloudlog +from openpilot.starpilot.models.receipt import process_start_ticks, read_receipt +from openpilot.starpilot.models.recovery import KILL_GRACE_NS, ModelRecoveryOwner, RecoveryEvidence +from openpilot.starpilot.models.status import ModelVariant +from openpilot.system.manager.process import PythonProcess, join_process, launcher + +SMALL_ONLY_ENV = 'STARPILOT_MODELD_RECOVERY_SMALL_ONLY' + + +def modeld_launcher(module, name, small_only): + # Set in this fresh child before importing modeld; never alter manager's env. + if small_only: + os.environ[SMALL_ONLY_ENV] = '1' + else: + os.environ.pop(SMALL_ONLY_ENV, None) + launcher(module, name) + + +class ModeldProcess(PythonProcess): + def __init__(self, name, module, should_run, enabled=True): + super().__init__(name, module, should_run, enabled=enabled) + self.recovery = ModelRecoveryOwner() + self.context_ready = False + + def observe(self, sm, started): + now = time.monotonic_ns() + alive = self.proc is not None and self.proc.is_alive() + pid = self.proc.pid if self.proc is not None else 0 + ticks = process_start_ticks(pid or 0) if alive else 0 + identity = (pid, ticks) if pid and ticks else None + load = read_receipt() if alive else None + matches = load is not None and identity == (load.pid, load.process_start_ticks) + cameras = [] + for service in ('narrowRoadCameraState', 'wideRoadCameraState'): + if sm.seen[service] and sm.valid[service] and sm.alive[service]: + cameras.append((int(sm.logMonoTime[service]), int(sm[service].frameId))) + evidence = RecoveryEvidence( + now, int(sm['deviceState'].startedMonoTime) if started else 0, identity, alive, + bool(matches and load.variant == ModelVariant.CHESTNUT), load.loaded_mono_ns if matches else 0, + int(sm.logMonoTime['modelV2']), int(sm.logMonoTime['drivingModelData']), + bool(sm.seen['modelV2'] and sm.seen['drivingModelData'] and sm.valid['modelV2'] and sm.valid['drivingModelData']), + bool(sm['modelV2'].big), tuple(cameras), + ) + device_fresh = (sm.seen['deviceState'] and sm.valid['deviceState'] and sm.alive['deviceState'] and + 0 < int(sm.logMonoTime['deviceState']) <= now <= int(sm.logMonoTime['deviceState']) + 1_000_000_000) + self.context_ready = bool(device_fresh and started) + if not device_fresh and self.recovery.phase == 'monitor': + self.recovery._withdraw() + return + action = self.recovery.step(evidence) + if action in ('interrupt', 'kill'): + # Owner identity is checked above; SIGKILL gets a bounded exit observation, + # never an unbounded join and never a concurrent Small publisher. + try: + self.signal(signal.SIGINT if action == 'interrupt' else signal.SIGKILL) + except ProcessLookupError: + pass # Exit is confirmed on the next owner observation. + cloudlog.event('modeld.recovery', action=action, pid=pid, drive=evidence.drive_id) + elif action == 'failed': + cloudlog.error('modeld recovery could not confirm process exit; Small restart blocked') + elif action in ('small', 'exit'): + assert self.proc is not None and not self.proc.is_alive() + self.proc.join(timeout=0) + self.proc = None + self.shutting_down = False + cloudlog.event('modeld.recovery', action='small-after-exit', pid=pid, drive=evidence.drive_id) + + def stop(self, retry=True, block=True, sig=None): + if self.recovery.phase in ('sigint', 'sigkill', 'failed'): + # Recovery owns teardown even when manager transitions offroad. Never + # enter the generic five-second/unbounded join for this known stalled PID. + if self.proc is None: + return None + if self.proc.is_alive() and block: + # manager_cleanup has no future observation tick. Kill this owned child + # now and bound the exit wait instead of abandoning a stalled process. + try: + self.signal(signal.SIGKILL) + except ProcessLookupError: + pass + join_process(self.proc, KILL_GRACE_NS / 1e9) + if not self.proc.is_alive(): + self.proc.join(timeout=0) + ret = self.proc.exitcode + self.proc = None + self.shutting_down = False + return ret + if block: + self.recovery.phase = 'failed' + cloudlog.error('modeld shutdown could not confirm stalled process exit') + return None + return super().stop(retry=retry, block=block, sig=sig) + + def start(self): + if not self.context_ready or self.recovery.phase in ('sigint', 'sigkill', 'failed'): + return + if self.shutting_down: + self.stop() + if self.proc is not None: + return + self.proc = Process(name=self.name, target=modeld_launcher, + args=(self.module, self.name, self.recovery.small_only)) + self.proc.start() + self.shutting_down = False diff --git a/openpilot/starpilot/models/runner.py b/openpilot/starpilot/models/runner.py new file mode 100644 index 0000000000..32ed204752 --- /dev/null +++ b/openpilot/starpilot/models/runner.py @@ -0,0 +1,294 @@ +"""Run catalog models using their declared history and output contracts.""" + +from collections.abc import Callable +from dataclasses import dataclass +import os +from pathlib import Path +import weakref + +import numpy as np +from tinygrad import Tensor, dtypes + +from openpilot.cereal import log +from openpilot.selfdrive.controls.lib.drive_helpers import MIN_SPEED, smooth_value +from openpilot.selfdrive.modeld.constants import ModelConstants, Plan +from openpilot.selfdrive.modeld.helpers import load_oob +from openpilot.starpilot.models.parser import Parser +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, COMPILER_REVISION +from openpilot.starpilot.models.receipt import PreparedArtifact, artifact_identity, prepare_artifact +from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + +SUPPORTED_VERSIONS = frozenset(f"v{version}" for version in range(8, 17)) +REQUIRED_OUTPUTS = frozenset(("plan", "lane_lines", "lane_lines_prob", "road_edges", "lead", "lead_prob", + "meta", "desire_pred", "pose", "wide_from_device_euler", "road_transform")) + + +@dataclass(frozen=True) +class CameraWarpDescriptor: + camera_size: tuple[int, int] + nv12_layout: tuple[int, int, int, int] + image_layouts: tuple[tuple[int, ...], tuple[int, ...]] + output_shape: tuple[int, ...] + dtype: object + device: str + + +@dataclass(frozen=True) +class SharedCameraWarp: + tensor: Tensor + descriptor: CameraWarpDescriptor + frame_pointers: tuple[int, int] + transforms: tuple[bytes, bytes] + producer: weakref.ReferenceType + + +def load_verified_model(cam_w: int, cam_h: int, artifact_path: Path, behavior_version: str, chestnut: bool, + model_id: str, expected_sha256: str | None, *, root: Path | None = None) -> tuple["CatalogModelState", PreparedArtifact]: + from openpilot.starpilot.models.manager import ROOT, state_lock + + if expected_sha256 is None: + raise ValueError("selected catalog model has no verified digest") + with state_lock(ROOT if root is None else root): + identity = artifact_identity(artifact_path) + prepared = prepare_artifact(artifact_path, identity) + if prepared.sha256 != expected_sha256: + raise ValueError("selected catalog model changed before loading") + model = CatalogModelState(cam_w, cam_h, artifact_path, behavior_version, chestnut) + model.warmup() + if artifact_identity(artifact_path) != identity: + raise ValueError("selected catalog model changed while loading") + model.model_id = model_id + return model, prepared + + +def validate_artifact(artifact: dict, version: str, camera_size: tuple[int, int]) -> None: + if (not isinstance(artifact, dict) or artifact.get("artifact_abi") != ARTIFACT_ABI or + artifact.get("compiler_revision") != COMPILER_REVISION or artifact.get("format_version") != 1): + raise ValueError("catalog model compiler/ABI mismatch") + if version not in SUPPORTED_VERSIONS or artifact.get("behavior_version") != version: + raise ValueError("catalog model behavior version mismatch") + if artifact.get("model_type") not in ("supercombo", "vision_policy", "vision_multi_policy"): + raise ValueError("unsupported catalog model layout") + if artifact.get("image_history_pipeline") not in ("policy", "warp"): + raise ValueError("unsupported catalog model history") + if type(artifact.get("frame_skip")) is not int or not 1 <= artifact["frame_skip"] <= 20: + raise ValueError("unsupported catalog model frame interval") + if camera_size not in artifact or not callable(artifact.get("run_policy")) or not callable(artifact[camera_size]): + raise ValueError("catalog model camera/runner unavailable") + metadata = artifact.get("metadata") + if not isinstance(metadata, dict): + raise ValueError("missing catalog model metadata") + keys = ["model"] if artifact["model_type"] == "supercombo" else ["vision", *artifact.get("policy_order", [])] + if not keys or (keys[0] == "vision" and not any(key in keys for key in ("on_policy", "policy"))): + raise ValueError("catalog model has no primary policy") + names = set() + for key in keys: + entry = metadata.get(key) + if not isinstance(entry, dict) or not isinstance(entry.get("input_shapes"), dict) or not isinstance(entry.get("output_slices"), dict): + raise ValueError("invalid catalog model metadata") + shape = entry.get("output_shapes", {}).get("outputs") + if (not isinstance(shape, (tuple, list)) or len(shape) != 2 or shape[0] != 1 or + type(shape[1]) is not int or shape[1] <= 0): + raise ValueError("catalog model output width unavailable") + width = shape[1] + for name, section in entry["output_slices"].items(): + if (not isinstance(section, slice) or section.step not in (None, 1) or + any(bound is not None and (type(bound) is not int or not -width <= bound <= width) + for bound in (section.start, section.stop))): + raise ValueError("catalog model output slice exceeds output") + # Metadata padding aliases use ordinary Python negative/open endpoints. + # Check explicit bounds before indices() so its clipping cannot hide a bad slice. + start, stop, _ = section.indices(width) + if not 0 <= start < stop <= width: + raise ValueError("catalog model output slice is empty or reversed") + names.add(name) + if not REQUIRED_OUTPUTS <= names or (version in ("v14", "v15", "v16") and "action" not in names): + raise ValueError("catalog model is missing required driving outputs") + + +def action_from_outputs(outputs: dict[str, np.ndarray], version: str, previous: log.ModelDataV2.Action, + lat_action_t: float, long_action_t: float, v_ego: float, + lat_smooth_seconds: float = 0.0, long_smooth_seconds: float = 0.3) -> log.ModelDataV2.Action: + """Preserve the catalog's generation-specific action units and v9 curvature.""" + if version not in SUPPORTED_VERSIONS or min(lat_action_t, long_action_t) <= 0: + raise ValueError("invalid catalog model action contract") + if version in ("v14", "v15", "v16"): + curvature, acceleration = outputs["action"][0] + curvature = curvature / (100.0 if version == "v14" else max(1.0, v_ego) ** 2) + stop = v_ego < 0.3 and acceleration < 0.1 + else: + plan = outputs["plan"][0] + if "planplus" in outputs: + plan = plan + outputs["planplus"][0] + speeds, accelerations = plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0] + acceleration = 2 * (np.interp(long_action_t, ModelConstants.T_IDXS, speeds) - speeds[0]) / long_action_t - accelerations[0] + stop = speeds[0] < 0.3 and acceleration < 0.1 + if version == "v9": + curvature = float(outputs["desired_curvature"][0, 0]) if "desired_curvature" in outputs else previous.desiredCurvature + else: + yaw = np.interp(lat_action_t, ModelConstants.T_IDXS, plan[:, Plan.T_FROM_CURRENT_EULER][:, 2]) + yaw_rate = plan[0, Plan.ORIENTATION_RATE][2] + curvature = 2 * yaw / (max(MIN_SPEED, v_ego) * lat_action_t) - yaw_rate / max(MIN_SPEED, v_ego) + acceleration = smooth_value(float(acceleration), previous.desiredAcceleration, long_smooth_seconds) + curvature = (smooth_value(float(curvature), previous.desiredCurvature, lat_smooth_seconds) + if v_ego > 0.3 else previous.desiredCurvature) + if not np.isfinite((acceleration, curvature)).all(): + raise ValueError("non-finite catalog model action") + return log.ModelDataV2.Action(desiredCurvature=float(curvature), desiredAcceleration=float(acceleration), shouldStop=bool(stop)) + + +class CatalogModelState: + """Current modeld interface around a current-pin unified catalog artifact.""" + + vision_input_names = ("img", "big_img") + + def __init__(self, cam_w: int, cam_h: int, artifact_path: Path, behavior_version: str, chestnut: bool): + from openpilot.starpilot.models.compile import _detect_vision_keys, stateful_image_shapes, stateful_host_shapes + + artifact = load_oob(artifact_path, chestnut) + validate_artifact(artifact, behavior_version, (cam_w, cam_h)) + self.chestnut = chestnut + self.behavior_version = behavior_version + self.model_type = artifact["model_type"] + self.metadata = artifact["metadata"] + self.policy_order = artifact.get("policy_order", []) + self.frame_skip = artifact["frame_skip"] + self.image_history_pipeline = artifact["image_history_pipeline"] + self.warp_input_keys = tuple(artifact["warp_input_keys"]) + self.policy_input_keys = tuple(artifact["policy_input_keys"]) + self.run_policy = artifact["run_policy"] + self.warp_enqueue = artifact[(cam_w, cam_h)] + self.onnx_history = self.model_type == "supercombo" and bool(self.metadata["model"].get("state_pairs")) + self.queue_device = artifact["execution_device"] + self.warp_device = artifact["warp_device"] + if (not isinstance(self.queue_device, str) or not isinstance(self.warp_device, str) or + chestnut != (self.queue_device.rsplit("+", 1)[-1].split(":", 1)[0] == "AMD")): + raise ValueError("catalog model execution hardware mismatch") + if self.onnx_history: + metadata = self.metadata["model"] + image_shapes = stateful_image_shapes(metadata) + self.policy_input_shapes = stateful_host_shapes(metadata) + self.output_slices = metadata["output_slices"] + elif self.model_type == "supercombo": + image_shapes = self.policy_input_shapes = self.metadata["model"]["input_shapes"] + self.output_slices = self.metadata["model"]["output_slices"] + else: + image_shapes = self.metadata["vision"]["input_shapes"] + primary = "on_policy" if "on_policy" in self.policy_order else "policy" + self.policy_input_shapes = self.metadata[primary]["input_shapes"] + self.road_key, self.wide_key = _detect_vision_keys(image_shapes) + self.desire_key = next(key for key in self.policy_input_shapes if key.startswith("desire")) + self.parser = Parser() + self.aux_parser = Parser(ignore_missing=True) + self.frame_buf_size = get_nv12_info(cam_w, cam_h)[3] + image_layouts = tuple(tuple(image_shapes[key][-2:]) for key in (self.road_key, self.wide_key)) + self.camera_warp_descriptor = (CameraWarpDescriptor((cam_w, cam_h), get_nv12_info(cam_w, cam_h), image_layouts, + (2, 6, *image_layouts[0]), dtypes.uint8, self.warp_device) + if self.image_history_pipeline == "policy" else None) + self._reset_state() + + def _reset_state(self) -> None: + from openpilot.starpilot.models.compile import make_stateful_input_queues, make_supercombo_input_queues, make_split_input_queues + + if self.onnx_history: + self.input_queues, self.npy = make_stateful_input_queues(self.metadata["model"], self.queue_device) + elif self.model_type == "supercombo": + self.input_queues, self.npy = make_supercombo_input_queues(self.policy_input_shapes, self.frame_skip, self.queue_device) + else: + self.input_queues, self.npy = make_split_input_queues(self.metadata["vision"]["input_shapes"], self.policy_input_shapes, + self.frame_skip, self.queue_device) + self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32) + self._blob_cache: dict[tuple[str, int], Tensor] = {} + self.last_warp: SharedCameraWarp | None = None + self._last_shared_warp: SharedCameraWarp | None = None + + def _validate_shared_warp(self, shared: SharedCameraWarp, pointers: tuple[int, int], transforms: tuple[bytes, bytes]) -> None: + if (not isinstance(shared, SharedCameraWarp) or self.camera_warp_descriptor is None or + shared.descriptor != self.camera_warp_descriptor): + raise ValueError("catalog models have incompatible camera warp descriptors/history") + tensor = shared.tensor + if (tuple(tensor.shape) != shared.descriptor.output_shape or tensor.dtype != shared.descriptor.dtype or + tensor.device != shared.descriptor.device): + raise ValueError("catalog shared camera warp tensor contract mismatch") + producer = shared.producer() + if producer is None or producer.last_warp is not shared or shared is self._last_shared_warp: + raise ValueError("catalog shared camera warp is stale or already consumed") + if shared.frame_pointers != pointers or shared.transforms != transforms: + raise ValueError("catalog shared camera warp frame/transforms mismatch") + + @staticmethod + def slice_outputs(output: np.ndarray, sections: dict[str, slice]) -> dict[str, np.ndarray]: + return {key: output[np.newaxis, section].copy() for key, section in sections.items()} + + def _parse(self, outputs: list[np.ndarray]) -> dict[str, np.ndarray]: + if self.model_type == "supercombo": + parsed = self.parser.parse_outputs(self.slice_outputs(outputs[0], self.output_slices)) + if "prev_feat" in self.npy and "hidden_state" in self.output_slices: + self.npy["prev_feat"][:] = outputs[0][self.output_slices["hidden_state"]] + return parsed + vision_output, *policies = outputs + parsed = self.parser.parse_vision_outputs(self.slice_outputs(vision_output, self.metadata["vision"]["output_slices"])) + results = {} + for key, output in zip(self.policy_order, policies, strict=True): + slices = self.slice_outputs(output, self.metadata[key]["output_slices"]) + results[key] = self.aux_parser.parse_off_policy_outputs(slices) if key == "off_policy" else self.parser.parse_policy_outputs(slices) + for key in self.policy_order: + if key not in ("on_policy", "policy"): + parsed.update(results[key]) + parsed.update(results["on_policy" if "on_policy" in results else "policy"]) + return parsed + + def warmup(self) -> None: + frames = {name: np.zeros(self.frame_buf_size, dtype=np.uint8) for name in self.vision_input_names} + self._blob_cache.update({(name, frame.ctypes.data): Tensor(frame, device=self.warp_device).realize() + for name, frame in frames.items()}) + inputs = {"desire_pulse": np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)} + self.run(frames, dict.fromkeys(self.vision_input_names, np.eye(3, dtype=np.float32)), inputs) + self._reset_state() + + def run(self, bufs: dict, transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], + after_enqueue: Callable[[], None] | None = None, *, shared_warp: SharedCameraWarp | None = None) -> dict[str, np.ndarray]: + self.last_warp = None + pointers = tuple(np.frombuffer(bufs[name].data, dtype=np.uint8).ctypes.data for name in self.vision_input_names) + transform_signature = tuple(np.asarray(transforms[name], dtype=np.float32).tobytes() for name in self.vision_input_names) + if shared_warp is not None: + self._validate_shared_warp(shared_warp, pointers, transform_signature) + self._last_shared_warp = shared_warp + desire = inputs["desire_pulse"].copy() + desire[0] = 0 + self.npy["desire"][:] = np.where(desire - self.prev_desire > 0.99, desire, 0) + self.prev_desire[:] = desire + for name, value in self.npy.items(): + if name not in ("desire", "tfm", "big_tfm", "prev_feat") and name in inputs: + value[:] = inputs[name] + self.npy["tfm"][:] = transforms["img"] + self.npy["big_tfm"][:] = transforms["big_img"] + if shared_warp is None: + frames = {} + for name, pointer in zip(self.vision_input_names, pointers, strict=True): + key = (name, pointer) + if key not in self._blob_cache: + self._blob_cache[key] = Tensor.from_blob(pointer, (self.frame_buf_size,), dtype="uint8", device=self.warp_device) + frames[name] = self._blob_cache[key] + warped = self.warp_enqueue(**{key: self.input_queues[key] for key in self.warp_input_keys}, + frame=frames["img"], big_frame=frames["big_img"]) + else: + warped = shared_warp.tensor + policy_inputs = {key: self.input_queues[key] for key in self.policy_input_keys} + output_tensors = (self.run_policy(**policy_inputs, warped=warped) if self.image_history_pipeline == "policy" else + self.run_policy(**policy_inputs, img=warped[0], big_img=warped[1])) + if after_enqueue is not None: + after_enqueue() + outputs = [output.numpy().flatten() for output in output_tensors] + if not all(np.isfinite(output).all() for output in outputs): + raise ValueError("catalog model output not finite") + parsed = self._parse(outputs) + if not REQUIRED_OUTPUTS <= parsed.keys() or not all(np.isfinite(value).all() for value in parsed.values()): + raise ValueError("catalog model parsed outputs invalid") + if os.getenv("SEND_RAW_PRED"): + parsed["raw_pred"] = np.concatenate(outputs) + if self.camera_warp_descriptor is not None: + self.last_warp = (shared_warp if shared_warp is not None else + SharedCameraWarp(warped, self.camera_warp_descriptor, pointers, transform_signature, weakref.ref(self))) + return parsed diff --git a/openpilot/starpilot/models/runtime.py b/openpilot/starpilot/models/runtime.py new file mode 100644 index 0000000000..7ed03bfdca --- /dev/null +++ b/openpilot/starpilot/models/runtime.py @@ -0,0 +1,95 @@ +"""One read-only model snapshot adapter for native UI and authenticated Galaxy.""" + +from collections.abc import Callable +from pathlib import Path +import time +from typing import Any + +from openpilot.cereal.services import SERVICE_LIST +from openpilot.starpilot.models.catalog import CATALOG +from openpilot.starpilot.models.receipt import process_start_ticks, read_receipt +from openpilot.starpilot.models.status import ModelLoad, ModelOutput, ModelProcess, ModelStatus, project_status + + +MODEL_SERVICES = ("managerState", "modelV2", "drivingModelData") + + +def _stamp(sm: Any, service: str, now_mono_ns: int) -> tuple[int, bool]: + try: + stamp = sm.logMonoTime[service] + valid = (type(stamp) is int and 0 < stamp <= now_mono_ns and bool(sm.valid[service]) and + bool(sm.alive[service]) and now_mono_ns - stamp <= int(2e9 / SERVICE_LIST[service].frequency)) + return stamp if type(stamp) is int else 0, valid + except (AttributeError, KeyError, TypeError, ValueError, ZeroDivisionError): + return 0, False + + +def _process(sm: Any, now_mono_ns: int, start_ticks: Callable[[int], int]) -> ModelProcess | None: + if not _stamp(sm, "managerState", now_mono_ns)[1]: + return None + try: + processes = [state for state in sm["managerState"].processes if state.name == "modeld"] + if len(processes) != 1: + return None + state = processes[0] + pid = int(state.pid) + if not state.running or pid <= 0: + return None + return ModelProcess(pid, start_ticks(pid), True) + except (AttributeError, KeyError, TypeError, ValueError, OSError): + return None + + +def _output(sm: Any, params: Any, now_mono_ns: int) -> ModelOutput | None: + model_stamp, model_valid = _stamp(sm, "modelV2", now_mono_ns) + driving_stamp, driving_valid = _stamp(sm, "drivingModelData", now_mono_ns) + if model_stamp == 0 and driving_stamp == 0: + return None + try: + raw_chestnut = params.get("ChestnutActive") + chestnut = (raw_chestnut if type(raw_chestnut) is bool else + None if raw_chestnut not in (None, b"0", b"1") else raw_chestnut == b"1") + big = bool(sm["modelV2"].big) if model_stamp else False + return ModelOutput(model_stamp, model_valid, driving_stamp, driving_valid, big, chestnut) + except (AttributeError, KeyError, TypeError, ValueError, OSError): + return None + + +def snapshot(sm: Any, params: Any, now_mono_ns: int, *, requested_id: str | None = None, + receipt_file: Path | None = None, start_ticks: Callable[[int], int] = process_start_ticks, + read_load: Callable[[Path | None], ModelLoad | None] = read_receipt) -> ModelStatus: + """Sample one manager process and two live model outputs; never infer a load from Params.""" + process = _process(sm, now_mono_ns, start_ticks) + output = _output(sm, params, now_mono_ns) if process is not None else None + load = read_load(receipt_file) if process is not None else None + return project_status(requested_id, process, load, output, now_mono_ns) + + +class ModelStatusSource: + """Independent optional subscriber; it does not affect UI/global service health.""" + + def __init__(self, params: Any, sm: Any | None = None, clock: Callable[[], int] = time.monotonic_ns): + if sm is None: + from openpilot.cereal import messaging + sm = messaging.SubMaster(list(MODEL_SERVICES)) + self.sm = sm + self.params = params + self.clock = clock + + def snapshot(self) -> ModelStatus: + self.sm.update(0) + from openpilot.selfdrive.modeld.helpers import chestnut_present + from openpilot.starpilot.models.manager import requested_runtime_id + return snapshot(self.sm, self.params, self.clock(), requested_id=requested_runtime_id(chestnut_present())) + + def json(self) -> dict[str, object]: + status = self.snapshot() + return {"schemaVersion": 1, "catalog": [{"id": entry.model_id, "name": entry.name, + "selectable": entry.selectable} for entry in CATALOG], + "requestedId": status.requested_id, "loadedId": status.loaded_id, + "variant": status.variant.value if status.variant is not None else None, + "health": status.health.value, "fallbackReason": status.fallback_reason, + "artifactSha256": status.artifact_sha256, "pendingNextStart": status.pending_next_start} + + def close(self) -> None: + self.sm = None diff --git a/openpilot/starpilot/models/startup.py b/openpilot/starpilot/models/startup.py new file mode 100644 index 0000000000..050af0619c --- /dev/null +++ b/openpilot/starpilot/models/startup.py @@ -0,0 +1,90 @@ +"""Wait for vehicle and external GPU power before opening the GPU runtime.""" + +from dataclasses import dataclass +import time + +from openpilot.cereal import messaging +from openpilot.common.swaglog import cloudlog +from openpilot.common.hardware import HARDWARE + + +POWER_MIN_MV = 10_000 +POWER_STABLE_NS = 3_000_000_000 +POWER_MAX_AGE_NS = 500_000_000 +VEHICLE_MAX_AGE_NS = 250_000_000 + + +@dataclass +class PowerReadiness: + stable_since_ns: int = 0 + previous_ns: int = 0 + sample_max_age_ns: int = POWER_MAX_AGE_NS + + def update(self, now_ns: int, sample_ns: int, voltage_mv: int, vehicle_ready: bool) -> bool: + continuous = self.previous_ns == 0 or 0 <= now_ns - self.previous_ns <= POWER_MAX_AGE_NS + self.previous_ns = now_ns + healthy = (0 < sample_ns <= now_ns and now_ns - sample_ns <= self.sample_max_age_ns and + voltage_mv >= POWER_MIN_MV and vehicle_ready) + if not continuous or not healthy: + self.stable_since_ns = 0 + if not healthy: + return False + if self.stable_since_ns == 0: + self.stable_since_ns = now_ns + return now_ns - self.stable_since_ns >= POWER_STABLE_NS + + +def paired_clocks_ns() -> tuple[int, int] | None: + before = time.monotonic_ns() + clock = getattr(time, 'CLOCK_BOOTTIME', None) + boot = time.clock_gettime_ns(clock) if clock is not None else before + after = time.monotonic_ns() + return ((before + after) // 2, boot) if 0 <= after - before <= 1_000_000 else None + + +def current_drive(sm, now_ns: int) -> bool: + device = sm['deviceState'] + stamp = sm.logMonoTime['deviceState'] + return (sm.seen['deviceState'] and sm.valid['deviceState'] and device.started and + 0 < device.startedMonoTime <= stamp <= now_ns <= stamp + 2_000_000_000) + + +def supply_voltage(states) -> int: + known = [state for state in states if str(state.pandaType) != 'unknown'] + return known[0].voltage if len(known) == 1 else 0 + + +def wait_for_chestnut_power(CP, timeout: float = 30.) -> None: + ready_bus = None + if CP.brand == "hyundai": + from opendbc.car.hyundai.model_startup import model_power_ready_bus, model_power_ready + ready_bus = model_power_ready_bus(CP) + + power_service = "peripheralState" if HARDWARE.get_device_type() == "tici" else "pandaStates" + sm = messaging.SubMaster([power_service, "deviceState"] + (["can"] if ready_bus is not None else [])) + readiness = PowerReadiness(sample_max_age_ns=1_000_000_000 if power_service == "peripheralState" else POWER_MAX_AGE_NS) + vehicle_ready, vehicle_sample_ns = ready_bus is None, 0 + started = time.monotonic() + cloudlog.info("waiting for Chestnut power stability") + while time.monotonic() - started < timeout: + sm.update(100) + clocks = paired_clocks_ns() + if clocks is None: + readiness.stable_since_ns = 0 + continue + now_mono_ns, now_ns = clocks + drive_ready = current_drive(sm, now_mono_ns) + if sm.seen["deviceState"] and not drive_ready: + raise RuntimeError("drive ended while waiting for Chestnut power") + if ready_bus is not None and sm.updated["can"]: + ready = model_power_ready(sm["can"], ready_bus) if sm.valid["can"] else False + if ready is not None: + vehicle_ready, vehicle_sample_ns = ready, sm.logMonoTime["can"] + fresh_vehicle = ready_bus is None or 0 < vehicle_sample_ns <= now_ns and now_ns - vehicle_sample_ns <= VEHICLE_MAX_AGE_NS + states = [sm[power_service]] if power_service == "peripheralState" else sm[power_service] + voltage = supply_voltage(states) + if readiness.update(now_ns, sm.logMonoTime[power_service] if sm.valid[power_service] else 0, + voltage, drive_ready and vehicle_ready and fresh_vehicle): + cloudlog.info("Chestnut power stable; starting GPU load") + return + raise TimeoutError("Chestnut power or vehicle READY did not stabilize") diff --git a/openpilot/starpilot/models/status.py b/openpilot/starpilot/models/status.py new file mode 100644 index 0000000000..496e812f79 --- /dev/null +++ b/openpilot/starpilot/models/status.py @@ -0,0 +1,112 @@ +"""Pure projection of requested model, loaded identity, and fresh modeld output.""" + +from dataclasses import dataclass +from enum import StrEnum + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, BY_ID, resolve_selection + + +OUTPUT_MAX_AGE_NS = 250_000_000 + + +class ModelHealth(StrEnum): + UNAVAILABLE = "unavailable" + LOADING = "loading" + IDENTITY_UNAVAILABLE = "identity-unavailable" + ACTIVE = "active" + STALE = "stale" + FAILED = "failed" + + +class ModelVariant(StrEnum): + SMALL = "small" + CHESTNUT = "chestnut" + + +@dataclass(frozen=True) +class ModelProcess: + pid: int + start_ticks: int + running: bool + + +@dataclass(frozen=True) +class ModelLoad: + """Identity recorded only after ModelState has loaded and initialized a runner.""" + pid: int + process_start_ticks: int + loaded_mono_ns: int + model_id: str + variant: ModelVariant + artifact_sha256: str + fallback_reason: str | None = None + + +@dataclass(frozen=True) +class ModelOutput: + model_v2_mono_ns: int + model_v2_valid: bool + driving_data_mono_ns: int + driving_data_valid: bool + model_v2_big: bool + chestnut_active: bool | None + + +@dataclass(frozen=True) +class ModelStatus: + requested_id: str + loaded_id: str | None + variant: ModelVariant | None + health: ModelHealth + pending_next_start: bool + fallback_reason: str | None + artifact_sha256: str | None + + +def project_status(requested_id: str | None, process: ModelProcess | None, load: ModelLoad | None, + output: ModelOutput | None, now_mono_ns: int) -> ModelStatus: + """Keep saved intent distinct from actual modeld identity and output health. + + The caller must obtain process and load records from the same owner. A stale + load receipt cannot make a new modeld process appear active. An output older + than the load boundary cannot establish health after a restart. + """ + requested = resolve_selection(requested_id).model_id + if not isinstance(now_mono_ns, int) or isinstance(now_mono_ns, bool) or now_mono_ns <= 0: + raise ValueError("invalid model-status clock") + + if process is None or not process.running or process.pid <= 0: + return ModelStatus(requested, None, None, ModelHealth.UNAVAILABLE, False, None, None) + if process.start_ticks <= 0: + return ModelStatus(requested, None, None, ModelHealth.IDENTITY_UNAVAILABLE, False, None, None) + output_is_fresh = (output is not None and output.model_v2_valid and output.driving_data_valid and + all(isinstance(stamp, int) and not isinstance(stamp, bool) and 0 < stamp <= now_mono_ns and + now_mono_ns - stamp <= OUTPUT_MAX_AGE_NS + for stamp in (output.model_v2_mono_ns, output.driving_data_mono_ns))) + if load is None: + health = ModelHealth.IDENTITY_UNAVAILABLE if output_is_fresh else ModelHealth.LOADING + return ModelStatus(requested, None, None, health, False, None, None) + if (load.pid != process.pid or load.process_start_ticks != process.start_ticks or + load.loaded_mono_ns <= 0 or load.loaded_mono_ns > now_mono_ns or + load.model_id not in BY_ID or not isinstance(load.variant, ModelVariant) or + not isinstance(load.artifact_sha256, str) or len(load.artifact_sha256) != 64 or + any(c not in "0123456789abcdef" for c in load.artifact_sha256)): + health = ModelHealth.IDENTITY_UNAVAILABLE if output_is_fresh else ModelHealth.LOADING + return ModelStatus(requested, None, None, health, False, None, None) + + if load.model_id != BUNDLED_CURRENT and BY_ID[load.model_id].uses_external_gpu != (load.variant is ModelVariant.CHESTNUT): + return ModelStatus(requested, None, None, ModelHealth.IDENTITY_UNAVAILABLE, False, None, None) + pending = requested != load.model_id + common = (requested, load.model_id, load.variant, pending, load.fallback_reason, load.artifact_sha256) + if output is None: + return ModelStatus(common[0], common[1], common[2], ModelHealth.LOADING, *common[3:]) + stamps = (output.model_v2_mono_ns, output.driving_data_mono_ns) + if (any(not isinstance(stamp, int) or isinstance(stamp, bool) or stamp < load.loaded_mono_ns or stamp > now_mono_ns + for stamp in stamps) or + not output.model_v2_valid or not output.driving_data_valid): + return ModelStatus(common[0], common[1], common[2], ModelHealth.STALE, *common[3:]) + if any(now_mono_ns - stamp > OUTPUT_MAX_AGE_NS for stamp in stamps): + return ModelStatus(common[0], common[1], common[2], ModelHealth.STALE, *common[3:]) + if output.chestnut_active is None or output.model_v2_big != output.chestnut_active or output.model_v2_big != (load.variant is ModelVariant.CHESTNUT): + return ModelStatus(common[0], common[1], common[2], ModelHealth.FAILED, *common[3:]) + return ModelStatus(common[0], common[1], common[2], ModelHealth.ACTIVE, *common[3:]) diff --git a/openpilot/starpilot/models/tests/__init__.py b/openpilot/starpilot/models/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json b/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json new file mode 100644 index 0000000000..7e0fa7b4f1 --- /dev/null +++ b/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json @@ -0,0 +1,111 @@ +{ + "original_source_sha256": "f0210b2ca26a1b9c4adbcf5825d0d01851254dbfbdbc54595c16d9d77d71731f", + "generator": "Original AST-selected pure composition; explicit longitudinal current-frame source; normalized float32 plan (1,33,15), action decoded separately.", + "cases": [ + { + "versions": [ + "v8", + "v9" + ], + "lateral_offset": 10, + "longitudinal_offset": 20, + "expected": { + "desired_curvature": "c2600e1170a6c8870bfa1db230512017ce5569d52a67906d2bd8c70edf57d4ed", + "desired_curvature_stds": "00c86d4ebf27bafc31acbbcd4aacae39e1642e5726d1b27858cfe6b127a65022", + "lat_planner_solution": "9d10c66e47cb0fc52a36e7e0be9497177f8df9d82fc6605a00fe73e147434dec", + "lat_planner_solution_stds": "d2046121607eb00f881f35e99e14a5d3164e550160b3f9eae5ef74d4d184cc48", + "lane_lines": "b1fa5fcbd1577014f692d8e633d7fb70f1dcd8ec95b3c4a6b734bd6e17f90d5c", + "lane_lines_stds": "4c0dcf842694d757c8200ba201d437506e54c4643cfa32f2511f0d9dfa3960ef", + "lane_lines_prob": "0374f5e2a29b996ebdb7fe2bd452ccd3b55290dc71fb5b3843d577f44655f9f3", + "road_edges": "a1177c7c6de49027ff21ad2fd78edbeef618a3000e1fff4c231074853b50180f", + "road_edges_stds": "d41173b5c2be5672a0abca35f34e0aac8cf2a6ff8ccdc472c88e00b45d271230", + "desire_state": "54a684f78cccd3f5bf7cdcfba9a25d2e1e979edf2a900f9064371bbe7ff12eaf", + "desire_pred": "93253864fb1cc58f3670d912c108b2c75165dd41f71deeead539c22b5bbafca9", + "pose": "db3ee5a345d0834277ff12d708791943e1ab9b33f829c6491789db5dd176fc9e", + "pose_stds": "e72a0a1259c6dfaa33587bbbb131a9ed8a5b1d8a7892a35973dfb6aa50dc341c", + "wide_from_device_euler": "e369af615af989f296ef62a9582e7c4bbb80e4a568d0b69ed93dfee0b7fb03b9", + "wide_from_device_euler_stds": "9dd8a25b398d69f4a2f5a14d16f2224918ab1a329d0c0b7fae3355a26247dd9e", + "road_transform": "45af797502186a93eb4632248488571f4c7d8e9d3a09febb4d96deee39a1e65e", + "road_transform_stds": "39d3270cfe99fb33d907c7a37451cdde0e2fa3f38683ceecd1a46ff1f447cd1b", + "lead": "66bfc2d5b98423fe06d351f687bef88be82f7119c409db82da3c87105c465fc6", + "lead_prob": "e45bb2ec3e63950b3e464a45bb8171c1209393c458e4991028ea42620f88b607", + "meta": "2befc666fccb379e084e3feed7c461eb095c2277b91733ae62333052a0be7196", + "planplus": "26d4b5717d6a9ed1a987c23e9cf779cfffee11e996be50688d90de9de49bace7", + "raw_pred": "51127f644b8fcf6b45f756098619960e20674e6dc726856e4c20da834253575a", + "unknown_extra": "19b7e0870c270281002ac440b752780a1ceba740b30bb62effca8d01b7b7ecbb", + "plan": "ca530c67c1613fe1cf9224ea8ed125fcdfe180da62341a181f84547ec17c3862", + "plan_stds": "f55f9fca10f6f0223c16d207242748ee3ac8bee75900f6703dae824e3387186b" + } + }, + { + "versions": [ + "v14", + "v15" + ], + "lateral_offset": 1000, + "longitudinal_offset": 2000, + "expected": { + "desired_curvature": "48270c20e131431ad33afbf76bae7286fbf4f0ad55373a4624f6957c0a7e5bd3", + "desired_curvature_stds": "7f54928b9e681540bf154d5e9c14f57da0d3e5b9654d5c3d573243d5ab168ac0", + "lat_planner_solution": "40b20dd85498461a7eb26e813eafc09ef8b784dd26f485f7a481e304d7f0614f", + "lat_planner_solution_stds": "acf086207faa6273da611964f94906db6696d2fd463a0bcfa4c650f9d80d2c6d", + "lane_lines": "a4eb06d857b4da55875b9f3455244d443abff2262da5bff5fb89ae056ba42214", + "lane_lines_stds": "41e3303c4a9f97ec24c23921c01adab75b0ad7ad3b71ba8d1412c289a87d5b75", + "lane_lines_prob": "6f8db4b04211c6c31bca3665e64193083078f673a0e3cbe6c3a8c473a2afe7a6", + "road_edges": "e43108080f9f9fab9d7729b018a50b0a60d1d1aff6a68d69320a91c30d29ac7a", + "road_edges_stds": "afa058f368a910c1a0124182a74f757ae28e55ef1ec1c3deffd6cba058070283", + "desire_state": "255106e4dfc96b87fe4f9f4738785df0d7ace598f89ee235222cf425f46cddd6", + "desire_pred": "30d67c7cc4f201e7b297b17f5f5d40ac9d538fc86003c1ef9939f2da43cad756", + "pose": "44478d7cfb3996843e47d12865b343435dceea321e4de85127c03e80143434c0", + "pose_stds": "9603324367b63793a975a24ab3db4356d339a0b47baea3544b3c4d760e2afcf3", + "wide_from_device_euler": "374258046cf62cce54400b1d0b2fc7fc4b454d54edb75a6767d31ee791981788", + "wide_from_device_euler_stds": "0e90893b1e358b79b82cb35ac987603190272e19a556ae84d56f4856f1744219", + "road_transform": "5bb150e05e927f66f67c1a45674521f9fb835e87f51a81c3367ef77820d337ce", + "road_transform_stds": "1f5a36d32fc56a69d52b66b3d341351c9786b61eda6002eb7a5f0cd049f99ee0", + "lead": "41ef548226148429978482c46219c6ead8b7f64eec5b4e0f26ab99cb79a7c524", + "lead_prob": "91ac1f3d1d11cc5e9a0cd33fcad703d177dd539cdfb2ba649241e1c2496d5ce0", + "meta": "fac9d42caf83ae32a0091f914b7b7cfa9be839f96675b64c5dd916698c5a6c19", + "planplus": "35eb89634463c1d99b1f6deaae2cb67b33512fe2798145c52e834c3c7932f51f", + "raw_pred": "6725fcf159c53c6e7896eeaa2a8cf1cb04d5e09e01438338678cbae35ddcfb6a", + "unknown_extra": "55f8e7ccecf5c95d544a093099e53cbd8e409942b6b73470d196beafd1f7b9a5", + "plan": "6d71aaf15c9646d847ddb04b2fb2718455ff998107ec111c54bc5575735a1ec3", + "plan_stds": "fa7d0ef0cad07421cb47b3d4c470238bda21673a59f3ed4653ab5e2ddd224118" + } + }, + { + "versions": [ + "v15", + "v16" + ], + "lateral_offset": -1000, + "longitudinal_offset": 1000, + "expected": { + "desired_curvature": "8e296dfc73dec27c9ae57442a6ef354b212d45a924187b11468316dae3984967", + "desired_curvature_stds": "d4c04057be63e1a687d71501cc123f7250e26cbbc7005dc181c05230c9cd089a", + "lat_planner_solution": "494e83454e5201015d521620cde3ff32e1d684963ae210ee995e4ea1d7ade207", + "lat_planner_solution_stds": "76ddcd6935a84860444f86c0e581153ed943f7b604791876ceee12fed6d43c3f", + "lane_lines": "316774c6fd093921c6862f6c0ca17ee1e97d2d061c061f9a424c7d72742bf4d7", + "lane_lines_stds": "cf2d8052f3a6fc55c9adc939b34207d1fab0a9b4906bb61dbeb74a2dcd22cb85", + "lane_lines_prob": "50f3cf87d50de3015734b51878f1c2e3025716fc5f763dc6ff9c06e028cee335", + "road_edges": "085b0e6d88f419906362cd94347b8d7c410d1af45a3b833a3860b299616c44df", + "road_edges_stds": "7df3713221372e1b5c4608b259e4127ee96d221d1aea7611e82b823ce0b11e8d", + "desire_state": "4ca63b15b26aedeafdd39ec2c2c00a3b8ff0a56b139ea7a8d07c7cd75544ff4a", + "desire_pred": "fa02d18f4158f326c6621437a79b95b7db666fedcaf7bbdca0ad8327ee57cdeb", + "pose": "45a99de0255b714a798c4d63f607c4d562a273d2a6587414a97c5fa3ed127b8c", + "pose_stds": "ddbc8c0795480fab91d495a81558ddcf9ce5852e9911edfeaa2fbd1a6cc0395f", + "wide_from_device_euler": "7e4a1fc996517d862e43dab9d5b38b32283726bc5cf1b88263fc6afd79cce0ba", + "wide_from_device_euler_stds": "c40c85d37bb3c50b863a25ad66e2342fbaab420e83a004ca1477f1052fec7d30", + "road_transform": "e234d71da99b30e8c1ca0c8ef8d4887785ecc8cffdd54aa13863de2051305799", + "road_transform_stds": "e5aecc961bf2b500458cae3023aa5d59aecefac735fc72b1ea98fae332383ff8", + "lead": "3db9b14a5f25b121c02137faff7b00d8263b99a267e017a4e35da2a8e9a728cb", + "lead_prob": "f9ee34aa05c8ee301ab80eb5303717023e4e1727992df63a81510d6e8e9b9a20", + "meta": "8e2bcca09d503d7472c56bf9f0c5345a40cbe055edde30db17f6eb705e245cd6", + "planplus": "24dcdc5ff3867ed237ad56915c7e9d1cf56db727a5787af327b0d7742d4629dd", + "raw_pred": "7bb224457c483a72c156f4b2705fabdaa305da839428baa00900a1f2df911388", + "unknown_extra": "9c9f0ed46eb94c2c6f896f0838eee44891aa4c8a938df4c544c80de400921ce2", + "plan": "efe95619a7b0efc4ab4e8975c49cf9cf883b8ee1994219f3e30b1d2d3b844b8f", + "plan_stds": "159b92b6f0d62985827f59be459a8c4a32fc828b705cade66fe14180fe4e633e" + } + } + ] +} diff --git a/openpilot/starpilot/models/tests/test_artifact_isolation.py b/openpilot/starpilot/models/tests/test_artifact_isolation.py new file mode 100644 index 0000000000..1df2cd2fb5 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_artifact_isolation.py @@ -0,0 +1,81 @@ +from itertools import combinations +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np +from tinygrad import Context, Tensor, TinyJit +from tinygrad.uop.ops import Ops +from tinygrad_repo.examples.openpilot.helpers import dump_pickle + +from openpilot.selfdrive.modeld.helpers import load_oob + + +def stateful_artifact(path, seed): + state = Tensor([seed, seed + 1], device="CPU").realize() + weight = Tensor([2., 3.], device="CPU").realize() + + @TinyJit + def step(x): + state.assign(state + x * weight).realize() + return state + + x = Tensor([1., 1.], device="CPU").realize() + for _ in range(3): + step(x) + artifact = {"state": state, "weight": weight, "step": step} + dump_pickle(artifact, path) + return artifact + + +def captured_buffers(artifact): + return {u.arg.buffer for u in artifact["step"].captured._linear.toposort() if u.op is Ops.BUFFER and u.arg.buffer is not None} + + +class TestArtifactIsolation(unittest.TestCase): + def setUp(self): + self.context = Context(DEV="CPU:LLVM") + self.context.__enter__() + self.addCleanup(self.context.__exit__, None, None, None) + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.paths = [Path(self.directory.name) / name for name in ("lateral.pkl", "longitudinal.pkl")] + self.originals = [stateful_artifact(path, seed) for path, seed in zip(self.paths, (1., 100.), strict=True)] + self.loads = [self.load(self.paths[0]), self.load(self.paths[1]), self.load(self.paths[0])] + + @staticmethod + def load(path): + # Exercise the production loader's CPU branch without opening Metal or any external device. + with patch("openpilot.selfdrive.modeld.helpers.sys.platform", "linux"), patch("openpilot.selfdrive.modeld.helpers.AGNOS", False): + return load_oob(path) + + def test_separate_load_arenas_and_captured_buffers_with_originals_alive(self): + artifacts = self.originals + self.loads + for artifact in self.loads: + self.assertIs(artifact["state"].uop.buffer.base, artifact["weight"].uop.buffer.base) + self.assertIsNot(artifact["state"].uop.buffer, artifact["weight"].uop.buffer) + for first, second in combinations(artifacts, 2): + self.assertIsNot(first["state"].uop.buffer.base, second["state"].uop.buffer.base) + self.assertTrue(captured_buffers(first).isdisjoint(captured_buffers(second))) + np.testing.assert_array_equal(self.loads[0]["state"].numpy(), self.loads[2]["state"].numpy()) + + def test_interleaved_execution_reset_and_repeat_load_do_not_change_other_state(self): + expected = [artifact["state"].numpy().copy() for artifact in self.loads] + originals = [artifact["state"].numpy().copy() for artifact in self.originals] + for index, scale in ((0, 1.), (1, 2.), (0, 3.), (2, 4.), (1, 5.)): + result = self.loads[index]["step"](Tensor([scale, scale], device="CPU").realize()) + expected[index] += np.array([2., 3.], dtype=np.float32) * scale + np.testing.assert_array_equal(result.numpy(), expected[index]) + for artifact, state in zip(self.loads, expected, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) + self.loads[0]["state"].uop.buffer.copy_from(Tensor(np.zeros(2, dtype=np.float32), device="CPU").realize().uop.buffer) + expected[0][:] = 0 + self.loads[0]["step"](Tensor([1., 1.], device="CPU").realize()) + expected[0] += [2., 3.] + fresh = self.load(self.paths[0]) + np.testing.assert_array_equal(fresh["state"].numpy(), originals[0]) + for artifact, state in zip(self.loads, expected, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) + for artifact, state in zip(self.originals, originals, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) diff --git a/openpilot/starpilot/models/tests/test_catalog_status.py b/openpilot/starpilot/models/tests/test_catalog_status.py new file mode 100644 index 0000000000..1e7cefff85 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_catalog_status.py @@ -0,0 +1,67 @@ +import hashlib +from pathlib import Path +import unittest + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, CATALOG, resolve_selection +from openpilot.starpilot.models.status import (ModelHealth, ModelLoad, ModelOutput, ModelProcess, + ModelVariant, project_status) + + +class TestModelCatalogStatus(unittest.TestCase): + def test_catalog_pins_current_source_and_lists_model_metadata(self): + self.assertEqual(len(CATALOG), 100) + entry = resolve_selection(None) + self.assertEqual(entry.model_id, BUNDLED_CURRENT) + source = Path(__file__).resolve().parents[4] / entry.source_path + self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), entry.source_sha256) + self.assertEqual(resolve_selection("rdf43").version, "v15") + with self.assertRaises(ValueError): + resolve_selection("unknown-model") + with self.assertRaises(ValueError): + resolve_selection("") + + def test_loaded_identity_requires_same_process_and_fresh_both_outputs(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + self.assertEqual(project_status(None, process, load, output, 1_200_000_000).health, ModelHealth.ACTIVE) + self.assertEqual(project_status(None, ModelProcess(422, 5001, True), load, output, + 1_200_000_000).health, ModelHealth.IDENTITY_UNAVAILABLE) + self.assertEqual(project_status(None, process, load, output, 1_500_000_001).health, ModelHealth.STALE) + self.assertEqual(project_status(None, process, load, ModelOutput(1_050_000_000, True, 1_185_000_000, + True, False, False), 1_200_000_000).health, + ModelHealth.STALE) + self.assertEqual(project_status(None, process, load, ModelOutput(1_180_000_000, True, 1_185_000_000, + True, True, True), 1_200_000_000).health, + ModelHealth.FAILED) + + def test_runtime_stall_preserves_requested_big_and_active_small_identity(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, 'sc23', ModelVariant.SMALL, 'a' * 64, 'chestnut-run-stalled') + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + status = project_status('cinquev3', process, load, output, 1_200_000_000) + self.assertEqual((status.requested_id, status.loaded_id, status.health), ('cinquev3', 'sc23', ModelHealth.ACTIVE)) + self.assertTrue(status.pending_next_start) + self.assertEqual(status.fallback_reason, 'chestnut-run-stalled') + + def test_catalog_selection_preserves_actual_fallback_identity(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, "sc23", ModelVariant.SMALL, "a" * 64, "chestnut-load-failed") + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + status = project_status("cinquev3", process, load, output, 1_200_000_000) + self.assertEqual((status.requested_id, status.loaded_id, status.health), ("cinquev3", "sc23", ModelHealth.ACTIVE)) + self.assertTrue(status.pending_next_start) + self.assertEqual(status.fallback_reason, "chestnut-load-failed") + impossible = ModelLoad(421, 5000, 1_100_000_000, "cinquev3", ModelVariant.SMALL, "a" * 64) + self.assertEqual(project_status("cinquev3", process, impossible, output, 1_200_000_000).health, ModelHealth.IDENTITY_UNAVAILABLE) + + def test_no_saved_choice_or_stale_receipt_claims_active(self): + process = ModelProcess(421, 5000, True) + self.assertEqual(project_status(None, None, None, None, 1_200_000_000).health, ModelHealth.UNAVAILABLE) + self.assertEqual(project_status(None, process, None, None, 1_200_000_000).health, ModelHealth.LOADING) + stale_load = ModelLoad(421, 4999, 1_100_000_000, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + self.assertEqual(project_status(None, process, stale_load, None, 1_200_000_000).health, ModelHealth.LOADING) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_compile.py b/openpilot/starpilot/models/tests/test_compile.py new file mode 100644 index 0000000000..1d14a31a3d --- /dev/null +++ b/openpilot/starpilot/models/tests/test_compile.py @@ -0,0 +1,154 @@ +import base64 +import json +import os +from pathlib import Path +import pickle +import subprocess +import sys +import tempfile +import unittest +from functools import partial +from unittest.mock import patch + +import numpy as np + +from scripts import model_compiler + + +def varint(value): + result = bytearray() + while value > 127: + result.append((value & 127) | 128) + value >>= 7 + return bytes(result) + bytes([value]) + + +def field(number, value): + if isinstance(value, int): + return varint(number << 3) + varint(value) + if isinstance(value, str): + value = value.encode() + return varint((number << 3) | 2) + varint(len(value)) + value + + +def value_info(name, shape): + dimensions = b"".join(field(1, field(1, dimension)) for dimension in shape) + return field(1, name) + field(2, field(1, field(1, 1) + field(2, dimensions))) + + +def stateful_onnx(): + inputs = {"new_img": (2, 6, 2, 2), "state": (1, 2), "desire": (1, 2)} + add = field(1, "state") + field(1, "desire") + field(2, "next_state") + field(4, "Add") + identity = field(1, "next_state") + field(2, "outputs") + field(4, "Identity") + graph = field(1, add) + field(1, identity) + field(2, "stateful-fixture") + graph += b"".join(field(11, value_info(name, shape)) for name, shape in inputs.items()) + graph += field(12, value_info("outputs", (1, 2))) + field(12, value_info("next_state", (1, 2))) + metadata = field(1, "output_slices") + field(2, base64.b64encode(pickle.dumps({"fixture": slice(0, 2)}))) + return field(1, 8) + field(7, graph) + field(8, field(2, 13)) + field(14, metadata) + + +class CompileTest(unittest.TestCase): + def test_legacy_split_and_supercombo_history_survive_arena_round_trip(self): + from tinygrad import Context, TinyJit + from openpilot.starpilot.models.compile import (compile_jit, make_random_images, make_run_split_policy, make_run_supercombo, + make_split_input_queues, make_supercombo_input_queues, FAST_POLICY_INPUTS) + image_shapes = {"input_imgs": (1, 12, 2, 2), "big_input_imgs": (1, 12, 2, 2)} + policy_shapes = {"features_buffer": (1, 3, 2), "desire": (1, 2, 2), "traffic_convention": (1, 2)} + def vision(inputs): + values = [inputs[name].cast("float32").mean().reshape(1, 1) for name in image_shapes] + return {"outputs": values[0].cat(values[1], dim=1)} + def policy(inputs): + return {"outputs": inputs["features_buffer"].sum(axis=1) + inputs["desire"].sum(axis=1) + inputs["traffic_convention"]} + def supercombo(inputs): + return {"outputs": policy(inputs)["outputs"] + vision(inputs)["outputs"]} + with Context(DEV="CPU:LLVM", IMAGE=0, FLOAT16=0, JIT_BATCH_SIZE=0): + for kind in ("split", "supercombo"): + with self.subTest(kind=kind): + if kind == "split": + metadata = {"vision": {"input_shapes": image_shapes, "output_slices": {"hidden_state": slice(0, 2)}}, + "policy": {"input_shapes": policy_shapes}} + run = make_run_split_policy(vision, {"policy": policy}, metadata, ["policy"], 2) + queues = partial(make_split_input_queues, image_shapes, policy_shapes, 2) + else: + metadata = {"model": {"input_shapes": image_shapes | policy_shapes}} + run = make_run_supercombo(supercombo, metadata, 2) + queues = partial(make_supercombo_input_queues, image_shapes | policy_shapes, 2) + compile_jit(TinyJit(run, prune=True), partial(make_random_images, ["warped"], (2, 6, 2, 2)), + FAST_POLICY_INPUTS, queues, validation_runs=5) + + def test_source_selection_rejects_ambiguity_and_missing_split_component(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "driving_supercombo.onnx").write_bytes(b"one") + self.assertEqual(set(model_compiler.source_files(root, "test")), {"supercombo"}) + (root / "supercombo.onnx").write_bytes(b"two") + with self.assertRaisesRegex(ValueError, "Ambiguous"): + model_compiler.source_files(root, "test") + with self.assertRaisesRegex(ValueError, "Split input requires"): + model_compiler.driving_compile_args({"vision": Path("vision.onnx")}, "split") + + def test_multi_policy_order_and_explicit_behavior_version(self): + kind, arguments = model_compiler.driving_compile_args({key: Path(key + ".onnx") for key in ("vision", "on-policy", "off-policy")}, "split") + self.assertEqual(kind, "vision_multi_policy") + self.assertEqual(arguments, ["--vision-onnx", "vision.onnx", "--on-policy-onnx", "on-policy.onnx", "--off-policy-onnx", "off-policy.onnx"]) + args = model_compiler.parse_args(["--example", "--version", "v15"]) + self.assertEqual((args.model, args.version, args.frame_skip), ("example", "v15", None)) + + def test_failed_compile_preserves_existing_output(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + (root / "driving_supercombo.onnx").write_bytes(stateful_onnx()) + output = root / "test_driving_tinygrad.pkl" + output.write_bytes(b"previous artifact") + with patch.object(model_compiler.subprocess, "run", side_effect=subprocess.CalledProcessError(1, "compiler")): + with self.assertRaises(subprocess.CalledProcessError): + model_compiler.main(["--model", "test", "--version", "v16", "--input-dir", str(root), "--output-dir", str(root)]) + self.assertEqual(output.read_bytes(), b"previous artifact") + + def test_build_receipt_hashes_exact_sources_and_artifact(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source = root / "driving_supercombo.onnx" + source.write_bytes(stateful_onnx()) + def compile_stub(command, **kwargs): + Path(command[command.index("--output") + 1]).write_bytes(b"compiled bytes") + with patch.object(model_compiler.subprocess, "run", side_effect=compile_stub): + model_compiler.main(["--model", "test", "--version", "v16", "--input-dir", str(root), "--output-dir", str(root)]) + output = root / "test_driving_tinygrad.pkl" + receipt = json.loads(output.with_suffix(".build.json").read_text()) + self.assertEqual(receipt["source_sha256"], {source.name: model_compiler.sha256_file(source)}) + self.assertEqual(receipt["artifact_sha256"], model_compiler.sha256_file(output)) + self.assertEqual(receipt["artifact_abi"], "tinygrad_single_v1_arena") + + def test_stateful_onnx_compiles_warp_and_replays_arena_on_cpu(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + source, artifact = root / "fixture.onnx", root / "fixture.pkl" + source.write_bytes(stateful_onnx()) + environment = dict(os.environ, DEV="CPU:LLVM", JIT_BATCH_SIZE="0", IMAGE="0", FLOAT16="0") + environment.pop("WARP_DEV", None) + command = [sys.executable, "-m", "openpilot.starpilot.models.compile", "--model-type", "supercombo", + "--model-size", "4x4", "--camera-resolutions", "8x8", "--behavior-version", "v16", + "--supercombo-onnx", str(source), "--output", str(artifact)] + result = subprocess.run(command, env=environment, capture_output=True, text=True, timeout=90) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + with patch.dict(os.environ, environment): + from tinygrad import Context + from tinygrad_repo.examples.openpilot.helpers import load_pickle + from openpilot.starpilot.models.compile import make_stateful_input_queues + with Context(DEV="CPU:LLVM"): + loaded = load_pickle(artifact, out_of_band=True) + queues, host = make_stateful_input_queues(loaded["metadata"]["model"], loaded["execution_device"]) + from tinygrad import Tensor + warped = Tensor(np.zeros((2, 6, 2, 2), dtype=np.uint8)).realize() + host["desire"][:] = (1, 2) + for step in range(1, 4): + output, = loaded["run_policy"](warped=warped, **{name: queues[name] for name in loaded["policy_input_keys"]}) + np.testing.assert_array_equal(output.numpy(), [[step, step * 2]]) + self.assertEqual(loaded["compiler_revision"], "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf") + self.assertEqual(loaded["frame_skip"], 1) + self.assertIn((8, 8), loaded) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_helper_import.py b/openpilot/starpilot/models/tests/test_helper_import.py new file mode 100644 index 0000000000..e029ceab50 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_helper_import.py @@ -0,0 +1,108 @@ +import importlib.util +from pathlib import Path +import subprocess +import sys +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np +from tinygrad import Context, Device +from tinygrad.helpers import OPENPILOT_HACKS, TC_MIN_GLOBALS, TC_OPT + + +class TestHelperImport(unittest.TestCase): + def test_catalog_compile_import_does_not_select_or_open_device(self): + # A fresh process covers the real transitive import, without a prior module + # cache or the host's selected device concealing import-time probing. + code = ''' +import importlib +from unittest.mock import patch +from tinygrad import Context, Device + +def unexpected_probe(*args): + raise AssertionError("catalog import attempted device auto-selection") + +with Context(DEV=""), patch.object(type(Device), "_select_device", property(unexpected_probe)), \\ + patch.object(type(Device), "__getitem__", side_effect=AssertionError("catalog import opened a device")): + importlib.import_module("openpilot.starpilot.models.compile") +''' + for module in ("tinygrad_repo.examples.openpilot.helpers", "openpilot.starpilot.models.compile"): + with self.subTest(module=module): + source = code.replace("openpilot.starpilot.models.compile", module) + result = subprocess.run([sys.executable, "-c", source], capture_output=True, text=True, check=False) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + @staticmethod + def load_helper(): + path = Path(__file__).resolve().parents[4] / "tinygrad_repo/examples/openpilot/helpers.py" + spec = importlib.util.spec_from_file_location("benchmark_import_regression", path) + helper = importlib.util.module_from_spec(spec) + # Import under QCOM, then call under another device to catch settings that + # were incorrectly captured by an import-time decorator. + with Context(DEV="QCOM"): + spec.loader.exec_module(helper) + return helper + + def test_benchmark_uses_call_time_device_and_preserves_results(self): + helper = self.load_helper() + for device, expected in (("AMD", (1, 2, 32)), ("QCOM", (1, 7, 11)), ("CPU", (1, 7, 11))): + with self.subTest(device=device): + events, elapsed = [], [] + array = np.array([1., 2.], dtype=np.float32) + output = SimpleNamespace(realize=lambda events=events: events.append("realize"), numpy=lambda array=array: array) + fake_device = SimpleNamespace(synchronize=lambda events=events: events.append("sync")) + + def run(value, expected=expected, events=events, output=output): + self.assertEqual(value, 9) + self.assertEqual((OPENPILOT_HACKS.value, TC_OPT.value, TC_MIN_GLOBALS.value), expected) + events.append("run") + return output + + with Context(DEV=device, OPENPILOT_HACKS=0, TC_OPT=7, TC_MIN_GLOBALS=11), \ + patch.object(type(Device), "default", property(lambda _, fake_device=fake_device: fake_device)), \ + patch.object(helper, "get_parameters", side_effect=lambda value: [value]), \ + patch.object(helper.time, "perf_counter", side_effect=(10., 10.25)): + result = helper.benchmark(run, cb=elapsed.append, value=9) + self.assertEqual((OPENPILOT_HACKS.value, TC_OPT.value, TC_MIN_GLOBALS.value), (0, 7, 11)) + self.assertEqual(events, ["sync", "run", "realize", "sync"]) + self.assertEqual(elapsed, [.25]) + np.testing.assert_array_equal(result[0], array) + self.assertIsNot(result[0], array) + + def test_benchmark_restores_context_on_failure(self): + helper = self.load_helper() + error = RuntimeError("benchmark failed") + fake_device = SimpleNamespace(synchronize=lambda: None) + + def fail(): + self.assertEqual((OPENPILOT_HACKS.value, TC_OPT.value, TC_MIN_GLOBALS.value), (1, 2, 32)) + raise error + + with Context(DEV="AMD", OPENPILOT_HACKS=0, TC_OPT=7, TC_MIN_GLOBALS=11), \ + patch.object(type(Device), "default", property(lambda _: fake_device)): + with self.assertRaises(RuntimeError) as caught: + helper.benchmark(fail) + self.assertIs(caught.exception, error) + self.assertEqual((OPENPILOT_HACKS.value, TC_OPT.value, TC_MIN_GLOBALS.value), (0, 7, 11)) + + def test_lazy_onnx_parser_preserves_metadata_without_gpu_probe(self): + from openpilot.starpilot.models.compile import make_metadata_dict + from openpilot.starpilot.models.tests.test_compile import stateful_onnx + + shapes = {"new_img": (2, 6, 2, 2), "state": (1, 2), "desire": (1, 2)} + runner = SimpleNamespace(graph_inputs={name: SimpleNamespace(shape=shape) for name, shape in shapes.items()}) + get_device = type(Device).__getitem__ + + def disk_only(device, name): + self.assertEqual(name.split(":", 1)[0], "DISK") + return get_device(device, name) + + with tempfile.TemporaryDirectory() as temporary, Context(DEV="CPU:LLVM"), \ + patch.object(type(Device), "__getitem__", disk_only): + path = Path(temporary) / "metadata.onnx" + path.write_bytes(stateful_onnx()) + metadata = make_metadata_dict(path, runner) + self.assertEqual(metadata, {"model_checkpoint": None, "output_slices": {"fixture": slice(0, 2)}, + "input_shapes": shapes, "output_shapes": {"outputs": (1, 2), "next_state": (1, 2)}}) diff --git a/openpilot/starpilot/models/tests/test_laboratory.py b/openpilot/starpilot/models/tests/test_laboratory.py new file mode 100644 index 0000000000..e46199d2da --- /dev/null +++ b/openpilot/starpilot/models/tests/test_laboratory.py @@ -0,0 +1,184 @@ +import copy +import hashlib +import io +import json +from pathlib import Path +import tempfile +import time +import unittest + +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, CATALOG_PATH, COMPILER_REVISION +from openpilot.starpilot.models.laboratory import ModelLaboratory, configuration +from openpilot.starpilot.models.manager import ModelError, ModelManager, artifact_path, atomic_json, validate_manifest + + +class LaboratoryTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.parked = True + self.gpu = False + self.manifest = json.loads(CATALOG_PATH.read_text()) + self.payloads = {} + for mid in ('gwm8223', 'sc23'): + row = next(row for row in self.manifest['models'] if row['id'] == mid) + for variant in ('standard', 'amd'): + content = f'{mid} compiled for {variant}'.encode() + path = artifact_path(mid, self.root, variant) + self.payloads[path.name] = content + metadata = {'artifact_format': ARTIFACT_ABI, 'artifact_sha256': hashlib.sha256(content).hexdigest(), + 'artifact_size': len(content), 'artifact_chunk_count': 0} + if variant == 'standard': + row.update(metadata) + else: + row['accelerator_artifacts'] = {'amd': {**metadata, 'execution_device': 'AMD', 'compiler_revision': COMPILER_REVISION}} + atomic_json(self.root / 'catalog.json', self.manifest) + + def opener(url, timeout): + if '/manifests/' in url: + return io.BytesIO(json.dumps(self.manifest).encode()) + return io.BytesIO(self.payloads[url.rsplit('/', 1)[-1]]) + + self.manager = ModelManager(root=self.root, parked=lambda: self.parked, gpu_present=lambda: self.gpu, opener=opener) + self.addCleanup(self.manager.close) + self.lab = ModelLaboratory(self.manager) + self.config = {'enabled': True, 'lateralModel': 'gwm8223', 'longitudinalModel': 'sc23'} + + def download(self, mid='gwm8223', variant='amd'): + if variant == 'amd': + self.lab.action('download', {'model': mid}) + else: + self.manager.action('download', {'model': mid}) + self.manager.worker.join(3) + self.assertFalse(self.manager.worker.is_alive()) + self.assertEqual(self.manager.snapshot()['progress'], 'Downloaded!') + + def ready_snapshot(self): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + result = self.lab.snapshot() + if not any(row['checking'] for row in result['models']): + return result + time.sleep(0.01) + self.fail('Laboratory verification stalled') + + def test_catalog_discovery_keeps_unpublished_and_big_rows_visible(self): + result = self.ready_snapshot() + rows = {row['value']: row for row in result['models']} + self.assertEqual(set(rows), {row['id'] for row in self.manifest['models']}) + self.assertEqual(result['summary']['catalog'], len(self.manifest['models'])) + self.assertEqual(result['summary']['published'], 2) + self.assertFalse(result['runtimeSupported']) + big = rows['cinquev3'] + self.assertFalse(big['small']) + self.assertFalse(big['modelLabEligible']) + self.assertEqual(big['modelLabStatus'], 'unsupported') + self.assertIn('shared camera warp', big['modelLabReason']) + unpublished = next(row for row in rows.values() if row['modelLabStatus'] == 'unpublished') + self.assertFalse(unpublished['modelLabArtifactAvailable']) + self.assertFalse(unpublished['modelLabArtifactInstalled']) + self.assertIn('Normal Small downloads', unpublished['modelLabReason']) + self.assertEqual(rows['gwm8223']['modelLabStatus'], 'missing') + self.download() + verified = next(row for row in self.ready_snapshot()['models'] if row['value'] == 'gwm8223') + self.assertEqual(verified['modelLabStatus'], 'runtime-unavailable') + self.assertFalse(self.lab.snapshot()['runtime']['active']) + + def test_amd_download_and_delete_preserve_normal_small_artifact(self): + self.download(variant='standard') + normal = artifact_path('gwm8223', self.root) + normal_bytes = normal.read_bytes() + self.assertEqual(self.ready_snapshot()['summary']['ready'], 0) + self.download() + amd = artifact_path('gwm8223', self.root, 'amd') + self.assertNotEqual(normal_bytes, amd.read_bytes()) + self.assertEqual(self.ready_snapshot()['summary']['ready'], 1) + self.assertFalse(self.lab.snapshot()['chestnutReady']) + result = self.lab.action('delete', {'model': 'gwm8223'}) + self.assertEqual(result['summary']['ready'], 0) + self.assertFalse(amd.exists()) + self.assertEqual(normal.read_bytes(), normal_bytes) + + def test_hash_mismatch_never_installs_amd_variant(self): + filename = artifact_path('gwm8223', self.root, 'amd').name + self.payloads[filename] = b'corrupt' + self.lab.action('download', {'model': 'gwm8223'}) + self.manager.worker.join(3) + self.assertIn('checksum', self.manager.snapshot()['progress']) + self.assertFalse(artifact_path('gwm8223', self.root, 'amd').exists()) + self.assertFalse(list(self.root.glob('*/.download-*'))) + + def test_wrong_runtime_hardware_and_model_class_are_rejected(self): + for field, value in (('compiler_revision', 'old-runtime'), ('execution_device', 'QCOM'), + ('artifact_sha256', 'invalid'), ('artifact_format', 'old-format')): + with self.subTest(field=field): + bad = copy.deepcopy(self.manifest) + row = next(row for row in bad['models'] if row['id'] == 'gwm8223') + row['accelerator_artifacts']['amd'][field] = value + with self.assertRaises(ModelError): + validate_manifest(bad) + bad = copy.deepcopy(self.manifest) + small = next(row for row in bad['models'] if row['id'] == 'gwm8223') + big = next(row for row in bad['models'] if row['id'] == 'cinquev3') + big['accelerator_artifacts'] = small['accelerator_artifacts'] + with self.assertRaises(ModelError): + validate_manifest(bad) + + def test_normal_models_and_old_amd_bytes_do_not_satisfy_pair_admission(self): + self.gpu = True + ready_runtime = ModelLaboratory(self.manager, runtime_supported=True) + for mid in ('gwm8223', 'sc23'): + self.download(mid, 'standard') + with self.assertRaisesRegex(ModelError, 'verify both eGPU'): + ready_runtime.action('configure', self.config) + self.download() + self.download('sc23') + artifact_path('sc23', self.root, 'amd').write_bytes(b'old compiled bytes') + with self.assertRaisesRegex(ModelError, 'verify both eGPU'): + ready_runtime.action('configure', self.config) + self.assertFalse(configuration(self.root)['enabled']) + + def test_enable_requires_runtime_hardware_distinct_models_and_verified_variants(self): + self.download() + self.download('sc23') + self.gpu = True + with self.assertRaisesRegex(ModelError, 'not available in this build'): + self.lab.action('configure', self.config) + ready_runtime = ModelLaboratory(self.manager, runtime_supported=True) + for invalid in ({**self.config, 'longitudinalModel': 'gwm8223'}, + {**self.config, 'lateralModel': 'cinquev3'}, {**self.config, 'enabled': 'true'}): + with self.assertRaises(ModelError): + ready_runtime.action('configure', invalid) + self.gpu = False + with self.assertRaisesRegex(ModelError, 'Chestnut'): + ready_runtime.action('configure', self.config) + self.gpu = True + saved = ready_runtime.action('configure', self.config) + self.assertTrue(saved['configuration']['enabled']) + self.assertFalse(saved['runtime']['active']) + with self.assertRaisesRegex(ModelError, 'Disable'): + ready_runtime.action('delete', {'model': 'sc23'}) + self.gpu = False + ready_runtime.action('configure', {**self.config, 'enabled': False}) + self.assertFalse(configuration(self.root)['enabled']) + + def test_parked_state_and_independent_progress_are_preserved(self): + self.parked = False + for action, payload in (('download', {'model': 'gwm8223'}), ('delete', {'model': 'gwm8223'}), + ('configure', {**self.config, 'enabled': False})): + with self.assertRaises(ModelError): + self.lab.action(action, payload) + self.parked = True + self.download() + observer = ModelManager(root=self.root, parked=lambda: True, gpu_present=lambda: False) + self.addCleanup(observer.close) + snapshot = ModelLaboratory(observer).snapshot() + self.assertEqual(snapshot['download']['variant'], 'amd') + self.assertEqual(snapshot['download']['progress'], 'Downloaded!') + self.assertEqual(snapshot['download']['jobId'], self.manager.job_id) + self.assertFalse(snapshot['runtimeSupported']) + + +if __name__ == '__main__': + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_laboratory_outputs.py b/openpilot/starpilot/models/tests/test_laboratory_outputs.py new file mode 100644 index 0000000000..b8ad770c73 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_laboratory_outputs.py @@ -0,0 +1,144 @@ +import hashlib +import json +from pathlib import Path +from types import SimpleNamespace +import unittest + +import numpy as np + +from openpilot.starpilot.models.laboratory_outputs import ( + CURRENT_FRAME_OUTPUT_KEYS, LATERAL_OUTPUT_KEYS, LATERAL_PLAN_COLUMNS, compose_model_outputs, hybrid_action_values, +) + + +def normalized_fixture(offset): + keys = (*LATERAL_OUTPUT_KEYS, *CURRENT_FRAME_OUTPUT_KEYS, "lead", "lead_prob", "meta", "planplus", "raw_pred", "unknown_extra") + outputs = {key: np.arange(6, dtype=np.float32).reshape(1, 2, 3) + offset + index for index, key in enumerate(keys)} + outputs["plan"] = np.arange(495, dtype=np.float32).reshape(1, 33, 15) + offset + outputs["plan_stds"] = outputs["plan"] / 100 + return outputs + + +def output_digest(array): + return hashlib.sha256(array.tobytes()).hexdigest() + + +class TestLaboratoryOutputs(unittest.TestCase): + def test_frozen_original_normalized_composition(self): + frozen = json.loads(Path(__file__).with_name("fixtures").joinpath("laboratory_outputs_original.json").read_text()) + self.assertEqual(frozen["original_source_sha256"], "f0210b2ca26a1b9c4adbcf5825d0d01851254dbfbdbc54595c16d9d77d71731f") + for case in frozen["cases"]: + with self.subTest(versions=case["versions"]): + result = compose_model_outputs(normalized_fixture(case["lateral_offset"]), normalized_fixture(case["longitudinal_offset"])) + self.assertEqual({key: output_digest(value) for key, value in result.items()}, case["expected"]) + + def test_every_axis_and_head_owner(self): + lateral, longitudinal = normalized_fixture(1000), normalized_fixture(2000) + result = compose_model_outputs(lateral, longitudinal) + for column in range(15): + owner = lateral if column in LATERAL_PLAN_COLUMNS else longitudinal + for key in ("plan", "plan_stds"): + np.testing.assert_array_equal(result[key][..., column], owner[key][..., column]) + for key in LATERAL_OUTPUT_KEYS: + np.testing.assert_array_equal(result[key], lateral[key]) + for key in (*CURRENT_FRAME_OUTPUT_KEYS, "lead", "lead_prob", "meta", "planplus", "raw_pred", "unknown_extra"): + np.testing.assert_array_equal(result[key], longitudinal[key]) + + def test_absent_lateral_and_frame_heads_remove_longitudinal_values(self): + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + for key in LATERAL_OUTPUT_KEYS: + del lateral[key] + result = compose_model_outputs(lateral, longitudinal, {}) + self.assertTrue(set(LATERAL_OUTPUT_KEYS + CURRENT_FRAME_OUTPUT_KEYS).isdisjoint(result)) + explicit_frame = {"pose": np.ones((1, 6))} + np.testing.assert_array_equal(compose_model_outputs(lateral, longitudinal, explicit_frame)["pose"], explicit_frame["pose"]) + + def test_arrays_independent_in_both_directions(self): + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + result = compose_model_outputs(lateral, longitudinal) + for key, value in result.items(): + for inputs in (lateral, longitudinal): + self.assertFalse(np.shares_memory(value, inputs[key])) + original = {key: value.copy() for key, value in result.items()} + for value in lateral.values(): + value.fill(-1) + for value in longitudinal.values(): + value.fill(-2) + for key in result: + np.testing.assert_array_equal(result[key], original[key]) + result[key].fill(-3) + self.assertTrue(all(np.all(value == -1) for value in lateral.values())) + self.assertTrue(all(np.all(value == -2) for value in longitudinal.values())) + + def test_missing_or_incompatible_plan_and_uncertainty_fail(self): + for key, value in (("plan", None), ("plan", np.zeros((1, 32, 15))), ("plan", np.zeros((15,))), + ("plan", np.zeros((1, 33, 14))), ("plan_stds", None), ("plan_stds", np.zeros((1, 32, 15)))): + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + if value is None: + del lateral[key] + else: + lateral[key] = value + with self.subTest(key=key, shape=None if value is None else value.shape), self.assertRaises(ValueError): + compose_model_outputs(lateral, longitudinal) + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + del lateral["plan_stds"], longitudinal["plan_stds"] + self.assertNotIn("plan_stds", compose_model_outputs(lateral, longitudinal)) + + def test_nonfinite_even_unselected_arrays_fail(self): + for role in (0, 1, 2): + for value in (np.nan, np.inf, -np.inf): + outputs = [normalized_fixture(10), normalized_fixture(20), normalized_fixture(30)] + outputs[role]["unknown_extra"].flat[0] = value + with self.subTest(role=role, value=value), self.assertRaises(ValueError): + compose_model_outputs(*outputs) + + def test_raw_actions_omitted_with_mixed_head_presence(self): + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + for outputs in (lateral, longitudinal): + outputs["action"] = np.array([[10, 20]], dtype=np.float32) + outputs["action_stds"] = np.ones((1, 2), dtype=np.float32) + for present in (True, False): + if not present: + del lateral["action"], lateral["action_stds"] + result = compose_model_outputs(lateral, longitudinal) + self.assertNotIn("action", result) + self.assertNotIn("action_stds", result) + + def test_decoded_actions_have_explicit_owners_and_finite_guard(self): + lateral = SimpleNamespace(desiredCurvature=.012, desiredAcceleration=99, shouldStop=False) + longitudinal = SimpleNamespace(desiredCurvature=99, desiredAcceleration=-.25, shouldStop=True) + self.assertEqual(hybrid_action_values(lateral, longitudinal), {"desiredCurvature": .012, "desiredAcceleration": -.25, "shouldStop": True}) + for action, field in ((lateral, "desiredCurvature"), (longitudinal, "desiredAcceleration")): + setattr(action, field, np.nan) + with self.assertRaises(ValueError): + hybrid_action_values(lateral, longitudinal) + setattr(action, field, 0) + + def test_mixed_v14_v15_actions_decoded_before_composition(self): + from openpilot.cereal import log + from openpilot.starpilot.models.runner import action_from_outputs + previous = log.ModelDataV2.Action() + lateral = action_from_outputs({"action": np.array([[2., 80.]])}, "v14", previous, .2, .2, 10, long_smooth_seconds=0) + longitudinal = action_from_outputs({"action": np.array([[40., -.5]])}, "v15", previous, .2, .2, 10, long_smooth_seconds=0) + result = hybrid_action_values(lateral, longitudinal) + self.assertAlmostEqual(result["desiredCurvature"], .02) + self.assertAlmostEqual(result["desiredAcceleration"], -.5) + self.assertFalse(result["shouldStop"]) + + def test_mixed_v15_v9_actions_allow_different_head_presence(self): + from openpilot.cereal import log + from openpilot.starpilot.models.runner import action_from_outputs + previous = log.ModelDataV2.Action() + lateral, longitudinal = normalized_fixture(10), normalized_fixture(20) + lateral["action"] = np.array([[3., 80.]]) + longitudinal["plan"] = np.zeros((1, 33, 15)) + longitudinal["plan"][..., 3] = 10 + longitudinal.pop("planplus") + longitudinal["desired_curvature"] = np.array([[.8]]) + lat_action = action_from_outputs(lateral, "v15", previous, .2, .2, 10, long_smooth_seconds=0) + long_action = action_from_outputs(longitudinal, "v9", previous, .2, .2, 10, long_smooth_seconds=0) + self.assertNotIn("action", compose_model_outputs(lateral, longitudinal)) + result = hybrid_action_values(lat_action, long_action) + self.assertAlmostEqual(result["desiredCurvature"], .03) + self.assertEqual(result["desiredAcceleration"], 0.) + self.assertFalse(result["shouldStop"]) diff --git a/openpilot/starpilot/models/tests/test_laboratory_runner.py b/openpilot/starpilot/models/tests/test_laboratory_runner.py new file mode 100644 index 0000000000..239fcb1089 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_laboratory_runner.py @@ -0,0 +1,183 @@ +import hashlib +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +import numpy as np + +from openpilot.cereal import log +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, CATALOG_PATH, COMPILER_REVISION +from openpilot.starpilot.models.laboratory_runner import LaboratoryPair, load_laboratory_pair +from openpilot.starpilot.models.manager import artifact_path, atomic_json +from openpilot.starpilot.models.receipt import PreparedArtifact +from openpilot.starpilot.models.tests.test_shared_camera_warp import frame_inputs, make_runner + + +def make_pair(versions=("v14", "v15")): + models = [make_runner(), make_runner()] + for model, mid, version in zip(models, ("lateral", "longitudinal"), versions, strict=True): + model.model_id, model.behavior_version, model.chestnut = mid, version, True + model._parse = Mock(return_value={ + **model._parse([model.run_policy.return_value[0].numpy()]), + "plan": np.full((1, 33, 15), 1. if mid == "lateral" else 2.), + "pose": np.full((1, 6), 3. if mid == "lateral" else 4.), + "action": np.array([[4., .5 if mid == "longitudinal" else 1.]]), + }) + prepared = PreparedArtifact((0, 0, 1, 0, 0), "a" * 64) + return LaboratoryPair(*models, prepared, prepared) + + +class TestLaboratoryFrame(unittest.TestCase): + @staticmethod + def run_frame(pair, **kwargs): + return pair.run_frame(*frame_inputs(pair.lateral), log.ModelDataV2.Action(), .2, .2, 20., + long_smooth_seconds=0., **kwargs) + + def test_sequential_frame_one_warp_independent_actions_and_current_frame_pose(self): + pair = make_pair() + callback = Mock() + frame = self.run_frame(pair, after_enqueue=callback) + pair.lateral.warp_enqueue.assert_called_once() + pair.longitudinal.warp_enqueue.assert_not_called() + callback.assert_called_once_with() + self.assertIs(pair.longitudinal.run_policy.call_args.kwargs["warped"], pair.lateral.last_warp.tensor) + self.assertAlmostEqual(frame.action.desiredCurvature, .04) + self.assertAlmostEqual(frame.action.desiredAcceleration, .5) + self.assertFalse(frame.action.shouldStop) + np.testing.assert_array_equal(frame.outputs["plan"][0, 0], [2, 1, 2, 2, 1, 2, 2, 1, 2, 2, 2, 1, 2, 2, 1]) + np.testing.assert_array_equal(frame.outputs["pose"], pair.longitudinal._parse.return_value["pose"]) + self.assertNotIn("action", frame.outputs) + frame.outputs["plan"][:] = 9 + self.assertEqual(pair.lateral._parse.return_value["plan"][0, 0, 1], 1) + + def test_mixed_generations_decode_before_combining_axes(self): + pair = make_pair(("v15", "v14")) + self.assertAlmostEqual(self.run_frame(pair).action.desiredCurvature, .01) + pair = make_pair(("v9", "v15")) + pair.lateral._parse.return_value.pop("desired_curvature") + previous = log.ModelDataV2.Action(desiredCurvature=.03, desiredAcceleration=.8) + frame = pair.run_frame(*frame_inputs(pair.lateral), previous, .2, .2, .2, long_smooth_seconds=0.) + self.assertAlmostEqual(frame.action.desiredCurvature, .03) + self.assertAlmostEqual(frame.action.desiredAcceleration, .5) + self.assertFalse(frame.action.shouldStop) + pair.longitudinal._parse.return_value["action"][0, 1] = -.5 + frame = pair.run_frame(*frame_inputs(pair.lateral), previous, .2, .2, .2, long_smooth_seconds=0.) + self.assertTrue(frame.action.shouldStop) + + def test_each_role_failure_latches_without_publishing_a_partial_frame(self): + for role in ("lateral", "longitudinal"): + with self.subTest(role=role): + pair = make_pair() + model = getattr(pair, role) + model.run_policy.side_effect = RuntimeError("execution failed") + with self.assertRaisesRegex(RuntimeError, "execution failed"): + self.run_frame(pair) + self.assertTrue(pair.failed) + self.assertIsNone(pair.lateral.last_warp) + self.assertIsNone(pair.longitudinal.last_warp) + if role == "lateral": + pair.longitudinal.run_policy.assert_not_called() + model.run_policy.side_effect = None + with self.assertRaisesRegex(RuntimeError, "fallback"): + self.run_frame(pair) + + def test_action_and_output_failures_are_inside_the_same_failure_boundary(self): + for key, invalid in (("plan", np.zeros((1, 32, 15))), ("action", np.zeros((1, 3))), + ("pose", np.full((1, 6), np.nan))): + with self.subTest(key=key): + pair = make_pair() + pair.longitudinal._parse.return_value[key] = invalid + with self.assertRaises(ValueError): + self.run_frame(pair) + self.assertTrue(pair.failed) + with self.assertRaisesRegex(RuntimeError, "fallback"): + self.run_frame(pair) + + def test_absent_warp_cannot_silently_run_second_preprocessor(self): + pair = make_pair() + pair.lateral.run = Mock(return_value=pair.lateral._parse.return_value) + with self.assertRaisesRegex(ValueError, "warp unavailable"): + self.run_frame(pair) + pair.longitudinal.run_policy.assert_not_called() + self.assertTrue(pair.failed) + + def test_pair_admission_rejects_same_runner_model_wrong_hardware_and_history(self): + pair = make_pair() + prepared = pair.lateral_role.artifact + with self.assertRaisesRegex(ValueError, "distinct"): + LaboratoryPair(pair.lateral, pair.lateral, prepared, prepared) + for key, value, message in (("model_id", "lateral", "distinct"), ("chestnut", False, "AMD"), + ("camera_warp_descriptor", None, "policy-history")): + with self.subTest(key=key): + pair = make_pair() + setattr(pair.longitudinal, key, value) + with self.assertRaisesRegex(ValueError, message): + LaboratoryPair(pair.lateral, pair.longitudinal, prepared, prepared) + + +class TestLaboratoryLoad(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = Path(directory.name) + self.config = {"enabled": True, "lateralModel": "gwm8223", "longitudinalModel": "sc23"} + manifest = json.loads(CATALOG_PATH.read_text()) + self.rows = {} + for mid in ("gwm8223", "sc23"): + content = f"verified AMD {mid}".encode() + path = artifact_path(mid, self.root, "amd") + path.parent.mkdir(parents=True) + path.write_bytes(content) + row = next(row for row in manifest["models"] if row["id"] == mid) + row["accelerator_artifacts"] = {"amd": { + "artifact_format": ARTIFACT_ABI, "execution_device": "AMD", "compiler_revision": COMPILER_REVISION, + "artifact_sha256": hashlib.sha256(content).hexdigest(), "artifact_size": len(content), "artifact_chunk_count": 0, + }} + self.rows[mid] = row + self.manifest = manifest + atomic_json(self.root / "catalog.json", manifest) + + def factory(self, width, height, path, version, chestnut): + self.assertEqual((width, height), (8, 8)) + self.assertTrue(chestnut) + self.assertIn("_amd_", path.name) + model = make_runner() + model.chestnut, model.behavior_version = chestnut, version + model.warmup = Mock() + return model + + def test_separate_verified_loads_preserve_both_identities_and_warmup(self): + with patch("openpilot.starpilot.models.runner.CatalogModelState", side_effect=self.factory) as loader: + pair = load_laboratory_pair(8, 8, self.config, root=self.root) + self.assertEqual(loader.call_count, 2) + self.assertIsNot(pair.lateral, pair.longitudinal) + for mid, model, role in (("gwm8223", pair.lateral, pair.lateral_role), ("sc23", pair.longitudinal, pair.longitudinal_role)): + self.assertEqual(role.model_id, mid) + self.assertEqual(role.behavior_version, self.rows[mid]["version"]) + self.assertEqual(role.artifact.sha256, self.rows[mid]["accelerator_artifacts"]["amd"]["artifact_sha256"]) + model.warmup.assert_called_once_with() + + def test_both_variants_must_be_declared_before_either_load(self): + del self.rows["sc23"]["accelerator_artifacts"] + atomic_json(self.root / "catalog.json", self.manifest) + with patch("openpilot.starpilot.models.runner.CatalogModelState") as loader: + with self.assertRaisesRegex(ValueError, "published AMD"): + load_laboratory_pair(8, 8, self.config, root=self.root) + loader.assert_not_called() + + def test_changed_second_artifact_cannot_return_a_half_loaded_pair(self): + artifact_path("sc23", self.root, "amd").write_bytes(b"changed bytes") + with patch("openpilot.starpilot.models.runner.CatalogModelState", side_effect=self.factory) as loader: + with self.assertRaisesRegex(ValueError, "changed before loading"): + load_laboratory_pair(8, 8, self.config, root=self.root) + self.assertEqual(loader.call_count, 1) + + def test_disabled_duplicate_and_big_requests_cannot_load(self): + for config in ({**self.config, "enabled": False}, {**self.config, "longitudinalModel": "gwm8223"}, + {**self.config, "lateralModel": "cinquev3"}): + with self.subTest(config=config), patch("openpilot.starpilot.models.runner.CatalogModelState") as loader: + with self.assertRaises(ValueError): + load_laboratory_pair(8, 8, config, root=self.root) + loader.assert_not_called() diff --git a/openpilot/starpilot/models/tests/test_manager.py b/openpilot/starpilot/models/tests/test_manager.py new file mode 100644 index 0000000000..04e77cf243 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_manager.py @@ -0,0 +1,539 @@ +import hashlib +import io +import json +import multiprocessing +from pathlib import Path +import tempfile +import threading +import time +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, BUNDLED_CURRENT, CATALOG_PATH +from openpilot.starpilot.models.manager import ( + ModelError, ModelManager, atomic_json, catalog, preferences, randomize_next_start, resolve_runtime, validate_manifest, +) + + +def run_download_owner(root, manifest, data, entered, release, result): + class BlockedPart(io.BytesIO): + def read(self, size=-1): + entered.set() + if not release.wait(5): + raise OSError('fixture stalled') + return super().read(size) + + def opener(url, timeout): + if '/manifests/' in url: + return io.BytesIO(json.dumps(manifest).encode()) + if url.endswith('chunk01of02'): + return io.BytesIO(data[:10]) + return BlockedPart(data[10:]) + + manager = ModelManager(root=Path(root), parked=lambda: True, gpu_present=lambda: False, opener=opener) + with patch('openpilot.starpilot.models.manager.shutil.disk_usage', return_value=SimpleNamespace(free=1024**3)): + try: + manager.action('download', {'model': 'gwm8223'}) + manager.worker.join(6) + result.put(manager.snapshot()) + finally: + manager.close() + + +class ModelManagerTest(unittest.TestCase): + def setUp(self): + disk_usage = patch('openpilot.starpilot.models.manager.shutil.disk_usage', return_value=SimpleNamespace(free=1024**3)) + self.disk_usage = disk_usage.start() + self.addCleanup(disk_usage.stop) + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.parked = True + self.data = b'a verified compiled artifact' + self.manifest = json.loads(CATALOG_PATH.read_text()) + self.row = next(x for x in self.manifest['models'] if x['id'] == 'gwm8223') + self.row.update(artifact_format=ARTIFACT_ABI, artifact_sha256=hashlib.sha256(self.data).hexdigest(), + artifact_size=len(self.data), artifact_chunk_count=2) + self.urls = [] + self.bad = False + + def request(url, timeout): + self.assertTrue(url.startswith('https://huggingface.co/buckets/StarPilot-Driving/StarPilot-Resources/resolve/')) + self.urls.append(url) + if '/manifests/' in url: + return io.BytesIO(json.dumps(self.manifest).encode()) + if url.endswith('chunk01of02'): + return io.BytesIO(self.data[:10]) + if url.endswith('chunk02of02'): + return io.BytesIO(b'corrupt' if self.bad else self.data[10:]) + raise OSError('missing fixture') + + self.manager = ModelManager(root=self.root, parked=lambda: self.parked, gpu_present=lambda: True, opener=request) + self.addCleanup(self.manager.close) + + def download(self): + self.manager.action('download', {'model': 'gwm8223'}) + self.manager.worker.join(3) + self.assertFalse(self.manager.worker.is_alive()) + + def model_status(self): + return next(x for x in self.manager.snapshot()['models'] if x['value'] == 'gwm8223') + + def wait_for_check(self): + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + row = self.model_status() + if not row['checking']: + return row + time.sleep(0.01) + self.fail('Artifact verification did not finish') + + def test_chunk_download_verifies_and_selects_next_start(self): + self.download() + self.assertEqual(self.manager.progress, 'Downloaded!') + self.assertEqual(preferences(self.root)['small'], BUNDLED_CURRENT) + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + selected = resolve_runtime(True, root=self.root) + self.assertEqual(selected.small_id, 'gwm8223') + self.assertEqual(selected.small_path.read_bytes(), self.data) + self.assertTrue(all('/v26/' in u or 'model_names_v26.json' in u for u in self.urls)) + + def test_corruption_never_publishes_or_selects(self): + self.bad = True + self.download() + self.assertIn('checksum', self.manager.progress) + self.assertFalse(list(self.root.rglob('*.pkl'))) + with self.assertRaises(ModelError): + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + + def test_insufficient_storage_preserves_installed_model_and_selection(self): + self.download() + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + other = next(row for row in self.manifest['models'] if row['id'] == 'pop223') + other.update({key: self.row[key] for key in ('artifact_format', 'artifact_sha256', 'artifact_size', 'artifact_chunk_count')}) + self.urls.clear() + self.disk_usage.return_value = SimpleNamespace(free=256 * 1024 * 1024 + len(self.data) - 1) + self.manager.action('download', {'model': 'pop223'}) + self.manager.worker.join(3) + self.assertFalse(self.manager.worker.is_alive()) + self.assertEqual(self.manager.progress, 'Not enough storage for this model') + self.assertTrue(self.urls) + self.assertTrue(all('/manifests/' in url for url in self.urls)) + self.assertEqual(preferences(self.root)['small'], 'gwm8223') + self.assertEqual(resolve_runtime(True, root=self.root).small_path.read_bytes(), self.data) + self.assertFalse(list((self.root / 'pop223').glob('.download-*'))) + + def test_changed_installed_bytes_fall_back_and_ui_rechecks(self): + self.download() + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + self.assertTrue(self.wait_for_check()['installed']) + (self.root/'gwm8223/gwm8223_driving_tinygrad.pkl').write_bytes(b'x' * len(self.data)) + self.assertEqual(resolve_runtime(True, root=self.root).small_id, BUNDLED_CURRENT) + self.assertFalse(self.model_status()['installed']) + self.assertFalse(self.wait_for_check()['installed']) + + def test_snapshot_does_not_wait_for_artifact_hash(self): + self.download() + self.manager.checked.clear() + entered, release = threading.Event(), threading.Event() + from openpilot.starpilot.models import manager as manager_module + original = manager_module.verified_artifact + + def slow_verify(*args, **kwargs): + entered.set() + self.assertTrue(release.wait(3)) + return original(*args, **kwargs) + + with patch.object(manager_module, 'verified_artifact', side_effect=slow_verify): + started = time.monotonic() + first = self.model_status() + self.assertLess(time.monotonic() - started, 0.5) + self.assertTrue(first['checking']) + self.assertFalse(first['installed']) + self.assertFalse(first['selectable']) + self.assertEqual(first['unavailableReason'], 'Checking artifact') + self.assertTrue(entered.wait(1)) + started = time.monotonic() + second = self.model_status() + self.assertLess(time.monotonic() - started, 0.5) + self.assertTrue(second['checking']) + release.set() + self.assertTrue(self.wait_for_check()['installed']) + + def test_replaced_artifact_invalidates_cached_validity(self): + self.download() + self.assertTrue(self.wait_for_check()['installed']) + path = self.root/'gwm8223/gwm8223_driving_tinygrad.pkl' + replacement = path.with_suffix('.replacement') + replacement.write_bytes(b'x' * len(self.data)) + replacement.replace(path) + row = self.model_status() + self.assertFalse(row['installed']) + self.assertFalse(row['selectable']) + self.assertFalse(self.wait_for_check()['installed']) + replacement.write_bytes(self.data) + replacement.replace(path) + self.assertFalse(self.model_status()['installed']) + self.assertTrue(self.wait_for_check()['installed']) + + def test_offroad_authority_rechecked_after_validation(self): + self.download() + samples = iter([True, False]) + self.manager.parked = lambda: next(samples) + with self.assertRaises(ModelError): + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + self.assertEqual(preferences(self.root)['small'], BUNDLED_CURRENT) + + def test_hardware_profiles_and_selected_delete(self): + self.download() + with self.assertRaises(ModelError): + self.manager.action('active', {'profile': 'big', 'model': 'gwm8223'}) + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + with self.assertRaises(ModelError): + self.manager.action('delete', {'model': 'gwm8223'}) + self.manager.action('active', {'profile': 'big', 'model': ''}) + self.assertFalse(resolve_runtime(True, root=self.root).allow_big) + + def test_manifest_rejects_old_generation_and_hardware_change(self): + self.manifest['generation'] = 'v25' + with self.assertRaises(ModelError): + validate_manifest(self.manifest) + self.manifest['generation'] = 'v26' + self.row['uses_external_gpu'] = True + with self.assertRaises(ModelError): + validate_manifest(self.manifest) + + def runtime_artifact(self, revision=2): + fields = ('artifact_format', 'artifact_sha256', 'artifact_size', 'artifact_chunk_count') + artifact = {name: self.row.pop(name) for name in fields} + self.row['runtime_artifacts'] = [dict(artifact, min_runner_revision=revision)] + return artifact + + def test_runner_revision_gates_download_and_runtime_selection(self): + self.runtime_artifact() + self.download() + self.manager.action('active', {'profile': 'small', 'model': 'gwm8223'}) + self.assertEqual(resolve_runtime(True, root=self.root).small_id, 'gwm8223') + with patch('openpilot.starpilot.models.manager.MODEL_RUNNER_REVISION', 1): + self.assertNotIn('artifact_sha256', validate_manifest(self.manifest)['gwm8223']) + self.assertEqual(resolve_runtime(True, root=self.root).small_id, BUNDLED_CURRENT) + self.assertFalse(self.model_status()['downloadAvailable']) + self.assertEqual(resolve_runtime(True, root=self.root).small_id, 'gwm8223') + + def test_future_runner_variant_preserves_existing_artifact(self): + previous = self.runtime_artifact(3) + self.assertNotIn('artifact_sha256', validate_manifest(self.manifest)['gwm8223']) + self.row.update(previous) + self.assertEqual(validate_manifest(self.manifest)['gwm8223']['artifact_sha256'], previous['artifact_sha256']) + self.download() + self.assertEqual(self.manager.progress, 'Downloaded!') + + def test_runtime_variant_rejects_invalid_contract(self): + self.runtime_artifact() + valid = dict(self.row['runtime_artifacts'][0]) + for field, value in [('min_runner_revision', True), ('min_runner_revision', 0), + ('artifact_sha256', 'bad'), ('artifact_size', 0), ('unknown_feature', True)]: + self.row['runtime_artifacts'] = [dict(valid, **{field: value})] + with self.subTest(field=field, value=value), self.assertRaises(ModelError): + validate_manifest(self.manifest) + self.row['runtime_artifacts'] = [valid, valid] + with self.assertRaises(ModelError): + validate_manifest(self.manifest) + + def test_runtime_variant_keeps_download_checksum_validation(self): + self.runtime_artifact() + self.bad = True + self.download() + self.assertIn('checksum', self.manager.progress) + self.assertFalse(list(self.root.rglob('*.pkl'))) + + def test_missing_big_download_uses_small_and_leaves_request(self): + atomic_json(self.root/'preferences.json', {'small': BUNDLED_CURRENT, 'big': 'cinquev3'}) + self.assertFalse(resolve_runtime(True, root=self.root).allow_big) + self.assertEqual(preferences(self.root)['big'], 'cinquev3') + + def test_preferences_and_unknown_payloads(self): + self.manager.action('preferences', {'userFavorites': ['gwm8223'], 'sortMode': 'series'}) + self.assertEqual(preferences(self.root)['userFavorites'], ['gwm8223']) + for action, payload in [('active', {'profile': 'small', 'model': []}), ('download', {'model': '../bad'}), + ('delete', {'model': BUNDLED_CURRENT}), ('preferences', {'userFavorites': ['unknown']})]: + with self.subTest(action=action), self.assertRaises(ModelError): + self.manager.action(action, payload) + + def other_manager(self, opener=None): + manager = ModelManager(root=self.root, parked=lambda: self.parked, gpu_present=lambda: True, + opener=opener or self.manager.opener) + self.addCleanup(manager.close) + return manager + + def test_another_manager_observes_progress_and_cancels_owner_job(self): + entered, release = threading.Event(), threading.Event() + original = self.manager.opener + + class BlockedPart(io.BytesIO): + def read(inner, size=-1): + entered.set() + if not release.wait(3): + raise OSError('fixture stalled') + return super().read(size) + + def opener(url, timeout): + return BlockedPart(self.data[10:]) if url.endswith('chunk02of02') else original(url, timeout) + + self.manager.opener = opener + observer = self.other_manager() + self.manager.action('download', {'model': 'gwm8223'}) + try: + self.assertTrue(entered.wait(1)) + view = observer.snapshot() + self.assertTrue(view['downloading']) + self.assertEqual(view['modelToDownload'], 'gwm8223') + self.assertIn('%', view['progress']) + self.assertEqual(view['jobId'], self.manager.job_id) + for action, payload in [('download', {'model': 'gwm8223'}), ('refresh_manifest', {}), + ('active', {'profile': 'small', 'model': BUNDLED_CURRENT}), ('delete', {'model': 'gwm8223'})]: + with self.subTest(action=action), self.assertRaises(ModelError): + observer.action(action, payload) + observer.action('cancel', {'jobId': view['jobId']}) + self.assertTrue(observer.snapshot()['cancelRequested']) + finally: + release.set() + self.manager.worker.join(3) + self.assertFalse(self.manager.worker.is_alive()) + self.assertEqual(observer.snapshot()['progress'], 'Download cancelled') + self.assertFalse(observer.snapshot()['downloading']) + self.assertFalse(list(self.root.rglob('*.pkl'))) + self.assertFalse(list((self.root / 'gwm8223').glob('.download-*'))) + observer.action('refresh_manifest', {}) + observer.worker.join(3) + self.assertEqual(observer.snapshot()['progress'], 'Catalog updated') + self.assertNotEqual(observer.snapshot()['jobId'], view['jobId']) + with self.assertRaises(ModelError): + observer.action('cancel', {'jobId': view['jobId']}) + + def test_another_process_observes_and_cancels_download(self): + context = multiprocessing.get_context('spawn') + entered, release, result = context.Event(), context.Event(), context.Queue() + owner = context.Process(target=run_download_owner, + args=(str(self.root), self.manifest, self.data, entered, release, result)) + owner.start() + try: + self.assertTrue(entered.wait(5)) + view = self.manager.snapshot() + self.assertTrue(view['downloading']) + self.assertEqual(view['modelToDownload'], 'gwm8223') + self.assertIn('%', view['progress']) + with self.assertRaises(ModelError): + self.manager.action('refresh_manifest', {}) + self.manager.action('cancel', {'jobId': view['jobId']}) + release.set() + terminal = result.get(timeout=5) + owner.join(5) + self.assertEqual(owner.exitcode, 0) + self.assertFalse(terminal['downloading']) + self.assertEqual(terminal['progress'], 'Download cancelled') + self.assertFalse(list(self.root.rglob('*.pkl'))) + self.assertFalse(self.manager.snapshot()['downloading']) + finally: + release.set() + owner.join(5) + if owner.is_alive(): + owner.terminate() + owner.join(5) + result.close() + + def test_simultaneous_managers_accept_exactly_one_job(self): + entered, release = threading.Event(), threading.Event() + original = self.manager.opener + + def opener(url, timeout): + entered.set() + if not release.wait(3): + raise OSError('fixture stalled') + return original(url, timeout) + + self.manager.opener = opener + other = self.other_manager(opener) + barrier = threading.Barrier(3) + accepted, rejected = [], [] + + def start(manager): + barrier.wait() + try: + manager.action('download', {'model': 'gwm8223'}) + accepted.append(manager) + except ModelError as error: + rejected.append(str(error)) + + callers = [threading.Thread(target=start, args=(manager,)) for manager in (self.manager, other)] + for caller in callers: + caller.start() + barrier.wait() + try: + for caller in callers: + caller.join(1) + self.assertFalse(caller.is_alive()) + self.assertTrue(entered.wait(1)) + self.assertEqual(len(accepted), 1) + self.assertEqual(len(rejected), 1) + self.assertIn('Another model manager', rejected[0]) + self.assertEqual(self.manager.snapshot()['jobId'], other.snapshot()['jobId']) + self.assertTrue(other.snapshot()['downloading']) + finally: + release.set() + for manager in (self.manager, other): + if manager.worker is not None: + manager.worker.join(3) + self.assertEqual(other.snapshot()['progress'], 'Downloaded!') + self.assertFalse(other.snapshot()['downloading']) + + def test_closing_observer_does_not_cancel_other_manager_job(self): + entered, release = threading.Event(), threading.Event() + original = self.manager.opener + + def opener(url, timeout): + if '/manifests/' in url: + entered.set() + if not release.wait(3): + raise OSError('fixture stalled') + return original(url, timeout) + + self.manager.opener = opener + observer = self.other_manager() + self.manager.action('download', {'model': 'gwm8223'}) + try: + self.assertTrue(entered.wait(1)) + observer.close() + self.assertFalse(self.manager.snapshot()['cancelRequested']) + self.assertFalse(self.manager.cancelled.is_set()) + finally: + release.set() + self.manager.worker.join(3) + self.assertEqual(self.manager.progress, 'Downloaded!') + self.assertTrue(self.wait_for_check()['installed']) + + def test_stale_journal_and_completed_job_do_not_block_new_downloads(self): + observer = self.other_manager() + atomic_json(self.root / '.download-job.json', { + 'schemaVersion': 1, 'jobId': 'a' * 32, 'state': 'running', 'model': 'gwm8223', + 'downloadAll': True, 'progress': 'Old download', 'cancelRequested': True, + }) + view = observer.snapshot() + self.assertFalse(view['downloading']) + self.assertEqual(view['progress'], 'Previous model download was interrupted') + self.assertEqual(observer.action('cancel', {})['message'], 'No model download is running') + observer.action('download', {'model': 'gwm8223'}) + observer.worker.join(3) + self.assertEqual(observer.snapshot()['progress'], 'Downloaded!') + self.assertFalse(observer.snapshot()['downloading']) + observer.action('refresh_manifest', {}) + observer.worker.join(3) + self.assertEqual(observer.snapshot()['progress'], 'Catalog updated') + self.assertFalse(observer.snapshot()['downloading']) + + def install_fixture(self, mid): + row = next(x for x in self.manifest['models'] if x['id'] == mid) + row.update(artifact_format=ARTIFACT_ABI, artifact_sha256=hashlib.sha256(self.data).hexdigest(), + artifact_size=len(self.data), artifact_chunk_count=0) + path = self.root / mid / f'{mid}_driving_tinygrad.pkl' + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(self.data) + atomic_json(self.root / 'catalog.json', self.manifest) + return path + + def test_randomizer_off_is_read_only_and_opt_in(self): + self.install_fixture('gwm8223') + self.assertEqual(resolve_runtime(False, root=self.root, randomize=True).small_id, BUNDLED_CURRENT) + self.assertFalse((self.root / 'preferences.json').exists()) + self.manager.action('preferences', {'randomizer': True}) + self.assertEqual(resolve_runtime(False, root=self.root).small_id, BUNDLED_CURRENT) + self.assertEqual(resolve_runtime(False, root=self.root, randomize=True).small_id, 'gwm8223') + self.assertEqual(preferences(self.root)['small'], 'gwm8223') + + def test_randomizer_avoids_repeat_ignores_favorites_and_respects_exclusions(self): + self.install_fixture('gwm8223') + self.install_fixture('rdf43') + self.manager.action('preferences', {'randomizer': True, 'userFavorites': ['gwm8223'], + 'blacklistedModels': ['gwm8223', BUNDLED_CURRENT]}) + self.assertEqual(randomize_next_start(False, root=self.root)['small'], 'rdf43') + self.assertEqual(randomize_next_start(False, root=self.root)['small'], 'rdf43') + self.manager.action('preferences', {'blacklistedModels': []}) + seen = [] + def choose(options): + seen.extend(options) + return options[0] + randomize_next_start(False, root=self.root, chooser=choose) + self.assertNotIn('rdf43', seen) + self.assertIn('gwm8223', seen) + self.assertIn(BUNDLED_CURRENT, seen) + + def test_randomizer_no_chestnut_changes_only_small(self): + self.install_fixture('gwm8223') + self.install_fixture('cinquev3') + atomic_json(self.root / 'preferences.json', {'randomizer': True, 'big': 'cinquev3'}) + selected = resolve_runtime(False, root=self.root, randomize=True) + self.assertEqual(selected.small_id, 'gwm8223') + self.assertFalse(selected.allow_big) + self.assertIsNone(selected.big_path) + self.assertEqual(preferences(self.root)['big'], 'cinquev3') + selected = resolve_runtime(True, root=self.root, randomize=True) + self.assertEqual(selected.big_id, 'cinquev3') + self.assertEqual(selected.big_path.read_bytes(), self.data) + self.assertTrue(selected.allow_big) + + def test_randomizer_empty_corrupt_pool_has_safe_fallback(self): + path = self.install_fixture('gwm8223') + path.write_bytes(b'x' * len(self.data)) + atomic_json(self.root / 'preferences.json', {'randomizer': True, 'small': 'gwm8223', + 'blacklistedModels': [BUNDLED_CURRENT]}) + self.assertEqual(resolve_runtime(False, root=self.root, randomize=True).small_id, BUNDLED_CURRENT) + self.manager.action('preferences', {'blacklistedModels': []}) + atomic_json(self.root / 'preferences.json', {'randomizer': True, 'big': 'cinquev3', 'small': 'gwm8223'}) + selected = resolve_runtime(True, root=self.root, randomize=True) + self.assertFalse(selected.allow_big) + self.assertEqual(selected.small_id, BUNDLED_CURRENT) + self.assertEqual(preferences(self.root)['big'], '') + + def test_selection_preferences_require_parked_and_strict_values(self): + for payload in ({'randomizer': 1}, {'blacklistedModels': ['unknown']}, {'blacklistedModels': 'gwm8223'}): + with self.subTest(payload=payload), self.assertRaises(ModelError): + self.manager.action('preferences', payload) + self.parked = False + for payload in ({'randomizer': True}, {'blacklistedModels': ['gwm8223']}): + with self.subTest(payload=payload), self.assertRaises(ModelError): + self.manager.action('preferences', payload) + self.manager.action('preferences', {'userFavorites': ['gwm8223']}) + self.parked = True + self.manager.action('preferences', {'randomizer': True}) + with self.assertRaises(ModelError): + self.manager.action('active', {'profile': 'small', 'model': BUNDLED_CURRENT}) + self.assertTrue(self.manager.snapshot()['randomizer']) + self.manager.parked = iter([True, False]).__next__ + with self.assertRaises(ModelError): + self.manager.action('preferences', {'randomizer': False}) + self.assertTrue(preferences(self.root)['randomizer']) + + def test_unpublished_catalog_is_visible_but_not_selectable(self): + self.assertEqual(len(catalog(self.root)), 99) + models = self.manager.snapshot()['models'] + self.assertEqual(sum(row['selectable'] for row in models), 1) + self.assertTrue(all(row.get('unavailableReason') for row in models if not row['selectable'])) + + def test_fresh_install_knows_published_downloads_without_network_or_gpu(self): + self.manager.gpu_present = lambda: False + models = {row['value']: row for row in self.manager.snapshot()['models']} + self.assertTrue(models['gwm8223']['downloadAvailable']) + self.assertEqual(models['gwm8223']['unavailableReason'], 'Download required') + self.assertFalse(models['gwm8223']['selectable']) + self.assertFalse(models['gwm6223']['downloadAvailable']) + self.assertEqual(models['gwm6223']['unavailableReason'], 'v26 download not published') + big = next(row for row in models.values() if row.get('requiresGpu') and row.get('downloadAvailable')) + self.assertFalse(big['gpuAvailable']) + self.assertEqual(big['unavailableReason'], 'Download required') + self.assertEqual(self.urls, []) + self.assertFalse((self.root / 'catalog.json').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_modeld_build.py b/openpilot/starpilot/models/tests/test_modeld_build.py new file mode 100644 index 0000000000..a84a5e90c8 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_modeld_build.py @@ -0,0 +1,105 @@ +"""Evaluate the actual SCons graph with fake nodes; never execute compilers.""" + +import ctypes.util +from pathlib import Path +import sys +from types import ModuleType, SimpleNamespace +from unittest.mock import Mock + +import pytest + + +SOURCE = Path(__file__).resolve().parents[3] / "selfdrive/modeld/SConscript" + + +class Returned(Exception): + pass + + +@pytest.fixture +def graph(monkeypatch, tmp_path): + script = ModuleType("SCons.Script") + script.Action = lambda command, description: command + script.Value = lambda value: value + monkeypatch.setitem(sys.modules, script.__name__, script) + helpers = ModuleType("openpilot.selfdrive.modeld.helpers") + helpers.chestnut_present = Mock(return_value=False) + monkeypatch.setitem(sys.modules, helpers.__name__, helpers) + env = Mock() + env.Clone.return_value = env + + def node(path): + relative = path.removeprefix("#") + return SimpleNamespace(abspath=str(tmp_path / relative), relpath=relative) + + env.Dir.side_effect = node + + def stop(): + raise Returned() + + def evaluate(arch): + scope = {"env": env, "arch": arch, "Import": Mock(), "Return": stop, "Dir": node, "File": node} + try: + exec(compile(SOURCE.read_text(), str(SOURCE), "exec"), scope) + except Returned: + pass + return scope + + return evaluate, env, helpers.chestnut_present + + +def test_missing_llvm_pc_skips_graph_before_probe_or_compilation(graph, monkeypatch, capsys): + evaluate, env, present = graph + llvm = Mock(return_value=None) + monkeypatch.setattr(ctypes.util, "find_library", llvm) + evaluate("aarch64") + llvm.assert_called_once_with("LLVM") + env.Clone.assert_not_called() + env.Command.assert_not_called() + present.assert_not_called() + assert "Install LLVM to compile models" in capsys.readouterr().out + + +@pytest.mark.parametrize("arch,llvm,backend", [ + ("aarch64", "libLLVM.so", "DEV=CPU:LLVM"), + ("Darwin", None, "DEV=METAL JIT=2"), + ("comma_arm64", None, "DEV=QCOM"), +]) +def test_supported_backend_retains_models_and_camera_warp_graph(graph, monkeypatch, arch, llvm, backend): + evaluate, env, present = graph + library = Mock(return_value=llvm) + monkeypatch.setattr(ctypes.util, "find_library", library) + evaluate(arch) + assert library.call_count == (1 if arch == "aarch64" else 0) + present.assert_called_once_with() + calls = env.Command.call_args_list + assert len(calls) == 6 + targets = [Path(call.args[0]).name for call in calls] + assert targets[:2] == ["dmonitoring_model_tinygrad.pkl", "driving_tinygrad.pkl"] + assert sum(name.startswith("driving_warp_") for name in targets) == 2 + assert sum(name.startswith("dm_warp_") for name in targets) == 2 + assert all(backend in call.args[2] for call in calls) + env.Execute.assert_not_called() + + +def test_llvm_probe_error_does_not_schedule_compilation(graph, monkeypatch): + evaluate, env, present = graph + monkeypatch.setattr(ctypes.util, "find_library", Mock(side_effect=OSError("probe failed"))) + with pytest.raises(OSError, match="probe failed"): + evaluate("aarch64") + env.Command.assert_not_called() + present.assert_not_called() + + +def test_chestnut_warp_graph_keeps_existing_serialization_without_compilation(graph, monkeypatch): + evaluate, env, present = graph + present.return_value = True + library = Mock() + monkeypatch.setattr(ctypes.util, "find_library", library) + evaluate("Darwin") + library.assert_not_called() + assert env.Command.call_count == 8 + assert sum(Path(call.args[0]).name.startswith("big_driving_warp_") for call in env.Command.call_args_list) == 2 + assert env.SideEffect.call_count == 2 + assert all(Path(call.args[0]).name == ".chestnut.lock" for call in env.SideEffect.call_args_list) + env.Execute.assert_not_called() diff --git a/openpilot/starpilot/models/tests/test_modeld_startup_fallback.py b/openpilot/starpilot/models/tests/test_modeld_startup_fallback.py new file mode 100644 index 0000000000..2b5d54492a --- /dev/null +++ b/openpilot/starpilot/models/tests/test_modeld_startup_fallback.py @@ -0,0 +1,258 @@ +"""CPU-only regressions for modeld startup scheduling and runner-owned fallback.""" + +import ast +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest + +from openpilot.starpilot.models.status import ModelVariant + + +MODEL_SOURCE = Path(__file__).resolve().parents[3] / "selfdrive/modeld/modeld.py" + + +def main_body(): + return next(node for node in ast.parse(MODEL_SOURCE.read_text()).body if isinstance(node, ast.FunctionDef) and node.name == "main").body + + +def test_realtime_starts_after_model_load_and_receipt(): + # Inspect startup without importing modeld's GPU runtime or loading artifacts. + calls = [(node.lineno, ast.unparse(node.func)) for statement in main_body() for node in ast.walk(statement) if isinstance(node, ast.Call)] + def line(name): + return [lineno for lineno, function in calls if function == name] + assert len(line("gc.disable")) == len(line("config_realtime_process")) == 1 + assert line("gc.disable")[0] < min(line("load_small") + line("loader.join")) + startup_receipt = next(statement for statement in main_body() if isinstance(statement, ast.Expr) + and isinstance(statement.value, ast.Call) and ast.unparse(statement.value.func) == "receipt_owner.loaded") + assert max(line("load_small") + line("loader.join")) < startup_receipt.lineno < line("config_realtime_process")[0] < line("PubMaster")[0] + + +@pytest.mark.parametrize("runner_chestnut,stored_active", [(True, False), (True, True), (False, True), (False, False)]) +def test_inference_failure_uses_runner_identity(runner_chestnut, stored_active): + # Run the production try/except statement with deliberately stale status Params. + fallback = next(node for statement in main_body() for node in ast.walk(statement) + if isinstance(node, ast.Try) and any(isinstance(child, ast.Call) and ast.unparse(child.func) == "model.run" + for child in ast.walk(node))) + failure = RuntimeError("inference failed") + failed_model = SimpleNamespace(chestnut=runner_chestnut, run=Mock(side_effect=failure)) + small_model = SimpleNamespace(chestnut=False, model_id="fallback-small") + params = Mock() + params.get_bool.return_value = stored_active + receipt = Mock() + chestnut_state = SimpleNamespace(big=True) + env = {"model": failed_model, "small_model": small_model, "params": params, "receipt_owner": receipt, + "chestnut_state": chestnut_state, "run_count": 1, "ModelConstants": SimpleNamespace(MODEL_RUN_FREQ=20), + "SERVICE_LIST": {"chestnutGpuState": SimpleNamespace(frequency=1)}, "bufs": {}, "transforms": {}, "inputs": {}, + "small_prepared": object(), "ModelVariant": ModelVariant, "cloudlog": Mock()} + code = compile(ast.fix_missing_locations(ast.Module(body=[fallback], type_ignores=[])), str(MODEL_SOURCE), "exec") + if runner_chestnut: + exec(code, env) + assert env["model"] is small_model + assert env["model_output"] is None + assert env["run_count"] == 0 + assert not chestnut_state.big + params.put_bool.assert_called_once_with("ChestnutActive", False) + receipt.loaded.assert_called_once_with(env["small_prepared"], ModelVariant.SMALL, "chestnut-load-failed", model_id="fallback-small") + else: + with pytest.raises(RuntimeError, match="inference failed"): + exec(code, env) + assert env["model"] is failed_model + params.put_bool.assert_not_called() + receipt.loaded.assert_not_called() + params.get_bool.assert_not_called() + + +def execute(statements, env): + env.setdefault("os", SimpleNamespace(getenv=lambda key: None)) + env.setdefault("recovery_small_only", False) + exec(compile(ast.fix_missing_locations(ast.Module(body=statements, type_ignores=[])), str(MODEL_SOURCE), "exec"), env) + + +@pytest.mark.parametrize("present,cable,allow_big,big_path,compiled,expected", [ + (True, False, True, "verified-big", False, True), + (False, True, True, "verified-big", False, True), + (False, True, True, None, True, True), + (False, False, False, None, True, False), + (True, False, False, "verified-big", True, False), + (False, True, True, None, False, False), +]) +def test_observed_availability_and_big_eligibility(monkeypatch, present, cable, allow_big, big_path, compiled, expected): + from types import ModuleType + import sys + + manager = ModuleType("openpilot.starpilot.models.manager") + manager.resolve_runtime = Mock(return_value=SimpleNamespace(allow_big=allow_big, big_path=big_path)) + manager.requested_runtime_id = Mock(return_value="selected") + amd = ModuleType("tinygrad.runtime.ops_amd") + amd.AMDDevice = type("FakeAMDDevice", (), {"wait_timeout_ms": 123}) + monkeypatch.setitem(sys.modules, manager.__name__, manager) + monkeypatch.setitem(sys.modules, amd.__name__, amd) + body = main_body() + start = next(i for i, node in enumerate(body) if isinstance(node, ast.ImportFrom) and node.module == manager.__name__) + end = next(i for i, node in enumerate(body) if ast.unparse(node) == "gc.disable()") + params = Mock() + env = {"chestnut_present": Mock(return_value=present), "cable_connected": Mock(return_value=cable), + "chestnut_compiled": Mock(return_value=compiled), "Params": lambda: params} + execute(body[start:end], env) + available = present or cable + manager.resolve_runtime.assert_called_once_with(chestnut_available=available, randomize=True) + manager.requested_runtime_id.assert_called_once_with(available) + env["chestnut_present"].assert_called_once_with() + assert env["cable_connected"].call_count == (0 if present else 1) + assert env["CHESTNUT"] is expected + assert amd.AMDDevice.wait_timeout_ms == (3000 if expected else 123) + params.put_bool.assert_called_once_with("ChestnutLoading", expected) + + +@pytest.mark.parametrize("timeout", [False, True]) +@pytest.mark.parametrize("custom", [False, True]) +def test_big_worker_waits_before_load_and_timeout_preserves_small_fallback(custom, timeout): + body = main_body() + start = next(i for i, node in enumerate(body) if isinstance(node, ast.Assign) and ast.unparse(node.targets[0]) == "st") + end = next(i for i, node in enumerate(body) if ast.unparse(node).startswith("config_realtime_process(")) + events = [] + worker = False + + class Thread: + def __init__(self, target, daemon): + assert daemon + self.target = target + + def start(self): + nonlocal worker + worker = True + self.target() + worker = False + + def join(self, duration): + events.append(("join", duration)) + + def wait(): + assert worker + events.append("wait") + if timeout: + raise TimeoutError("chestnut did not enumerate") + + def state(width, height, chestnut): + events.append("big" if chestnut else "small") + return SimpleNamespace(chestnut=chestnut, model_id="big" if chestnut else "small", warmup=Mock()) + + def verified(width, height, path, version, chestnut, model_id, sha): + return state(width, height, chestnut), "prepared-" + model_id + + params, receipt = Mock(), Mock() + selection = SimpleNamespace(big_path="selected-big" if custom else None, small_path="selected-small" if custom else None, + small_version=1, small_id="small", small_sha256="small-hash", + big_version=1, big_id="big", big_sha256="hash") + env = {"time": SimpleNamespace(monotonic=lambda: 0), "cloudlog": Mock(), "selection": selection, + "CHESTNUT": True, "vipc_client_main": SimpleNamespace(width=1928, height=1208), "wait_for_chestnut": wait, + "ModelState": state, "load_verified_model": verified, "modeld_pkl_path": lambda big: "bundled-big" if big else "bundled-small", + "receipt_owner": receipt, "threading": SimpleNamespace(Thread=Thread), "BIG_MODEL_TIMEOUT": 30, + "params": params, "requested_model_id": "big", "ModelVariant": ModelVariant, + "CP": SimpleNamespace(brand="mock"), "demo": False, + "wait_for_chestnut_power": lambda CP, timeout: events.append("power")} + wrapper = ast.parse("def startup():\n pass").body[0] + wrapper.body = body[start:end] + [ast.Return(value=ast.Call(func=ast.Name(id="locals", ctx=ast.Load()), args=[], keywords=[]))] + execute([wrapper], env) + env.update(env["startup"]()) + assert events[0] == ("small" if custom else "wait") + assert events.count("small") == 1 + assert ("big" in events) is not timeout + assert env["model"].chestnut is not timeout + assert env["small_model"].chestnut is False + assert events.index("wait") < events.index(("join", 30)) + if not timeout: + assert events.index("wait") < events.index("power") < events.index("big") + params.put_bool.assert_any_call("ChestnutActive", not timeout) + params.put_bool.assert_any_call("ChestnutLoading", False) + assert receipt.loaded.call_args.args[1] == (ModelVariant.SMALL if timeout else ModelVariant.CHESTNUT) + assert receipt.loaded.call_args.args[2] == ("chestnut-load-failed" if timeout else None) + + +def test_wait_for_enumeration_succeeds_without_real_sleep(monkeypatch): + from openpilot.selfdrive.modeld import helpers + present = Mock(side_effect=[False, False, True]) + sleep = Mock() + monkeypatch.setattr(helpers, "chestnut_present", present) + monkeypatch.setattr(helpers, "time", SimpleNamespace(monotonic=Mock(side_effect=[0, 0, 0.1]), sleep=sleep)) + helpers.wait_for_chestnut(timeout=1) + assert present.call_count == 3 + assert sleep.call_args_list == [((0.1,),), ((0.1,),)] + + +def test_wait_for_enumeration_times_out_without_loading_hardware(monkeypatch): + from openpilot.selfdrive.modeld import helpers + sleep = Mock() + monkeypatch.setattr(helpers, "chestnut_present", lambda: False) + monkeypatch.setattr(helpers, "time", SimpleNamespace(monotonic=Mock(side_effect=[0, 0, 0.2]), sleep=sleep)) + with pytest.raises(TimeoutError, match="chestnut did not enumerate"): + helpers.wait_for_chestnut(timeout=0.1) + sleep.assert_called_once_with(0.1) + + +@pytest.mark.parametrize("previous", ["READY", "FAILED"]) +def test_native_ui_keeps_loading_during_enumeration_then_reports_failure(previous): + from enum import Enum + source = MODEL_SOURCE.parents[1] / "ui/ui_state.py" + body = ast.parse(source.read_text()).body + chestnut_enum = next(node for node in body if isinstance(node, ast.ClassDef) and node.name == "ChestnutState") + ui_class = next(node for node in body if isinstance(node, ast.ClassDef) and node.name == "UIState") + update = next(node for node in ui_class.body if isinstance(node, ast.FunctionDef) and node.name == "_update_chestnut_state") + env = {"Enum": Enum} + execute([chestnut_enum, update], env) + states = env["ChestnutState"] + + class Messages(dict): + recv_frame = {"modelV2": 2} + alive = {"modelV2": True} + + ui = SimpleNamespace(sm=Messages(deviceState=SimpleNamespace(chestnutPresent=False), modelV2=SimpleNamespace(big=False)), + started=True, started_frame=1, chestnut_present=True, chestnut_compiled=True, + chestnut_loading=True, chestnut_active=False, chestnut_state=states[previous]) + env["_update_chestnut_state"](ui) + assert ui.chestnut_state is states.LOADING + ui.chestnut_loading = False + env["_update_chestnut_state"](ui) + assert ui.chestnut_state is states.FAILED + ui.chestnut_present = False + env["_update_chestnut_state"](ui) + assert ui.chestnut_state is states.DISCONNECTED + ui.chestnut_present, ui.chestnut_compiled = True, False + env["_update_chestnut_state"](ui) + assert ui.chestnut_state is states.UNCOMPILED + + +def test_recovery_child_resolves_small_without_randomizer_or_amd(monkeypatch): + from types import ModuleType + import sys + manager = ModuleType("openpilot.starpilot.models.manager") + manager.resolve_runtime = Mock(return_value=SimpleNamespace(allow_big=False, big_path=None)) + manager.requested_runtime_id = Mock(return_value="requested-big") + monkeypatch.setitem(sys.modules, manager.__name__, manager) + body = main_body() + start = next(i for i, node in enumerate(body) if isinstance(node, ast.ImportFrom) and node.module == manager.__name__) + end = next(i for i, node in enumerate(body) if ast.unparse(node) == "gc.disable()") + params = Mock() + env = {"chestnut_present": lambda: True, "cable_connected": lambda: True, + "chestnut_compiled": Mock(side_effect=AssertionError("Big compiled probe forbidden")), + "Params": lambda: params, "os": SimpleNamespace(getenv=lambda key: "1")} + execute(body[start:end], env) + manager.resolve_runtime.assert_called_once_with(chestnut_available=False, randomize=False) + manager.requested_runtime_id.assert_called_once_with(True) + assert env["CHESTNUT"] is False + assert "AMDDevice" not in env + assert params.put_bool.call_args_list == [(("ChestnutLoading", False),)] + + +def test_recovery_small_receipt_reports_runtime_stall_not_load_failure(): + statement = next(node for node in main_body() if isinstance(node, ast.Expr) and + isinstance(node.value, ast.Call) and ast.unparse(node.value.func) == 'receipt_owner.loaded') + receipt = Mock() + env = {'model': SimpleNamespace(chestnut=False, model_id='bundled-current'), 'small_prepared': object(), + 'receipt_owner': receipt, 'ModelVariant': ModelVariant, 'initial_chestnut_fallback': False, + 'recovery_small_only': True, 'selected_small_failed': False, 'requested_model_id': 'requested-big'} + execute([statement], env) + assert receipt.loaded.call_args.args[1:3] == (ModelVariant.SMALL, 'chestnut-run-stalled') + assert receipt.loaded.call_args.kwargs['model_id'] == 'bundled-current' diff --git a/openpilot/starpilot/models/tests/test_receipt.py b/openpilot/starpilot/models/tests/test_receipt.py new file mode 100644 index 0000000000..08b13fffaa --- /dev/null +++ b/openpilot/starpilot/models/tests/test_receipt.py @@ -0,0 +1,165 @@ +from dataclasses import replace +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT +from openpilot.starpilot.models.receipt import (ModelReceiptOwner, ReceiptUnavailable, artifact_digest, artifact_identity, + clear_receipt, prepare_artifact, process_start_ticks, read_receipt, + record_prepared, + write_receipt, logged_model_load, log_loaded_model) +from openpilot.starpilot.models.status import ModelLoad, ModelVariant + + +class TestModelReceipt(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.path = self.root / "owned" / "load-v1.json" + self.load = ModelLoad(os.getpid(), 12345, 987654321, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + + def test_atomic_roundtrip_clear_and_malformed(self): + self.assertIsNone(read_receipt(self.path)) + write_receipt(self.load, self.path) + self.assertEqual(read_receipt(self.path), self.load) + self.assertEqual(self.path.stat().st_mode & 0o777, 0o600) + self.assertEqual(self.path.parent.stat().st_mode & 0o777, 0o700) + raw = self.path.read_text() + self.path.write_text(raw.replace('"pid":', '"pid":1,"pid":')) + self.assertIsNone(read_receipt(self.path)) + self.path.write_bytes(b"{" * 1025) + self.assertIsNone(read_receipt(self.path)) + self.path.write_text(json.dumps({"version": 1})) + self.assertIsNone(read_receipt(self.path)) + clear_receipt(self.path) + self.assertFalse(self.path.exists()) + + def test_catalog_id_roundtrip_and_wrong_hardware_rejection(self): + for model_id, variant in (("sc23", ModelVariant.SMALL), ("cinquev3", ModelVariant.CHESTNUT)): + load = replace(self.load, model_id=model_id, variant=variant) + write_receipt(load, self.path) + self.assertEqual(read_receipt(self.path), load) + with self.assertRaises(ReceiptUnavailable): + write_receipt(replace(self.load, model_id="cinquev3"), self.path) + with self.assertRaises(ReceiptUnavailable): + write_receipt(replace(self.load, model_id="unknown"), self.path) + + def test_symlink_fifo_and_unowned_directory_are_unavailable(self): + target = self.root / "outside" + target.write_bytes(b"untouched") + self.path.parent.symlink_to(self.root, target_is_directory=True) + self.assertIsNone(read_receipt(self.path)) + with self.assertRaises(OSError): + write_receipt(self.load, self.path) + clear_receipt(self.path) + self.assertEqual(target.read_bytes(), b"untouched") + self.path.parent.unlink() + self.path.parent.mkdir(mode=0o700) + os.mkfifo(self.path) + self.assertIsNone(read_receipt(self.path)) + clear_receipt(self.path) + self.path.parent.chmod(0o755) + self.assertIsNone(read_receipt(self.path)) + with self.assertRaises(ReceiptUnavailable): + write_receipt(self.load, self.path) + + def test_invalid_namespace_is_unavailable_to_reader(self): + with mock.patch.dict(os.environ, {"OPENPILOT_PREFIX": "../../escape"}): + self.assertIsNone(read_receipt()) + + def test_installed_artifact_same_file_and_regular(self): + artifact = self.root / "model.pkl" + artifact.write_bytes(b"compiled-artifact") + identity = artifact_identity(artifact) + self.assertEqual(len(artifact_digest(artifact, identity)), 64) + prepared = prepare_artifact(artifact, identity) + recorded = record_prepared(prepared, ModelVariant.SMALL, receipt_file=self.path, + pid=421, start_ticks=12345, now_mono_ns=987654321) + self.assertEqual(read_receipt(self.path), recorded) + artifact.write_bytes(b"changed") + with self.assertRaises(ReceiptUnavailable): + artifact_digest(artifact, identity) + artifact.unlink() + artifact.symlink_to(self.root / "outside") + with self.assertRaises(ReceiptUnavailable): + artifact_identity(artifact) + artifact.unlink() + os.mkfifo(artifact) + with self.assertRaises(ReceiptUnavailable): + artifact_identity(artifact) + + def test_proc_start_parser(self): + path = self.root / "421" / "stat" + path.parent.mkdir() + fields = ["S"] + ["0"] * 18 + ["98765"] + ["0"] * 30 + path.write_text("421 (modeld with ) parens) " + " ".join(fields)) + self.assertEqual(process_start_ticks(421, proc_root=self.root), 98765) + self.assertEqual(process_start_ticks(422, proc_root=self.root), 0) + + def test_best_effort_start_and_fallback_transitions(self): + artifact = self.root / "compiled.pkl" + artifact.write_bytes(b"compiled-small") + reasons = [] + recorded = [] + owner = ModelReceiptOwner(self.path, reasons.append, record=recorded.append) + owner.start() + identity = owner.capture(artifact) + self.assertIsNotNone(identity) + if identity is None: + self.fail("regular fixture artifact had no identity") + # A diagnostic identity failure leaves model operation available. + self.assertIsNone(owner.prepare(artifact, None)) + self.assertEqual(reasons, ["artifact-identity-unavailable"]) + prepared = owner.prepare(artifact, identity) + self.assertIsNotNone(prepared) + # Runtime fallback only emits a tiny receipt: even a broken hash/open path + # cannot be reached from loaded() after startup preparation. + with mock.patch('openpilot.starpilot.models.receipt.artifact_digest', side_effect=AssertionError('bulk I/O')): + with mock.patch('openpilot.starpilot.models.receipt.process_start_ticks', return_value=12345): + self.assertTrue(owner.loaded(prepared, ModelVariant.SMALL, "chestnut-load-failed")) + self.assertEqual(read_receipt(self.path).fallback_reason, "chestnut-load-failed") + self.assertEqual(recorded, [read_receipt(self.path)]) + owner.start() + self.assertIsNone(read_receipt(self.path)) + + def test_runtime_stall_reason_roundtrips_without_altering_loaded_identity(self): + load = replace(self.load, fallback_reason='chestnut-run-stalled') + write_receipt(load, self.path) + self.assertEqual(read_receipt(self.path), load) + with mock.patch('openpilot.common.swaglog.cloudlog.event') as record: + log_loaded_model(load) + payload = {'event': record.call_args.args[0], **record.call_args.kwargs} + self.assertEqual(logged_model_load(json.dumps({'process': load.pid, 'msg': payload})), load) + + def test_log_identity_roundtrip_is_actual_loaded_variant_and_bound_to_process(self): + with mock.patch('openpilot.common.swaglog.cloudlog.event') as record: + log_loaded_model(self.load) + payload = {'event': record.call_args.args[0], **record.call_args.kwargs} + raw = {'process': self.load.pid, 'msg': payload} + self.assertEqual(logged_model_load(json.dumps(raw)), self.load) + raw['process'] += 1 + self.assertIsNone(logged_model_load(json.dumps(raw))) + raw['process'] = self.load.pid + payload['modelId'] = 'cinquev3' # A big selection cannot masquerade as the small runner. + self.assertIsNone(logged_model_load(json.dumps(raw))) + for value in ('not json', '{' * 8193, '[]', json.dumps({'msg': 'loading model'})): + self.assertIsNone(logged_model_load(value)) + + def test_log_failure_does_not_invalidate_successful_live_receipt(self): + reasons = [] + owner = ModelReceiptOwner(self.path, reasons.append, record=mock.Mock(side_effect=OSError('logger absent'))) + artifact = self.root / 'compiled.pkl' + artifact.write_bytes(b'compiled') + prepared = prepare_artifact(artifact, artifact_identity(artifact)) + with mock.patch('openpilot.starpilot.models.receipt.process_start_ticks', return_value=12345): + self.assertTrue(owner.loaded(prepared, ModelVariant.SMALL)) + self.assertIsNotNone(read_receipt(self.path)) + self.assertEqual(reasons, ['log-failed']) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_recovery.py b/openpilot/starpilot/models/tests/test_recovery.py new file mode 100644 index 0000000000..bd1d566ee2 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_recovery.py @@ -0,0 +1,152 @@ +"""Process recovery requires established Big output and confirmed owner exit.""" +import ast +from dataclasses import replace +from pathlib import Path +import unittest + +from openpilot.starpilot.models.recovery import ModelRecoveryOwner, RecoveryEvidence + +BASE = 10_000_000_000 +IDENTITY = (123, 456) + + +def evidence(t, *, output=True, alive=True): + now = BASE + int(t * 1e9) + stamp = now if output else BASE + 2_000_000_000 + return RecoveryEvidence(now, BASE, IDENTITY if alive else None, alive, True, BASE, + stamp, stamp, output, True, ((now, int(t * 20)), (now, int(t * 20)))) + + +def arm(owner): + for t in (.1, .6, 1.1, 1.6, 2.): + assert owner.step(evidence(t)) is None + assert owner.armed + + +class TestModelRecovery(unittest.TestCase): + def test_stall_escalates_once_and_small_waits_for_exit(self): + owner = ModelRecoveryOwner() + arm(owner) + self.assertIsNone(owner.step(evidence(3., output=False))) + self.assertEqual(owner.step(evidence(4., output=False)), 'interrupt') + self.assertTrue(owner.small_only) + self.assertIsNone(owner.step(evidence(4.2, output=False))) + self.assertEqual(owner.step(evidence(4.5, output=False)), 'kill') + self.assertEqual(owner.step(evidence(4.6, output=False, alive=False)), 'small') + for t in (5., 10., 100.): + self.assertIsNone(owner.step(evidence(t, output=False))) + + def test_separated_big_outputs_do_not_count_as_sustained_arming(self): + owner = ModelRecoveryOwner() + for t in (.1, 3., 6., 9.): + self.assertIsNone(owner.step(evidence(t))) + self.assertFalse(owner.armed) + self.assertIsNone(owner.step(evidence(12., output=False))) + self.assertFalse(owner.small_only) + + def test_uncertain_exit_blocks_small_and_new_drive(self): + owner = ModelRecoveryOwner() + arm(owner) + self.assertEqual(owner.step(evidence(4., output=False)), 'interrupt') + self.assertEqual(owner.step(evidence(4.5, output=False)), 'kill') + self.assertEqual(owner.step(evidence(5.5, output=False)), 'failed') + self.assertEqual(owner.phase, 'failed') + self.assertIsNone(owner.step(replace(evidence(6.), drive_id=BASE + 1))) + self.assertEqual(owner.phase, 'failed') + + def test_startup_small_wrong_receipt_old_output_cannot_arm(self): + mutations = ({'big_receipt': False}, {'identity': None}, {'output_big': False}, + {'loaded_ns': BASE - 1}, {'model_ns': BASE - 1}, {'driving_ns': BASE - 1}, + {'output_valid': False}) + for mutation in mutations: + owner = ModelRecoveryOwner() + for t in (1., 2., 3., 10.): + self.assertIsNone(owner.step(replace(evidence(t), **mutation))) + self.assertFalse(owner.small_only) + + def test_camera_starvation_repeated_frame_and_frozen_timestamp_block_recovery(self): + for cameras in ((), ((BASE, 1), (BASE, 1)), ((BASE + 4_000_000_000, 40),) * 2): + owner = ModelRecoveryOwner() + arm(owner) + for t in (3., 4., 5.): + self.assertIsNone(owner.step(replace(evidence(t, output=False), cameras=cameras))) + self.assertFalse(owner.small_only) + + def test_camera_reacquisition_requires_sustained_progress_then_recovers(self): + owner = ModelRecoveryOwner() + arm(owner) + self.assertIsNone(owner.step(replace(evidence(3., output=False), cameras=()))) + for t in (3.5, 4., 4.5, 5., 5.5): + self.assertIsNone(owner.step(evidence(t, output=False))) + self.assertEqual(owner.step(evidence(6., output=False)), 'interrupt') + + def test_owner_change_and_output_resume_rearm_cleanly(self): + owner = ModelRecoveryOwner() + arm(owner) + changed = replace(evidence(4., output=False), identity=(124, 457), loaded_ns=BASE + 3_000_000_000) + self.assertIsNone(owner.step(changed)) + self.assertFalse(owner.armed) + owner = ModelRecoveryOwner() + arm(owner) + self.assertIsNone(owner.step(evidence(3., output=False))) + self.assertIsNone(owner.step(evidence(3.5))) + self.assertIsNone(owner.step(evidence(4., output=False))) + + def test_offroad_resets_after_failed_owner_exit(self): + owner = ModelRecoveryOwner() + arm(owner) + owner.step(evidence(4., output=False)) + self.assertIsNone(owner.step(replace(evidence(4.1), drive_id=0))) + self.assertTrue(owner.small_only) + self.assertEqual(owner.step(replace(evidence(4.2, alive=False), drive_id=0)), 'exit') + self.assertFalse(owner.small_only) + self.assertEqual(owner.phase, 'monitor') + + def test_offroad_and_new_drive_keep_bounded_escalation(self): + for drive in (0, BASE + 5_000_000_000): + owner = ModelRecoveryOwner() + arm(owner) + owner.step(evidence(4., output=False)) + self.assertEqual(owner.step(replace(evidence(4.5, output=False), drive_id=drive)), 'kill') + self.assertEqual(owner.step(replace(evidence(4.6, output=False, alive=False), drive_id=drive)), 'exit') + self.assertFalse(owner.small_only) + + def test_real_child_exit_is_required_before_small_action(self): + import subprocess + import sys + import signal + import select + child = subprocess.Popen([sys.executable, '-c', + 'import signal,time; signal.signal(signal.SIGINT,signal.SIG_IGN); print("ready",flush=True); time.sleep(30)'], + stdout=subprocess.PIPE, text=True) + try: + self.assertTrue(select.select([child.stdout], [], [], 2)[0]) + self.assertEqual(child.stdout.readline().strip(), "ready") + owner = ModelRecoveryOwner() + arm(owner) + self.assertEqual(owner.step(evidence(4., output=False)), 'interrupt') + child.send_signal(signal.SIGINT) + self.assertIsNone(child.poll()) + self.assertIsNone(owner.step(evidence(4.2, output=False))) + self.assertEqual(owner.step(evidence(4.5, output=False)), "kill") + child.send_signal(signal.SIGKILL) + child.wait(timeout=2) + self.assertIsNotNone(child.poll()) + self.assertEqual(owner.step(evidence(4.6, output=False, alive=False)), 'small') + finally: + if child.poll() is None: + child.kill() + child.wait(timeout=2) + if child.stdout is not None: + child.stdout.close() + + def test_child_restriction_precedes_amd_and_preserves_randomizer_selection(self): + source = Path(__file__).parents[3] / 'selfdrive/modeld/modeld.py' + tree = ast.parse(source.read_text()) + main = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == 'main') + statements = [ast.unparse(n) for n in main.body] + selection = next(s for s in statements if 'selection = resolve_runtime' in s) + self.assertIn('not recovery_small_only', selection) + self.assertIn('randomize=not recovery_small_only', selection) + self.assertLess(next(i for i, s in enumerate(statements) if 'selection = resolve_runtime' in s), + next(i for i, s in enumerate(statements) if 'from tinygrad.runtime.ops_amd import AMDDevice' in s)) diff --git a/openpilot/starpilot/models/tests/test_recovery_process.py b/openpilot/starpilot/models/tests/test_recovery_process.py new file mode 100644 index 0000000000..9fa5607dd5 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_recovery_process.py @@ -0,0 +1,114 @@ +from types import SimpleNamespace as NS +from unittest.mock import Mock, patch +import unittest + +from openpilot.starpilot.models.recovery_process import ModeldProcess, modeld_launcher, SMALL_ONLY_ENV +from openpilot.starpilot.models.tests.test_recovery import BASE +from openpilot.starpilot.models.status import ModelVariant + + +class Feed(dict): + def __init__(self, t, output=True): + now = BASE + int(t * 1e9) + super().__init__(deviceState=NS(startedMonoTime=BASE), modelV2=NS(big=True), + narrowRoadCameraState=NS(frameId=int(t * 20)), wideRoadCameraState=NS(frameId=int(t * 20))) + self.seen = dict.fromkeys(('deviceState', 'modelV2', 'drivingModelData', 'narrowRoadCameraState', 'wideRoadCameraState'), True) + self.valid = self.seen.copy() + self.valid.update(modelV2=output, drivingModelData=output) + self.alive = self.seen.copy() + self.logMonoTime = dict.fromkeys(self.seen, now) + if not output: + self.logMonoTime.update(modelV2=BASE + 2_000_000_000, drivingModelData=BASE + 2_000_000_000) + + +class TestRecoveryProcess(unittest.TestCase): + def test_adapter_never_spawns_before_confirmed_exit(self): + owner = ModeldProcess('modeld', 'openpilot.selfdrive.modeld.modeld', lambda *args: True) + child = Mock(pid=123) + child.is_alive.return_value = True + owner.proc = child + load = NS(pid=123, process_start_ticks=456, loaded_mono_ns=BASE, variant=ModelVariant.CHESTNUT) + with patch('openpilot.starpilot.models.recovery_process.process_start_ticks', return_value=456), \ + patch('openpilot.starpilot.models.recovery_process.read_receipt', return_value=load), \ + patch('openpilot.starpilot.models.recovery_process.time.monotonic_ns') as clock, \ + patch.object(owner, 'signal') as send, \ + patch('openpilot.starpilot.models.recovery_process.Process') as spawn: + for t in (.1, .6, 1.1, 1.6, 2., 3., 4., 4.5): + clock.return_value = BASE + int(t * 1e9) + owner.observe(Feed(t, output=t <= 2.), True) + owner.start() + self.assertEqual(send.call_count, 2) + spawn.assert_not_called() + child.is_alive.return_value = False + clock.return_value = BASE + 4_600_000_000 + owner.observe(Feed(4.6, output=False), True) + child.join.assert_called_once_with(timeout=0) + owner.start() + spawn.assert_called_once() + self.assertEqual(spawn.call_args.kwargs['args'][-1], True) + spawn.return_value.start.assert_called_once() + owner.start() + spawn.assert_called_once() + + def test_offroad_teardown_never_uses_generic_stop_or_waits(self): + owner = ModeldProcess('modeld', 'module', lambda *args: True) + child = Mock(pid=123) + child.is_alive.return_value = True + owner.proc = child + owner.recovery.phase = 'sigint' + with patch('openpilot.starpilot.models.recovery_process.PythonProcess.stop') as generic: + self.assertIsNone(owner.stop(block=False)) + generic.assert_not_called() + child.join.assert_not_called() + + def test_final_cleanup_kills_with_bounded_wait_and_reports_uncertain_exit(self): + for exits in (False, True): + owner = ModeldProcess('modeld', 'module', lambda *args: True) + child = Mock(pid=123, exitcode=-9) + child.is_alive.side_effect = [True, not exits] + owner.proc = child + owner.recovery.phase = 'sigint' + with patch.object(owner, 'signal') as send, patch('openpilot.starpilot.models.recovery_process.PythonProcess.stop') as generic, \ + patch('openpilot.starpilot.models.recovery_process.join_process') as wait: + result = owner.stop(block=True) + generic.assert_not_called() + send.assert_called_once() + wait.assert_called_once_with(child, 1.0) + self.assertTrue(all(call.kwargs['timeout'] == 0 for call in child.join.call_args_list)) + self.assertEqual(result, -9 if exits else None) + self.assertEqual(owner.recovery.phase, 'sigint' if exits else 'failed') + self.assertEqual(owner.proc is None, exits) + + def test_existing_exit_wait_polls_without_multiprocessing_join(self): + from openpilot.system.manager.process import join_process + child = Mock(exitcode=None) + clock = NS(monotonic=Mock(side_effect=[0., 0., .5, 1.]), sleep=Mock()) + with patch('openpilot.system.manager.process.time', clock): + join_process(child, 1.) + self.assertEqual(clock.sleep.call_count, 2) + child.join.assert_not_called() + child.exitcode = -9 + clock = NS(monotonic=Mock(side_effect=[0., 0.]), sleep=Mock()) + with patch('openpilot.system.manager.process.time', clock): + join_process(child, 1.) + clock.sleep.assert_not_called() + child.join.assert_not_called() + + def test_invalid_device_context_cannot_arm_or_start(self): + owner = ModeldProcess('modeld', 'module', lambda *args: True) + feed = Feed(2.) + feed.valid['deviceState'] = False + with patch('openpilot.starpilot.models.recovery_process.Process') as spawn: + owner.observe(feed, True) + owner.start() + spawn.assert_not_called() + self.assertFalse(owner.recovery.armed) + + def test_child_environment_only_and_before_import(self): + with patch('openpilot.starpilot.models.recovery_process.os.environ', {}) as environment, \ + patch('openpilot.starpilot.models.recovery_process.launcher') as launch: + modeld_launcher('module', 'modeld', True) + self.assertEqual(environment[SMALL_ONLY_ENV], '1') + launch.assert_called_once_with('module', 'modeld') + modeld_launcher('module', 'modeld', False) + self.assertNotIn(SMALL_ONLY_ENV, environment) diff --git a/openpilot/starpilot/models/tests/test_runner.py b/openpilot/starpilot/models/tests/test_runner.py new file mode 100644 index 0000000000..7e5105991b --- /dev/null +++ b/openpilot/starpilot/models/tests/test_runner.py @@ -0,0 +1,241 @@ +from copy import deepcopy +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +import numpy as np +from tinygrad import Context, Device + +from openpilot.cereal import log +from openpilot.starpilot.models.runner import (ARTIFACT_ABI, COMPILER_REVISION, CatalogModelState, + action_from_outputs, load_verified_model, validate_artifact) +from openpilot.starpilot.models.parser import Parser +from openpilot.selfdrive.modeld.fill_model_msg import PublishState, fill_model_msg, fill_driving_model_data, fill_pose_msg + + +def output_fixture(mixture=False): + lengths = {"lane_lines": 528, "lane_lines_prob": 8, "road_edges": 264, "meta": 55, "desire_pred": 32, + "pose": 12, "wide_from_device_euler": 6, "road_transform": 12, + "plan": 4955 if mixture else 990, "lead": 102 if mixture else 144, + "lead_prob": 3, "desire_state": 8, "action": 4, "desired_curvature": 2} + sections, end = {}, 0 + for name, width in lengths.items(): + sections[name] = slice(end, end + width) + end += width + return np.zeros(end, dtype=np.float32), sections + + +def artifact_fixture(): + outputs, sections = output_fixture() + shapes = {"img": (1, 12, 2, 2), "big_img": (1, 12, 2, 2), "features_buffer": (1, 2, 4), + "desire_pulse": (1, 3, 8), "traffic_convention": (1, 2), "prev_action": (1, 2), + "lateral_control_params": (1, 2)} + return {"artifact_abi": ARTIFACT_ABI, "compiler_revision": COMPILER_REVISION, "format_version": 1, + "behavior_version": "v15", "model_type": "supercombo", "image_history_pipeline": "policy", + "frame_skip": 4, "execution_device": "CPU", "warp_device": "CPU", + "warp_input_keys": ("tfm", "big_tfm"), + "policy_input_keys": ("img_q", "big_img_q", "feat_q", "desire_q", "packed_npy_inputs"), + "metadata": {"model": {"input_shapes": shapes, "output_shapes": {"outputs": (1, len(outputs))}, "output_slices": sections}}, + "run_policy": Mock(return_value=(Mock(numpy=lambda: outputs.copy()),)), (8, 8): Mock(return_value="warped")} + + +class TestCatalogRunner(unittest.TestCase): + def test_warmup_uses_declared_device_without_autoselection(self): + runner = CatalogModelState.__new__(CatalogModelState) + runner.frame_buf_size = 256 + runner.warp_device = "NPY" + runner._blob_cache = {} + runner._reset_state = Mock() + + def verify_frames(frames, transforms, inputs): + for name, frame in frames.items(): + tensor = runner._blob_cache[name, frame.ctypes.data] + self.assertEqual(tensor.device, "NPY") + self.assertEqual(tensor.shape, (256,)) + np.testing.assert_array_equal(tensor.numpy(), np.zeros(256, dtype=np.uint8)) + np.testing.assert_array_equal(transforms[name], np.eye(3, dtype=np.float32)) + self.assertEqual(set(frames), {"img", "big_img"}) + np.testing.assert_array_equal(inputs["desire_pulse"], np.zeros(8, dtype=np.float32)) + + def unexpected_default(_): + raise AssertionError("warmup attempted device auto-selection") + + runner.run = Mock(side_effect=verify_frames) + with Context(DEV=""), patch.object(type(Device), "_select_device", property(unexpected_default)): + runner.warmup() + runner.run.assert_called_once() + runner._reset_state.assert_called_once() + + def test_artifact_identity_camera_version_and_output_contract(self): + artifact = artifact_fixture() + validate_artifact(artifact, "v15", (8, 8)) + for field, value in (("artifact_abi", "tinygrad_single_v1"), ("compiler_revision", "e" * 40), + ("behavior_version", "v14"), ("frame_skip", 0), ("image_history_pipeline", "guess")): + with self.subTest(field=field), self.assertRaises(ValueError): + validate_artifact({**artifact, field: value}, "v15", (8, 8)) + with self.assertRaises(ValueError): + validate_artifact(artifact, "v15", (1928, 1208)) + changed = deepcopy(artifact) + changed["metadata"]["model"]["output_slices"]["plan"] = slice(0, 1_000_000) + with self.assertRaises(ValueError): + validate_artifact(changed, "v15", (8, 8)) + + def test_both_plan_and_lead_generations_normalize_for_current_messages(self): + for mixture in (False, True): + raw, sections = output_fixture(mixture) + parsed = Parser().parse_outputs(CatalogModelState.slice_outputs(raw, sections)) + self.assertEqual(parsed["plan"].shape, (1, 33, 15)) + self.assertEqual(parsed["plan_stds"].shape, (1, 33, 15)) + self.assertEqual(parsed["lead"].shape, (1, 3, 6, 4)) + self.assertEqual(parsed["action"].shape, (1, 2)) + self.assertEqual(parsed["lane_lines"].shape, (1, 4, 33, 2)) + self.assertTrue(all(np.isfinite(value).all() for value in parsed.values())) + model_msg = log.Event.new_message() + model_msg.init("modelV2") + driving_msg = log.Event.new_message() + driving_msg.init("drivingModelData") + pose_msg = log.Event.new_message() + pose_msg.init("cameraOdometry") + fill_model_msg(model_msg, parsed, log.ModelDataV2.Action(), PublishState(), 10, 10, 10, 0., 123, .02, True) + fill_driving_model_data(driving_msg, model_msg) + fill_pose_msg(pose_msg, parsed, 10, 0, 123, True) + self.assertEqual(driving_msg.drivingModelData.frameId, 10) + self.assertEqual(len(model_msg.modelV2.leadsV3), 3) + self.assertTrue(pose_msg.valid) + + def test_missing_desire_state_uses_original_publisher_fallback(self): + artifact = artifact_fixture() + artifact["metadata"]["model"]["output_slices"].pop("desire_state") + validate_artifact(artifact, "v15", (8, 8)) + raw, sections = output_fixture() + sections.pop("desire_state") + parsed = Parser().parse_outputs(CatalogModelState.slice_outputs(raw, sections)) + self.assertNotIn("desire_state", parsed) + event = log.Event.new_message() + event.init("modelV2") + fill_model_msg(event, parsed, log.ModelDataV2.Action(), PublishState(), 10, 10, 10, 0., 123, .02, True) + self.assertEqual(list(event.modelV2.meta.desireState), [0.0] * 8) + + parsed["desire_state"] = np.arange(8, dtype=np.float32)[np.newaxis, :] + event = log.Event.new_message() + event.init("modelV2") + fill_model_msg(event, parsed, log.ModelDataV2.Action(), PublishState(), 10, 10, 10, 0., 123, .02, True) + self.assertEqual(list(event.modelV2.meta.desireState), list(range(8))) + + for missing in ("plan", "lead", "pose", "desire_pred", "meta"): + altered = deepcopy(artifact) + altered["metadata"]["model"]["output_slices"].pop(missing) + with self.subTest(missing=missing), self.assertRaisesRegex(ValueError, "missing required driving outputs"): + validate_artifact(altered, "v15", (8, 8)) + + def test_overlapping_slices_are_isolated_before_parser_mutation(self): + raw = np.arange(8, dtype=np.float32) + sections = CatalogModelState.slice_outputs(raw, {"state": slice(0, 8), "pad": slice(0, 8)}) + sections["state"][:] = 0 + np.testing.assert_array_equal(sections["pad"], [np.arange(8)]) + np.testing.assert_array_equal(raw, np.arange(8)) + + def test_gwm_padding_aliases_use_declared_output_width(self): + artifact = artifact_fixture() + artifact.update(model_type="vision_policy", behavior_version="v11", policy_order=["policy"]) + artifact["metadata"] = { + "vision": {"input_shapes": {"img": (1, 12, 128, 256), "big_img": (1, 12, 128, 256)}, + "output_shapes": {"outputs": (1, 1576)}, "output_slices": { + "meta": slice(0, 55), "desire_pred": slice(55, 87), "pose": slice(87, 99), + "wide_from_device_euler": slice(99, 105), "road_transform": slice(105, 117), + "lane_lines": slice(117, 645), "lane_lines_prob": slice(645, 653), "road_edges": slice(653, 917), + "lead": slice(917, 1061), "lead_prob": slice(1061, 1064), "hidden_state": slice(1064, 1576), + "pad": slice(0, None)}}, + "policy": {"input_shapes": {"features_buffer": (1, 25, 512)}, "output_shapes": {"outputs": (1, 1000)}, + "output_slices": {"plan": slice(0, 990), "desire_state": slice(990, 998), "pad": slice(-2, None)}}, + } + validate_artifact(artifact, "v11", (8, 8)) + slices = artifact["metadata"]["policy"]["output_slices"] + np.testing.assert_array_equal(CatalogModelState.slice_outputs(np.arange(1000), slices)["pad"], [[998, 999]]) + for invalid in (slice(-1001, None), slice(0, 1001), slice(None, -1001), slice(10, 9), slice(0, None, -1)): + with self.subTest(invalid=invalid), self.assertRaises(ValueError): + altered = deepcopy(artifact) + altered["metadata"]["policy"]["output_slices"]["pad"] = invalid + validate_artifact(altered, "v11", (8, 8)) + for shape in (None, (), (1, 0), (1, "1000"), (2, 1000)): + with self.subTest(shape=shape), self.assertRaisesRegex(ValueError, "width unavailable"): + altered = deepcopy(artifact) + altered["metadata"]["policy"]["output_shapes"]["outputs"] = shape + validate_artifact(altered, "v11", (8, 8)) + + def test_split_primary_policy_wins_over_auxiliary(self): + runner = CatalogModelState.__new__(CatalogModelState) + runner.model_type = "vision_multi_policy" + runner.policy_order = ["on_policy", "off_policy"] + runner.metadata = {key: {"output_slices": {"plan": slice(0, 1)}} for key in ("vision", "on_policy", "off_policy")} + runner.parser = Mock() + runner.parser.parse_vision_outputs.return_value = {"pose": np.array([1])} + runner.parser.parse_policy_outputs.return_value = {"plan": np.array([2])} + runner.aux_parser = Mock() + runner.aux_parser.parse_off_policy_outputs.return_value = {"plan": np.array([3]), "aux": np.array([4])} + parsed = runner._parse([np.zeros(1)] * 3) + np.testing.assert_array_equal(parsed["plan"], [2]) + np.testing.assert_array_equal(parsed["pose"], [1]) + np.testing.assert_array_equal(parsed["aux"], [4]) + + def test_generation_action_units_and_v9_curvature(self): + previous = log.ModelDataV2.Action(desiredCurvature=0.02) + outputs = {"action": np.array([[4., 0.5]])} + self.assertAlmostEqual(action_from_outputs(outputs, "v14", previous, .2, .2, 20).desiredCurvature, .04) + for version in ("v15", "v16"): + self.assertAlmostEqual(action_from_outputs(outputs, version, previous, .2, .2, 20).desiredCurvature, .01) + plan = np.zeros((1, 33, 15)) + plan[0, :, 3] = 20 + outputs = {"plan": plan, "desired_curvature": np.array([[.03]])} + self.assertAlmostEqual(action_from_outputs(outputs, "v9", previous, .2, .2, 20).desiredCurvature, .03) + del outputs["desired_curvature"] + self.assertAlmostEqual(action_from_outputs(outputs, "v9", previous, .2, .2, 20).desiredCurvature, .02) + self.assertAlmostEqual(action_from_outputs(outputs, "v11", previous, .2, .2, 20).desiredCurvature, 0.) + + def test_runtime_inputs_desire_edges_reset_and_nonfinite_failure(self): + artifact = artifact_fixture() + with Context(DEV="CPU:LLVM"), patch("openpilot.starpilot.models.runner.load_oob", return_value=artifact): + runner = CatalogModelState(8, 8, Path("verified.pkl"), "v15", False) + frames = {name: np.zeros(runner.frame_buf_size, dtype=np.uint8) for name in runner.vision_input_names} + transforms = dict.fromkeys(runner.vision_input_names, np.eye(3, dtype=np.float32)) + inputs = {"desire_pulse": np.array([0, 1, 0, 0, 0, 0, 0, 0], dtype=np.float32), + "traffic_convention": np.array([1, 0]), "prev_action": np.array([2, 3]), + "lateral_control_params": np.array([20, .2])} + callback = Mock() + runner.run(frames, transforms, inputs, callback) + callback.assert_called_once_with() + np.testing.assert_array_equal(runner.npy["desire"], inputs["desire_pulse"]) + np.testing.assert_array_equal(runner.npy["prev_action"], [[2, 3]]) + np.testing.assert_allclose(runner.npy["lateral_control_params"], [[20, .2]]) + runner.run(frames, transforms, inputs) + np.testing.assert_array_equal(runner.npy["desire"], np.zeros(8)) + runner.warmup() + np.testing.assert_array_equal(runner.prev_desire, np.zeros(8)) + np.testing.assert_array_equal(runner.npy["prev_action"], [[0, 0]]) + artifact["run_policy"].return_value = (Mock(numpy=lambda: np.array([np.nan])),) + with self.assertRaisesRegex(ValueError, "not finite"): + runner.run(frames, transforms, inputs) + + def test_verified_load_rejects_changed_bytes_before_deserialization(self): + from openpilot.starpilot.models.receipt import artifact_digest, artifact_identity + + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + artifact = root / "model.pkl" + artifact.write_bytes(b"trusted compiled artifact") + digest = artifact_digest(artifact, artifact_identity(artifact)) + with patch("openpilot.starpilot.models.runner.CatalogModelState") as state: + loaded, prepared = load_verified_model(8, 8, artifact, "v11", False, "sc23", digest, root=root) + self.assertEqual(loaded.model_id, "sc23") + self.assertEqual(prepared.sha256, digest) + loaded.warmup.assert_called_once_with() + state.reset_mock() + artifact.write_bytes(b"changed artifact") + with self.assertRaisesRegex(ValueError, "changed before loading"): + load_verified_model(8, 8, artifact, "v11", False, "sc23", digest, root=root) + state.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_runtime.py b/openpilot/starpilot/models/tests/test_runtime.py new file mode 100644 index 0000000000..4f40064066 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_runtime.py @@ -0,0 +1,102 @@ +from types import SimpleNamespace +import time +import tempfile +import unittest +import uuid + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT +from openpilot.starpilot.models.runtime import snapshot +from openpilot.starpilot.models.status import ModelHealth, ModelLoad, ModelVariant + + +class FakeSubMaster: + def __init__(self): + self.logMonoTime = {"managerState": 1_900_000_000, "modelV2": 1_990_000_000, + "drivingModelData": 1_995_000_000} + self.valid = dict.fromkeys(self.logMonoTime, True) + self.alive = dict(self.valid) + self.messages = { + "managerState": SimpleNamespace(processes=[SimpleNamespace(name="modeld", pid=421, running=True)]), + "modelV2": SimpleNamespace(big=False), + "drivingModelData": SimpleNamespace(), + } + + def __getitem__(self, service): + return self.messages[service] + + +class FakeParams: + def __init__(self, chestnut=None): + self.chestnut = chestnut + + def get(self, key): + assert key == "ChestnutActive" + return self.chestnut + + +class TestRuntimeSnapshot(unittest.TestCase): + def setUp(self): + self.sm = FakeSubMaster() + self.load = ModelLoad(421, 54321, 1_800_000_000, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + + def status(self, *, load=None, params=None, process_ticks=54321, now=2_000_000_000): + return snapshot(self.sm, FakeParams() if params is None else params, now, + start_ticks=lambda _pid: process_ticks, read_load=lambda _path: self.load if load is None else load) + + def test_actual_manager_and_two_fresh_outputs_are_required(self): + self.assertEqual(self.status().health, ModelHealth.ACTIVE) + self.sm.valid["drivingModelData"] = False + self.assertEqual(self.status().health, ModelHealth.STALE) + self.sm.valid["drivingModelData"] = True + self.sm.logMonoTime["managerState"] = 100 + self.assertEqual(self.status().health, ModelHealth.UNAVAILABLE) + + def test_pid_reuse_or_corrupt_chestnut_does_not_claim_identity(self): + self.assertEqual(self.status(process_ticks=54322).health, ModelHealth.IDENTITY_UNAVAILABLE) + self.assertEqual(self.status(process_ticks=0).health, ModelHealth.IDENTITY_UNAVAILABLE) + self.assertEqual(self.status(params=FakeParams(b"broken")).health, ModelHealth.FAILED) + self.sm.messages["modelV2"].big = True + self.assertEqual(self.status(params=FakeParams(b"1")).health, ModelHealth.FAILED) + + def test_native_typed_chestnut_parameter_matches_loaded_runner(self): + from openpilot.common.params import Params + with tempfile.TemporaryDirectory() as directory: + params = Params(directory) + params.put_bool("ChestnutActive", False, block=True) + self.assertIs(params.get("ChestnutActive"), False) + self.assertEqual(self.status(params=params).health, ModelHealth.ACTIVE) + params.put_bool("ChestnutActive", True, block=True) + self.sm.messages["modelV2"].big = True + self.load = ModelLoad(421, 54321, 1_800_000_000, BUNDLED_CURRENT, ModelVariant.CHESTNUT, "a" * 64) + self.assertEqual(self.status(params=params).health, ModelHealth.ACTIVE) + params.put_bool("ChestnutActive", False, block=True) + self.assertEqual(self.status(params=params).health, ModelHealth.FAILED) + + def test_real_manager_and_model_output_ipc(self): + from openpilot.cereal import messaging + messaging.set_fake_prefix("model_status_" + uuid.uuid4().hex) + publisher = messaging.PubMaster(["managerState", "modelV2", "drivingModelData"]) + subscriber = messaging.SubMaster(["managerState", "modelV2", "drivingModelData"]) + time.sleep(0.05) + for name in ("managerState", "modelV2", "drivingModelData"): + message = messaging.new_message(name) + message.valid = True + if name == "managerState": + state = message.managerState.init("processes", 1)[0] + state.name = "modeld" + state.pid = 421 + state.running = True + elif name == "modelV2": + message.modelV2.big = False + publisher.send(name, message) + subscriber.update(100) + self.assertTrue(all(subscriber.seen[name] for name in ("managerState", "modelV2", "drivingModelData"))) + load = ModelLoad(421, 54321, time.monotonic_ns() - 100_000_000, BUNDLED_CURRENT, + ModelVariant.SMALL, "a" * 64) + observed = snapshot(subscriber, FakeParams(), time.monotonic_ns(), + start_ticks=lambda _pid: 54321, read_load=lambda _path: load) + self.assertEqual(observed.health, ModelHealth.ACTIVE) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_shared_camera_warp.py b/openpilot/starpilot/models/tests/test_shared_camera_warp.py new file mode 100644 index 0000000000..a06769b2ae --- /dev/null +++ b/openpilot/starpilot/models/tests/test_shared_camera_warp.py @@ -0,0 +1,167 @@ +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace +import unittest +from unittest.mock import Mock, patch + +import numpy as np +from tinygrad import Context, dtypes + +from openpilot.starpilot.models.runner import CatalogModelState +from openpilot.starpilot.models.tests.test_runner import artifact_fixture + + +def fake_warp(shape=(2, 6, 2, 2), dtype=dtypes.uint8, device="CPU"): + return SimpleNamespace(shape=shape, dtype=dtype, device=device) + + +def make_runner(artifact=None, camera=(8, 8)): + artifact = artifact_fixture() if artifact is None else artifact + artifact[camera] = Mock(return_value=fake_warp()) + with Context(DEV="CPU:LLVM"), patch("openpilot.starpilot.models.runner.load_oob", return_value=artifact): + runner = CatalogModelState(*camera, Path("verified.pkl"), "v15", False) + return runner + + +def frame_inputs(runner): + frames = {name: np.zeros(runner.frame_buf_size, dtype=np.uint8) for name in runner.vision_input_names} + transforms = {name: np.eye(3, dtype=np.float32) for name in runner.vision_input_names} + inputs = {"desire_pulse": np.array([0, 1, 0, 0, 0, 0, 0, 0], dtype=np.float32), + "prev_action": np.array([2, 3], dtype=np.float32)} + return frames, transforms, inputs + + +class TestSharedCameraWarp(unittest.TestCase): + def test_shared_frame_runs_one_warp_and_two_independent_policies(self): + lateral, longitudinal, ordinary = (make_runner() for _ in range(3)) + frames, transforms, inputs = frame_inputs(lateral) + lat_output = lateral.run(frames, transforms, inputs) + packet = lateral.last_warp + callback = Mock() + long_output = longitudinal.run(frames, transforms, inputs, callback, shared_warp=packet) + normal_output = ordinary.run(frames, transforms, inputs) + lateral.warp_enqueue.assert_called_once() + longitudinal.warp_enqueue.assert_not_called() + ordinary.warp_enqueue.assert_called_once() + callback.assert_called_once_with() + self.assertIs(longitudinal.run_policy.call_args.kwargs["warped"], packet.tensor) + self.assertIs(longitudinal.last_warp, packet) + for key in lat_output: + np.testing.assert_array_equal(long_output[key], normal_output[key]) + for key in lateral.input_queues: + self.assertIsNot(lateral.input_queues[key], longitudinal.input_queues[key]) + for key in lateral.npy: + self.assertFalse(np.shares_memory(lateral.npy[key], longitudinal.npy[key])) + longitudinal.run(frames, transforms, {**inputs, "desire_pulse": np.zeros(8)}) + np.testing.assert_array_equal(lateral.prev_desire, inputs["desire_pulse"]) + np.testing.assert_array_equal(longitudinal.prev_desire, np.zeros(8)) + self.assertIs(lateral.last_warp, packet) + + def test_history_length_and_sampling_remain_private(self): + artifact = artifact_fixture() + artifact["metadata"]["model"]["input_shapes"]["img"] = (1, 6, 2, 2) + artifact["metadata"]["model"]["input_shapes"]["big_img"] = (1, 6, 2, 2) + artifact["frame_skip"] = 1 + lateral, longitudinal = make_runner(), make_runner(artifact) + self.assertEqual(lateral.camera_warp_descriptor, longitudinal.camera_warp_descriptor) + self.assertNotEqual(lateral.input_queues["img_q"].shape, longitudinal.input_queues["img_q"].shape) + args = frame_inputs(lateral) + lateral.run(*args) + longitudinal.run(*args, shared_warp=lateral.last_warp) + longitudinal.warp_enqueue.assert_not_called() + + def test_descriptor_and_actual_tensor_mismatches_rejected_before_state_update(self): + lateral, longitudinal = make_runner(), make_runner() + args = frame_inputs(lateral) + lateral.run(*args) + packet = lateral.last_warp + descriptor = packet.descriptor + mismatches = [replace(packet, descriptor=replace(descriptor, camera_size=(16, 8))), + replace(packet, descriptor=replace(descriptor, nv12_layout=(1, 2, 3, 4))), + replace(packet, descriptor=replace(descriptor, image_layouts=((4, 4), (4, 4)))), + replace(packet, descriptor=replace(descriptor, device="AMD")), + replace(packet, tensor=fake_warp(shape=(2, 6, 4, 4))), + replace(packet, tensor=fake_warp(dtype=dtypes.float32)), + replace(packet, tensor=fake_warp(device="AMD"))] + for mismatch in mismatches: + with self.subTest(packet=mismatch), self.assertRaises(ValueError): + longitudinal.run(*args, shared_warp=mismatch) + longitudinal.run_policy.assert_not_called() + np.testing.assert_array_equal(longitudinal.prev_desire, np.zeros(8)) + + def test_new_buffers_or_transforms_cannot_reuse_camera_warp(self): + lateral, longitudinal = make_runner(), make_runner() + frames, transforms, inputs = frame_inputs(lateral) + lateral.run(frames, transforms, inputs) + packet = lateral.last_warp + other_frames = {key: value.copy() for key, value in frames.items()} + with self.assertRaisesRegex(ValueError, "frame/transforms"): + longitudinal.run(other_frames, transforms, inputs, shared_warp=packet) + changed = {key: value.copy() for key, value in transforms.items()} + changed["big_img"][0, 2] = 1 + with self.assertRaisesRegex(ValueError, "frame/transforms"): + longitudinal.run(frames, changed, inputs, shared_warp=packet) + longitudinal.run_policy.assert_not_called() + + def test_old_packet_and_repeat_consumption_rejected(self): + lateral, longitudinal = make_runner(), make_runner() + args = frame_inputs(lateral) + lateral.run(*args) + packet = lateral.last_warp + longitudinal.run(*args, shared_warp=packet) + with self.assertRaisesRegex(ValueError, "already consumed"): + longitudinal.run(*args, shared_warp=packet) + lateral.run(*args) + with self.assertRaisesRegex(ValueError, "stale"): + make_runner().run(*args, shared_warp=packet) + + def test_failed_output_and_warmup_invalidate_exposed_packet(self): + producer = make_runner() + args = frame_inputs(producer) + producer.run(*args) + packet = producer.last_warp + producer.run_policy.return_value = (Mock(numpy=lambda: np.array([np.nan])),) + with self.assertRaisesRegex(ValueError, "not finite"): + producer.run(*args) + self.assertIsNone(producer.last_warp) + with self.assertRaisesRegex(ValueError, "stale"): + make_runner().run(*args, shared_warp=packet) + producer = make_runner() + producer.run(*frame_inputs(producer)) + producer.warmup() + self.assertIsNone(producer.last_warp) + self.assertIsNone(producer._last_shared_warp) + + def test_borrowed_packet_relay_keeps_original_producer_lifetime(self): + for invalidate in ("advance", "reset"): + producer, relay, consumer = (make_runner() for _ in range(3)) + args = frame_inputs(producer) + producer.run(*args) + packet = producer.last_warp + relay.run(*args, shared_warp=packet) + self.assertIs(relay.last_warp, packet) + self.assertIs(relay.last_warp.producer(), producer) + if invalidate == "advance": + producer.run(*args) + else: + producer._reset_state() + with self.subTest(invalidate=invalidate), self.assertRaisesRegex(ValueError, "stale"): + consumer.run(*args, shared_warp=relay.last_warp) + consumer.run_policy.assert_not_called() + consumer.warp_enqueue.assert_not_called() + + def test_history_in_warp_cannot_share_but_default_path_is_preserved(self): + artifact = artifact_fixture() + artifact["image_history_pipeline"] = "warp" + consumer = make_runner(artifact) + producer = make_runner() + args = frame_inputs(producer) + producer.run(*args) + with self.assertRaisesRegex(ValueError, "history"): + consumer.run(*args, shared_warp=producer.last_warp) + consumer.warp_enqueue.return_value = ("road-history", "wide-history") + consumer.run(*args) + consumer.warp_enqueue.assert_called_once() + self.assertEqual(consumer.run_policy.call_args.kwargs["img"], "road-history") + self.assertEqual(consumer.run_policy.call_args.kwargs["big_img"], "wide-history") + self.assertIsNone(consumer.last_warp) diff --git a/openpilot/starpilot/models/tests/test_validation.py b/openpilot/starpilot/models/tests/test_validation.py new file mode 100644 index 0000000000..fd9daa7426 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_validation.py @@ -0,0 +1,115 @@ +import json +from pathlib import Path +import tempfile +from types import SimpleNamespace +import unittest +from unittest.mock import patch + +import numpy as np + +from scripts import validate_driving_model as validator +from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + +def outputs(step=0): + values = {name: np.full(validator.REQUIRED_SHAPES.get(name, (1, 2)), step / 100, dtype=np.float32) + for name in validator.REQUIRED_KEYS} + values["action"] = np.array([[step / 1000, step / 100]], dtype=np.float32) + return values + + +class FixtureRunner: + vision_input_names = ("img", "big_img") + queue_device = warp_device = "CPU" + + def __init__(self, width, height, *_args): + self.frame_buf_size = get_nv12_info(width, height)[3] + self.step = 0 + self.frame_hashes = [] + self.inputs = [] + + def warmup(self): + self.step = 0 + + def run(self, frames, transforms, inputs): + self.frame_hashes.append(hash(frames["img"].tobytes())) + self.inputs.append(inputs) + self.step += 1 + return outputs(self.step) + + +def update_action(values, *_args): + return SimpleNamespace(desiredCurvature=values["action"][0, 0], desiredAcceleration=values["action"][0, 1]) + + +class TestDrivingValidation(unittest.TestCase): + def test_changed_frames_recurrent_actions_shapes_and_timing(self): + runner = FixtureRunner(8, 8) + result = validator.validate_camera(Path("fixture.pkl"), "v15", (8, 8), 5, 42, False, + runner_factory=lambda *_: runner, action_update=update_action, + initial_action=SimpleNamespace(desiredCurvature=0., desiredAcceleration=0.)) + self.assertTrue(result["passed"]) + self.assertEqual(len(set(runner.frame_hashes)), 5) + self.assertEqual(result["output_shapes"]["plan"], [1, 33, 15]) + self.assertIn("pose", result["changed_outputs"]) + self.assertGreater(result["inference_ms"]["p95"], 0) + self.assertGreater(runner.inputs[1]["prev_action"][0], 0) + self.assertEqual(runner.inputs[0]["desire_pulse"][1], 1) + self.assertEqual(runner.inputs[1]["desire_pulse"].sum(), 0) + + def test_nv12_stride_padding_and_chroma_are_well_formed(self): + width, height = 1344, 760 + stride, y_height, uv_height, size = get_nv12_info(width, height) + frame = np.zeros(size, dtype=np.uint8) + validator.fill_nv12(frame, width, height, 3, 42) + y = frame[:stride * y_height].reshape(y_height, stride) + uv = frame[stride * y_height:stride * (y_height + uv_height)].reshape(uv_height, stride) + self.assertTrue(np.all((y[:height, :width] >= 16) & (y[:height, :width] <= 235))) + self.assertTrue(np.all(y[height:] == 16)) + self.assertTrue(np.all(uv[:height // 2, :width:2] == 147)) + self.assertTrue(np.all(uv[:height // 2, 1:width:2] == 159)) + + def test_missing_nonfinite_wrong_shape_and_frozen_outputs_fail(self): + for field, value in (("plan", np.full((1, 33, 15), np.nan)), ("lead", np.zeros((1, 2, 6, 4)))): + with self.subTest(field=field), self.assertRaises(ValueError): + validator.check_outputs(outputs() | {field: value}, "v15") + missing_action = outputs() + del missing_action["action"] + with self.assertRaisesRegex(ValueError, "Missing"): + validator.check_outputs(missing_action, "v15") + validator.check_outputs(missing_action, "v11") + runner = FixtureRunner(8, 8) + runner.run = lambda *_: outputs() + with self.assertRaisesRegex(ValueError, "unchanged"): + validator.validate_camera(Path("fixture.pkl"), "v15", (8, 8), 3, 42, False, + runner_factory=lambda *_: runner, action_update=update_action, + initial_action=SimpleNamespace(desiredCurvature=0., desiredAcceleration=0.)) + + def test_parent_binds_artifact_sources_and_runs_both_camera_processes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + artifact, source, report = root / "test.pkl", root / "driving_supercombo.onnx", root / "report.json" + artifact.write_bytes(b"compiled") + source.write_bytes(b"onnx") + receipt = {"artifact_sha256": validator.sha256_file(artifact), "behavior_version": "v15", + "source_sha256": {source.name: validator.sha256_file(source)}} + artifact.with_suffix(".build.json").write_text(json.dumps(receipt)) + def child(command, **_kwargs): + camera = list(map(int, command[command.index("--camera") + 1].split("x"))) + Path(command[command.index("--report") + 1]).write_text(json.dumps({"passed": True, "camera": camera})) + return SimpleNamespace(returncode=0) + with patch.object(validator.subprocess, "run", side_effect=child) as child_run: + self.assertEqual(validator.main([str(artifact), "--version", "v15", "--source-dir", str(root), "--report", str(report)]), 0) + self.assertEqual(child_run.call_count, 2) + result = json.loads(report.read_text()) + self.assertTrue(result["sources_verified"]) + self.assertEqual([camera["camera"] for camera in result["cameras"]], [list(camera) for camera in validator.CAMERAS]) + source.write_bytes(b"changed") + with patch.object(validator.subprocess, "run") as child_run: + self.assertEqual(validator.main([str(artifact), "--version", "v15", "--source-dir", str(root), "--report", str(report)]), 1) + child_run.assert_not_called() + self.assertFalse(json.loads(report.read_text())["passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py new file mode 100755 index 0000000000..1214272243 --- /dev/null +++ b/scripts/model_compiler.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Compile staged ONNX sources without installing or selecting the result.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +REPO_ROOT = Path(__file__).resolve().parents[1] +COMPONENTS = { + "supercombo": ("driving_supercombo", "supercombo"), + "off-policy": ("driving_off_policy", "off_policy", "offpolicy"), + "on-policy": ("driving_on_policy", "on_policy", "onpolicy"), + "policy": ("driving_policy", "policy"), + "vision": ("driving_vision", "vision"), +} + + +def source_files(directory: Path, model: str) -> dict[str, Path]: + selected = directory / model if (directory / model).is_dir() else directory + found = {} + for path in sorted(selected.glob("*.onnx")): + stem = path.stem.lower() + for component, aliases in COMPONENTS.items(): + if any(stem == alias or stem == f"{model}_{alias}" for alias in aliases): + if component in found: + raise ValueError(f"Ambiguous {component} source: {found[component]} and {path}") + found[component] = path.resolve() + break + return found + + +def driving_compile_args(files: dict[str, Path], requested: str) -> tuple[str, list[str]]: + selected = "supercombo" if requested == "auto" and "supercombo" in files else requested + if selected == "supercombo": + if "supercombo" not in files: + raise ValueError("Supercombo input requires driving_supercombo.onnx") + return "supercombo", ["--supercombo-onnx", str(files["supercombo"])] + if "vision" not in files or not ({"policy", "on-policy"} & files.keys()): + raise ValueError("Split input requires driving_vision.onnx and driving_policy.onnx or driving_on_policy.onnx") + result = ["--vision-onnx", str(files["vision"])] + primary = files.get("on-policy", files.get("policy")) + if "off-policy" in files: + return "vision_multi_policy", [*result, "--on-policy-onnx", str(primary), "--off-policy-onnx", str(files["off-policy"])] + return "vision_policy", [*result, "--policy-onnx", str(primary)] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def compile_env(gpu: bool = False) -> dict[str, str]: + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(REPO_ROOT), str(REPO_ROOT / "tinygrad_repo"), env.get("PYTHONPATH", ""))) + defaults = {"DEV": "QCOM", "IMAGE": "1", "FLOAT16": "1", "NOLOCALS": "1", "JIT_BATCH_SIZE": "0", "OPENPILOT_HACKS": "1"} + for key, value in defaults.items(): + env.setdefault(key, value) + if gpu: + for key in ("IMAGE", "NOLOCALS", "OPENPILOT_HACKS"): + env.pop(key, None) + env.update(DEV="USB+AMD:LLVM", WARP_DEV="QCOM", GMMU="0", TC_OPT="2", TC_MIN_GLOBALS="32") + return env + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", help="Stable output model ID") + parser.add_argument("--version", required=True, choices=tuple(f"v{value}" for value in range(8, 17))) + parser.add_argument("--input-dir", type=Path, default=Path("/data/openpilot/uncompiledmodels")) + parser.add_argument("--output-dir", type=Path, default=Path("/data/openpilot/compiledmodels")) + parser.add_argument("--input-format", choices=("auto", "supercombo", "split"), default="auto") + parser.add_argument("--gpu", action="store_true") + parser.add_argument("--image-history-pipeline", choices=("policy", "warp"), default="policy") + parser.add_argument("--frame-skip", type=int) + args, unknown = parser.parse_known_args(argv) + if unknown: + if len(unknown) != 1 or not unknown[0].startswith("--") or args.model: + parser.error("Use --model ID or one --ID flag") + args.model = unknown[0][2:] + if not args.model or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", args.model): + parser.error("A lowercase model ID containing letters, digits, hyphens or underscores is required") + if args.frame_skip is not None and args.frame_skip < 1: + parser.error("--frame-skip must be positive") + return args + + +def main(argv=None): + args = parse_args(argv) + model_type, source_args = driving_compile_args(source_files(args.input_dir, args.model), args.input_format) + sources = [Path(source_args[index]) for index in range(1, len(source_args), 2)] + for source in sources: + with source.open("rb") as file: + prefix = file.read(128) + if not prefix or prefix.startswith(b"version https://git-lfs.github.com/spec/v1"): + raise ValueError(f"Expected ONNX model bytes, not an empty file or Git LFS pointer: {source}") + before = {path.name: sha256_file(path) for path in sources} + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + output = output_dir / f"{args.model}_driving_tinygrad.pkl" + env = compile_env(args.gpu) + with tempfile.TemporaryDirectory(prefix=f".{args.model}-compile-", dir=output_dir) as temporary: + staged = Path(temporary) / output.name + command = [sys.executable, "-m", "openpilot.starpilot.models.compile", "--model-type", model_type, + "--model-size", "512x256", "--camera-resolutions", "1928x1208", "1344x760", + "--behavior-version", args.version, "--output", str(staged), "--out-of-band", + "--image-history-pipeline", args.image_history_pipeline, *source_args] + if args.frame_skip is not None: + command += ["--frame-skip", str(args.frame_skip)] + subprocess.run(command, cwd=REPO_ROOT, env=env, check=True) + if {path.name: sha256_file(path) for path in sources} != before: + raise ValueError("ONNX source changed during compilation") + revision = next(item["commit"] for item in json.loads((REPO_ROOT / "upstream-sync.json").read_text())["dependencies"] + if item["path"] == "tinygrad_repo") + receipt = {"model_id": args.model, "behavior_version": args.version, "model_type": model_type, + "artifact_abi": "tinygrad_single_v1_arena", "compiler_revision": revision, + "source_sha256": before, "artifact_sha256": sha256_file(staged), "artifact_size": staged.stat().st_size, + "execution_device": env["DEV"], "warp_device": env.get("WARP_DEV", env["DEV"]), + "image_history_pipeline": args.image_history_pipeline, "camera_resolutions": [[1928, 1208], [1344, 760]]} + staged.replace(output) + receipt_path = output.with_suffix(".build.json") + temporary_receipt = Path(temporary) / receipt_path.name + temporary_receipt.write_text(json.dumps(receipt, indent=2) + "\n") + temporary_receipt.replace(receipt_path) + print(f"Compiled {output}; build receipt {receipt_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/model_rebuild.py b/scripts/model_rebuild.py new file mode 100644 index 0000000000..1cdf979603 --- /dev/null +++ b/scripts/model_rebuild.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Rebuild a model generation on an explicitly selected compiler device.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shlex +import subprocess +import sys +import time +from urllib.request import urlopen +from urllib.error import HTTPError + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, CATALOG_PATH, COMPILER_REVISION, GENERATION +from openpilot.starpilot.models.manager import atomic_json, validate_manifest + +BUCKET = "StarPilot-Driving/StarPilot-Resources" +REMOTE_WORK = "/data/starpilot-v26-rebuild" +CHUNK_BYTES = 45 * 1024 * 1024 + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def run(args: list[str], *, output=None) -> None: + subprocess.run(args, check=True, stdout=output, stderr=subprocess.STDOUT if output is not None else None) + + +def remote(device: str, command: list[str], *, output=None) -> None: + run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=30", + device, shlex.join(command)], output=output) + + +def inventory() -> list[dict]: + result = [] + url = f"https://huggingface.co/api/buckets/{BUCKET}/tree?recursive=true&limit=1000" + while url: + with urlopen(url, timeout=40) as response: + result.extend(json.load(response)) + links = response.headers.get("Link", "") + url = next((p.split(">")[0].strip("< ") for p in links.split(",") if 'rel="next"' in p), "") + return result + + +def source_plan(model_id: str, source_map: dict, objects: list[dict]) -> list[dict]: + source_id = source_map[model_id]["source_id"] + files = [obj for obj in objects if obj["type"] == "file" and obj["path"].startswith(f"onnx/{source_id}/") and + obj["path"].endswith(".onnx")] + if not files: + raise ValueError(f"No archived ONNX source for {model_id} ({source_id})") + return files + + +def prepare_sources(model_id: str, files: list[dict], workspace: Path, hf: str) -> Path: + directory = workspace / "onnx" / model_id + directory.mkdir(parents=True, exist_ok=True) + for obj in files: + name = Path(obj["path"]).name + if "driving_" not in name: + raise ValueError(f"Unrecognized model source filename: {name}") + name = model_id + "_" + name[name.index("driving_"):] + destination = directory / name + if not destination.is_file() or destination.stat().st_size != obj["size"]: + run([hf, "buckets", "cp", f"hf://buckets/{BUCKET}/{obj['path']}", str(destination), "--format", "quiet"]) + if destination.stat().st_size != obj["size"]: + raise ValueError("Source download size mismatch") + return directory + + +def split_artifact(artifact: Path, directory: Path) -> tuple[list[Path], int]: + directory.mkdir(parents=True, exist_ok=True) + count = (artifact.stat().st_size + CHUNK_BYTES - 1) // CHUNK_BYTES + if count == 1: + return [artifact], 0 + parts = [] + with artifact.open("rb") as stream: + for index in range(1, count + 1): + part = directory / f"{artifact.name}.chunk{index:02d}of{count:02d}" + part.write_bytes(stream.read(CHUNK_BYTES)) + parts.append(part) + return parts, count + + +def publish_artifact(model_id: str, artifact: Path, receipt: dict, workspace: Path, hf: str) -> dict: + if receipt["compiler_revision"] != COMPILER_REVISION or receipt["artifact_abi"] != ARTIFACT_ABI or digest(artifact) != receipt["artifact_sha256"]: + raise ValueError("Compiled artifact identity changed") + parts, count = split_artifact(artifact, workspace / "release" / model_id) + for part in parts: + run([hf, "buckets", "cp", str(part), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{part.name}", "--format", "quiet"]) + if count: + marker = workspace / "release" / model_id / f"{artifact.name}.chunkmanifest" + marker.write_text(str(count)) + run([hf, "buckets", "cp", str(marker), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{marker.name}", "--format", "quiet"]) + return {"artifact_format": ARTIFACT_ABI, "artifact_size": artifact.stat().st_size, + "artifact_sha256": receipt["artifact_sha256"], "artifact_chunk_count": count, + "source_sha256": receipt["source_sha256"]} + + +def compile_one(model: dict, args, source_map: dict, objects: list[dict]) -> dict: + mid = model["id"] + work = args.workspace + result_path = work / "results" / f"{mid}.json" + output_dir = work / "compiled" / mid + artifact = output_dir / f"{mid}_driving_tinygrad.pkl" + if result_path.is_file(): + previous = json.loads(result_path.read_text()) + if (previous.get("status") == "passed" and previous.get("compiler_revision") == COMPILER_REVISION and artifact.is_file() and + digest(artifact) == previous["artifact_sha256"] and (not args.publish or previous.get("published"))): + return previous + sources = prepare_sources(mid, source_plan(mid, source_map, objects), work, args.hf) + remote_dir = f"{REMOTE_WORK}/work/{mid}" + remote(args.device, ["mkdir", "-p", f"{remote_dir}/onnx", f"{remote_dir}/compiled"]) + run(["rsync", "-az", str(sources) + "/", f"{args.device}:{remote_dir}/onnx/"]) + command = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/model_compiler.py", "--model", mid, + "--version", model["version"], "--input-format", source_map[mid]["input_format"], + "--input-dir", f"{remote_dir}/onnx", "--output-dir", f"{remote_dir}/compiled"] + if model.get("uses_external_gpu"): + command.append("--gpu") + with (work / "logs" / f"{mid}.log").open("ab") as log: + remote(args.device, ["env", "PYTHONUNBUFFERED=1", *command], output=log) + validate = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/validate_driving_model.py", + f"{remote_dir}/compiled/{artifact.name}", "--version", model["version"], + "--source-dir", f"{remote_dir}/onnx", "--report", f"{remote_dir}/compiled/validation.json", "--max-p95-ms", "50"] + if model.get("uses_external_gpu"): + validate.append("--gpu") + remote(args.device, validate, output=log) + output_dir.mkdir(parents=True, exist_ok=True) + run(["rsync", "-az", f"{args.device}:{remote_dir}/compiled/", str(output_dir) + "/"]) + receipt = json.loads(artifact.with_suffix(".build.json").read_text()) + validation = json.loads((output_dir / "validation.json").read_text()) + if (digest(artifact) != receipt["artifact_sha256"] or not validation.get("passed") or + validation.get("artifact_sha256") != receipt["artifact_sha256"]): + raise ValueError("Compiled model failed runtime validation") + result = {**receipt, "status": "passed", "published": False, "validation": validation} + if args.publish: + result["release"] = publish_artifact(mid, artifact, receipt, work, args.hf) + result["published"] = True + atomic_json(result_path, result) + # Only this invocation's sources and output are removed after the verified host copy exists. + remote(args.device, ["/usr/local/venv/bin/python", "-c", "import shutil,sys; shutil.rmtree(sys.argv[1])", remote_dir]) + return result + + +def publish_manifest(workspace: Path, hf: str) -> None: + manifest = json.loads(CATALOG_PATH.read_text()) + try: + with urlopen(f"https://huggingface.co/buckets/{BUCKET}/resolve/manifests/model_names_{GENERATION}.json", timeout=30) as response: + live = json.load(response) + existing = validate_manifest(live) + for row in manifest["models"]: + if row["id"] in existing: + row.update(existing[row["id"]]) + except HTTPError as error: + if error.code != 404: + raise + for model in manifest["models"]: + path = workspace / "results" / f"{model['id']}.json" + if path.is_file(): + result = json.loads(path.read_text()) + if result.get("status") == "passed" and result.get("published") and result.get("compiler_revision") == COMPILER_REVISION: + model.update(result["release"]) + path = workspace / "manifests" / f"model_names_{GENERATION}.json" + atomic_json(path, manifest) + run([hf, "buckets", "cp", str(path), f"hf://buckets/{BUCKET}/manifests/{path.name}", "--format", "quiet"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", required=True, help="Explicit SSH target, for example comma@comma-device.local") + parser.add_argument("--workspace", required=True, type=Path) + parser.add_argument("--models", nargs="+", help="Stable model IDs; defaults to the full catalog, favorites first") + parser.add_argument("--hf", default=str(Path.home()/".local/bin/hf")) + parser.add_argument("--publish", action="store_true", help="Upload validated v26 artifacts and manifest to Hugging Face") + args = parser.parse_args() + args.workspace = args.workspace.resolve() + for directory in ("results", "logs", "manifests"): + (args.workspace / directory).mkdir(parents=True, exist_ok=True) + rows = json.loads(CATALOG_PATH.read_text())["models"] + source_map = json.loads((REPO / "scripts/model_source_map_v26.json").read_text()) + if args.models: + by_id = {row["id"]: row for row in rows} + rows = [by_id[mid] for mid in args.models] + else: + rows.sort(key=lambda row: (not row.get("community_favorite", False), row["id"] != "cinquev3")) + objects = inventory() + atomic_json(args.workspace / "source-inventory.json", {"objects": objects}) + failures = [] + for model in rows: + mid = model["id"] + print(f"START {mid}", flush=True) + started = time.monotonic() + try: + compile_one(model, args, source_map, objects) + if args.publish: + publish_manifest(args.workspace, args.hf) + print(f"PASS {mid} {time.monotonic()-started:.0f}s", flush=True) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + failures.append(mid) + atomic_json(args.workspace / "results" / f"{mid}.failure.json", {"model_id": mid, "error": str(error)}) + print(f"FAIL {mid}: {error}", flush=True) + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/model_source_map_v26.json b/scripts/model_source_map_v26.json new file mode 100644 index 0000000000..c483414441 --- /dev/null +++ b/scripts/model_source_map_v26.json @@ -0,0 +1,487 @@ +{ + "tr14223": { + "source_id": "tr1422", + "input_format": "split", + "ref": "64fd3f986081ba6cb488d02e799845367add29e2" + }, + "tr15223": { + "source_id": "tr1522", + "input_format": "split", + "ref": "ed38ca8cc998f0a2b47cde29da11b782e4551e8b" + }, + "tr16223": { + "source_id": "tr16", + "input_format": "split", + "ref": "e4b6a2736580dc427473bcf8a6e3b08eb352edd4" + }, + "letr223": { + "source_id": "letr22", + "input_format": "split", + "ref": "c2fa07b82c2e09964e4b6dccb3e4465a2e4ad138" + }, + "spacelab22223": { + "source_id": "spacelab2222", + "input_format": "split", + "ref": "d3a0378ae601f61307a53d1173c251ee36c56317" + }, + "vikander223": { + "source_id": "vikander22", + "input_format": "split", + "ref": "d134fdd7d51ba84848866c0265e37508c38f9366" + }, + "fof223": { + "source_id": "fof22", + "input_format": "split", + "ref": "d807b5c476658dba7760d30949baa619074c4a4e" + }, + "vfof223": { + "source_id": "vfof22", + "input_format": "split", + "ref": "c790d3f58ad32cf15f384e6018d942b2348f0c5b" + }, + "dtr223": { + "source_id": "dtr22", + "input_format": "split", + "ref": "5b405920ec9511e146432de08171d6f93295248b" + }, + "dtr6223": { + "source_id": "dtr622", + "input_format": "split", + "ref": "a6cf39c07a67dbc346af03010093fe4788c402d0" + }, + "uvdtr6223": { + "source_id": "uvdtr622", + "input_format": "split", + "ref": "079d0461604632719e73dc5b6a5eb9897c624da2" + }, + "sp2223": { + "source_id": "sp222", + "input_format": "split", + "ref": "f1b8f510b2c554191ca15cebe35f2c5b304cf23c" + }, + "fp223": { + "source_id": "fp22", + "input_format": "split", + "ref": "95aff72e351e935221ec965aa0384dfd502d383d" + }, + "kv223": { + "source_id": "kv22", + "input_format": "split", + "ref": "16e87d4c72e5efec18032f6b4a5f6ec35d0df4e4" + }, + "gwm3223": { + "source_id": "gwm322", + "input_format": "split", + "ref": "93b26a91a62b69b809162ad2bf65e36be2158dc1" + }, + "gwm5223": { + "source_id": "gwm522", + "input_format": "split", + "ref": "e3ee440f7d5ac12c50db6381e292a55a69b745a7" + }, + "gwm6223": { + "source_id": "gwm622", + "input_format": "split", + "ref": "91a1cea814f76d605d5b15de9634a3c8f79df509" + }, + "gwm8223": { + "source_id": "gwm822", + "input_format": "split", + "ref": "96e7b310b164b55fb73abdd505b66fb36630718e" + }, + "gwm9223": { + "source_id": "gwm922", + "input_format": "split", + "ref": "f2413040a8560c0c17c18353a3c75124a2f09d17" + }, + "cgwm223": { + "source_id": "cgwm22", + "input_format": "split", + "ref": "94dd3bd9107cd413ed2418f9e6c0cf805a9ca495" + }, + "bd223": { + "source_id": "bd22", + "input_format": "split", + "ref": "b74f5189a74446015c0cf78a4a9f0134a347ae3b" + }, + "nr223": { + "source_id": "nr22", + "input_format": "split", + "ref": "266b642180f0e9278e1db46ccf8c1a2a8dae4bb3" + }, + "tcp2223": { + "source_id": "tcp222", + "input_format": "split", + "ref": "19084132cd3956413a0fdfcdc18971a55df5e0ff" + }, + "tcp3223": { + "source_id": "tcp322", + "input_format": "split", + "ref": "5eb912c025a2207c7d428deee74ded49e5905527" + }, + "fbw223": { + "source_id": "fbw22", + "input_format": "split", + "ref": "c4488e3411285f6fc3d008bca885e05731fd75c2" + }, + "nevada223": { + "source_id": "nevada22", + "input_format": "split", + "ref": "3193eac5e385aa010694a8ac192ff38ffe000193" + }, + "wmi4223": { + "source_id": "wmi422", + "input_format": "split", + "ref": "2d8e596a0208dafe4ed2d945b07d6a7d72f70fb8" + }, + "wmi523": { + "source_id": "wmi52", + "input_format": "split", + "ref": "545e0ed13f85e4c744fa1f69095863ea9dbe6d6b" + }, + "wmi623": { + "source_id": "wmi62", + "input_format": "split", + "ref": "54c6c5776a37dc56fde626314f53ec3316d88a48" + }, + "wmi723": { + "source_id": "wmi72", + "input_format": "split", + "ref": "aa3d90c92a7bd7a4e7db1b3fdee3d3e522358823" + }, + "wmi7223": { + "source_id": "wmi722", + "input_format": "split", + "ref": "a2ace1ed6b84d7c7a2dbf990c1b566f9a9df167f" + }, + "wmi823": { + "source_id": "wmi82", + "input_format": "split", + "ref": "4c438f59e7dcf5fddc769032882b62d0cb805e9b" + }, + "wmi923": { + "source_id": "wmi92", + "input_format": "split", + "ref": "8950897d7e4d2ba2426b2b31cf29f25e87a3d4ba" + }, + "wmi1023": { + "source_id": "wmi102", + "input_format": "split", + "ref": "855f5e4ddefd69a20cc4e9da004eb53f3e00d950" + }, + "wmi1123": { + "source_id": "wmi112", + "input_format": "split", + "ref": "7e3fd7a63c5a09c3fabe108b1b62bba3cf684878" + }, + "cd21023": { + "source_id": "cd2102", + "input_format": "split", + "ref": "55f66e2246359c6593605399a0199d94d13ad90d" + }, + "op223": { + "source_id": "op22", + "input_format": "split", + "ref": "ae34a24c59d85df7efad5fde2b760fb6d0b9dd6e" + }, + "op323": { + "source_id": "op32", + "input_format": "split", + "ref": "7e548dd765873bed301c0a19cfe10c3ca6be2bbe" + }, + "op423": { + "source_id": "op42", + "input_format": "split", + "ref": "25abcc49fce2f6268d8cabf759e31c626edbf584" + }, + "op523": { + "source_id": "op52", + "input_format": "split", + "ref": "b390b98c5a359c293fdc2501af79b19867fcdac4" + }, + "op623": { + "source_id": "op62", + "input_format": "split", + "ref": "831f13da61d937bb40ea0f50590cdc3b5380f313" + }, + "opv73": { + "source_id": "opv7", + "input_format": "split", + "ref": "cb327933002bc1a00bf60a3c20af2eb7a5f653e5" + }, + "opv83": { + "source_id": "opv8", + "input_format": "split", + "ref": "052692b25d63c5ddda276b5c2271383b6aff129f" + }, + "opv93": { + "source_id": "opv9", + "input_format": "split", + "ref": "eae2a73e0ac600d7f0342fc4397568e0733fe6e2" + }, + "opv103": { + "source_id": "opv10", + "input_format": "split", + "ref": "5faf14e04e4038ec4673d119c950b46051630205" + }, + "opv113": { + "source_id": "opv11", + "input_format": "split", + "ref": "bbde5ddb5a0ee35c98a962029355f9a95403a825" + }, + "opv123": { + "source_id": "opv12", + "input_format": "split", + "ref": "7d4c295c27bc3f72727c49ec73b98e45745b588d" + }, + "opv133": { + "source_id": "opv13", + "input_format": "split", + "ref": "faeeaad3a5841694b4c3bd6d37e41e3db4b4873e" + }, + "op163": { + "source_id": "op16", + "input_format": "split", + "ref": "72101c6e50ea594f24e75d3b605abf8b5cab1d6b" + }, + "op16d3": { + "source_id": "op16d", + "input_format": "split", + "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4" + }, + "op16dv23": { + "source_id": "op16dv2", + "input_format": "split", + "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4" + }, + "rlv1dl3": { + "source_id": "rlv1dl", + "input_format": "split", + "ref": "38c0e3c95de5b63b56ecbbe507abc49b9b0091a3" + }, + "deeprl33": { + "source_id": "deeprl3", + "input_format": "split", + "ref": "a1060427b9b3ac1d11cbad05380929ea8235840a" + }, + "deeprl333": { + "source_id": "deeprl33", + "input_format": "supercombo", + "ref": "4604c98f21c9a22230e417d17d03c402b9b5f869" + }, + "rl343": { + "source_id": "rl34", + "input_format": "supercombo", + "ref": "4e6b072404110056ccf9bcc232c718ac71d45478" + }, + "drl3": { + "source_id": "drl", + "input_format": "supercombo", + "ref": "a881acfbd59cb9ed18c54535b835721e2bc96def" + }, + "nopp3": { + "source_id": "nopp", + "input_format": "supercombo", + "ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8" + }, + "tobyrl3": { + "source_id": "tobyrl", + "input_format": "supercombo", + "ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746" + }, + "michael-rl3": { + "source_id": "michael-rl", + "input_format": "supercombo", + "ref": "968fd3c989b993cf03187aa0e05cd2088e275551" + }, + "michael-rl23": { + "source_id": "michael-rl2", + "input_format": "supercombo", + "ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258" + }, + "karnbir3": { + "source_id": "karnbir", + "input_format": "supercombo", + "ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb" + }, + "karnbir23": { + "source_id": "karnbir2", + "input_format": "supercombo", + "ref": "682d33d0414d7cef5a7286c24298003503ea13f4" + }, + "ms23": { + "source_id": "ms2", + "input_format": "split", + "ref": "d70b13931793bc0e4d8efd36128dd66f227a3f81" + }, + "pp2223": { + "source_id": "pp222", + "input_format": "split", + "ref": "50c78a9dd670a305ca898b96245cadbc2ecdc1a6" + }, + "ds2223": { + "source_id": "ds222", + "input_format": "split", + "ref": "5ff0e48f90d662d8c9d5061adc63e90c198c9b81" + }, + "nn2223": { + "source_id": "nn222", + "input_format": "split", + "ref": "83b81b83cdf76a577946a1567f4644e1a018443c" + }, + "sc23": { + "source_id": "sc2", + "input_format": "split", + "ref": "e4a4b4b1adf2d19fedab4d195faa382b061fa754" + }, + "pop23": { + "source_id": "pop2", + "input_format": "split", + "ref": "6f71783a8a8faa07ddaeef5bbb6809b4f4f44a15" + }, + "pop223": { + "source_id": "pop22", + "input_format": "split", + "ref": "62bf6fb072880905a4c490f0f4f4a6b3c23346ec" + }, + "nid223": { + "source_id": "nid22", + "input_format": "split", + "ref": "13e79e9fad60c19e751e4f9ab0538d39f1bb54dd" + }, + "kerrygold223": { + "source_id": "kerrygold22", + "input_format": "split", + "ref": "dc7a92ea630e4f6053082fc25eca83938927b91e" + }, + "deeprl3v23": { + "source_id": "deeprl3v2", + "input_format": "supercombo", + "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9" + }, + "rh3": { + "source_id": "rh", + "input_format": "supercombo", + "ref": "93f5aa469a72b7621aef7da7901c100e0113e4d9" + }, + "gyhu3": { + "source_id": "gyhu", + "input_format": "supercombo", + "ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b" + }, + "rdfv23": { + "source_id": "rdf", + "input_format": "supercombo", + "ref": "a95e2c25cae5fbf1afba7628bfb7acc4af59e0cc" + }, + "rdf23": { + "source_id": "rdf2", + "input_format": "supercombo", + "ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620" + }, + "rdf43": { + "source_id": "rdf4", + "input_format": "supercombo", + "ref": "a5a6412d08474cffb49a69afb910756afdee123e" + }, + "rdf33": { + "source_id": "rdf3", + "input_format": "supercombo", + "ref": "ea2151ba4b82854277f37f03b949f15fe2733dc8" + }, + "rdf53": { + "source_id": "rdf5", + "input_format": "supercombo", + "ref": "7fb03ca474f03e95e59ec0c8a6c5fba831bd5fd1" + }, + "rdf63": { + "source_id": "rdf6", + "input_format": "supercombo", + "ref": "35703097905a122c9f3ddf0d12889b4873d7e2a2" + }, + "tsf": { + "source_id": "tsf", + "input_format": "supercombo", + "ref": "4d911346cde4e0d2978a625f31679808284cc19d" + }, + "tsfdo": { + "source_id": "tsfdo", + "input_format": "supercombo", + "ref": "0394d7284cfc229f772ee4fa73ff724ba8245110" + }, + "lebowski3": { + "source_id": "lebowski", + "input_format": "supercombo", + "ref": "fa0c6876d3cf070e91e25e5353ceadc68a5b3285" + }, + "tt-driving": { + "source_id": "tt-driving", + "input_format": "supercombo" + }, + "tee-time": { + "source_id": "tee-time", + "input_format": "supercombo" + }, + "bmrlnapv2": { + "source_id": "bmrlnapv2", + "input_format": "supercombo", + "ref": "ba37c02c7c64225b8aec79411d1a8c2f5c1aa343" + }, + "happybirthday": { + "source_id": "happybirthday", + "input_format": "supercombo", + "ref": "db06f5f36caa379e112bafd38423e37b66fa4149" + }, + "berightthere": { + "source_id": "berightthere", + "input_format": "supercombo", + "ref": "bf74ce5448306088b4e9bc843472c39402930bc7" + }, + "remove-avgpoolv4": { + "source_id": "remove-avgpoolv4", + "input_format": "supercombo" + }, + "remove-avgpoolv3": { + "source_id": "remove-avgpoolv3", + "input_format": "supercombo" + }, + "remove-avgpoolv2": { + "source_id": "remove-avgpoolv2", + "input_format": "supercombo" + }, + "remove-avgpoolv1": { + "source_id": "remove-avgpoolv1", + "input_format": "supercombo" + }, + "rdf-driving": { + "source_id": "rdf-driving", + "input_format": "supercombo" + }, + "sad": { + "source_id": "sad", + "input_format": "supercombo", + "ref": "30de303d5ffb63957f8acbdb256fd4c7d360455a" + }, + "bmrlnapv6": { + "source_id": "bmrlnapv6", + "input_format": "supercombo", + "ref": "9d683c06518c0358fb402f38468a7030700c38ac" + }, + "deeprl3v2": { + "source_id": "deeprl3v2", + "input_format": "supercombo", + "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9" + }, + "cinque-driving": { + "source_id": "cinque-driving", + "input_format": "supercombo" + }, + "cdm": { + "source_id": "cdm", + "input_format": "supercombo" + }, + "cinquev3": { + "source_id": "cinquev3", + "input_format": "supercombo" + } +} diff --git a/scripts/validate_driving_model.py b/scripts/validate_driving_model.py new file mode 100755 index 0000000000..e62edba548 --- /dev/null +++ b/scripts/validate_driving_model.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Exercise a compiled catalog model on synthetic NV12 frames without starting modeld.""" +import argparse +import gc +import hashlib +import json +import math +from pathlib import Path +import subprocess +import sys +import tempfile +import time + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +CAMERAS = ((1928, 1208), (1344, 760)) +REQUIRED_SHAPES = {"plan": (1, 33, 15), "lane_lines": (1, 4, 33, 2), "road_edges": (1, 2, 33, 2), + "lead": (1, 3, 6, 4), "pose": (1, 6)} +REQUIRED_KEYS = frozenset((*REQUIRED_SHAPES, "lane_lines_prob", "lead_prob", "meta", "desire_state", "desire_pred", + "wide_from_device_euler", "road_transform")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for block in iter(lambda: file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def check_outputs(outputs: dict[str, np.ndarray], version: str) -> dict[str, str]: + required = REQUIRED_KEYS | ({"action"} if version in ("v14", "v15", "v16") else set()) + if missing := required - outputs.keys(): + raise ValueError(f"Missing parsed outputs: {sorted(missing)}") + shapes = REQUIRED_SHAPES | ({"action": (1, 2)} if "action" in outputs else {}) + hashes = {} + for name, value in outputs.items(): + array = np.asarray(value) + if not array.size or not np.isfinite(array).all(): + raise ValueError(f"Empty or non-finite parsed output: {name}") + if name in shapes and array.shape != shapes[name]: + raise ValueError(f"Incorrect parsed {name} shape: {array.shape}, expected {shapes[name]}") + hashes[name] = hashlib.sha256(array.tobytes()).hexdigest() + return hashes + + +def fill_nv12(frame: np.ndarray, width: int, height: int, step: int, seed: int) -> None: + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + stride, y_height, uv_height, _ = get_nv12_info(width, height) + frame.fill(16) + y = frame[:stride * y_height].reshape(y_height, stride) + x_coordinate = np.arange(width, dtype=np.uint32)[None, :] + y_coordinate = np.arange(height, dtype=np.uint32)[:, None] + y[:height, :width] = 16 + ((x_coordinate * 3 + y_coordinate * 5 + step * 17 + seed) % 220) + uv = frame[stride * y_height:stride * (y_height + uv_height)].reshape(uv_height, stride) + uv.fill(128) + uv[:height // 2, :width:2] = 96 + ((step * 3 + seed) % 64) + uv[:height // 2, 1:width:2] = 96 + ((step * 7 + seed) % 64) + + +def frame_transform(width: int, height: int, step: int) -> np.ndarray: + return np.array([[0.8 * width / 512, 0, 0.1 * width + math.sin(step) * 2], + [0, 0.8 * height / 256, 0.1 * height + math.cos(step) * 2], [0, 0, 1]], dtype=np.float32) + + +def upload_frames(runner, frames: dict[str, np.ndarray]) -> None: + from tinygrad import Device, Tensor + + for name, frame in frames.items(): + runner._blob_cache[(name, frame.ctypes.data)] = Tensor(frame, device=runner.warp_device).realize() + Device[runner.warp_device].synchronize() + if runner.queue_device != runner.warp_device: + Device[runner.queue_device].synchronize() + + +def validate_camera(artifact: Path, version: str, camera: tuple[int, int], steps: int, seed: int, + gpu: bool, max_p95_ms: float | None = None, runner_factory=None, action_update=None, initial_action=None) -> dict: + device_frames = runner_factory is None + if runner_factory is None: + from openpilot.starpilot.models.runner import CatalogModelState, action_from_outputs + from openpilot.cereal import log + runner_factory, action_update = CatalogModelState, action_from_outputs + initial_action = log.ModelDataV2.Action() + width, height = camera + start = time.perf_counter() + runner = runner_factory(width, height, artifact, version, gpu) + load_ms = (time.perf_counter() - start) * 1000 + start = time.perf_counter() + runner.warmup() + warmup_ms = (time.perf_counter() - start) * 1000 + frames = {name: np.zeros(runner.frame_buf_size, dtype=np.uint8) for name in runner.vision_input_names} + previous_action = initial_action + latencies, changed, baseline = [], set(), None + output_shapes = {} + actions = [] + for step in range(steps): + for index, frame in enumerate(frames.values()): + fill_nv12(frame, width, height, step, seed + index * 97) + transforms = {name: frame_transform(width, height, step + index) for index, name in enumerate(frames)} + desire = np.zeros(8, dtype=np.float32) + if step % 4 == 0: + desire[1 + (step // 4) % 2] = 1 + speed = 5.0 + step % 20 + inputs = {"desire_pulse": desire, "traffic_convention": np.array([step % 2, 1 - step % 2], dtype=np.float32), + "action_t": np.array([.2, .4], dtype=np.float32), + "prev_action": np.array([previous_action.desiredCurvature * max(1., speed) ** 2, + previous_action.desiredAcceleration], dtype=np.float32), + "lateral_control_params": np.array([speed, .2], dtype=np.float32)} + if device_frames: + upload_frames(runner, frames) + start = time.perf_counter() + outputs = runner.run(frames, transforms, inputs) + latencies.append((time.perf_counter() - start) * 1000) + hashes = check_outputs(outputs, version) + if baseline is None: + baseline = hashes + output_shapes = {name: list(np.asarray(value).shape) for name, value in outputs.items()} + else: + if hashes.keys() != baseline.keys(): + raise ValueError("Parsed output names changed between frames") + changed.update(name for name, value in hashes.items() if value != baseline[name]) + previous_action = action_update(outputs, version, previous_action, .2, .4, speed) + action = [float(previous_action.desiredCurvature), float(previous_action.desiredAcceleration)] + if not np.isfinite(action).all(): + raise ValueError("Non-finite derived driving action") + actions.append(action) + if not changed.intersection(REQUIRED_KEYS | {"action"}): + raise ValueError("Driving outputs stayed unchanged across changing frames and inputs") + timing = {"min": float(np.min(latencies)), "median": float(np.median(latencies)), + "p95": float(np.percentile(latencies, 95)), "max": float(np.max(latencies))} + if max_p95_ms is not None and timing["p95"] > max_p95_ms: + raise ValueError(f"p95 latency {timing['p95']:.2f} ms exceeds {max_p95_ms:.2f} ms") + result = {"passed": True, "camera": [width, height], "steps": steps, "seed": seed, + "load_ms": load_ms, "warmup_ms": warmup_ms, "inference_ms": timing, + "changed_outputs": sorted(changed), "output_shapes": output_shapes, "last_output_sha256": hashes, + "last_action": {"desired_curvature": actions[-1][0], "desired_acceleration": actions[-1][1]}, + "execution_device": getattr(runner, "queue_device", None), "warp_device": getattr(runner, "warp_device", None)} + del runner + gc.collect() + return result + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact", type=Path) + parser.add_argument("--version", required=True, choices=tuple(f"v{number}" for number in range(8, 17))) + parser.add_argument("--steps", type=int, default=24) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--gpu", action="store_true") + parser.add_argument("--max-p95-ms", type=float) + parser.add_argument("--build-receipt", type=Path) + parser.add_argument("--source-dir", type=Path, help="Also verify the build receipt's ONNX source hashes") + parser.add_argument("--report", type=Path) + parser.add_argument("--camera", choices=("1928x1208", "1344x760"), help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if args.steps < 2: + parser.error("--steps must be at least 2") + if args.max_p95_ms is not None and (not math.isfinite(args.max_p95_ms) or args.max_p95_ms <= 0): + parser.error("--max-p95-ms must be positive and finite") + if args.seed < 0: + parser.error("--seed must be nonnegative") + return args + + +def main(argv=None): + args = parse_args(argv) + artifact = args.artifact.resolve() + report_path = args.report or artifact.with_suffix(".validation.json") + receipt_path = args.build_receipt or artifact.with_suffix(".build.json") + protected = {artifact, receipt_path.resolve()} + report = {"passed": False, "validation": "synthetic_nv12_catalog_runner", "artifact": str(artifact), + "behavior_version": args.version, "steps": args.steps, "seed": args.seed, "cameras": []} + try: + if args.camera: + camera = tuple(map(int, args.camera.split("x"))) + report.update(validate_camera(artifact, args.version, camera, args.steps, args.seed, args.gpu, args.max_p95_ms)) + else: + before = sha256_file(artifact) + report.update(artifact_sha256=before, artifact_size=artifact.stat().st_size, sources_verified=False) + if receipt_path.is_file(): + receipt = json.loads(receipt_path.read_text()) + if receipt.get("artifact_sha256") != before or receipt.get("behavior_version") != args.version: + raise ValueError("Build receipt does not match artifact hash/behavior version") + source_hashes = receipt.get("source_sha256", {}) + report.update(build_receipt_sha256=sha256_file(receipt_path), source_sha256=source_hashes, + compiler_revision=receipt.get("compiler_revision"), artifact_abi=receipt.get("artifact_abi")) + if args.source_dir: + if not source_hashes or any(Path(name).name != name for name in source_hashes): + raise ValueError("Build receipt has no bounded source filenames") + protected.update((args.source_dir / name).resolve() for name in source_hashes) + if {name: sha256_file(args.source_dir / name) for name in source_hashes} != source_hashes: + raise ValueError("Source hashes do not match build receipt") + report["sources_verified"] = True + elif args.build_receipt or args.source_dir: + raise ValueError("Build receipt required for requested source verification") + # Separate processes release all device allocations between camera formats. + with tempfile.TemporaryDirectory(prefix="starpilot-model-validation-") as temporary: + for width, height in CAMERAS: + child_report = Path(temporary) / f"{width}x{height}.json" + command = [sys.executable, str(Path(__file__).resolve()), str(artifact), "--version", args.version, + "--steps", str(args.steps), "--seed", str(args.seed), "--camera", f"{width}x{height}", + "--report", str(child_report)] + if args.gpu: + command.append("--gpu") + if args.max_p95_ms is not None: + command += ["--max-p95-ms", str(args.max_p95_ms)] + completed = subprocess.run(command, cwd=REPO_ROOT, check=False) + result = json.loads(child_report.read_text()) if child_report.is_file() else { + "passed": False, "camera": [width, height], "error": "Validation process exited without a report"} + if completed.returncode: + result["passed"] = False + report["cameras"].append(result) + if sha256_file(artifact) != before: + raise ValueError("Artifact changed during validation") + report["passed"] = all(camera["passed"] for camera in report["cameras"]) + except Exception as error: + report.update(passed=False, error=f"{type(error).__name__}: {error}") + if report_path.resolve() in protected: + print("Refusing to overwrite a validation input with its report", file=sys.stderr) + return 1 + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + print(f"{'PASS' if report['passed'] else 'FAIL'}: {report_path}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())