From dc0f73c63be6a40abcee1cf7eb0384a57e9f2081 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 08:54:36 -0700 Subject: [PATCH 01/57] modeld_v2: safe model validation --- sunnypilot/modeld_v2/modeld.py | 27 +-- sunnypilot/modeld_v2/models/README.md | 62 ------ sunnypilot/modeld_v2/models/__init__.py | 0 sunnypilot/modeld_v2/tests/dmon_lag/repro.cc | 101 --------- sunnypilot/modeld_v2/tests/tf_test/build.sh | 2 - sunnypilot/modeld_v2/tests/tf_test/main.cc | 69 ------ .../modeld_v2/tests/tf_test/pb_loader.py | 8 - .../modeld_v2/tests/timing/benchmark.py | 38 ---- sunnypilot/models/helpers.py | 202 ++++++++++-------- sunnypilot/models/manager.py | 7 +- 10 files changed, 132 insertions(+), 384 deletions(-) delete mode 100644 sunnypilot/modeld_v2/models/README.md delete mode 100644 sunnypilot/modeld_v2/models/__init__.py delete mode 100644 sunnypilot/modeld_v2/tests/dmon_lag/repro.cc delete mode 100755 sunnypilot/modeld_v2/tests/tf_test/build.sh delete mode 100644 sunnypilot/modeld_v2/tests/tf_test/main.cc delete mode 100755 sunnypilot/modeld_v2/tests/tf_test/pb_loader.py delete mode 100755 sunnypilot/modeld_v2/tests/timing/benchmark.py diff --git a/sunnypilot/modeld_v2/modeld.py b/sunnypilot/modeld_v2/modeld.py index dfff2e6c22..b124e3645a 100755 --- a/sunnypilot/modeld_v2/modeld.py +++ b/sunnypilot/modeld_v2/modeld.py @@ -62,11 +62,6 @@ def _find_driving_pkl(bundle): if _pkl_exists(pkl_path): return pkl_path - fallback = os.path.join(model_root, 'driving_tinygrad.pkl') - if _pkl_exists(fallback): - return fallback - return None - class FrameMeta: frame_id: int = 0 @@ -125,7 +120,7 @@ class ModelState(ModelStateBase): self._vision_input_names = [k for k in model_metadata['input_shapes'] if 'img' in k] from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) - self.input_queues, self.npy = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.DEV) + self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.DEV) else: vision_metadata = metadata['vision'] policy_keys = [k for k in metadata if k != 'vision'] @@ -143,7 +138,7 @@ class ModelState(ModelStateBase): policy_input_shapes = first_policy_metadata['input_shapes'] self._vision_input_names = [k for k in vision_input_shapes if 'img' in k] frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes) - self.input_queues, self.npy = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.DEV) + self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.DEV) from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser @@ -183,7 +178,7 @@ class ModelState(ModelStateBase): @property def desire_key(self) -> str: - return next(k for k in self.npy if k.startswith('desire')) + return next(k for k in self.numpy_inputs if k.startswith('desire')) def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None: @@ -199,16 +194,16 @@ class ModelState(ModelStateBase): desire_key = self.desire_key inputs[desire_key][0] = 0 - self.npy[desire_key][:] = np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0) + self.numpy_inputs[desire_key][:] = np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0) self.prev_desire[:] = inputs[desire_key] for key in ('traffic_convention', 'lateral_control_params'): - if key in self.npy and key in inputs: - self.npy[key][:] = inputs[key] + if key in self.numpy_inputs and key in inputs: + self.numpy_inputs[key][:] = inputs[key] road_key = next(n for n in bufs if 'big' not in n) wide_key = next(n for n in bufs if 'big' in n) - self.npy['tfm'][:, :] = transforms[road_key].reshape(3, 3) - self.npy['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3) + self.numpy_inputs['tfm'][:, :] = transforms[road_key].reshape(3, 3) + self.numpy_inputs['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3) if prepare_only: self._warp_enqueue(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key]) @@ -236,8 +231,8 @@ class ModelState(ModelStateBase): if 'planplus' in outputs and 'plan' in outputs: outputs['plan'] = outputs['plan'] + outputs['planplus'] - if 'desired_curvature' in outputs and 'prev_desired_curv' in self.npy: - buf = self.npy['prev_desired_curv'] + if 'desired_curvature' in outputs and 'prev_desired_curv' in self.numpy_inputs: + buf = self.numpy_inputs['prev_desired_curv'] buf[0, :-1] = buf[0, 1:] buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0 @@ -409,7 +404,7 @@ def main(demo=False): 'traffic_convention': traffic_convention, } - if 'lateral_control_params' in model.npy: + if 'lateral_control_params' in model.numpy_inputs: inputs['lateral_control_params'] = np.array([v_ego, lat_delay], dtype=np.float32) mt1 = time.perf_counter() diff --git a/sunnypilot/modeld_v2/models/README.md b/sunnypilot/modeld_v2/models/README.md deleted file mode 100644 index 9e11ca8255..0000000000 --- a/sunnypilot/modeld_v2/models/README.md +++ /dev/null @@ -1,62 +0,0 @@ -## Neural networks in openpilot -To view the architecture of the ONNX networks, you can use [netron](https://netron.app/) - -## Supercombo -### Supercombo input format (Full size: 799906 x float32) -* **image stream** - * Two consecutive images (256 * 512 * 3 in RGB) recorded at 20 Hz : 393216 = 2 * 6 * 128 * 256 - * Each 256 * 512 image is represented in YUV420 with 6 channels : 6 * 128 * 256 - * Channels 0,1,2,3 represent the full-res Y channel and are represented in numpy as Y[::2, ::2], Y[::2, 1::2], Y[1::2, ::2], and Y[1::2, 1::2] - * Channel 4 represents the half-res U channel - * Channel 5 represents the half-res V channel -* **wide image stream** - * Two consecutive images (256 * 512 * 3 in RGB) recorded at 20 Hz : 393216 = 2 * 6 * 128 * 256 - * Each 256 * 512 image is represented in YUV420 with 6 channels : 6 * 128 * 256 - * Channels 0,1,2,3 represent the full-res Y channel and are represented in numpy as Y[::2, ::2], Y[::2, 1::2], Y[1::2, ::2], and Y[1::2, 1::2] - * Channel 4 represents the half-res U channel - * Channel 5 represents the half-res V channel -* **desire** - * one-hot encoded buffer to command model to execute certain actions, bit needs to be sent for the past 5 seconds (at 20FPS) : 100 * 8 -* **traffic convention** - * one-hot encoded vector to tell model whether traffic is right-hand or left-hand traffic : 2 -* **feature buffer** - * A buffer of intermediate features that gets appended to the current feature to form a 5 seconds temporal context (at 20FPS) : 99 * 512 - - -### Supercombo output format (Full size: XXX x float32) -Read [here](https://github.com/commaai/openpilot/blob/90af436a121164a51da9fa48d093c29f738adf6a/selfdrive/modeld/models/driving.h#L236) for more. - - -## Driver Monitoring Model -* .onnx model can be run with onnx runtimes -* .dlc file is a pre-quantized model and only runs on qualcomm DSPs - -### input format -* single image W = 1440 H = 960 luminance channel (Y) from the planar YUV420 format: - * full input size is 1440 * 960 = 1382400 - * normalized ranging from 0.0 to 1.0 in float32 (onnx runner) or ranging from 0 to 255 in uint8 (snpe runner) -* camera calibration angles (roll, pitch, yaw) from liveCalibration: 3 x float32 inputs - -### output format -* 84 x float32 outputs = 2 + 41 * 2 ([parsing example](https://github.com/commaai/openpilot/blob/22ce4e17ba0d3bfcf37f8255a4dd1dc683fe0c38/selfdrive/modeld/models/dmonitoring.cc#L33)) - * for each person in the front seats (2 * 41) - * face pose: 12 = 6 + 6 - * face orientation [pitch, yaw, roll] in camera frame: 3 - * face position [dx, dy] relative to image center: 2 - * normalized face size: 1 - * standard deviations for above outputs: 6 - * face visible probability: 1 - * eyes: 20 = (8 + 1) + (8 + 1) + 1 + 1 - * eye position and size, and their standard deviations: 8 - * eye visible probability: 1 - * eye closed probability: 1 - * wearing sunglasses probability: 1 - * face occluded probability: 1 - * touching wheel probability: 1 - * paying attention probability: 1 - * (deprecated) distracted probabilities: 2 - * using phone probability: 1 - * distracted probability: 1 - * common outputs 2 - * poor camera vision probability: 1 - * left hand drive probability: 1 diff --git a/sunnypilot/modeld_v2/models/__init__.py b/sunnypilot/modeld_v2/models/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/sunnypilot/modeld_v2/tests/dmon_lag/repro.cc b/sunnypilot/modeld_v2/tests/dmon_lag/repro.cc deleted file mode 100644 index c4c1c65cbe..0000000000 --- a/sunnypilot/modeld_v2/tests/dmon_lag/repro.cc +++ /dev/null @@ -1,101 +0,0 @@ -// clang++ -O2 repro.cc && ./a.out - -#include -#include -#include - -#include -#include -#include -#include -#include - -static inline double millis_since_boot() { - struct timespec t; - clock_gettime(CLOCK_BOOTTIME, &t); - return t.tv_sec * 1000.0 + t.tv_nsec * 1e-6; -} - -#define MODEL_WIDTH 320 -#define MODEL_HEIGHT 640 - -// null function still breaks it -#define input_lambda(x) x - -// this is copied from models/dmonitoring.cc, and is the code that triggers the issue -void inner(uint8_t *resized_buf, float *net_input_buf) { - int resized_width = MODEL_WIDTH; - int resized_height = MODEL_HEIGHT; - - // one shot conversion, O(n) anyway - // yuvframe2tensor, normalize - for (int r = 0; r < MODEL_HEIGHT/2; r++) { - for (int c = 0; c < MODEL_WIDTH/2; c++) { - // Y_ul - net_input_buf[(c*MODEL_HEIGHT/2) + r] = input_lambda(resized_buf[(2*r*resized_width) + (2*c)]); - // Y_ur - net_input_buf[(c*MODEL_HEIGHT/2) + r + (2*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width) + (2*c+1)]); - // Y_dl - net_input_buf[(c*MODEL_HEIGHT/2) + r + ((MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c)]); - // Y_dr - net_input_buf[(c*MODEL_HEIGHT/2) + r + (3*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c+1)]); - // U - net_input_buf[(c*MODEL_HEIGHT/2) + r + (4*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + (r*resized_width/2) + c]); - // V - net_input_buf[(c*MODEL_HEIGHT/2) + r + (5*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + ((resized_width/2)*(resized_height/2)) + (r*resized_width/2) + c]); - } - } -} - -float trial() { - int resized_width = MODEL_WIDTH; - int resized_height = MODEL_HEIGHT; - - int yuv_buf_len = (MODEL_WIDTH/2) * (MODEL_HEIGHT/2) * 6; // Y|u|v -> y|y|y|y|u|v - - // allocate the buffers - uint8_t *resized_buf = (uint8_t*)malloc(resized_width*resized_height*3/2); - float *net_input_buf = (float*)malloc(yuv_buf_len*sizeof(float)); - printf("allocate -- %p 0x%x -- %p 0x%lx\n", resized_buf, resized_width*resized_height*3/2, net_input_buf, yuv_buf_len*sizeof(float)); - - // test for bad buffers - static int CNT = 20; - float avg = 0.0; - for (int i = 0; i < CNT; i++) { - double s4 = millis_since_boot(); - inner(resized_buf, net_input_buf); - double s5 = millis_since_boot(); - avg += s5-s4; - } - avg /= CNT; - - // once it's bad, it's reliably bad - if (avg > 10) { - printf("HIT %f\n", avg); - printf("BAD\n"); - - for (int i = 0; i < 200; i++) { - double s4 = millis_since_boot(); - inner(resized_buf, net_input_buf); - double s5 = millis_since_boot(); - printf("%.2f ", s5-s4); - } - printf("\n"); - - exit(0); - } - - // don't free so we get a different buffer each time - //free(resized_buf); - //free(net_input_buf); - - return avg; -} - -int main() { - while (true) { - float ret = trial(); - printf("got %f\n", ret); - } -} - diff --git a/sunnypilot/modeld_v2/tests/tf_test/build.sh b/sunnypilot/modeld_v2/tests/tf_test/build.sh deleted file mode 100755 index df1d24761e..0000000000 --- a/sunnypilot/modeld_v2/tests/tf_test/build.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/usr/bin/env bash -clang++ -I /home/batman/one/external/tensorflow/include/ -L /home/batman/one/external/tensorflow/lib -Wl,-rpath=/home/batman/one/external/tensorflow/lib main.cc -ltensorflow diff --git a/sunnypilot/modeld_v2/tests/tf_test/main.cc b/sunnypilot/modeld_v2/tests/tf_test/main.cc deleted file mode 100644 index b00f7f95e8..0000000000 --- a/sunnypilot/modeld_v2/tests/tf_test/main.cc +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include -#include -#include "tensorflow/c/c_api.h" - -void* read_file(const char* path, size_t* out_len) { - FILE* f = fopen(path, "r"); - if (!f) { - return NULL; - } - fseek(f, 0, SEEK_END); - long f_len = ftell(f); - rewind(f); - - char* buf = (char*)calloc(f_len, 1); - assert(buf); - - size_t num_read = fread(buf, f_len, 1, f); - fclose(f); - - if (num_read != 1) { - free(buf); - return NULL; - } - - if (out_len) { - *out_len = f_len; - } - - return buf; -} - -static void DeallocateBuffer(void* data, size_t) { - free(data); -} - -int main(int argc, char* argv[]) { - TF_Buffer* buf; - TF_Graph* graph; - TF_Status* status; - char *path = argv[1]; - - // load model - { - size_t model_size; - char tmp[1024]; - snprintf(tmp, sizeof(tmp), "%s.pb", path); - printf("loading model %s\n", tmp); - uint8_t *model_data = (uint8_t *)read_file(tmp, &model_size); - buf = TF_NewBuffer(); - buf->data = model_data; - buf->length = model_size; - buf->data_deallocator = DeallocateBuffer; - printf("loaded model of size %d\n", model_size); - } - - // import graph - status = TF_NewStatus(); - graph = TF_NewGraph(); - TF_ImportGraphDefOptions *opts = TF_NewImportGraphDefOptions(); - TF_GraphImportGraphDef(graph, buf, opts, status); - TF_DeleteImportGraphDefOptions(opts); - TF_DeleteBuffer(buf); - if (TF_GetCode(status) != TF_OK) { - printf("FAIL: %s\n", TF_Message(status)); - } else { - printf("SUCCESS\n"); - } -} diff --git a/sunnypilot/modeld_v2/tests/tf_test/pb_loader.py b/sunnypilot/modeld_v2/tests/tf_test/pb_loader.py deleted file mode 100755 index 3e476628eb..0000000000 --- a/sunnypilot/modeld_v2/tests/tf_test/pb_loader.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -import sys -import tensorflow as tf - -with open(sys.argv[1], "rb") as f: - graph_def = tf.compat.v1.GraphDef() - graph_def.ParseFromString(f.read()) - #tf.io.write_graph(graph_def, '', sys.argv[1]+".try") diff --git a/sunnypilot/modeld_v2/tests/timing/benchmark.py b/sunnypilot/modeld_v2/tests/timing/benchmark.py deleted file mode 100755 index 3e81f73fa3..0000000000 --- a/sunnypilot/modeld_v2/tests/timing/benchmark.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -import os -import time -import numpy as np - -import cereal.messaging as messaging -from openpilot.system.manager.process_config import managed_processes - - -N = int(os.getenv("N", "5")) -TIME = int(os.getenv("TIME", "30")) - -if __name__ == "__main__": - sock = messaging.sub_sock('modelV2', conflate=False, timeout=1000) - - execution_times = [] - - for _ in range(N): - os.environ['LOGPRINT'] = 'debug' - managed_processes['modeld'].start() - time.sleep(5) - - t = [] - start = time.monotonic() - while time.monotonic() - start < TIME: - msgs = messaging.drain_sock(sock, wait_for_one=True) - for m in msgs: - t.append(m.modelV2.modelExecutionTime) - - execution_times.append(np.array(t[10:]) * 1000) - managed_processes['modeld'].stop() - - print("\n\n") - print(f"ran modeld {N} times for {TIME}s each") - for _, t in enumerate(execution_times): - print(f"\tavg: {sum(t)/len(t):0.2f}ms, min: {min(t):0.2f}ms, max: {max(t):0.2f}ms") - print("\n\n") diff --git a/sunnypilot/models/helpers.py b/sunnypilot/models/helpers.py index c9344f54ef..9a8bd7d478 100644 --- a/sunnypilot/models/helpers.py +++ b/sunnypilot/models/helpers.py @@ -6,80 +6,138 @@ See the LICENSE.md file in the root directory for more details. """ import hashlib +import os import pickle +from pathlib import Path import numpy as np -from openpilot.common.params import Params from cereal import custom -from openpilot.sunnypilot.models.constants import Meta, MetaTombRaider, MetaSimPose +from openpilot.common.params import Params +from openpilot.common.swaglog import cloudlog +from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRaider from openpilot.system.hardware.hw import Paths -from pathlib import Path - -# see the README.md for more details on the model selector versioning -CURRENT_SELECTOR_VERSION = 15 -REQUIRED_MIN_SELECTOR_VERSION = 14 +# SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO +REQUIRED_JSON_VERSION = 15 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' - ModelManager = custom.ModelManagerSP +_LAST_VALIDATED_RAW = None + + +def _compute_hash(file_path: str) -> str | None: + from openpilot.common.file_chunker import read_file_chunked + try: + return hashlib.sha256(read_file_chunked(file_path)).hexdigest().lower() + except FileNotFoundError: + return None async def verify_file(file_path: str, expected_hash: str) -> bool: - from openpilot.common.file_chunker import read_file_chunked - try: - data = read_file_chunked(file_path) - except FileNotFoundError: - return False - return hashlib.sha256(data).hexdigest().lower() == expected_hash.lower() + file_hash = _compute_hash(file_path) + return file_hash == expected_hash.lower() if file_hash else False + + +def _verify_file(file_path: str, expected_hash: str) -> bool: + file_hash = _compute_hash(file_path) + return file_hash == expected_hash.lower() if file_hash else False def is_bundle_version_compatible(bundle: dict) -> bool: """ - Checks whether the model bundle is compatible with the current selector version constraints. - - The bundle specifies a `minimum_selector_version`, which defines the minimum selector version + The bundle parsed from the json specifies a `minimum_selector_version`, which defines the minimum selector version required to load the model. This function ensures that: - - 1. The model is not too old: the bundle must require at least `REQUIRED_MIN_SELECTOR_VERSION`. - 2. The model is not too new: it must support the current selector version (`CURRENT_SELECTOR_VERSION`). - - This allows the selector to enforce both a minimum and maximum range of supported models, - even if a model would otherwise be compatible. - - :param bundle: Dictionary containing `minimum_selector_version`, as defined by the model bundle. - :type bundle: Dict - :return: True if the selector version is within the accepted range for the bundle; otherwise False. - :rtype: Bool + the bundle MUST match the `REQUIRED_JSON_VERSION` set here in helpers. """ - return bool(REQUIRED_MIN_SELECTOR_VERSION <= bundle.get("minimumSelectorVersion", 0) <= CURRENT_SELECTOR_VERSION) + return bundle.get("minimumSelectorVersion", 0) == REQUIRED_JSON_VERSION -def get_active_bundle(params: Params = None) -> custom.ModelManagerSP.ModelBundle: - """Gets the active model bundle from cache""" - if params is None: - params = Params() +def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[str, str]]: + artifacts = [] + for model in getattr(bundle, 'models', []) or []: + for artifact in (getattr(model, 'artifact', None), getattr(model, 'metadata', None)): + if artifact and getattr(artifact, 'fileName', None) and getattr(artifact, 'downloadUri', None): + sha256 = getattr(artifact.downloadUri, 'sha256', None) + if sha256: + artifacts.append((artifact.fileName, sha256)) + return artifacts + +def _bundle_is_valid_locally(bundle: custom.ModelManagerSP.ModelBundle) -> bool: + model_root = Paths.model_root() + return all(_verify_file(os.path.join(model_root, file_name), expected_hash) + for file_name, expected_hash in _bundle_artifacts(bundle)) + + +def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None) -> bool: + if active_bundle is None: + return False + + if available_bundles is not None: + matching_bundle = None + for bundle in available_bundles: + if getattr(active_bundle, 'ref', None) and getattr(bundle, 'ref', None): + if active_bundle.ref == bundle.ref: + matching_bundle = bundle + break + elif getattr(active_bundle, 'internalName', None) == getattr(bundle, 'internalName', None): + matching_bundle = bundle + break + + if matching_bundle is None: + return True + if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion: + return True + + active_runner = getattr(active_bundle, 'runner', None) + matching_runner = getattr(matching_bundle, 'runner', None) + if active_runner is not None and matching_runner is not None: + if getattr(active_runner, 'raw', active_runner) != getattr(matching_runner, 'raw', matching_runner): + return True + if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)): + return True + + return not _bundle_is_valid_locally(active_bundle) + + +def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None: + global _LAST_VALIDATED_RAW + + raw_bundle = params.get("ModelManager_ActiveBundle") + if not raw_bundle: + return + + if raw_bundle == _LAST_VALIDATED_RAW: + return + + active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle) + if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles): + cloudlog.warning("Active model bundle invalid; resetting to default") + params.remove("ModelManager_ActiveBundle") + params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True) + _LAST_VALIDATED_RAW = None + else: + _LAST_VALIDATED_RAW = raw_bundle + + +def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> custom.ModelManagerSP.ModelBundle | None: + params = params or Params() try: - if (active_bundle := params.get("ModelManager_ActiveBundle") or {}) and is_bundle_version_compatible(active_bundle): - return custom.ModelManagerSP.ModelBundle(**active_bundle) + active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {}) + if active_bundle_dict and is_bundle_version_compatible(active_bundle_dict): + return custom.ModelManagerSP.ModelBundle(**active_bundle_dict) except Exception: pass - return None -def get_active_model_runner(params: Params = None, force_check=False) -> int: - if params is None: - params = Params() - +def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int: + params = params or Params() cached_runner_type = params.get("ModelRunnerTypeCache") if cached_runner_type is not None and not force_check: return cached_runner_type - runner_type = custom.ModelManagerSP.Runner.stock - if active_bundle := get_active_bundle(params): runner_type = active_bundle.runner.raw @@ -88,66 +146,40 @@ def get_active_model_runner(params: Params = None, force_check=False) -> int: return runner_type + def _get_model(): if bundle := get_active_bundle(): drive_model = next(model for model in bundle.models if model.type == ModelManager.Model.Type.supercombo) return drive_model - return None -def load_metadata(): - metadata_path = METADATA_PATH - if model := _get_model(): - metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" +def load_metadata(): + model = _get_model() + metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" if model else METADATA_PATH with open(metadata_path, 'rb') as f: return pickle.load(f) -def prepare_inputs(model_metadata) -> dict[str, np.ndarray]: - # img buffers are managed in openCL transform code so we don't pass them as inputs - inputs = { - k: np.zeros(v, dtype=np.float32).flatten() - for k, v in model_metadata['input_shapes'].items() - if 'img' not in k +def prepare_inputs(model_metadata: dict) -> dict[str, np.ndarray]: + return { + key: np.zeros(shape, dtype=np.float32).flatten() + for key, shape in model_metadata['input_shapes'].items() + if 'img' not in key } - return inputs +def load_meta_constants(model_metadata: dict): + """ Loads the appropriate meta model class based on key shapes""" + if 'sim_pose' in model_metadata['input_shapes']: + return MetaSimPose -def load_meta_constants(model_metadata): - """ - Determines and loads the appropriate meta model class based on the metadata provided. The function checks - specific keys and conditions within the provided metadata dictionary to identify the corresponding meta - model class to return. + meta_slice = model_metadata['output_slices']['meta'] + if (meta_slice.start, meta_slice.stop, meta_slice.step) == (5868, 5921, None): + return MetaTombRaider - :param model_metadata: Dictionary containing metadata about the model. It includes - details such as input shapes, output slices, and other configurations for identifying - metadata-dependent meta model classes. - :type model_metadata: dict - :return: The appropriate meta model class (Meta, MetaSimPose, or MetaTombRaider) - based on the conditions and metadata provided. - :rtype: type - """ - meta = Meta # Default Meta - - if 'sim_pose' in model_metadata['input_shapes'].keys(): - # Meta for models with sim_pose input - meta = MetaSimPose - else: - # Meta for Tomb Raider, it does not include sim_pose input but has the same meta slice as previous models - meta_slice = model_metadata['output_slices']['meta'] - meta_tf_slice = slice(5868, 5921, None) - - if ( - meta_slice.start == meta_tf_slice.start and - meta_slice.stop == meta_tf_slice.stop and - meta_slice.step == meta_tf_slice.step - ): - meta = MetaTombRaider - - return meta + return Meta # The following method(s) are modeld helper methods diff --git a/sunnypilot/models/manager.py b/sunnypilot/models/manager.py index b5ccb736c1..518671181e 100644 --- a/sunnypilot/models/manager.py +++ b/sunnypilot/models/manager.py @@ -17,7 +17,7 @@ from openpilot.system.hardware.hw import Paths from cereal import messaging, custom from openpilot.sunnypilot.models.fetcher import ModelFetcher -from openpilot.sunnypilot.models.helpers import verify_file, get_active_bundle +from openpilot.sunnypilot.models.helpers import get_active_bundle, validate_active_bundle, verify_file class ModelManagerSP: @@ -239,6 +239,7 @@ class ModelManagerSP: while True: try: self.available_models = self.model_fetcher.get_available_bundles() + validate_active_bundle(self.params, self.available_models) self.active_bundle = get_active_bundle(self.params) if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None: @@ -252,8 +253,8 @@ class ModelManagerSP: self.selected_bundle = None if self.params.get("ModelManager_ClearCache"): - self.clear_model_cache() - self.params.remove("ModelManager_ClearCache") + self.clear_model_cache() + self.params.remove("ModelManager_ClearCache") self._report_status() rk.keep_time() From 8611e08dc6ea08196dea5c6d7347b3e583be53b6 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 09:03:36 -0700 Subject: [PATCH 02/57] fix string --- sunnypilot/models/helpers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sunnypilot/models/helpers.py b/sunnypilot/models/helpers.py index 9a8bd7d478..0fa94e39ff 100644 --- a/sunnypilot/models/helpers.py +++ b/sunnypilot/models/helpers.py @@ -121,7 +121,7 @@ def validate_active_bundle(params: Params, available_bundles: list[custom.ModelM _LAST_VALIDATED_RAW = raw_bundle -def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> custom.ModelManagerSP.ModelBundle | None: +def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None": params = params or Params() try: active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {}) From dc5116c7188ffedaf78b2e1856552f685c64812a Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 09:15:51 -0700 Subject: [PATCH 03/57] numpy --- .../modeld_v2/tests/test_combined_pkl_loader.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index d4ef0d476b..c2fe887e28 100644 --- a/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -84,8 +84,8 @@ class TestStockEquivalence: skip_keys = {'action_t'} assert set(state.input_queues.keys()) == set(stock_queues.keys()) - skip_keys, \ f"Queue keys differ: v2={set(state.input_queues.keys())}, stock={set(stock_queues.keys())}" - assert set(state.npy.keys()) == set(stock_npy.keys()) - skip_keys, \ - f"Npy keys differ: v2={set(state.npy.keys())}, stock={set(stock_npy.keys())}" + assert set(state.numpy_inputs.keys()) == set(stock_npy.keys()) - skip_keys, \ + f"Npy keys differ: v2={set(state.numpy_inputs.keys())}, stock={set(stock_npy.keys())}" def test_split_queue_keys_work_with_desire_key(self, model_state_factory): from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues @@ -188,16 +188,16 @@ class TestInputQueueCreation: def test_npy_contains_transforms(self, archetype_name, model_state_factory): arch = ARCHETYPES[archetype_name] state = model_state_factory(arch) - assert 'tfm' in state.npy, f"{arch.name}: 'tfm' missing from npy" - assert 'big_tfm' in state.npy, f"{arch.name}: 'big_tfm' missing from npy" - assert state.npy['tfm'].shape == (3, 3) - assert state.npy['big_tfm'].shape == (3, 3) + assert 'tfm' in state.numpy_inputs, f"{arch.name}: 'tfm' missing from npy" + assert 'big_tfm' in state.numpy_inputs, f"{arch.name}: 'big_tfm' missing from npy" + assert state.numpy_inputs['tfm'].shape == (3, 3) + assert state.numpy_inputs['big_tfm'].shape == (3, 3) @pytest.mark.parametrize("archetype_name", ARCHETYPE_NAMES) def test_npy_contains_desire(self, archetype_name, model_state_factory): arch = ARCHETYPES[archetype_name] state = model_state_factory(arch) - assert arch.expected_desire_key in state.npy, \ + assert arch.expected_desire_key in state.numpy_inputs, \ f"{arch.name}: '{arch.expected_desire_key}' missing from npy" From 1083f5bf2120fdbf3eac046bd34261e6c415d567 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 09:16:36 -0700 Subject: [PATCH 04/57] dumb --- sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index c2fe887e28..38f6cd446b 100644 --- a/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -46,16 +46,6 @@ class TestFindDrivingPkl: assert result is not None assert 'driving_fof_tinygrad.pkl' in result - def test_finds_fallback_driving_tinygrad(self, tmp_path, monkeypatch): - (tmp_path / 'driving_tinygrad.pkl').write_bytes(b'fake') - from openpilot.system.hardware import hw - monkeypatch.setattr(hw.Paths, 'model_root', staticmethod(lambda: str(tmp_path))) - - bundle = DummyBundle(models=[DummyModel('vision', 'nonexistent.pkl')]) - result = _find_driving_pkl(bundle) - assert result is not None - assert 'driving_tinygrad.pkl' in result - # Init — assertion guard From 6c1e0f370b3f5527381b1363135c528766c216fb Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 09:25:50 -0700 Subject: [PATCH 05/57] god use full attribute names please --- sunnypilot/modeld_v2/compile_modeld.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 387d3378e2..9c8cdb2c0a 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -188,7 +188,7 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device): n_frames = img_shape[1] // 6 img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img_shape[2], img_shape[3]) - npy_keys = {} + numpy_keys = {} queue_keys = {} for key, shape in input_shapes.items(): @@ -196,7 +196,7 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device): continue if len(shape) == 3 and shape[1] > 1: if key.startswith('desire'): - npy_keys[key] = np.zeros(shape[2], dtype=np.float32) + numpy_keys[key] = np.zeros(shape[2], dtype=np.float32) queue_keys[f'{key}_q'] = Tensor( np.zeros((frame_skip * shape[1], shape[0], shape[2]), dtype=np.float32), device=device).contiguous().realize() @@ -205,24 +205,24 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device): np.zeros((frame_skip * (shape[1] - 1) + 1, shape[0], shape[2]), dtype=np.float32), device=device).contiguous().realize() else: - npy_keys[key] = np.zeros(shape, dtype=np.float32) + numpy_keys[key] = np.zeros(shape, dtype=np.float32) elif len(shape) == 2: - npy_keys[key] = np.zeros(shape, dtype=np.float32) + numpy_keys[key] = np.zeros(shape, dtype=np.float32) - if 'traffic_convention' not in npy_keys: + if 'traffic_convention' not in numpy_keys: tc_shape = input_shapes.get('traffic_convention', (1, 2)) - npy_keys['traffic_convention'] = np.zeros(tc_shape, dtype=np.float32) + numpy_keys['traffic_convention'] = np.zeros(tc_shape, dtype=np.float32) - npy_keys['tfm'] = np.zeros((3, 3), dtype=np.float32) - npy_keys['big_tfm'] = np.zeros((3, 3), dtype=np.float32) + numpy_keys['tfm'] = np.zeros((3, 3), dtype=np.float32) + numpy_keys['big_tfm'] = np.zeros((3, 3), dtype=np.float32) input_queues = { 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), **queue_keys, - **{k: Tensor(v, device='NPY').realize() for k, v in npy_keys.items()}, + **{k: Tensor(v, device='NPY').realize() for k, v in numpy_keys.items()}, } - return input_queues, npy_keys + return input_queues, numpy_keys def make_run_supercombo(model_runner, nv12: NV12Frame, model_w, model_h, From ad5abd242ada0bcd89e279658dafb7d3663d358b Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 09:58:48 -0700 Subject: [PATCH 06/57] modeld_v2: refactor compile_modeld --- sunnypilot/modeld_v2/compile_modeld.py | 551 ++++++++----------------- 1 file changed, 167 insertions(+), 384 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 9c8cdb2c0a..b9a392631a 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -10,372 +10,199 @@ import argparse import os import pickle import time -from functools import partial from collections import defaultdict - +from functools import partial import numpy as np -from tinygrad.tensor import Tensor + +from tinygrad import dtypes from tinygrad.device import Device from tinygrad.engine.jit import TinyJit +from tinygrad.tensor import Tensor + +from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare, sample_desire, sample_skip, shift_and_sample -from openpilot.selfdrive.modeld.compile_modeld import ( - NV12Frame, make_frame_prepare, - shift_and_sample, sample_skip, sample_desire, -) MODEL_TYPES = ('vision_policy', 'supercombo', 'vision_multi_policy') -def _detect_desire_key(policy_input_shapes): - for k in policy_input_shapes: - if k.startswith('desire'): - return k - return None +def _detect_desire_key(shapes: dict) -> str | None: + return next((key for key in shapes if key.startswith('desire')), None) -def _detect_vision_keys(vision_input_shapes): - img_keys = sorted([k for k in vision_input_shapes if 'img' in k]) - road_key = next((k for k in img_keys if 'big' not in k), None) - wide_key = next((k for k in img_keys if 'big' in k), None) - if road_key is None or wide_key is None: - raise ValueError(f"Cannot determine road/wide image keys from {list(vision_input_shapes.keys())}") - return road_key, wide_key +def _detect_vision_keys(shapes: dict) -> tuple[str | None, str | None]: + img_keys = sorted(key for key in shapes if 'img' in key) + return ( + next((key for key in img_keys if 'big' not in key), None), + next((key for key in img_keys if 'big' in key), None) + ) -def make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device): - road_key, _ = _detect_vision_keys(vision_input_shapes) - img = vision_input_shapes[road_key] - n_frames = img[1] // 6 - img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img[2], img[3]) - - fb = policy_input_shapes['features_buffer'] - desire_key = _detect_desire_key(policy_input_shapes) - dp = policy_input_shapes[desire_key] - tc = policy_input_shapes.get('traffic_convention', (1, 2)) - - npy = { - 'desire': np.zeros(dp[2], dtype=np.float32), - 'traffic_convention': np.zeros(tc, dtype=np.float32), - 'tfm': np.zeros((3, 3), dtype=np.float32), - 'big_tfm': np.zeros((3, 3), dtype=np.float32), - } - - handled = {'features_buffer', desire_key, 'traffic_convention'} - for key, shape in policy_input_shapes.items(): - if key in handled: - continue - npy[key] = np.zeros(shape, dtype=np.float32) - - input_queues = { - 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'feat_q': Tensor(np.zeros((frame_skip * (fb[1] - 1) + 1, fb[0], fb[2]), dtype=np.float32), device=device).contiguous().realize(), - 'desire_q': Tensor(np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32), device=device).contiguous().realize(), - **{k: Tensor(v, device='NPY').realize() for k, v in npy.items()}, - } - return input_queues, npy +def derive_frame_skip(vision_input_shapes: dict, policy_input_shapes: dict) -> int: + features_buffer = policy_input_shapes.get('features_buffer') + return 1 if not features_buffer or features_buffer[1] >= 99 else 4 -def make_run_split_policy(vision_runner, policy_runner, nv12: NV12Frame, model_w, model_h, - vision_features_slice, frame_skip, desire_key, extra_policy_keys, - vision_road_key, vision_wide_key, prepare_only=False): - frame_prepare = make_frame_prepare(nv12, model_w, model_h) - sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) - sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) - - def run_policy(img_q, big_img_q, feat_q, desire_q, desire, traffic_convention, tfm, big_tfm, frame, big_frame, **extra): - npy_tensors = [tfm.to(Device.DEFAULT), big_tfm.to(Device.DEFAULT), - desire.to(Device.DEFAULT), traffic_convention.to(Device.DEFAULT)] - extra_device = {k: extra[k].to(Device.DEFAULT) for k in extra_policy_keys} - Tensor.realize(*npy_tensors, *extra_device.values()) - tfm, big_tfm, desire, traffic_convention = npy_tensors - - img = shift_and_sample(img_q, frame_prepare(frame, tfm).unsqueeze(0), sample_skip_fn) - big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm).unsqueeze(0), sample_skip_fn) - - if prepare_only: - return img, big_img - - vision_out = next(iter(vision_runner({vision_road_key: img, vision_wide_key: big_img}).values())).cast('float32') - - new_feat = vision_out[:, vision_features_slice].reshape(1, -1).unsqueeze(0) - feat_buf = shift_and_sample(feat_q, new_feat, sample_skip_fn) - desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn) - - inputs = {'features_buffer': feat_buf, desire_key: desire_buf, 'traffic_convention': traffic_convention, **extra_device} - policy_out = next(iter(policy_runner(inputs).values())).cast('float32') - - return vision_out, policy_out - return run_policy - - -def compile_split_policy(nv12: NV12Frame, model_w, model_h, prepare_only, frame_skip, - vision_runner, policy_runner, vision_metadata, policy_metadata): - print(f"Compiling combined policy JIT for {nv12.width}x{nv12.height} (prepare_only={prepare_only})...") - - vision_features_slice = vision_metadata['output_slices']['hidden_state'] - vision_input_shapes = vision_metadata['input_shapes'] - policy_input_shapes = policy_metadata['input_shapes'] - desire_key = _detect_desire_key(policy_input_shapes) - extra_policy_keys = [k for k in policy_input_shapes if k not in ('features_buffer', desire_key, 'traffic_convention')] - vision_road_key, vision_wide_key = _detect_vision_keys(vision_input_shapes) - - _run = make_run_split_policy(vision_runner, policy_runner, nv12, model_w, model_h, - vision_features_slice, frame_skip, desire_key, extra_policy_keys, - vision_road_key, vision_wide_key, prepare_only) - run_policy_jit = TinyJit(_run, prune=True) - - SEED = 42 - - def random_inputs_run_fn(fn, seed, test_val=None, test_buffers=None, expect_match=True): - input_queues, npy = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, Device.DEFAULT) - np.random.seed(seed) - Tensor.manual_seed(seed) - - testing = test_val is not None or test_buffers is not None - n_runs = 1 if testing else 3 - - for i in range(n_runs): - frame = Tensor.randint(nv12.size, low=0, high=256, dtype='uint8').realize() - big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype='uint8').realize() - for v in npy.values(): - v[:] = np.random.randn(*v.shape).astype(v.dtype) - Device.default.synchronize() - st = time.perf_counter() - outs = fn(**input_queues, frame=frame, big_frame=big_frame) - mt = time.perf_counter() - Device.default.synchronize() - et = time.perf_counter() - print(f" [{i+1}/{n_runs}] enqueue {(mt-st)*1e3:6.2f} ms -- total {(et-st)*1e3:6.2f} ms") - - if i == 0: - val = [np.copy(v.numpy()) for v in outs] - buffers = [np.copy(v.numpy().copy()) for v in input_queues.values()] - - if test_val is not None: - match = all(np.array_equal(a, b) for a, b in zip(val, test_val, strict=True)) - assert match == expect_match, f"outputs {'differ from' if expect_match else 'match'} baseline (seed={seed})" - if test_buffers is not None: - match = all(np.array_equal(a, b) for a, b in zip(buffers, test_buffers, strict=True)) - assert match == expect_match, f"buffers {'differ from' if expect_match else 'match'} baseline (seed={seed})" - return fn, val, buffers - - print('capture + replay') - run_policy_jit, test_val, test_buffers = random_inputs_run_fn(run_policy_jit, SEED) - - print('pickle round trip') - run_policy_jit = pickle.loads(pickle.dumps(run_policy_jit)) - random_inputs_run_fn(run_policy_jit, SEED, test_val, test_buffers, expect_match=True) - random_inputs_run_fn(run_policy_jit, SEED+1, test_val, test_buffers, expect_match=False) - return run_policy_jit - - -def derive_frame_skip(vision_input_shapes, policy_input_shapes): - fb = policy_input_shapes.get('features_buffer') - if fb is None: - return 1 - fb_history = fb[1] - if fb_history >= 99: - return 1 - return 4 - - -def make_supercombo_input_queues(input_shapes, frame_skip, device): - img_shape = input_shapes.get('img', input_shapes.get('input_imgs')) - if img_shape is None: - raise ValueError("No img input found in model shapes") +def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: + road_key, _ = _detect_vision_keys(input_shapes) + if not road_key: + raise ValueError("Vision road key missing from input shapes.") + img_shape = input_shapes[road_key] n_frames = img_shape[1] // 6 img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img_shape[2], img_shape[3]) - numpy_keys = {} - queue_keys = {} + desire_key = _detect_desire_key(input_shapes) + if not desire_key: + raise ValueError("Desire key missing from input shapes.") + + desire_shape = input_shapes[desire_key] + features_buffer = input_shapes.get('features_buffer') + + npy_arrays = { + 'desire': np.zeros(desire_shape[2], dtype=np.float32), + 'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32) + } for key, shape in input_shapes.items(): - if 'img' in key: - continue - if len(shape) == 3 and shape[1] > 1: - if key.startswith('desire'): - numpy_keys[key] = np.zeros(shape[2], dtype=np.float32) - queue_keys[f'{key}_q'] = Tensor( - np.zeros((frame_skip * shape[1], shape[0], shape[2]), dtype=np.float32), - device=device).contiguous().realize() - elif key == 'features_buffer': - queue_keys['feat_q'] = Tensor( - np.zeros((frame_skip * (shape[1] - 1) + 1, shape[0], shape[2]), dtype=np.float32), - device=device).contiguous().realize() - else: - numpy_keys[key] = np.zeros(shape, dtype=np.float32) - elif len(shape) == 2: - numpy_keys[key] = np.zeros(shape, dtype=np.float32) + if key not in npy_arrays and 'img' not in key and key not in ('features_buffer', desire_key): + npy_arrays[key] = np.zeros(shape, dtype=np.float32) - if 'traffic_convention' not in numpy_keys: - tc_shape = input_shapes.get('traffic_convention', (1, 2)) - numpy_keys['traffic_convention'] = np.zeros(tc_shape, dtype=np.float32) - - numpy_keys['tfm'] = np.zeros((3, 3), dtype=np.float32) - numpy_keys['big_tfm'] = np.zeros((3, 3), dtype=np.float32) - - input_queues = { + queues = { 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - **queue_keys, - **{k: Tensor(v, device='NPY').realize() for k, v in numpy_keys.items()}, + 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), + dtype=np.float32), device=device).contiguous().realize() } - return input_queues, numpy_keys + + if features_buffer: + queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), + dtype=np.float32), device=device).contiguous().realize() + + queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) + return queues, npy_arrays -def make_run_supercombo(model_runner, nv12: NV12Frame, model_w, model_h, - features_slice, frame_skip, input_shapes, prepare_only=False): - frame_prepare = make_frame_prepare(nv12, model_w, model_h) +def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: + return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device) + + +def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: + return generate_queues_and_npy(input_shapes, frame_skip, device) + + +def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], + features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): + frame_prepare = make_frame_prepare(nv12, *model_size) sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) desire_key = _detect_desire_key(input_shapes) - if desire_key is None: - raise ValueError(f"No desire* key found in input_shapes: {list(input_shapes.keys())}") - road_img_key, wide_img_key = _detect_vision_keys(input_shapes) - extra_policy_keys = [k for k in input_shapes - if k not in (desire_key, 'features_buffer', 'traffic_convention') - and 'img' not in k] + road_key, wide_key = _detect_vision_keys(input_shapes) - def run_supercombo(img_q, big_img_q, feat_q, desire_q, - frame, big_frame, **kwargs): - desire = kwargs.get(desire_key) + if not desire_key or not road_key or not wide_key: + raise ValueError("Missing required vision or desire keys in input shapes.") + + extra_keys = [key for key in input_shapes if key not in (desire_key, 'features_buffer', 'traffic_convention') and 'img' not in key] + + def runner(img_q, big_img_q, feat_q, frame, big_frame, tfm, big_tfm, **kwargs): + desire_q = kwargs['desire_q'] + desire = kwargs['desire'] traffic_convention = kwargs.get('traffic_convention') - tfm = kwargs['tfm'] - big_tfm = kwargs['big_tfm'] - tfm = tfm.to(Device.DEFAULT) - big_tfm = big_tfm.to(Device.DEFAULT) - desire = desire.to(Device.DEFAULT) - traffic_convention = traffic_convention.to(Device.DEFAULT) - Tensor.realize(tfm, big_tfm, desire, traffic_convention) + npys = [tfm.to(Device.DEFAULT), big_tfm.to(Device.DEFAULT), desire.to(Device.DEFAULT)] + if traffic_convention is not None: + npys.append(traffic_convention.to(Device.DEFAULT)) - img = shift_and_sample(img_q, frame_prepare(frame, tfm).unsqueeze(0), sample_skip_fn) - big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm).unsqueeze(0), sample_skip_fn) + extra_tensors = {key: kwargs[key].to(Device.DEFAULT) for key in extra_keys if key in kwargs} + Tensor.realize(*npys, *extra_tensors.values()) + + tfm_dev, big_tfm_dev, desire_dev = npys[:3] + traffic_conv_dev = npys[3] if traffic_convention is not None else None + + img = shift_and_sample(img_q, frame_prepare(frame, tfm_dev).unsqueeze(0), sample_skip_fn) + big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm_dev).unsqueeze(0), sample_skip_fn) if prepare_only: return img, big_img - desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn) - feat_buf = sample_skip_fn(feat_q) + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) + inputs = {desire_key: desire_buf, **extra_tensors} + if traffic_conv_dev is not None: + inputs['traffic_convention'] = traffic_conv_dev - inputs = {road_img_key: img, wide_img_key: big_img, - desire_key: desire_buf, 'features_buffer': feat_buf, - 'traffic_convention': traffic_convention} - for k in extra_policy_keys: - if k in kwargs: - inputs[k] = kwargs[k].to(Device.DEFAULT) + if vision_runner: + vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32') + new_feat = vision_out[:, features_slice].reshape(1, -1).unsqueeze(0) + inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn) + policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] + return (vision_out, *policy_outs) if len(policy_outs) > 1 else (vision_out, policy_outs[0]) - model_out = next(iter(model_runner(inputs).values())).cast('float32') - - new_feat = model_out[:, features_slice].reshape(1, -1).unsqueeze(0) + inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) + policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') + new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) shift_and_sample(feat_q, new_feat, sample_skip_fn) + return policy_out - return model_out - - return run_supercombo + return runner -def make_run_vision_multi_policy(vision_runner, policy_runners, nv12: NV12Frame, model_w, model_h, - vision_features_slice, frame_skip, desire_key, extra_policy_keys, - vision_road_key, vision_wide_key, prepare_only=False): - frame_prepare = make_frame_prepare(nv12, model_w, model_h) - sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) - sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) +def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_only: bool, frame_skip: int, vision_runner, policy_runners: list, metadata: dict): + print(f"Compiling combined JIT for {nv12.width}x{nv12.height} (prepare_only={prepare_only})...") - def run_multi_policy(img_q, big_img_q, feat_q, desire_q, desire, - traffic_convention, tfm, big_tfm, frame, big_frame, **extra): - npy_tensors = [tfm.to(Device.DEFAULT), big_tfm.to(Device.DEFAULT), - desire.to(Device.DEFAULT), traffic_convention.to(Device.DEFAULT)] - extra_device = {k: extra[k].to(Device.DEFAULT) for k in extra_policy_keys} - Tensor.realize(*npy_tensors, *extra_device.values()) - tfm, big_tfm, desire, traffic_convention = npy_tensors + all_shapes = {key: value for meta in metadata.values() for key, value in meta['input_shapes'].items()} - img = shift_and_sample(img_q, frame_prepare(frame, tfm).unsqueeze(0), sample_skip_fn) - big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm).unsqueeze(0), sample_skip_fn) + feat_meta = metadata.get('vision') or metadata.get('model') or metadata.get('policy') + if not feat_meta: + raise ValueError("Could not find vision, model, or policy metadata.") - if prepare_only: - return img, big_img + features_slice = feat_meta['output_slices']['hidden_state'] - vision_out = next(iter(vision_runner({vision_road_key: img, vision_wide_key: big_img}).values())).cast('float32') + run_func = create_jit_runner(vision_runner, policy_runners, nv12, model_size, features_slice, frame_skip, all_shapes, prepare_only) + run_jit = TinyJit(run_func, prune=True) + queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT) - new_feat = vision_out[:, vision_features_slice].reshape(1, -1).unsqueeze(0) - feat_buf = shift_and_sample(feat_q, new_feat, sample_skip_fn) - desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn) - - inputs = {'features_buffer': feat_buf, desire_key: desire_buf, 'traffic_convention': traffic_convention, **extra_device} - - policy_outputs = [] - for runner in policy_runners: - policy_out = next(iter(runner(inputs).values())).cast('float32') - policy_outputs.append(policy_out) - - return (vision_out, *policy_outputs) - - return run_multi_policy - - -def _warmup_and_serialize(run_jit, input_queues, npy, nv12): for i in range(3): np.random.seed(42 + i) - frame = Tensor.randint(nv12.size, low=0, high=256, dtype='uint8').realize() - big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype='uint8').realize() - for v in npy.values(): - v[:] = np.random.randn(*v.shape).astype(v.dtype) + frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() + big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() + for arr in npy_arrays.values(): + arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) + Device.default.synchronize() - st = time.perf_counter() - run_jit(**input_queues, frame=frame, big_frame=big_frame) - mt = time.perf_counter() + start_time = time.perf_counter() + run_jit(**queues, frame=frame, big_frame=big_frame) + mid_time = time.perf_counter() Device.default.synchronize() - et = time.perf_counter() - print(f" [{i + 1}/3] enqueue {(mt - st) * 1e3:6.2f} ms -- total {(et - st) * 1e3:6.2f} ms") - return pickle.loads(pickle.dumps(run_jit)) + print(f" [{i + 1}/3] enqueue {(mid_time - start_time) * 1e3:6.2f} ms -- total {(time.perf_counter() - start_time) * 1e3:6.2f} ms") + + return pickle.loads(pickle.dumps(run_jit)) if not prepare_only else run_jit -def compile_supercombo(nv12: NV12Frame, model_w, model_h, prepare_only, frame_skip, - model_runner, metadata): - print(f"Compiling combined supercombo JIT for {nv12.width}x{nv12.height} (prepare_only={prepare_only})...") - - features_slice = metadata['output_slices']['hidden_state'] - input_shapes = metadata['input_shapes'] - - _run = make_run_supercombo(model_runner, nv12, model_w, model_h, - features_slice, frame_skip, input_shapes, prepare_only) - run_jit = TinyJit(_run, prune=True) - - input_queues, npy = make_supercombo_input_queues(input_shapes, frame_skip, Device.DEFAULT) - - run_jit = _warmup_and_serialize(run_jit, input_queues, npy, nv12) - return run_jit +def _parse_size(size_str: str) -> tuple[int, int]: + width, height = size_str.lower().split('x') + return int(width), int(height) -def compile_multi_policy(nv12: NV12Frame, model_w, model_h, prepare_only, frame_skip, - vision_runner, policy_runners, vision_metadata, policy_metadata): - print(f"Compiling combined multi-policy JIT for {nv12.width}x{nv12.height} (prepare_only={prepare_only})...") - - vision_features_slice = vision_metadata['output_slices']['hidden_state'] - vision_input_shapes = vision_metadata['input_shapes'] - policy_input_shapes = policy_metadata['input_shapes'] - desire_key = _detect_desire_key(policy_input_shapes) - extra_policy_keys = [k for k in policy_input_shapes if k not in ('features_buffer', desire_key, 'traffic_convention')] - vision_road_key, vision_wide_key = _detect_vision_keys(vision_input_shapes) - - _run = make_run_vision_multi_policy(vision_runner, policy_runners, nv12, model_w, model_h, - vision_features_slice, frame_skip, desire_key, extra_policy_keys, - vision_road_key, vision_wide_key, prepare_only) - run_jit = TinyJit(_run, prune=True) - - input_queues, npy = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, Device.DEFAULT) - - run_jit = _warmup_and_serialize(run_jit, input_queues, npy, nv12) - return run_jit +def _compile_for_resolutions(camera_resolutions: list, model_size: tuple[int, int], frame_skip: int, + vision_runner, policy_runners: list, metadata: dict) -> dict: + return { + (cam_w, cam_h): { + name: compile_and_warmup(NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)), model_size, prepare_only, + frame_skip, vision_runner, policy_runners, metadata) + for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] + } + for cam_w, cam_h in camera_resolutions + } -def _parse_size(s): - w, h = s.lower().split('x') - return int(w), int(h) +def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: + runners, keys = [], [] + for name, onnx_arg in [('policy', args.policy_onnx), ('off_policy', args.off_policy_onnx), ('on_policy', args.on_policy_onnx)]: + if onnx_arg: + runners.append(OnnxRunner(onnx_arg)) + keys.append(name) + return runners, keys if __name__ == "__main__": @@ -383,98 +210,54 @@ if __name__ == "__main__": from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict - p = argparse.ArgumentParser(description="Compile combined JIT pkl for sunnypilot modeld_v2") - p.add_argument('--model-type', choices=MODEL_TYPES, required=True) - p.add_argument('--model-size', type=_parse_size, required=True, help='model input WxH') - p.add_argument('--camera-resolutions', type=_parse_size, nargs='+', required=True) - p.add_argument('--frame-skip', type=int, default=None, help='frame skip value (auto-derived if not provided)') - p.add_argument('--output', required=True) + parser = argparse.ArgumentParser(description="Compile combined JIT pkl for sunnypilot modeld_v2") + parser.add_argument('--model-type', choices=MODEL_TYPES, required=True) + parser.add_argument('--model-size', type=_parse_size, required=True, help='model input WxH') + parser.add_argument('--camera-resolutions', type=_parse_size, nargs='+', required=True) + parser.add_argument('--frame-skip', type=int, default=None, help='frame skip value (auto-derived if not provided)') + parser.add_argument('--output', required=True) - p.add_argument('--vision-onnx', help='vision ONNX (for split models)') - p.add_argument('--policy-onnx', help='policy ONNX (for vision_policy)') - p.add_argument('--off-policy-onnx', help='off-policy ONNX (for vision_multi_policy)') - p.add_argument('--on-policy-onnx', help='on-policy ONNX (for vision_multi_policy)') - p.add_argument('--supercombo-onnx', help='supercombo ONNX (for supercombo)') + parser.add_argument('--vision-onnx', help='vision ONNX (for split models)') + parser.add_argument('--policy-onnx', help='policy ONNX (for vision_policy)') + parser.add_argument('--off-policy-onnx', help='off-policy ONNX (for vision_multi_policy)') + parser.add_argument('--on-policy-onnx', help='on-policy ONNX (for vision_multi_policy)') + parser.add_argument('--supercombo-onnx', help='supercombo ONNX (for supercombo)') - args = p.parse_args() - out = defaultdict(dict) + args = parser.parse_args() + output_data = defaultdict(dict) + + vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None if args.model_type == 'vision_policy': - assert args.vision_onnx and args.policy_onnx - vision_runner = OnnxRunner(args.vision_onnx) - policy_runner = OnnxRunner(args.policy_onnx) - out['metadata']['vision'] = make_metadata_dict(args.vision_onnx) - out['metadata']['policy'] = make_metadata_dict(args.policy_onnx) - - frame_skip = args.frame_skip if args.frame_skip is not None else derive_frame_skip(out['metadata']['vision']['input_shapes'], - out['metadata']['policy']['input_shapes']) - - for cam_w, cam_h in args.camera_resolutions: - nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) - model_w, model_h = args.model_size - out[(cam_w, cam_h)] = { - name: compile_split_policy(nv12, model_w, model_h, prepare_only, frame_skip, - vision_runner, policy_runner, - out['metadata']['vision'], out['metadata']['policy']) - for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] - } - + assert vision_runner and args.policy_onnx + policy_runners = [OnnxRunner(args.policy_onnx)] + output_data['metadata'] = {'vision': make_metadata_dict(args.vision_onnx), 'policy': make_metadata_dict(args.policy_onnx)} elif args.model_type == 'supercombo': assert args.supercombo_onnx - model_runner = OnnxRunner(args.supercombo_onnx) - out['metadata']['model'] = make_metadata_dict(args.supercombo_onnx) - - frame_skip = args.frame_skip if args.frame_skip is not None else derive_frame_skip({}, out['metadata']['model']['input_shapes']) - - for cam_w, cam_h in args.camera_resolutions: - nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) - model_w, model_h = args.model_size - out[(cam_w, cam_h)] = { - name: compile_supercombo(nv12, model_w, model_h, prepare_only, frame_skip, - model_runner, out['metadata']['model']) - for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] - } - + policy_runners = [OnnxRunner(args.supercombo_onnx)] + output_data['metadata'] = {'model': make_metadata_dict(args.supercombo_onnx)} elif args.model_type == 'vision_multi_policy': - assert args.vision_onnx - vision_runner = OnnxRunner(args.vision_onnx) - out['metadata']['vision'] = make_metadata_dict(args.vision_onnx) + assert vision_runner + policy_runners, policy_names = _load_policy_runners(args) + output_data['metadata'] = {'vision': make_metadata_dict(args.vision_onnx)} + for name, runner_arg in zip(policy_names, [args.policy_onnx, args.off_policy_onnx, args.on_policy_onnx], strict=True): + if runner_arg: + output_data['metadata'][name] = make_metadata_dict(runner_arg) - policy_runners = [] - policy_onnxes = [] - if args.policy_onnx: - policy_onnxes.append(('policy', args.policy_onnx)) - if args.off_policy_onnx: - policy_onnxes.append(('off_policy', args.off_policy_onnx)) - if args.on_policy_onnx: - policy_onnxes.append(('on_policy', args.on_policy_onnx)) + first_policy_meta = output_data['metadata'].get('policy', output_data['metadata'].get('model', output_data['metadata'].get('off_policy', {}))) + vision_meta = output_data['metadata'].get('vision', {}) - for name, onnx_path in policy_onnxes: - runner = OnnxRunner(onnx_path) - policy_runners.append(runner) - out['metadata'][name] = make_metadata_dict(onnx_path) + derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {})) + output_data.update(_compile_for_resolutions(args.camera_resolutions, args.model_size, derived_frame_skip, + vision_runner, policy_runners, output_data['metadata'])) - first_policy_key = policy_onnxes[0][0] - frame_skip = args.frame_skip if args.frame_skip is not None else derive_frame_skip(out['metadata']['vision']['input_shapes'], - out['metadata'][first_policy_key]['input_shapes']) + with open(args.output, "wb") as file: + pickle.dump(output_data, file) - for cam_w, cam_h in args.camera_resolutions: - nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) - model_w, model_h = args.model_size - out[(cam_w, cam_h)] = { - name: compile_multi_policy(nv12, model_w, model_h, prepare_only, frame_skip, - vision_runner, policy_runners, - out['metadata']['vision'], out['metadata'][first_policy_key]) - for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] - } - - with open(args.output, "wb") as f: - pickle.dump(out, f) pkl_size = os.path.getsize(args.output) print(f"Saved combined JIT to {args.output} ({pkl_size / 1e6:.2f} MB)") from openpilot.common.file_chunker import chunk_file, get_chunk_targets chunk_targets = get_chunk_targets(args.output, pkl_size) chunk_file(args.output, chunk_targets) - num_chunks = len(chunk_targets) - 1 - print(f"Chunked into {num_chunks} file(s)") + print(f"Chunked into {len(chunk_targets) - 1} file(s)") From 2697008aa736af2aeb74ad68b41453ecb56e0aaf Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 6 Jun 2026 10:09:07 -0700 Subject: [PATCH 07/57] redundant --- sunnypilot/modeld_v2/tests/test_warp.py | 103 -------------- sunnypilot/modeld_v2/warp.py | 171 ------------------------ 2 files changed, 274 deletions(-) delete mode 100644 sunnypilot/modeld_v2/tests/test_warp.py delete mode 100644 sunnypilot/modeld_v2/warp.py diff --git a/sunnypilot/modeld_v2/tests/test_warp.py b/sunnypilot/modeld_v2/tests/test_warp.py deleted file mode 100644 index 49dc634a4d..0000000000 --- a/sunnypilot/modeld_v2/tests/test_warp.py +++ /dev/null @@ -1,103 +0,0 @@ -import os -os.environ['DEV'] = 'CPU' -import pytest -import numpy as np -from openpilot.system.camerad.cameras.nv12_info import get_nv12_info -from openpilot.sunnypilot.modeld_v2.warp import CAMERA_CONFIGS -from openpilot.sunnypilot.modeld_v2.warp import Warp, MODEL_W, MODEL_H - -VISION_NAME_PAIRS = [ # needed to account for supercombos input_imgs - ('img', 'big_img'), - ('input_imgs', 'big_input_imgs'), -] - - -class MockVisionBuf: - def __init__(self, w, h): - self.width = w - self.height = h - _, _, _, yuv_size = get_nv12_info(w, h) - self.data = np.zeros(yuv_size, dtype=np.uint8) - - -@pytest.mark.parametrize("buffer_length", [2, 5]) -def test_warp_initialization(buffer_length): - warp = Warp(buffer_length) - assert warp.buffer_length == buffer_length - assert warp.img_buffer_shape == (buffer_length * 6, MODEL_H // 2, MODEL_W // 2) - - -@pytest.mark.parametrize("buffer_length", [2, 5]) -@pytest.mark.parametrize("cam_w, cam_h", CAMERA_CONFIGS) -@pytest.mark.parametrize("road, wide", VISION_NAME_PAIRS) -def test_warp_process(buffer_length, cam_w, cam_h, road, wide): - warp = Warp(buffer_length) - mock_buf = MockVisionBuf(cam_w, cam_h) - transform = np.eye(3, dtype=np.float32).flatten() - bufs = {road: mock_buf, wide: mock_buf} - transforms = {road: transform, wide: transform} - - out = warp.process(bufs, transforms) - assert isinstance(out, dict) - assert road in out and wide in out - assert out[road].shape == (1, 12, MODEL_H // 2, MODEL_W // 2) - assert out[wide].shape == (1, 12, MODEL_H // 2, MODEL_W // 2) - - key = (cam_w, cam_h) - assert key in warp.jit_cache - - out2 = warp.process(bufs, transforms) - assert out2[road].shape == out[road].shape - - -@pytest.mark.parametrize("road, wide", VISION_NAME_PAIRS) -def test_warp_buffer_shift(road, wide): - warp = Warp(2) - cam_w, cam_h = CAMERA_CONFIGS[1] - transform = np.eye(3, dtype=np.float32).flatten() - - buf1 = MockVisionBuf(cam_w, cam_h) - buf1.data[0] = 255 - bufs1 = {road: buf1, wide: buf1} - transforms = {road: transform, wide: transform} - out1 = warp.process(bufs1, transforms) - road1 = out1[road].numpy().copy() - - buf2 = MockVisionBuf(cam_w, cam_h) - buf2.data[0] = 128 - bufs2 = {road: buf2, wide: buf2} - out2 = warp.process(bufs2, transforms) - assert not np.array_equal(road1, out2[road].numpy()) - - -@pytest.mark.parametrize("buffer_length", [2, 5]) -@pytest.mark.parametrize("road, wide", VISION_NAME_PAIRS) -def test_warp_buffer_accumulation(buffer_length, road, wide): - warp = Warp(buffer_length) - cam_w, cam_h = CAMERA_CONFIGS[0] - transform = np.eye(3, dtype=np.float32).flatten() - transforms = {road: transform, wide: transform} - outputs = [] - - for i in range(buffer_length + 1): - buf = MockVisionBuf(cam_w, cam_h) - buf.data[:] = i * 10 - out = warp.process({road: buf, wide: buf}, transforms) - outputs.append(out[road].numpy().copy()) - - assert warp.full_buffers['img'].shape == (buffer_length * 6, MODEL_H // 2, MODEL_W // 2) - for i in range(1, len(outputs)): - assert not np.array_equal(outputs[i - 1], outputs[i]) - - -def test_warp_different_cameras_same_instance(): - warp = Warp(2) - transform = np.eye(3, dtype=np.float32).flatten() - - buf1 = MockVisionBuf(*CAMERA_CONFIGS[0]) - warp.process({'img': buf1, 'big_img': buf1}, {'img': transform, 'big_img': transform}) - assert len(warp.jit_cache) == 1 - - buf2 = MockVisionBuf(*CAMERA_CONFIGS[1]) - warp.process({'img': buf2, 'big_img': buf2}, {'img': transform, 'big_img': transform}) - assert len(warp.jit_cache) == 2 diff --git a/sunnypilot/modeld_v2/warp.py b/sunnypilot/modeld_v2/warp.py deleted file mode 100644 index f91e456c00..0000000000 --- a/sunnypilot/modeld_v2/warp.py +++ /dev/null @@ -1,171 +0,0 @@ -import pickle -import time -import numpy as np -from pathlib import Path -from tinygrad.tensor import Tensor -from tinygrad.engine.jit import TinyJit -from tinygrad.device import Device - -from openpilot.system.camerad.cameras.nv12_info import get_nv12_info -from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE -from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye -from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare as _make_frame_prepare - -CAMERA_CONFIGS = [ - (_ar_ox_fisheye.width, _ar_ox_fisheye.height), - (_os_fisheye.width, _os_fisheye.height), -] - - -def make_frame_prepare(cam_w, cam_h, model_w, model_h): - nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) - return _make_frame_prepare(nv12, model_w, model_h) - - -def warp_pkl_path(w, h): - from openpilot.selfdrive.modeld.helpers import MODELS_DIR - return MODELS_DIR / f'warp_{w}x{h}_tinygrad.pkl' - - -def make_update_img_input(frame_prepare, model_w, model_h): - def update_img_input_tinygrad(tensor, frame, M_inv): - M_inv = M_inv.to(Device.DEFAULT) - new_img = frame_prepare(frame, M_inv) - tensor.assign(tensor[6:].cat(new_img, dim=0).contiguous()) - return Tensor.cat(tensor[:6], tensor[-6:], dim=0).contiguous().reshape(1, 12, model_h//2, model_w//2) - return update_img_input_tinygrad - - -def make_update_both_imgs(frame_prepare, model_w, model_h): - update_img = make_update_img_input(frame_prepare, model_w, model_h) - def update_both_imgs_tinygrad(calib_img_buffer, new_img, M_inv, - calib_big_img_buffer, new_big_img, M_inv_big): - calib_img_pair = update_img(calib_img_buffer, new_img, M_inv) - calib_big_img_pair = update_img(calib_big_img_buffer, new_big_img, M_inv_big) - return calib_img_pair, calib_big_img_pair - return update_both_imgs_tinygrad - -MODELS_DIR = Path(__file__).parent / 'models' -MODEL_W, MODEL_H = MEDMODEL_INPUT_SIZE -UPSTREAM_BUFFER_LENGTH = 5 - - -def v2_warp_pkl_path(cam_w, cam_h, buffer_length): - return MODELS_DIR / f'warp_{cam_w}x{cam_h}_b{buffer_length}_tinygrad.pkl' - - -def compile_v2_warp(cam_w, cam_h, buffer_length): - _, _, _, yuv_size = get_nv12_info(cam_w, cam_h) - img_buffer_shape = (buffer_length * 6, MODEL_H // 2, MODEL_W // 2) - - print(f"Compiling v2 warp for {cam_w}x{cam_h} buffer_length={buffer_length}...") - - frame_prepare = make_frame_prepare(cam_w, cam_h, MODEL_W, MODEL_H) - update_both_imgs = make_update_both_imgs(frame_prepare, MODEL_W, MODEL_H) - update_img_jit = TinyJit(update_both_imgs, prune=True) - - full_buffer = Tensor.zeros(img_buffer_shape, dtype='uint8').contiguous().realize() - big_full_buffer = Tensor.zeros(img_buffer_shape, dtype='uint8').contiguous().realize() - new_frame_np = np.random.randint(0, 256, yuv_size, dtype=np.uint8) - new_big_frame_np = np.random.randint(0, 256, yuv_size, dtype=np.uint8) - for i in range(10): - img_inputs = [full_buffer, - Tensor.from_blob(new_frame_np.ctypes.data, (yuv_size,), dtype='uint8').realize(), - Tensor(Tensor.randn(3, 3).mul(8).realize().numpy(), device='NPY')] - big_img_inputs = [big_full_buffer, - Tensor.from_blob(new_big_frame_np.ctypes.data, (yuv_size,), dtype='uint8').realize(), - Tensor(Tensor.randn(3, 3).mul(8).realize().numpy(), device='NPY')] - inputs = img_inputs + big_img_inputs - Device.default.synchronize() - - st = time.perf_counter() - _ = update_img_jit(*inputs) - mt = time.perf_counter() - Device.default.synchronize() - et = time.perf_counter() - print(f" [{i+1}/10] enqueue {(mt-st)*1e3:6.2f} ms -- total {(et-st)*1e3:6.2f} ms") - - pkl_path = v2_warp_pkl_path(cam_w, cam_h, buffer_length) - with open(pkl_path, "wb") as f: - pickle.dump(update_img_jit, f) - print(f" Saved to {pkl_path}") - - jit = pickle.load(open(pkl_path, "rb")) - verify_frame = np.random.randint(0, 256, yuv_size, dtype=np.uint8) - verify_big_frame = np.random.randint(0, 256, yuv_size, dtype=np.uint8) - fresh_inputs = [ - Tensor.zeros(img_buffer_shape, dtype='uint8').contiguous().realize(), - Tensor.from_blob(verify_frame.ctypes.data, (yuv_size,), dtype='uint8').realize(), - Tensor(Tensor.randn(3, 3).mul(8).realize().numpy(), device='NPY'), - Tensor.zeros(img_buffer_shape, dtype='uint8').contiguous().realize(), - Tensor.from_blob(verify_big_frame.ctypes.data, (yuv_size,), dtype='uint8').realize(), - Tensor(Tensor.randn(3, 3).mul(8).realize().numpy(), device='NPY'), - ] - jit(*fresh_inputs) - - -class Warp: - def __init__(self, buffer_length=2): - self.buffer_length = buffer_length - self.img_buffer_shape = (buffer_length * 6, MODEL_H // 2, MODEL_W // 2) - - self.jit_cache = {} - self.full_buffers = {k: Tensor.zeros(self.img_buffer_shape, dtype='uint8').contiguous().realize() for k in ['img', 'big_img']} - self._blob_cache: dict[int, Tensor] = {} - self._nv12_cache: dict[tuple[int, int], int] = {} - self.transforms_np = {k: np.zeros((3, 3), dtype=np.float32) for k in ['img', 'big_img']} - self.transforms = {k: Tensor(v, device='NPY').realize() for k, v in self.transforms_np.items()} - - def process(self, bufs, transforms): - if not bufs: - return {} - road = next(n for n in bufs if 'big' not in n) - wide = next(n for n in bufs if 'big' in n) - cam_w, cam_h = bufs[road].width, bufs[road].height - key = (cam_w, cam_h) - - if key not in self.jit_cache: - v2_pkl = v2_warp_pkl_path(cam_w, cam_h, self.buffer_length) - if v2_pkl.exists(): - with open(v2_pkl, 'rb') as f: - self.jit_cache[key] = pickle.load(f) - elif self.buffer_length == UPSTREAM_BUFFER_LENGTH: - upstream_pkl = warp_pkl_path(cam_w, cam_h) - if upstream_pkl.exists(): - with open(upstream_pkl, 'rb') as f: - self.jit_cache[key] = pickle.load(f) - if key not in self.jit_cache: - frame_prepare = make_frame_prepare(cam_w, cam_h, MODEL_W, MODEL_H) - update_both_imgs = make_update_both_imgs(frame_prepare, MODEL_W, MODEL_H) - self.jit_cache[key] = TinyJit(update_both_imgs, prune=True) - - if key not in self._nv12_cache: - self._nv12_cache[key] = get_nv12_info(cam_w, cam_h)[3] - yuv_size = self._nv12_cache[key] - - road_ptr = bufs[road].data.ctypes.data - wide_ptr = bufs[wide].data.ctypes.data - if road_ptr not in self._blob_cache: - self._blob_cache[road_ptr] = Tensor.from_blob(road_ptr, (yuv_size,), dtype='uint8') - if wide_ptr not in self._blob_cache: - self._blob_cache[wide_ptr] = Tensor.from_blob(wide_ptr, (yuv_size,), dtype='uint8') - road_blob = self._blob_cache[road_ptr] - wide_blob = self._blob_cache[wide_ptr] if wide_ptr != road_ptr else Tensor.from_blob(wide_ptr, (yuv_size,), dtype='uint8') - np.copyto(self.transforms_np['img'], transforms[road].reshape(3, 3)) - np.copyto(self.transforms_np['big_img'], transforms[wide].reshape(3, 3)) - - Device.default.synchronize() - res = self.jit_cache[key]( - self.full_buffers['img'], road_blob, self.transforms['img'], - self.full_buffers['big_img'], wide_blob, self.transforms['big_img'], - ) - out_road = res[0].realize() - out_wide = res[1].realize() - - return {road: out_road, wide: out_wide} - - -if __name__ == "__main__": - for cam_w, cam_h in CAMERA_CONFIGS: - for bl in [2, 5]: - compile_v2_warp(cam_w, cam_h, bl) From a232f54e2d63f6928ad321dfcd4e4850ebb50eff Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 01:45:42 -0700 Subject: [PATCH 08/57] CREAM AND SUGAR --- sunnypilot/modeld_v2/compile_modeld.py | 35 +++++++++++++++++++ sunnypilot/modeld_v2/modeld.py | 32 ++++++++++++----- .../modeld_v2/parse_model_outputs_split.py | 2 ++ sunnypilot/models/split_model_constants.py | 1 + 4 files changed, 62 insertions(+), 8 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index b9a392631a..5e980295fe 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -14,6 +14,22 @@ from collections import defaultdict from functools import partial import numpy as np +def _patch_tinygrad_fetch_fw(): + import hashlib + import pathlib + import zstandard + from tinygrad import helpers + _orig_fetch_fw = helpers.fetch_fw + def fetch_fw(path, name, sha256): + p = pathlib.Path(f"/lib/firmware/{path}/{name}.zst") + if p.is_file(): + blob = zstandard.ZstdDecompressor().stream_reader(p.read_bytes()).read() + if hashlib.sha256(blob).hexdigest() == sha256: + return blob + return _orig_fetch_fw(path, name, sha256) + helpers.fetch_fw = fetch_fw +_patch_tinygrad_fetch_fw() + from tinygrad import dtypes from tinygrad.device import Device from tinygrad.engine.jit import TinyJit @@ -184,6 +200,19 @@ def _parse_size(size_str: str) -> tuple[int, int]: return int(width), int(height) +def read_file_chunked_to_shm(path): + if not path: + return None + from openpilot.common.file_chunker import read_file_chunked + from openpilot.system.hardware.hw import Paths + import atexit + shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) + atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) + with open(shm_path, 'wb') as f: + f.write(read_file_chunked(path)) + return shm_path + + def _compile_for_resolutions(camera_resolutions: list, model_size: tuple[int, int], frame_skip: int, vision_runner, policy_runners: list, metadata: dict) -> dict: return { @@ -226,6 +255,12 @@ if __name__ == "__main__": args = parser.parse_args() output_data = defaultdict(dict) + args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx) + args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx) + args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx) + args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx) + args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx) + vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None if args.model_type == 'vision_policy': diff --git a/sunnypilot/modeld_v2/modeld.py b/sunnypilot/modeld_v2/modeld.py index b124e3645a..7557715dd4 100755 --- a/sunnypilot/modeld_v2/modeld.py +++ b/sunnypilot/modeld_v2/modeld.py @@ -196,7 +196,7 @@ class ModelState(ModelStateBase): inputs[desire_key][0] = 0 self.numpy_inputs[desire_key][:] = np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0) self.prev_desire[:] = inputs[desire_key] - for key in ('traffic_convention', 'lateral_control_params'): + for key in ('traffic_convention', 'lateral_control_params', 'action_t'): if key in self.numpy_inputs and key in inputs: self.numpy_inputs[key][:] = inputs[key] @@ -240,13 +240,20 @@ class ModelState(ModelStateBase): def get_action_from_model(self, model_output: dict[str, np.ndarray], prev_action: log.ModelDataV2.Action, lat_action_t: float, long_action_t: float, v_ego: float) -> log.ModelDataV2.Action: - plan = model_output['plan'][0] - desired_accel, should_stop = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0], self.constants.T_IDXS, - action_t=long_action_t) - desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) + if 'action' not in model_output: + plan = model_output['plan'][0] + desired_accel, should_stop = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0], self.constants.T_IDXS, + action_t=long_action_t) + desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) + + curvature_plan = (plan + (self.PLANPLUS_CONTROL - 1.0) * model_output['planplus'][0] + if 'planplus' in model_output and self.PLANPLUS_CONTROL != 1.0 else plan) + desired_curvature = get_curvature_from_output(model_output, curvature_plan, v_ego, lat_action_t, self.mlsim) + else: + desired_accel = model_output['action'][0, 1] + desired_curvature = model_output['action'][0, 0] / (max(1.0, v_ego))**2 + should_stop = (v_ego < 0.3 and desired_accel < 0.1) - curvature_plan = plan + (self.PLANPLUS_CONTROL - 1.0) * model_output['planplus'][0] if 'planplus' in model_output and self.PLANPLUS_CONTROL != 1.0 else plan - desired_curvature = get_curvature_from_output(model_output, curvature_plan, v_ego, lat_action_t, self.mlsim) if self.generation is not None and self.generation >= 10: # smooth curvature for post FOF models if v_ego > self.MIN_LAT_CONTROL_SPEED: desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, self.LAT_SMOOTH_SECONDS) @@ -399,6 +406,12 @@ def main(demo=False): bufs = {name: buf_extra if 'big' in name else buf_main for name in model.vision_input_names} transforms = {name: model_transform_extra if 'big' in name else model_transform_main for name in model.vision_input_names} + + frame_delay = DT_MDL # compensate for time passed since the frame was captured: current_time - timestamp_eof is 50ms on average + action_delay = DT_MDL / 2 # middle of the interval between model output (current state) and next frame (expected state) + lat_action_t = lat_delay + frame_delay + action_delay + long_action_t = long_delay + frame_delay + action_delay + inputs:dict[str, np.ndarray] = { model.desire_key: vec_desire, 'traffic_convention': traffic_convention, @@ -407,6 +420,9 @@ def main(demo=False): if 'lateral_control_params' in model.numpy_inputs: inputs['lateral_control_params'] = np.array([v_ego, lat_delay], dtype=np.float32) + if 'action_t' in model.numpy_inputs: + inputs['action_t'] = np.array([lat_action_t, long_action_t], dtype=np.float32) + mt1 = time.perf_counter() model_output = model.run(bufs, transforms, inputs, prepare_only) mt2 = time.perf_counter() @@ -418,7 +434,7 @@ def main(demo=False): posenet_send = messaging.new_message('cameraOdometry') mdv2sp_send = messaging.new_message('modelDataV2SP') - action = model.get_action_from_model(model_output, prev_action, lat_delay + DT_MDL, long_delay + DT_MDL, v_ego) + action = model.get_action_from_model(model_output, prev_action, lat_action_t, long_action_t, v_ego) prev_action = action fill_model_msg(drivingdata_send, modelv2_send, model_output, action, publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id, diff --git a/sunnypilot/modeld_v2/parse_model_outputs_split.py b/sunnypilot/modeld_v2/parse_model_outputs_split.py index 831649e3c1..7848e3e185 100644 --- a/sunnypilot/modeld_v2/parse_model_outputs_split.py +++ b/sunnypilot/modeld_v2/parse_model_outputs_split.py @@ -134,6 +134,8 @@ class Parser: out_shape=(SplitModelConstants.NUM_ROAD_EDGES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) if 'sim_pose' in outs: self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) + if 'action' in outs: + self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.ACTION_WIDTH,)) def parse_vision_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) diff --git a/sunnypilot/models/split_model_constants.py b/sunnypilot/models/split_model_constants.py index a3e1dce8f6..a5f57e5453 100644 --- a/sunnypilot/models/split_model_constants.py +++ b/sunnypilot/models/split_model_constants.py @@ -43,6 +43,7 @@ class SplitModelConstants: LANE_LINES_WIDTH = 2 ROAD_EDGES_WIDTH = 2 PLAN_WIDTH = 15 + ACTION_WIDTH = 2 DESIRE_PRED_WIDTH = 8 LAT_PLANNER_SOLUTION_WIDTH = 4 DESIRED_CURV_WIDTH = 1 From a8ef55bfaadb76f7407d658be25eb4594e71e219 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 02:14:25 -0700 Subject: [PATCH 09/57] gpu stuffs --- sunnypilot/modeld_v2/compile_modeld.py | 6 ++++-- sunnypilot/modeld_v2/modeld.py | 26 ++++++++++++++++---------- 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 5e980295fe..a163dfc9bd 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -8,6 +8,7 @@ See the LICENSE.md file in the root directory for more details. import argparse import os +os.environ['GMMU'] = '0' import pickle import time from collections import defaultdict @@ -173,6 +174,7 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl raise ValueError("Could not find vision, model, or policy metadata.") features_slice = feat_meta['output_slices']['hidden_state'] + WARP_DEV = 'CPU' if "USBGPU" in os.environ else Device.DEFAULT run_func = create_jit_runner(vision_runner, policy_runners, nv12, model_size, features_slice, frame_skip, all_shapes, prepare_only) run_jit = TinyJit(run_func, prune=True) @@ -180,8 +182,8 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl for i in range(3): np.random.seed(42 + i) - frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() - big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() + frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() + big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() for arr in npy_arrays.values(): arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) diff --git a/sunnypilot/modeld_v2/modeld.py b/sunnypilot/modeld_v2/modeld.py index 7557715dd4..1f28e2d16e 100755 --- a/sunnypilot/modeld_v2/modeld.py +++ b/sunnypilot/modeld_v2/modeld.py @@ -7,6 +7,7 @@ See the LICENSE.md file in the root directory for more details. """ import os +os.environ['GMMU'] = '0' from openpilot.system.hardware import TICI os.environ['DEV'] = 'QCOM' if TICI else 'CPU' USBGPU = "USBGPU" in os.environ @@ -109,6 +110,8 @@ class ModelState(ModelStateBase): jits = pickle.loads(read_file_chunked(pkl_path)) self.DEV = Device.DEFAULT + self.WARP_DEV = 'CPU' if USBGPU else self.DEV + self.QUEUE_DEV = self.DEV metadata = jits['metadata'] if 'model' in metadata: @@ -120,7 +123,7 @@ class ModelState(ModelStateBase): self._vision_input_names = [k for k in model_metadata['input_shapes'] if 'img' in k] from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) - self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.DEV) + self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.QUEUE_DEV) else: vision_metadata = metadata['vision'] policy_keys = [k for k in metadata if k != 'vision'] @@ -138,7 +141,11 @@ class ModelState(ModelStateBase): policy_input_shapes = first_policy_metadata['input_shapes'] self._vision_input_names = [k for k in vision_input_shapes if 'img' in k] frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes) - self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.DEV) + self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.QUEUE_DEV) + + self._desire_key = next(key for key in self.numpy_inputs if key.startswith('desire')) + self._road_key = next(key for key in self._vision_input_names if 'big' not in key) + self._wide_key = next(key for key in self._vision_input_names if 'big' in key) from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser @@ -160,12 +167,11 @@ class ModelState(ModelStateBase): self._run_policy = jits[(cam_w, cam_h)]['run_policy'] self._warp_enqueue = jits[(cam_w, cam_h)]['warp_enqueue'] - road_name = next(k for k in self._vision_input_names if 'big' not in k) - yuv_size = self.frame_buf_params[road_name][3] + yuv_size = self.frame_buf_params[self._road_key][3] self._warp_enqueue( **self.input_queues, - frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.DEV).contiguous().realize(), - big_frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.DEV).contiguous().realize()) + frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize(), + big_frame=Tensor(np.zeros(yuv_size, dtype=np.uint8), device=self.WARP_DEV).contiguous().realize()) @property @@ -178,7 +184,7 @@ class ModelState(ModelStateBase): @property def desire_key(self) -> str: - return next(k for k in self.numpy_inputs if k.startswith('desire')) + return self._desire_key def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None: @@ -189,7 +195,7 @@ class ModelState(ModelStateBase): yuv_size = self.frame_buf_params[key][3] cache_key = (key, ptr) if cache_key not in self._blob_cache: - self._blob_cache[cache_key] = Tensor.from_blob(ptr, (yuv_size,), dtype='uint8', device=self.DEV) + self._blob_cache[cache_key] = Tensor.from_blob(ptr, (yuv_size,), dtype='uint8', device=self.WARP_DEV) self.full_frames[key] = self._blob_cache[cache_key] desire_key = self.desire_key @@ -200,8 +206,8 @@ class ModelState(ModelStateBase): if key in self.numpy_inputs and key in inputs: self.numpy_inputs[key][:] = inputs[key] - road_key = next(n for n in bufs if 'big' not in n) - wide_key = next(n for n in bufs if 'big' in n) + road_key = self._road_key + wide_key = self._wide_key self.numpy_inputs['tfm'][:, :] = transforms[road_key].reshape(3, 3) self.numpy_inputs['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3) From fba521dcff1f4afbb68abe1d63a2c4949aebe5ad Mon Sep 17 00:00:00 2001 From: James Vecellio-Grant <159560811+Discountchubbs@users.noreply.github.com> Date: Sun, 7 Jun 2026 12:06:02 +0200 Subject: [PATCH 10/57] Update fetcher.py --- sunnypilot/models/fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py index 6484f2b444..5461666c71 100644 --- a/sunnypilot/models/fetcher.py +++ b/sunnypilot/models/fetcher.py @@ -116,7 +116,7 @@ class ModelCache: class ModelFetcher: """Handles fetching and caching of model data from remote source""" - MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v17.json" + MODEL_URL = "https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_v18.json" def __init__(self, params: Params): self.params = params From e1fe30fd3e5b3ac03cc4503022191e383c873400 Mon Sep 17 00:00:00 2001 From: James Vecellio-Grant <159560811+Discountchubbs@users.noreply.github.com> Date: Sun, 7 Jun 2026 12:19:36 +0200 Subject: [PATCH 11/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index b9a392631a..4b4d0c2c3a 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -240,9 +240,9 @@ if __name__ == "__main__": assert vision_runner policy_runners, policy_names = _load_policy_runners(args) output_data['metadata'] = {'vision': make_metadata_dict(args.vision_onnx)} - for name, runner_arg in zip(policy_names, [args.policy_onnx, args.off_policy_onnx, args.on_policy_onnx], strict=True): - if runner_arg: - output_data['metadata'][name] = make_metadata_dict(runner_arg) + for name in policy_names: + runner_arg = getattr(args, f"{name}_onnx") + output_data['metadata'][name] = make_metadata_dict(runner_arg) first_policy_meta = output_data['metadata'].get('policy', output_data['metadata'].get('model', output_data['metadata'].get('off_policy', {}))) vision_meta = output_data['metadata'].get('vision', {}) From f1ab6c8dfb29e20acfc9c537baeea915b8c479e6 Mon Sep 17 00:00:00 2001 From: James Vecellio-Grant <159560811+Discountchubbs@users.noreply.github.com> Date: Sun, 7 Jun 2026 12:21:49 +0200 Subject: [PATCH 12/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index a163dfc9bd..588d1b91f5 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -277,9 +277,9 @@ if __name__ == "__main__": assert vision_runner policy_runners, policy_names = _load_policy_runners(args) output_data['metadata'] = {'vision': make_metadata_dict(args.vision_onnx)} - for name, runner_arg in zip(policy_names, [args.policy_onnx, args.off_policy_onnx, args.on_policy_onnx], strict=True): - if runner_arg: - output_data['metadata'][name] = make_metadata_dict(runner_arg) + for name in policy_names: + runner_arg = getattr(args, f"{name}_onnx") + output_data['metadata'][name] = make_metadata_dict(runner_arg) first_policy_meta = output_data['metadata'].get('policy', output_data['metadata'].get('model', output_data['metadata'].get('off_policy', {}))) vision_meta = output_data['metadata'].get('vision', {}) From 159140e64ead7f5aaffb64134968926f6942b9bb Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 03:41:09 -0700 Subject: [PATCH 13/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 588d1b91f5..b611664e85 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -144,14 +144,15 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode return img, big_img desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) - inputs = {desire_key: desire_buf, **extra_tensors} + inputs = {desire_key: desire_buf.realize(), **extra_tensors} if traffic_conv_dev is not None: inputs['traffic_convention'] = traffic_conv_dev if vision_runner: vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32') + vision_out = vision_out.realize() new_feat = vision_out[:, features_slice].reshape(1, -1).unsqueeze(0) - inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn) + inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] return (vision_out, *policy_outs) if len(policy_outs) > 1 else (vision_out, policy_outs[0]) From dd35c27981e9cf1559db0ee640cbaf16df6449c4 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 03:44:27 -0700 Subject: [PATCH 14/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index b611664e85..3c53d71710 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -159,7 +159,7 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) - shift_and_sample(feat_q, new_feat, sample_skip_fn) + shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() return policy_out return runner @@ -282,7 +282,8 @@ if __name__ == "__main__": runner_arg = getattr(args, f"{name}_onnx") output_data['metadata'][name] = make_metadata_dict(runner_arg) - first_policy_meta = output_data['metadata'].get('policy', output_data['metadata'].get('model', output_data['metadata'].get('off_policy', {}))) + policy_keys = [key for key in output_data['metadata'].keys() if key != 'vision'] + first_policy_meta = output_data['metadata'][policy_keys[0]] if policy_keys else {} vision_meta = output_data['metadata'].get('vision', {}) derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {})) From 74692d0b5fae566c702372068121075942703ee0 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 03:56:09 -0700 Subject: [PATCH 15/57] summary --- .github/workflows/build-single-tinygrad-model.yaml | 13 ++++++------- sunnypilot/modeld_v2/compile_modeld.py | 10 +++++----- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-single-tinygrad-model.yaml b/.github/workflows/build-single-tinygrad-model.yaml index f10f1b71a3..cf2d870802 100644 --- a/.github/workflows/build-single-tinygrad-model.yaml +++ b/.github/workflows/build-single-tinygrad-model.yaml @@ -12,11 +12,11 @@ on: required: false type: string recompiled_dir: - description: 'Existing recompiled directory number (e.g. 3 for recompiled3)' + description: 'Existing recompiled directory number (e.g. 1 for recompiled1)' required: true type: string json_version: - description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)' + description: 'driving_models version number to update (e.g. 18 for driving_models_v18.json)' required: true type: string artifact_suffix: @@ -63,12 +63,11 @@ on: default: 'None' options: - None - - Simple Plan Models - - Space Lab Models - - TR Models - - DTR Models + - Master Models + - Release Models + - 2025 World Models + - 2026 World Models - Custom Merge Models - - FOF series models - Other custom_model_folder: description: 'Custom model folder name (if "Other" selected)' diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 3c53d71710..f7eb35f42e 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -149,13 +149,13 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode inputs['traffic_convention'] = traffic_conv_dev if vision_runner: - vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32') - vision_out = vision_out.realize() - new_feat = vision_out[:, features_slice].reshape(1, -1).unsqueeze(0) + vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())) + vision_out = vision_out.cast('float32').realize().numpy() + vision_out_tensor = Tensor(vision_out, device=Device.DEFAULT) + new_feat = vision_out_tensor[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - return (vision_out, *policy_outs) if len(policy_outs) > 1 else (vision_out, policy_outs[0]) - + return (vision_out_tensor, *policy_outs) if len(policy_outs) > 1 else (vision_out_tensor, policy_outs[0]) inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) From b21c70b1ba70593f16c71919ed5bd558212c6987 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 04:05:54 -0700 Subject: [PATCH 16/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index f7eb35f42e..0d2ad56780 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -144,19 +144,18 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode return img, big_img desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) - inputs = {desire_key: desire_buf.realize(), **extra_tensors} + inputs = {desire_key: desire_buf.realize(), **{key: value.realize() for key, value in extra_tensors.items()}} if traffic_conv_dev is not None: - inputs['traffic_convention'] = traffic_conv_dev + inputs['traffic_convention'] = traffic_conv_dev.realize() if vision_runner: vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())) - vision_out = vision_out.cast('float32').realize().numpy() - vision_out_tensor = Tensor(vision_out, device=Device.DEFAULT) - new_feat = vision_out_tensor[:, features_slice].reshape(1, -1).unsqueeze(0) + vision_out_realized = vision_out.cast('float32').realize() + new_feat = vision_out_realized[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - return (vision_out_tensor, *policy_outs) if len(policy_outs) > 1 else (vision_out_tensor, policy_outs[0]) - inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) + return (vision_out_realized, *policy_outs) if len(policy_outs) > 1 else (vision_out_realized, policy_outs[0]) + inputs.update({road_key: img.realize(), wide_key: big_img.realize(), 'features_buffer': sample_skip_fn(feat_q).realize()}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() From be20848487db9837a6ea7d3b88d726b1aab92ff8 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 04:17:14 -0700 Subject: [PATCH 17/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 0d2ad56780..bae8f93fcc 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -154,7 +154,9 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode new_feat = vision_out_realized[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - return (vision_out_realized, *policy_outs) if len(policy_outs) > 1 else (vision_out_realized, policy_outs[0]) + vision_out_kernel = vision_out_realized + 0 + policy_out_kernels = [p + 0 for p in policy_outs] + return (vision_out_kernel, *policy_out_kernels) if len(policy_out_kernels) > 1 else (vision_out_kernel, policy_out_kernels[0]) inputs.update({road_key: img.realize(), wide_key: big_img.realize(), 'features_buffer': sample_skip_fn(feat_q).realize()}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) From 049dfd2eaa707047d089969e518f2b94d017567f Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 09:47:40 -0700 Subject: [PATCH 18/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index bae8f93fcc..2582699364 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -154,9 +154,9 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode new_feat = vision_out_realized[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - vision_out_kernel = vision_out_realized + 0 - policy_out_kernels = [p + 0 for p in policy_outs] - return (vision_out_kernel, *policy_out_kernels) if len(policy_out_kernels) > 1 else (vision_out_kernel, policy_out_kernels[0]) + vision_out_final = vision_out_realized.detach() + policy_out_final = [p.detach() for p in policy_outs] + return (vision_out_final, *policy_out_final) if len(policy_out_final) > 1 else (vision_out_final, policy_out_final[0]) inputs.update({road_key: img.realize(), wide_key: big_img.realize(), 'features_buffer': sample_skip_fn(feat_q).realize()}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) From fb5cb7a1cc15d193669e586f098c6f4108439fdf Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 10:06:02 -0700 Subject: [PATCH 19/57] i could lie say --- sunnypilot/modeld_v2/compile_modeld.py | 43 ++++++++++++-------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 2582699364..3140c04ff1 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -8,12 +8,12 @@ See the LICENSE.md file in the root directory for more details. import argparse import os -os.environ['GMMU'] = '0' import pickle import time from collections import defaultdict from functools import partial import numpy as np +os.environ['GMMU'] = '0' def _patch_tinygrad_fetch_fw(): import hashlib @@ -31,14 +31,12 @@ def _patch_tinygrad_fetch_fw(): helpers.fetch_fw = fetch_fw _patch_tinygrad_fetch_fw() +from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare, sample_desire, sample_skip, shift_and_sample from tinygrad import dtypes from tinygrad.device import Device from tinygrad.engine.jit import TinyJit from tinygrad.tensor import Tensor -from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare, sample_desire, sample_skip, shift_and_sample - - MODEL_TYPES = ('vision_policy', 'supercombo', 'vision_multi_policy') @@ -89,12 +87,12 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize() + dtype=np.float32), device=device).contiguous().realize() } if features_buffer: queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), - dtype=np.float32), device=device).contiguous().realize() + dtype=np.float32), device=device).contiguous().realize() queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) return queues, npy_arrays @@ -109,7 +107,7 @@ def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: st def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], - features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): + features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): frame_prepare = make_frame_prepare(nv12, *model_size) sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) @@ -144,20 +142,19 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode return img, big_img desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) - inputs = {desire_key: desire_buf.realize(), **{key: value.realize() for key, value in extra_tensors.items()}} + inputs = {desire_key: desire_buf, **extra_tensors} + if traffic_conv_dev is not None: - inputs['traffic_convention'] = traffic_conv_dev.realize() + inputs['traffic_convention'] = traffic_conv_dev if vision_runner: vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())) - vision_out_realized = vision_out.cast('float32').realize() - new_feat = vision_out_realized[:, features_slice].reshape(1, -1).unsqueeze(0) + vision_out_cast = vision_out.cast('float32') + new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() - policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - vision_out_final = vision_out_realized.detach() - policy_out_final = [p.detach() for p in policy_outs] - return (vision_out_final, *policy_out_final) if len(policy_out_final) > 1 else (vision_out_final, policy_out_final[0]) - inputs.update({road_key: img.realize(), wide_key: big_img.realize(), 'features_buffer': sample_skip_fn(feat_q).realize()}) + policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32') for pol_runner in policy_runners] + return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0]) + inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() @@ -187,7 +184,7 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() for arr in npy_arrays.values(): - arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) + arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) Device.default.synchronize() start_time = time.perf_counter() @@ -207,9 +204,9 @@ def _parse_size(size_str: str) -> tuple[int, int]: def read_file_chunked_to_shm(path): if not path: return None + import atexit from openpilot.common.file_chunker import read_file_chunked from openpilot.system.hardware.hw import Paths - import atexit shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) with open(shm_path, 'wb') as f: @@ -218,11 +215,12 @@ def read_file_chunked_to_shm(path): def _compile_for_resolutions(camera_resolutions: list, model_size: tuple[int, int], frame_skip: int, - vision_runner, policy_runners: list, metadata: dict) -> dict: + vision_runner, policy_runners: list, metadata: dict) -> dict: + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info return { (cam_w, cam_h): { name: compile_and_warmup(NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)), model_size, prepare_only, - frame_skip, vision_runner, policy_runners, metadata) + frame_skip, vision_runner, policy_runners, metadata) for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] } for cam_w, cam_h in camera_resolutions @@ -239,9 +237,8 @@ def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: if __name__ == "__main__": - from tinygrad.nn.onnx import OnnxRunner - from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict + from tinygrad.nn.onnx import OnnxRunner parser = argparse.ArgumentParser(description="Compile combined JIT pkl for sunnypilot modeld_v2") parser.add_argument('--model-type', choices=MODEL_TYPES, required=True) @@ -289,7 +286,7 @@ if __name__ == "__main__": derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {})) output_data.update(_compile_for_resolutions(args.camera_resolutions, args.model_size, derived_frame_skip, - vision_runner, policy_runners, output_data['metadata'])) + vision_runner, policy_runners, output_data['metadata'])) with open(args.output, "wb") as file: pickle.dump(output_data, file) From 6a4c59c3e03c417526496605cb55bfbaf165a512 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 10:22:07 -0700 Subject: [PATCH 20/57] needed --- tinygrad_repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad_repo b/tinygrad_repo index ac1632ab96..bf656d50c9 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit ac1632ab966c77ba96a7048b893a30f1a714dc87 +Subproject commit bf656d50c9d1bb47840dc3aee0a0c70d692e21a2 From 7d7b6ee3063de0d81fa59c4eba33052ef277fdd9 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 10:32:55 -0700 Subject: [PATCH 21/57] i could --- tinygrad_repo | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tinygrad_repo b/tinygrad_repo index bf656d50c9..2fecac4e4a 160000 --- a/tinygrad_repo +++ b/tinygrad_repo @@ -1 +1 @@ -Subproject commit bf656d50c9d1bb47840dc3aee0a0c70d692e21a2 +Subproject commit 2fecac4e4ac32fe369c41f8400b6e7b9adb18683 From ae573c7c3f04be6729934f27aab80b50203e370b Mon Sep 17 00:00:00 2001 From: James Vecellio-Grant <159560811+Discountchubbs@users.noreply.github.com> Date: Sun, 7 Jun 2026 10:52:39 -0700 Subject: [PATCH 22/57] Update compile_modeld.py --- sunnypilot/modeld_v2/compile_modeld.py | 81 +++++++++++++++++++------- 1 file changed, 59 insertions(+), 22 deletions(-) diff --git a/sunnypilot/modeld_v2/compile_modeld.py b/sunnypilot/modeld_v2/compile_modeld.py index 4b4d0c2c3a..3140c04ff1 100755 --- a/sunnypilot/modeld_v2/compile_modeld.py +++ b/sunnypilot/modeld_v2/compile_modeld.py @@ -13,15 +13,30 @@ import time from collections import defaultdict from functools import partial import numpy as np +os.environ['GMMU'] = '0' +def _patch_tinygrad_fetch_fw(): + import hashlib + import pathlib + import zstandard + from tinygrad import helpers + _orig_fetch_fw = helpers.fetch_fw + def fetch_fw(path, name, sha256): + p = pathlib.Path(f"/lib/firmware/{path}/{name}.zst") + if p.is_file(): + blob = zstandard.ZstdDecompressor().stream_reader(p.read_bytes()).read() + if hashlib.sha256(blob).hexdigest() == sha256: + return blob + return _orig_fetch_fw(path, name, sha256) + helpers.fetch_fw = fetch_fw +_patch_tinygrad_fetch_fw() + +from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare, sample_desire, sample_skip, shift_and_sample from tinygrad import dtypes from tinygrad.device import Device from tinygrad.engine.jit import TinyJit from tinygrad.tensor import Tensor -from openpilot.selfdrive.modeld.compile_modeld import NV12Frame, make_frame_prepare, sample_desire, sample_skip, shift_and_sample - - MODEL_TYPES = ('vision_policy', 'supercombo', 'vision_multi_policy') @@ -72,12 +87,12 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize() + dtype=np.float32), device=device).contiguous().realize() } if features_buffer: queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), - dtype=np.float32), device=device).contiguous().realize() + dtype=np.float32), device=device).contiguous().realize() queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) return queues, npy_arrays @@ -92,7 +107,7 @@ def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: st def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], - features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): + features_slice: slice, frame_skip: int, input_shapes: dict, prepare_only: bool): frame_prepare = make_frame_prepare(nv12, *model_size) sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) @@ -128,20 +143,21 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) inputs = {desire_key: desire_buf, **extra_tensors} + if traffic_conv_dev is not None: inputs['traffic_convention'] = traffic_conv_dev if vision_runner: - vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32') - new_feat = vision_out[:, features_slice].reshape(1, -1).unsqueeze(0) - inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn) - policy_outs = [next(iter(runner(inputs).values())).cast('float32') for runner in policy_runners] - return (vision_out, *policy_outs) if len(policy_outs) > 1 else (vision_out, policy_outs[0]) - + vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())) + vision_out_cast = vision_out.cast('float32') + new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0) + inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() + policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32') for pol_runner in policy_runners] + return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0]) inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) - shift_and_sample(feat_q, new_feat, sample_skip_fn) + shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() return policy_out return runner @@ -157,6 +173,7 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl raise ValueError("Could not find vision, model, or policy metadata.") features_slice = feat_meta['output_slices']['hidden_state'] + WARP_DEV = 'CPU' if "USBGPU" in os.environ else Device.DEFAULT run_func = create_jit_runner(vision_runner, policy_runners, nv12, model_size, features_slice, frame_skip, all_shapes, prepare_only) run_jit = TinyJit(run_func, prune=True) @@ -164,10 +181,10 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl for i in range(3): np.random.seed(42 + i) - frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() - big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8).realize() + frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() + big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() for arr in npy_arrays.values(): - arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) + arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) Device.default.synchronize() start_time = time.perf_counter() @@ -184,12 +201,26 @@ def _parse_size(size_str: str) -> tuple[int, int]: return int(width), int(height) +def read_file_chunked_to_shm(path): + if not path: + return None + import atexit + from openpilot.common.file_chunker import read_file_chunked + from openpilot.system.hardware.hw import Paths + shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) + atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) + with open(shm_path, 'wb') as f: + f.write(read_file_chunked(path)) + return shm_path + + def _compile_for_resolutions(camera_resolutions: list, model_size: tuple[int, int], frame_skip: int, - vision_runner, policy_runners: list, metadata: dict) -> dict: + vision_runner, policy_runners: list, metadata: dict) -> dict: + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info return { (cam_w, cam_h): { name: compile_and_warmup(NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)), model_size, prepare_only, - frame_skip, vision_runner, policy_runners, metadata) + frame_skip, vision_runner, policy_runners, metadata) for name, prepare_only in [('warp_enqueue', True), ('run_policy', False)] } for cam_w, cam_h in camera_resolutions @@ -206,9 +237,8 @@ def _load_policy_runners(args: argparse.Namespace) -> tuple[list, list]: if __name__ == "__main__": - from tinygrad.nn.onnx import OnnxRunner - from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict + from tinygrad.nn.onnx import OnnxRunner parser = argparse.ArgumentParser(description="Compile combined JIT pkl for sunnypilot modeld_v2") parser.add_argument('--model-type', choices=MODEL_TYPES, required=True) @@ -226,6 +256,12 @@ if __name__ == "__main__": args = parser.parse_args() output_data = defaultdict(dict) + args.vision_onnx = read_file_chunked_to_shm(args.vision_onnx) + args.policy_onnx = read_file_chunked_to_shm(args.policy_onnx) + args.off_policy_onnx = read_file_chunked_to_shm(args.off_policy_onnx) + args.on_policy_onnx = read_file_chunked_to_shm(args.on_policy_onnx) + args.supercombo_onnx = read_file_chunked_to_shm(args.supercombo_onnx) + vision_runner = OnnxRunner(args.vision_onnx) if args.vision_onnx else None if args.model_type == 'vision_policy': @@ -244,12 +280,13 @@ if __name__ == "__main__": runner_arg = getattr(args, f"{name}_onnx") output_data['metadata'][name] = make_metadata_dict(runner_arg) - first_policy_meta = output_data['metadata'].get('policy', output_data['metadata'].get('model', output_data['metadata'].get('off_policy', {}))) + policy_keys = [key for key in output_data['metadata'].keys() if key != 'vision'] + first_policy_meta = output_data['metadata'][policy_keys[0]] if policy_keys else {} vision_meta = output_data['metadata'].get('vision', {}) derived_frame_skip = args.frame_skip or derive_frame_skip(vision_meta.get('input_shapes', {}), first_policy_meta.get('input_shapes', {})) output_data.update(_compile_for_resolutions(args.camera_resolutions, args.model_size, derived_frame_skip, - vision_runner, policy_runners, output_data['metadata'])) + vision_runner, policy_runners, output_data['metadata'])) with open(args.output, "wb") as file: pickle.dump(output_data, file) From 6e7d9e5e52072f6069df65d75c6347dc46bf0b45 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 7 Jun 2026 11:37:30 -0700 Subject: [PATCH 23/57] done done done --- .github/workflows/sunnypilot-build-model.yaml | 29 ++- cereal/custom.capnp | 6 + release/ci/model_generator.py | 46 +++-- sunnypilot/models/fetcher.py | 13 ++ sunnypilot/models/manager.py | 37 ++-- sunnypilot/models/runners/helpers.py | 28 --- sunnypilot/models/runners/model_runner.py | 174 ----------------- .../models/runners/tinygrad/model_types.py | 91 --------- .../runners/tinygrad/tinygrad_runner.py | 179 ------------------ 9 files changed, 95 insertions(+), 508 deletions(-) delete mode 100644 sunnypilot/models/runners/helpers.py delete mode 100644 sunnypilot/models/runners/model_runner.py delete mode 100644 sunnypilot/models/runners/tinygrad/model_types.py delete mode 100644 sunnypilot/models/runners/tinygrad/tinygrad_runner.py diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 3c5554fcc4..507ee484dc 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -30,6 +30,11 @@ on: required: false type: string default: '' + target_hardware: + description: 'Hardware target to compile for (qcom or usbgpu)' + required: false + type: string + default: 'qcom' workflow_dispatch: inputs: upstream_branch: @@ -46,6 +51,14 @@ on: required: false type: boolean default: true + target_hardware: + description: 'Hardware target to compile for' + required: true + type: choice + options: + - qcom + - usbgpu + default: 'qcom' run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}] @@ -169,7 +182,17 @@ jobs: COMPILE_MODELD="${{ github.workspace }}/sunnypilot/modeld_v2/compile_modeld.py" MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')") CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')") - TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + + if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then + echo "USBGPU build" + export USBGPU=1 + TG_FLAGS="DEV=AMD USBGPU=1 IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl" + else + echo "QCOM build" + TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1" + OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl" + fi # Generate metadata for all ONNX files find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do @@ -203,13 +226,13 @@ jobs: fi if [ -n "$MODEL_TYPE" ]; then - echo "Detected: $MODEL_TYPE -> driving_tinygrad.pkl" + echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL" env ${TG_FLAGS} python3 "$COMPILE_MODELD" \ --model-type $MODEL_TYPE \ --model-size $MODEL_SIZE \ --camera-resolutions $CAMERA_RES \ $ONNX_ARGS \ - --output "${{ env.MODELS_DIR }}/driving_tinygrad.pkl" + --output "$OUTPUT_PKL" fi - name: Validate Model Outputs diff --git a/cereal/custom.capnp b/cereal/custom.capnp index fe3ed9196f..1a6090a8b0 100644 --- a/cereal/custom.capnp +++ b/cereal/custom.capnp @@ -137,10 +137,16 @@ struct ModelManagerSP @0xaedffd8f31e7b55d { eta @2 :UInt32; } + struct Chunk { + fileName @0 :Text; + sha256 @1 :Text; + } + struct Artifact { fileName @0 :Text; downloadUri @1 :DownloadUri; downloadProgress @2 :DownloadProgress; + chunks @3 :List(Chunk); } struct Model { diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index afee782beb..9881bf3e14 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -53,7 +53,7 @@ def validate_model_outputs(metadata_paths: list[Path]) -> None: print(f"Optional output keys detected: {detected_optional}") -def create_short_name(full_name): +def create_short_name(full_name: str) -> str: # Remove parentheses and extract alphanumeric words clean_name = re.sub(r'\([^)]*\)', '', full_name) words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)] @@ -121,7 +121,7 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path: return old_pkl.rename(new_pkl) -def generate_metadata(model_path: Path, output_dir: Path, short_name: str, driving_pkl: Path): +def generate_metadata(model_path: Path, output_dir: Path, short_name: str, driving_pkl: Path) -> dict | None: base = model_path.stem metadata_file = output_dir / f"{base}_metadata.pkl" @@ -134,7 +134,7 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str, drivi if not metadata_file.exists(): print(f"Warning: Missing metadata for {base} ({metadata_file}), skipping", file=sys.stderr) - return + return None tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest() @@ -143,15 +143,33 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str, drivi model_type = "offPolicy" if "off_policy" in base else "onPolicy" if "on_policy" in base else base.split("_")[-1] + chunks_config = [] + manifest_file = Path(f"{driving_pkl}.chunkmanifest") + if manifest_file.exists(): + num_chunks = int(manifest_file.read_text().strip()) + for i in range(num_chunks): + chunk_path = Path(f"{driving_pkl}.chunk{i + 1:02d}of{num_chunks:02d}") + if chunk_path.exists(): + chunk_hash = hashlib.sha256(chunk_path.read_bytes()).hexdigest() + chunks_config.append({ + "file_name": chunk_path.name, + "sha256": chunk_hash + }) + + artifact_data = { + "file_name": driving_pkl.name, + "download_uri": { + "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/", + "sha256": tinygrad_hash + } + } + + if chunks_config: + artifact_data["chunks"] = chunks_config + return { "type": model_type, - "artifact": { - "file_name": driving_pkl.name, - "download_uri": { - "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/", - "sha256": tinygrad_hash - } - }, + "artifact": artifact_data, "metadata": { "file_name": metadata_file.name, "download_uri": { @@ -162,8 +180,8 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str, drivi } -def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown"): - metadata_json = { +def create_metadata_json(models: list, output_dir: Path, custom_name=None, short_name=None, is_20hz=False, upstream_branch="unknown") -> None: + bundle_json = { "short_name": short_name, "display_name": custom_name or upstream_branch, "is_20hz": is_20hz, @@ -179,6 +197,10 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short } # Write metadata to output_dir + metadata_json = { + "bundles": [bundle_json] + } + with open(output_dir / "metadata.json", "w") as f: json.dump(metadata_json, f, indent=2) diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py index 5461666c71..fb51961d15 100644 --- a/sunnypilot/models/fetcher.py +++ b/sunnypilot/models/fetcher.py @@ -26,11 +26,22 @@ class ModelParser: download_uri.sha256 = download_uri_data.get("sha256") return download_uri + @staticmethod + def _parse_chunk(chunk_data) -> custom.ModelManagerSP.Chunk: + chunk = custom.ModelManagerSP.Chunk() + chunk.fileName = chunk_data.get("file_name") + chunk.sha256 = chunk_data.get("sha256") + return chunk + @staticmethod def _parse_artifact(artifact_data) -> custom.ModelManagerSP.Artifact: artifact = custom.ModelManagerSP.Artifact() artifact.fileName = artifact_data.get("file_name") artifact.downloadUri = ModelParser._parse_download_uri(artifact_data.get("download_uri", {})) + + if "chunks" in artifact_data: + artifact.chunks = [ModelParser._parse_chunk(chunk_data) for chunk_data in artifact_data["chunks"]] + return artifact @staticmethod @@ -184,4 +195,6 @@ if __name__ == "__main__": # Print artifact details print(f"Artifact: {model.artifact.fileName}, Download URI: {model.artifact.downloadUri.uri}") # Print metadata details + if model.artifact.chunks: + print(f"Contains {len(model.artifact.chunks)} chunks.") print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") diff --git a/sunnypilot/models/manager.py b/sunnypilot/models/manager.py index 518671181e..b3765d12d8 100644 --- a/sunnypilot/models/manager.py +++ b/sunnypilot/models/manager.py @@ -89,20 +89,16 @@ class ModelManagerSP: del self._download_start_times[model.fileName] async def _download_chunked(self, base_url: str, base_path: str, artifact) -> None: - from openpilot.common.file_chunker import get_manifest_path, get_chunk_name - manifest_url = get_manifest_path(base_url) + from openpilot.common.file_chunker import get_chunk_name, get_manifest_path + + num_chunks = len(artifact.chunks) + if num_chunks == 0: + raise ValueError("No chunks defined in artifact") + manifest_path = get_manifest_path(base_path) - - async with aiohttp.ClientSession() as session: - async with session.get(manifest_url) as resp: - if resp.status == 404: - raise FileNotFoundError - resp.raise_for_status() - num_chunks = int((await resp.read()).strip()) - self._download_start_times[artifact.fileName] = time.monotonic() - for i in range(num_chunks): + for i, _ in enumerate(artifact.chunks): chunk_url = get_chunk_name(base_url, i, num_chunks) chunk_path = get_chunk_name(base_path, i, num_chunks) chunk_downloaded = 0 @@ -117,7 +113,7 @@ class ModelManagerSP: if self.params.get("ModelManager_DownloadIndex") is None: raise Exception("Download cancelled") intra = chunk_downloaded / max(chunk_size, 1) - progress = min(99, (i + intra) / num_chunks * 100) + progress = min(99.0, ((i + intra) / num_chunks) * 100) artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloading artifact.downloadProgress.progress = progress artifact.downloadProgress.eta = self._calculate_eta(artifact.fileName, progress) @@ -148,9 +144,9 @@ class ModelManagerSP: self._report_status() return - try: + if len(artifact.chunks) > 0: await self._download_chunked(url, full_path, artifact) - except (FileNotFoundError, aiohttp.ClientResponseError): + else: await self._download_file(url, full_path, artifact) if not await verify_file(full_path, expected_hash): @@ -170,18 +166,16 @@ class ModelManagerSP: artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.failed artifact.downloadProgress.eta = 0 self._sync_artifact_progress(artifact) - self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + if self.selected_bundle: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed self._report_status() self._download_start_times.pop(artifact.fileName, None) raise async def _process_model(self, model, destination_path: str) -> None: """Processes a single model download including verification""" - model_artifact = model.artifact - metadata_artifact = model.metadata - - await self._process_artifact(metadata_artifact, destination_path) - await self._process_artifact(model_artifact, destination_path) + await self._process_artifact(model.metadata, destination_path) + await self._process_artifact(model.artifact, destination_path) def _report_status(self) -> None: """Reports current status through messaging system""" @@ -222,7 +216,8 @@ class ModelManagerSP: self.selected_bundle = None except Exception: - self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed + if self.selected_bundle: + self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed raise finally: diff --git a/sunnypilot/models/runners/helpers.py b/sunnypilot/models/runners/helpers.py deleted file mode 100644 index b34a62132b..0000000000 --- a/sunnypilot/models/runners/helpers.py +++ /dev/null @@ -1,28 +0,0 @@ -from openpilot.sunnypilot.models.helpers import get_active_bundle -from openpilot.sunnypilot.models.runners.model_runner import ModelRunner -from openpilot.sunnypilot.models.runners.tinygrad.tinygrad_runner import TinygradRunner, TinygradSplitRunner -from openpilot.sunnypilot.models.runners.constants import ModelType - - -def get_model_runner() -> ModelRunner: - """ - Factory function to create and return the appropriate ModelRunner instance. - - Selects TinygradRunner, choosing TinygradSplitRunner if separate vision/policy - models are detected in the active bundle. - - :return: An instance of a ModelRunner subclass (ONNXRunner, TinygradRunner, or TinygradSplitRunner). - """ - bundle = get_active_bundle() - if bundle and bundle.models: - model_types = {m.type.raw for m in bundle.models} - # Check if the bundle uses separate vision and policy models (legacy or new split format) - split_types = {ModelType.vision, ModelType.policy, ModelType.offPolicy, ModelType.onPolicy} - if model_types & split_types: - return TinygradSplitRunner() - # Otherwise, assume a single model (likely supercombo) - if bundle.models: - return TinygradRunner(bundle.models[0].type.raw) - - # Default fallback to TinygradRunner with the supercombo type if bundle info is missing/incomplete - return TinygradRunner(ModelType.supercombo) diff --git a/sunnypilot/models/runners/model_runner.py b/sunnypilot/models/runners/model_runner.py deleted file mode 100644 index 051fa349db..0000000000 --- a/sunnypilot/models/runners/model_runner.py +++ /dev/null @@ -1,174 +0,0 @@ -from abc import abstractmethod, ABC - -import numpy as np -from openpilot.sunnypilot.models.helpers import get_active_bundle -from openpilot.sunnypilot.models.runners.constants import NumpyDict, ShapeDict, Model, SliceDict, SEND_RAW_PRED -from openpilot.system.hardware.hw import Paths -import pickle - -CUSTOM_MODEL_PATH = Paths.model_root() - - -class ModelData: - """ - Stores metadata and configuration for a specific machine learning model. - - This class loads model metadata (like input shapes and output slices) - from a pickle file associated with a model instance. - - :param model: The machine learning model object containing metadata. - """ - def __init__(self, model: Model): - self.model = model - self.metadata = model.metadata - self.input_shapes: ShapeDict = {} - self.output_slices: SliceDict = {} - if self.metadata: - self._load_metadata() - - def _load_metadata(self) -> None: - """Loads input shapes and output slices from the model's metadata pickle file.""" - metadata_path = f"{CUSTOM_MODEL_PATH}/{self.metadata.fileName}" - with open(metadata_path, 'rb') as f: - model_metadata = pickle.load(f) - self.input_shapes = model_metadata.get('input_shapes', {}) - self.output_slices = model_metadata.get('output_slices', {}) - - -class ModularRunner(ABC): - """ - Represents a modular runner for handling and slicing model outputs. - - This abstract base class is designed to provide an interface for modular - parsing and processing of model outputs. Classes inheriting from it must - implement the specified abstract methods, defining how model outputs - should be handled and stored. The primary goal is to enable structured - parsing of outputs through a dictionary-based method mapping. - - :ivar parser_method_dict: Mapping dictionary containing parser methods - for handling specific types of outputs. - :type parser_method_dict: dict - """ - - @property - @abstractmethod - def parser_method_dict(self) -> dict: - pass - - @parser_method_dict.setter - @abstractmethod - def parser_method_dict(self, value: dict) -> None: - pass - - @abstractmethod - def _slice_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - pass - - -class ModelRunner(ModularRunner): - """ - Abstract base class for managing and executing machine learning models. - - Provides a common interface for loading models, preparing inputs, running - inference, and slicing/parsing outputs based on model metadata. Derived - classes implement the specifics of input preparation and model execution - for different frameworks (e.g., Tinygrad, ONNX). - """ - - def __init__(self): - """Initializes the model runner, loading the active model bundle.""" - self.is_20hz: bool | None = None - self.is_20hz_3d: bool | None = None - self.models: dict[int, ModelData] = {} - self._model_data: ModelData | None = None # Active model data for current operation - self._parser_method_dict: dict = {} - self.inputs: dict = {} - self._parser = None - self._load_models() - self._constants = None - - @property - def constants(self): - return self._constants - - @property - def parser_method_dict(self) -> dict: - """Returns the dictionary mapping model types to their respective parsing methods.""" - return self._parser_method_dict - - @parser_method_dict.setter - def parser_method_dict(self, value: dict) -> None: - """Sets the dictionary mapping model types to their respective parsing methods.""" - self._parser_method_dict = value - - def _load_models(self) -> None: - """Loads the active model bundle configuration and sets up ModelData.""" - bundle = get_active_bundle() - if not bundle: - raise ValueError("No active model bundle found, why are we being executed?") - - self.models = {model.type.raw: ModelData(model) for model in bundle.models} - self.is_20hz = bundle.is20hz - self.is_20hz_3d = False - - @property - def input_shapes(self) -> ShapeDict: - """Returns the input shapes for the currently active model.""" - if self._model_data: - return self._model_data.input_shapes - raise ValueError("Model data is not available. Ensure the model is loaded correctly.") - - @property - def output_slices(self) -> SliceDict: - """Returns the output slices for the currently active model.""" - if self._model_data: - return self._model_data.output_slices - raise ValueError("Model data is not available. Ensure the model is loaded correctly.") - - @property - def vision_input_names(self) -> list[str]: - """Returns the list of vision input names from the input shapes.""" - if self._model_data: - return list(self._model_data.input_shapes.keys()) - raise ValueError("Model data is not available. Ensure the model is loaded correctly.") - - @abstractmethod - def prepare_inputs(self, numpy_inputs: NumpyDict) -> dict: - """ - Abstract method to prepare inputs for model inference. - - :param numpy_inputs: Dictionary of numpy arrays for non-image inputs. - :return: Dictionary of prepared inputs ready for the model. - """ - raise NotImplementedError - - @abstractmethod - def _run_model(self) -> NumpyDict: - """ - Abstract method to execute model inference with prepared inputs. - - :return: Dictionary containing the model's raw output arrays. - """ - raise NotImplementedError - - def _slice_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """ - Slices the raw model output array based on the output_slices metadata. - - :param model_outputs: The raw numpy array output from the model. - :return: A dictionary where keys are output names and values are sliced numpy arrays. - """ - if not self._model_data: - raise ValueError("Model data is not available. Ensure the model is loaded correctly.") - sliced_outputs = {k: model_outputs[np.newaxis, v] for k, v in self._model_data.output_slices.items()} - if SEND_RAW_PRED: - sliced_outputs['raw_pred'] = model_outputs.copy() # Optionally include the full raw output - return sliced_outputs - - def run_model(self) -> NumpyDict: - """ - Executes the model inference pipeline: runs the model and parses outputs. - - :return: Dictionary containing the final parsed model outputs. - """ - return self._run_model() # Parsing is handled within specific runner implementations diff --git a/sunnypilot/models/runners/tinygrad/model_types.py b/sunnypilot/models/runners/tinygrad/model_types.py deleted file mode 100644 index 015adc035f..0000000000 --- a/sunnypilot/models/runners/tinygrad/model_types.py +++ /dev/null @@ -1,91 +0,0 @@ -import os -from abc import ABC - -import numpy as np -from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser -from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser -from openpilot.sunnypilot.models.runners.constants import ModelType, NumpyDict -from openpilot.sunnypilot.models.runners.model_runner import ModularRunner -from openpilot.system.hardware.hw import Paths - - -SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') -CUSTOM_MODEL_PATH = Paths.model_root() - - -class OffPolicyTinygrad(ModularRunner, ABC): - """ - A TinygradRunner specialized for off-policy models. - - Uses a SplitParser to handle outputs specific to the off-policy part of a split model setup. - """ - def __init__(self): - self._off_policy_parser = SplitParser() - self.parser_method_dict[ModelType.offPolicy] = self._parse_off_policy_outputs - - def _parse_off_policy_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses off-policy model outputs using SplitParser.""" - result: NumpyDict = self._off_policy_parser.parse_policy_outputs(self._slice_outputs(model_outputs)) - return result - - -class OnPolicyTinygrad(ModularRunner, ABC): - """ - A TinygradRunner specialized for on-policy models. - - Uses a SplitParser to handle outputs specific to the on-policy part of a split model setup. - """ - def __init__(self): - self._on_policy_parser = SplitParser() - self.parser_method_dict[ModelType.onPolicy] = self._parse_on_policy_outputs - - def _parse_on_policy_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses on-policy model outputs using SplitParser.""" - result: NumpyDict = self._on_policy_parser.parse_policy_outputs(self._slice_outputs(model_outputs)) - return result - - -class PolicyTinygrad(ModularRunner, ABC): - """ - A TinygradRunner specialized for policy-only models. - - Uses a SplitParser to handle outputs specific to the policy part of a split model setup. - """ - def __init__(self): - self._policy_parser = SplitParser() - self.parser_method_dict[ModelType.policy] = self._parse_policy_outputs - - def _parse_policy_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses policy model outputs using SplitParser.""" - result: NumpyDict = self._policy_parser.parse_policy_outputs(self._slice_outputs(model_outputs)) - return result - -class VisionTinygrad(ModularRunner, ABC): - """ - A TinygradRunner specialized for vision-only models. - - Uses a SplitParser to handle outputs specific to the vision part of a split model setup. - """ - def __init__(self): - self._vision_parser = SplitParser() - self.parser_method_dict[ModelType.vision] = self._parse_vision_outputs - - def _parse_vision_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses vision model outputs using SplitParser.""" - result: NumpyDict = self._vision_parser.parse_vision_outputs(self._slice_outputs(model_outputs)) - return result - -class SupercomboTinygrad(ModularRunner, ABC): - """ - A TinygradRunner specialized for vision-only models. - - Uses a SplitParser to handle outputs specific to the vision part of a split model setup. - """ - def __init__(self): - self._supercombo_parser = CombinedParser() - self.parser_method_dict[ModelType.supercombo] = self._parse_supercombo_outputs - - def _parse_supercombo_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses vision model outputs using SplitParser.""" - result: NumpyDict = self._supercombo_parser.parse_outputs(self._slice_outputs(model_outputs)) - return result diff --git a/sunnypilot/models/runners/tinygrad/tinygrad_runner.py b/sunnypilot/models/runners/tinygrad/tinygrad_runner.py deleted file mode 100644 index 4e17bd5ead..0000000000 --- a/sunnypilot/models/runners/tinygrad/tinygrad_runner.py +++ /dev/null @@ -1,179 +0,0 @@ -import pickle - -import numpy as np -from openpilot.sunnypilot.models.runners.constants import NumpyDict, ModelType, ShapeDict, CUSTOM_MODEL_PATH, SliceDict -from openpilot.sunnypilot.models.runners.model_runner import ModelRunner -from openpilot.sunnypilot.models.runners.tinygrad.model_types import PolicyTinygrad, VisionTinygrad, SupercomboTinygrad, OffPolicyTinygrad, OnPolicyTinygrad -from openpilot.sunnypilot.models.split_model_constants import SplitModelConstants -from openpilot.sunnypilot.modeld_v2.constants import ModelConstants - -from tinygrad.tensor import Tensor - - -class TinygradRunner(ModelRunner, SupercomboTinygrad, PolicyTinygrad, VisionTinygrad, OffPolicyTinygrad, OnPolicyTinygrad): - """ - A ModelRunner implementation for executing Tinygrad models. - - Handles loading Tinygrad model artifacts (.pkl), preparing inputs as Tinygrad - Tensors (potentially using QCOM extensions on TICI), running inference, - and parsing the outputs. - - :param model_type: The type of model (e.g., supercombo) to load and run. - """ - def __init__(self, model_type: int = ModelType.supercombo): - ModelRunner.__init__(self) - SupercomboTinygrad.__init__(self) - PolicyTinygrad.__init__(self) - VisionTinygrad.__init__(self) - OffPolicyTinygrad.__init__(self) - OnPolicyTinygrad.__init__(self) - self._constants = ModelConstants - self._model_data = self.models.get(model_type) - if not self._model_data or not self._model_data.model: - raise ValueError(f"Model data for type {model_type} not available.") - - artifact_filename = self._model_data.model.artifact.fileName - assert artifact_filename.endswith('_tinygrad.pkl'), \ - f"Invalid model file {artifact_filename} for TinygradRunner" - - model_pkl_path = f"{CUSTOM_MODEL_PATH}/{artifact_filename}" - with open(model_pkl_path, "rb") as f: - try: - # Load the compiled Tinygrad model runner function - self.model_run = pickle.load(f) - except FileNotFoundError as e: - # Provide a helpful error message if the model was built for a different platform - assert "/dev/kgsl-3d0" not in str(e), "Model was built on C3 or C3X, but is being loaded on PC" - raise - - # Map input names to their required dtype and device from the loaded model - self.input_to_dtype = {} - self.input_to_device = {} - for idx, name in enumerate(self.model_run.captured.expected_names): - info = self.model_run.captured.expected_input_info[idx] - self.input_to_dtype[name] = info[2] # dtype - self.input_to_device[name] = info[3] # device - self._policy_cached = False - - @property - def vision_input_names(self) -> list[str]: - """Returns the list of vision input names from the input shapes.""" - return [name for name in self.input_shapes.keys() if 'img' in name] - - - def prepare_policy_inputs(self, numpy_inputs: NumpyDict): - if not self._policy_cached: - for key, value in numpy_inputs.items(): - self.inputs[key] = Tensor(value, device='NPY').realize() - self._policy_cached = True - - def prepare_inputs(self, numpy_inputs: NumpyDict) -> dict: - """Prepares all vision and policy inputs for the model.""" - self.prepare_policy_inputs(numpy_inputs) - for key in self.vision_input_names: - if key in self.inputs: - self.inputs[key] = self.inputs[key].cast(self.input_to_dtype[key]) - return self.inputs - - def _run_model(self) -> NumpyDict: - """Runs the Tinygrad model inference and parses the outputs.""" - outputs = self.model_run(**self.inputs).contiguous().realize().uop.base.buffer.numpy().flatten() - return self._parse_outputs(outputs) - - def _parse_outputs(self, model_outputs: np.ndarray) -> NumpyDict: - """Parses the raw model outputs using the standard Parser.""" - if self._model_data is None: - raise ValueError("Model data is not available. Ensure the model is loaded correctly.") - - result: NumpyDict = self.parser_method_dict[self._model_data.model.type.raw](model_outputs) - return result - - -class TinygradSplitRunner(ModelRunner): - """ - A ModelRunner that coordinates separate TinygradVisionRunner and TinygradPolicyRunner instances. - - Manages the execution of split vision and policy models, combining their inputs and outputs. - """ - def __init__(self): - super().__init__() - self.is_20hz_3d = True - self.vision_runner = TinygradRunner(ModelType.vision) - self.policy_runner = TinygradRunner(ModelType.policy) if self.models.get(ModelType.policy) else None - self.off_policy_runner = TinygradRunner(ModelType.offPolicy) if self.models.get(ModelType.offPolicy) else None - self.on_policy_runner = TinygradRunner(ModelType.onPolicy) if self.models.get(ModelType.onPolicy) else None - self._constants = SplitModelConstants - - def _run_model(self) -> NumpyDict: - """Runs both vision and policy models and merges their parsed outputs.""" - vision_output = self.vision_runner.run_model() - outputs = {**vision_output} - - if self.policy_runner: - policy_output = self.policy_runner.run_model() - outputs.update(policy_output) - - if self.off_policy_runner: - off_policy_output = self.off_policy_runner.run_model() - if self.on_policy_runner: - off_policy_output.pop('plan', None) - outputs.update(off_policy_output) - - if self.on_policy_runner: - on_policy_output = self.on_policy_runner.run_model() - outputs.update(on_policy_output) - - if 'planplus' in outputs and 'plan' in outputs: - outputs['plan'] = outputs['plan'] + outputs['planplus'] - - return outputs - - @property - def vision_input_names(self) -> list[str]: - """Returns the list of vision input names from the vision runner.""" - return list(self.vision_runner.vision_input_names) - - @property - def input_shapes(self) -> ShapeDict: - """Returns the combined input shapes from both vision and policy models.""" - shapes = {**self.vision_runner.input_shapes} - if self.policy_runner: - shapes.update(self.policy_runner.input_shapes) - if self.off_policy_runner: - shapes.update(self.off_policy_runner.input_shapes) - if self.on_policy_runner: - shapes.update(self.on_policy_runner.input_shapes) - return shapes - - @property - def output_slices(self) -> SliceDict: - """Returns the combined output slices from both vision and policy models.""" - slices = {**self.vision_runner.output_slices} - if self.policy_runner: - slices.update(self.policy_runner.output_slices) - if self.off_policy_runner: - slices.update(self.off_policy_runner.output_slices) - if self.on_policy_runner: - slices.update(self.on_policy_runner.output_slices) - return slices - - def prepare_inputs(self, numpy_inputs: NumpyDict) -> dict: - """Prepares inputs for both vision and policy models.""" - if self.policy_runner: - self.policy_runner.prepare_policy_inputs(numpy_inputs) - - for key in self.vision_input_names: - if key in self.inputs: - self.vision_runner.inputs[key] = self.inputs[key].cast(self.vision_runner.input_to_dtype[key]) - - inputs = {**self.vision_runner.inputs} - if self.policy_runner: - inputs.update(self.policy_runner.inputs) - - if self.off_policy_runner: - self.off_policy_runner.prepare_policy_inputs(numpy_inputs) - inputs.update(self.off_policy_runner.inputs) - if self.on_policy_runner: - self.on_policy_runner.prepare_policy_inputs(numpy_inputs) - inputs.update(self.on_policy_runner.inputs) - return inputs From fa284be7e6ed80ea0d271e34dd05eea10c54343f Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Fri, 12 Jun 2026 03:12:13 -0700 Subject: [PATCH 24/57] bye metadata --- cereal/custom.capnp | 1 + release/ci/model_generator.py | 9 +-------- sunnypilot/models/fetcher.py | 3 ++- sunnypilot/models/manager.py | 1 - 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/cereal/custom.capnp b/cereal/custom.capnp index 1a6090a8b0..df8b420568 100644 --- a/cereal/custom.capnp +++ b/cereal/custom.capnp @@ -161,6 +161,7 @@ struct ModelManagerSP @0xaedffd8f31e7b55d { policy @3; offPolicy @4; onPolicy @5; + chunked @6; } } diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 9881bf3e14..38a6386cfb 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -168,15 +168,8 @@ def generate_metadata(model_path: Path, output_dir: Path, short_name: str, drivi artifact_data["chunks"] = chunks_config return { - "type": model_type, + "type": "chunked", "artifact": artifact_data, - "metadata": { - "file_name": metadata_file.name, - "download_uri": { - "url": "https://gitlab.com/sunnypilot/public/docs.sunnypilot.ai/-/raw/main/", - "sha256": metadata_hash - } - } } diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py index fb51961d15..e3d785b84e 100644 --- a/sunnypilot/models/fetcher.py +++ b/sunnypilot/models/fetcher.py @@ -197,4 +197,5 @@ if __name__ == "__main__": # Print metadata details if model.artifact.chunks: print(f"Contains {len(model.artifact.chunks)} chunks.") - print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") + if model.metadata.fileName: + print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") diff --git a/sunnypilot/models/manager.py b/sunnypilot/models/manager.py index b3765d12d8..f4db071c7b 100644 --- a/sunnypilot/models/manager.py +++ b/sunnypilot/models/manager.py @@ -174,7 +174,6 @@ class ModelManagerSP: async def _process_model(self, model, destination_path: str) -> None: """Processes a single model download including verification""" - await self._process_artifact(model.metadata, destination_path) await self._process_artifact(model.artifact, destination_path) def _report_status(self) -> None: From 91316c8cb59d4365a80155a7244e15c58c9a47f4 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Fri, 12 Jun 2026 03:38:33 -0700 Subject: [PATCH 25/57] simplify --- release/ci/model_generator.py | 98 +++-------------------------------- 1 file changed, 6 insertions(+), 92 deletions(-) diff --git a/release/ci/model_generator.py b/release/ci/model_generator.py index 38a6386cfb..76935f3627 100755 --- a/release/ci/model_generator.py +++ b/release/ci/model_generator.py @@ -5,8 +5,6 @@ This file is part of sunnypilot and is licensed under the MIT License. See the LICENSE.md file in the root directory for more details. """ -import os -import pickle import sys import hashlib import json @@ -14,44 +12,6 @@ import re from pathlib import Path from datetime import datetime, UTC -REQUIRED_OUTPUT_KEYS = frozenset({ - "plan", - "lane_lines", - "road_edges", - "lead", - "desire_state", - "desire_pred", - "meta", - "lead_prob", - "lane_lines_prob", - "pose", - "wide_from_device_euler", - "road_transform", - "hidden_state", -}) -OPTIONAL_OUTPUT_KEYS = frozenset({ - "planplus", - "sim_pose", - "desired_curvature", -}) - - -def validate_model_outputs(metadata_paths: list[Path]) -> None: - combined_keys: set[str] = set() - for path in metadata_paths: - if path.stat().st_size == 0: - print(f"skipping empty metadata: {path}") - continue - with open(path, "rb") as f: - metadata = pickle.load(f) - combined_keys.update(metadata.get("output_slices", {}).keys()) - missing = REQUIRED_OUTPUT_KEYS - combined_keys - if missing: - raise ValueError(f"Combined model metadata is missing required output keys: {sorted(missing)}") - detected_optional = sorted(OPTIONAL_OUTPUT_KEYS & combined_keys) - if detected_optional: - print(f"Optional output keys detected: {detected_optional}") - def create_short_name(full_name: str) -> str: # Remove parentheses and extract alphanumeric words @@ -121,28 +81,9 @@ def _rename_pkl_with_chunks(old_pkl: Path, new_pkl: Path) -> Path: return old_pkl.rename(new_pkl) -def generate_metadata(model_path: Path, output_dir: Path, short_name: str, driving_pkl: Path) -> dict | None: - base = model_path.stem - metadata_file = output_dir / f"{base}_metadata.pkl" - - if short_name: - renamed_meta = output_dir / f"{base}_{short_name.lower()}_metadata.pkl" - if metadata_file.exists() and not renamed_meta.exists(): - metadata_file = metadata_file.rename(renamed_meta) - elif renamed_meta.exists(): - metadata_file = renamed_meta - - if not metadata_file.exists(): - print(f"Warning: Missing metadata for {base} ({metadata_file}), skipping", file=sys.stderr) - return None - +def generate_chunked_model(driving_pkl: Path) -> dict: tinygrad_hash = hashlib.sha256(_read_pkl_bytes(driving_pkl)).hexdigest() - with open(metadata_file, 'rb') as f: - metadata_hash = hashlib.sha256(f.read()).hexdigest() - - model_type = "offPolicy" if "off_policy" in base else "onPolicy" if "on_policy" in base else base.split("_")[-1] - chunks_config = [] manifest_file = Path(f"{driving_pkl}.chunkmanifest") if manifest_file.exists(): @@ -196,38 +137,20 @@ def create_metadata_json(models: list, output_dir: Path, custom_name=None, short with open(output_dir / "metadata.json", "w") as f: json.dump(metadata_json, f, indent=2) - - print(f"Generated metadata.json with {len(models)} models.") + print("Generated metadata.json") if __name__ == "__main__": import argparse - import glob - parser = argparse.ArgumentParser(description="Generate metadata for model files") - parser.add_argument("--model-dir", default="./models", help="Directory containing ONNX model files") + parser = argparse.ArgumentParser(description="Generate metadata JSON for the compiled JIT model") + parser.add_argument("--model-dir", default="./models", help="Directory containing the model files") parser.add_argument("--output-dir", default="./output", help="Output directory for metadata") parser.add_argument("--custom-name", help="Custom display name for the model") parser.add_argument("--is-20hz", action="store_true", help="Whether this is a 20Hz model") - parser.add_argument("--validate-only", action="store_true") parser.add_argument("--upstream-branch", default="unknown", help="Upstream branch name") args = parser.parse_args() - if args.validate_only: - metadata_paths = glob.glob(os.path.join(args.model_dir, "*_metadata.pkl")) - if not metadata_paths: - print(f"No metadata files found in {args.model_dir}", file=sys.stderr) - sys.exit(1) - validate_model_outputs([Path(p) for p in metadata_paths]) - print(f"Validated {len(metadata_paths)} metadata files successfully.") - sys.exit(0) - - # Find all ONNX files in the given directory - model_paths = glob.glob(os.path.join(args.model_dir, "*.onnx")) - if not model_paths: - print(f"No ONNX files found in {args.model_dir}", file=sys.stderr) - sys.exit(1) - _output_dir = Path(args.output_dir) _output_dir.mkdir(exist_ok=True, parents=True) _short_name = create_short_name(args.custom_name) if args.custom_name else None @@ -244,14 +167,5 @@ if __name__ == "__main__": else: _driving_pkl = new_pkl - _models = [] - - for _model_path in model_paths: - _model_metadata = generate_metadata(Path(_model_path), _output_dir, _short_name, _driving_pkl) - if _model_metadata: - _models.append(_model_metadata) - - if _models: - create_metadata_json(_models, _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch) - else: - print("No models processed.", file=sys.stderr) + _model_metadata = generate_chunked_model(_driving_pkl) + create_metadata_json([_model_metadata], _output_dir, args.custom_name, _short_name, args.is_20hz, args.upstream_branch) From d215eab1d4baa198eaa0274af9c9e4ee16eb8cdf Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Fri, 12 Jun 2026 03:49:03 -0700 Subject: [PATCH 26/57] deeeeep --- sunnypilot/models/fetcher.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py index e3d785b84e..e0829c8909 100644 --- a/sunnypilot/models/fetcher.py +++ b/sunnypilot/models/fetcher.py @@ -6,11 +6,12 @@ See the LICENSE.md file in the root directory for more details. """ import time - +import os import requests from requests.exceptions import (SSLError, RequestException, HTTPError) from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog +from openpilot.system.hardware.hw import Paths from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible from cereal import custom @@ -42,6 +43,19 @@ class ModelParser: if "chunks" in artifact_data: artifact.chunks = [ModelParser._parse_chunk(chunk_data) for chunk_data in artifact_data["chunks"]] + try: + model_dir = Paths.model_root() + os.makedirs(model_dir, exist_ok=True) + manifest_path = os.path.join(model_dir, f"{artifact.fileName}.chunkmanifest") + num_chunks = str(len(artifact.chunks)) + + if not os.path.exists(manifest_path) or open(manifest_path).read().strip() != num_chunks: + with open(manifest_path, "w") as f: + f.write(num_chunks) + cloudlog.info(f"Wrote chunk manifest for {artifact.fileName}: {num_chunks} chunks") + except Exception as e: + cloudlog.warning(f"Failed to write chunk manifest for {artifact.fileName}: {e}") + return artifact @staticmethod @@ -197,5 +211,5 @@ if __name__ == "__main__": # Print metadata details if model.artifact.chunks: print(f"Contains {len(model.artifact.chunks)} chunks.") - if model.metadata.fileName: + if hasattr(model, 'metadata') and model.metadata and model.metadata.fileName: print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") From 70424bd66190bbe7d7d60c1e9687a259d6861d14 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Fri, 12 Jun 2026 04:08:47 -0700 Subject: [PATCH 27/57] oopsie --- .github/workflows/sunnypilot-build-model.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 507ee484dc..df485fa6a1 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -235,15 +235,6 @@ jobs: --output "$OUTPUT_PKL" fi - - name: Validate Model Outputs - run: | - source /etc/profile - export UV_PROJECT_ENVIRONMENT=${HOME}/venv - export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT - python3 "${{ github.workspace }}/release/ci/model_generator.py" \ - --validate-only \ - --model-dir "${{ env.MODELS_DIR }}" - - name: Prepare Output run: | sudo rm -rf ${{ env.OUTPUT_DIR }} From 76b21c72cff24bb3a062fb30206aca003f01e722 Mon Sep 17 00:00:00 2001 From: nayan Date: Fri, 24 Jul 2026 22:40:40 -0400 Subject: [PATCH 28/57] i don't know what i'm doing --- sunnypilot/modeld_v2/modeld.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sunnypilot/modeld_v2/modeld.py b/sunnypilot/modeld_v2/modeld.py index 1f28e2d16e..6edc919ff7 100755 --- a/sunnypilot/modeld_v2/modeld.py +++ b/sunnypilot/modeld_v2/modeld.py @@ -230,8 +230,12 @@ class ModelState(ModelStateBase): policy_output = raw_outputs[i + 1].numpy().flatten() policy_sliced = {k: policy_output[np.newaxis, v] for k, v in policy_slices.items()} parsed = self.parser.parse_policy_outputs(policy_sliced) - if 'off' in self._policy_keys[i] and self._has_on_policy: + if ('off' in self._policy_keys[i] + and self._has_on_policy + and any('plan' in self._policy_slices_list[j] for j, k in enumerate(self._policy_keys) if 'on' in k.lower())): + parsed.pop('plan', None) + outputs.update(parsed) if 'planplus' in outputs and 'plan' in outputs: From 110568a9d117571774a08a829c72a66975335166 Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 16:28:47 -0400 Subject: [PATCH 29/57] it's a supercombo --- .github/workflows/sunnypilot-build-model.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index dd04ab88b9..f5e7027274 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -174,7 +174,7 @@ jobs: name: models-${{ env.REF }}${{ inputs.artifact_suffix }} path: ${{ env.MODELS_DIR }} - run: | - rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx + rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big-driving-supercombo}.onnx - name: Build Model run: | @@ -209,7 +209,13 @@ jobs: POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx" OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx" ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx" - SUPERCOMBO_ONNX="${{ env.MODELS_DIR }}/supercombo.onnx" + SUPERCOMBO_ONNX="" + for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving-supercombo.onnx"; do + if [ -f "$f" ]; then + SUPERCOMBO_ONNX="$f" + break + fi + done MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME="" if [ -f "$VISION_ONNX" ]; then @@ -224,7 +230,7 @@ jobs: MODEL_TYPE=vision_policy ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX" fi - elif [ -f "$SUPERCOMBO_ONNX" ]; then + elif [ -n "$SUPERCOMBO_ONNX" ]; then MODEL_TYPE=supercombo ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX" fi From 91e40b80d89a925fdefcfe43d6421545de32a5ba Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 16:34:01 -0400 Subject: [PATCH 30/57] wtf. ghostwriter --- .github/workflows/sunnypilot-build-model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index f5e7027274..09d47d6614 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -230,7 +230,7 @@ jobs: MODEL_TYPE=vision_policy ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX" fi - elif [ -n "$SUPERCOMBO_ONNX" ]; then + elif [ -f "$SUPERCOMBO_ONNX" ]; then MODEL_TYPE=supercombo ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX" fi From bee1cdd45d1a34abd0dce3e08277589ad2ba3619 Mon Sep 17 00:00:00 2001 From: Nayan Date: Sat, 25 Jul 2026 16:47:01 -0400 Subject: [PATCH 31/57] i might be blind --- .github/workflows/sunnypilot-build-model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 09d47d6614..756c7bcee2 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -174,7 +174,7 @@ jobs: name: models-${{ env.REF }}${{ inputs.artifact_suffix }} path: ${{ env.MODELS_DIR }} - run: | - rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big-driving-supercombo}.onnx + rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big_driving_supercombo}.onnx - name: Build Model run: | From 98254867a9db2633c1ed5f7d5899e22596e2fef6 Mon Sep 17 00:00:00 2001 From: Nayan Date: Sat, 25 Jul 2026 16:49:07 -0400 Subject: [PATCH 32/57] fuck. i AM blind. or dumb. or both. --- .github/workflows/sunnypilot-build-model.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 756c7bcee2..ccc2e784c4 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -210,7 +210,7 @@ jobs: OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx" ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx" SUPERCOMBO_ONNX="" - for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving-supercombo.onnx"; do + for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do if [ -f "$f" ]; then SUPERCOMBO_ONNX="$f" break From 4cbe97fd200b955f83ff18b184c8e900714f6952 Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 17:29:18 -0400 Subject: [PATCH 33/57] hmmmm --- .github/workflows/sunnypilot-build-model.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sunnypilot-build-model.yaml b/.github/workflows/sunnypilot-build-model.yaml index 09d47d6614..e6b0e66f09 100644 --- a/.github/workflows/sunnypilot-build-model.yaml +++ b/.github/workflows/sunnypilot-build-model.yaml @@ -181,6 +181,7 @@ jobs: source /etc/profile export UV_PROJECT_ENVIRONMENT=${HOME}/venv export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT + source ${UV_PROJECT_ENVIRONMENT}/bin/activate export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}" COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py" From 8092f134382362d67cc3c551e8daf20a1527c0f0 Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 17:43:59 -0400 Subject: [PATCH 34/57] read/open - what's the difference --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 3140c04ff1..68c33985bd 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -205,12 +205,13 @@ def read_file_chunked_to_shm(path): if not path: return None import atexit - from openpilot.common.file_chunker import read_file_chunked - from openpilot.system.hardware.hw import Paths + import shutil + from openpilot.common.file_chunker import open_file_chunked + from openpilot.common.hardware.hw import Paths shm_path = os.path.join(Paths.shm_path(), os.path.basename(path)) atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path)) - with open(shm_path, 'wb') as f: - f.write(read_file_chunked(path)) + with open(shm_path, 'wb') as dst, open_file_chunked(path) as src: + shutil.copyfileobj(src, dst) return shm_path From e143888eb1425cf2b6b84abe13ca31f298f2c3ac Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 18:08:17 -0400 Subject: [PATCH 35/57] realize. that i don't know shit --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 68c33985bd..200ad1db1b 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -135,13 +135,13 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode tfm_dev, big_tfm_dev, desire_dev = npys[:3] traffic_conv_dev = npys[3] if traffic_convention is not None else None - img = shift_and_sample(img_q, frame_prepare(frame, tfm_dev).unsqueeze(0), sample_skip_fn) - big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm_dev).unsqueeze(0), sample_skip_fn) + img = shift_and_sample(img_q, frame_prepare(frame, tfm_dev).unsqueeze(0), sample_skip_fn).realize() + big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm_dev).unsqueeze(0), sample_skip_fn).realize() if prepare_only: return img, big_img - desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn) + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() inputs = {desire_key: desire_buf, **extra_tensors} if traffic_conv_dev is not None: From 273eb9f9974136bd8a72bbc7d7969904e4cce2bd Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 18:30:29 -0400 Subject: [PATCH 36/57] whatever --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 200ad1db1b..d4ea49bb45 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -148,14 +148,13 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode inputs['traffic_convention'] = traffic_conv_dev if vision_runner: - vision_out = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())) - vision_out_cast = vision_out.cast('float32') + vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize() new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0) inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() - policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32') for pol_runner in policy_runners] + policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32').realize() for pol_runner in policy_runners] return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0]) inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) - policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32') + policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize() new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() return policy_out From 3ec226474a46ff299f4dcdf4097d2722a10cbf6e Mon Sep 17 00:00:00 2001 From: nayan Date: Sat, 25 Jul 2026 20:22:04 -0400 Subject: [PATCH 37/57] fix paths --- openpilot/sunnypilot/models/fetcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index 8b545e14a6..655c4163a3 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -11,7 +11,7 @@ import requests from requests.exceptions import (SSLError, RequestException, HTTPError) from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog -from openpilot.system.hardware.hw import Paths +from openpilot.common.hardware.hw import Paths from openpilot.sunnypilot.models.helpers import is_bundle_version_compatible from openpilot.cereal import custom From 8c3cd2575f8ad3c6921414b89cebcedc6f8cdfa2 Mon Sep 17 00:00:00 2001 From: nayan Date: Mon, 27 Jul 2026 16:14:34 -0400 Subject: [PATCH 38/57] fuckit. dynamic everything. --- .../sunnypilot/modeld_v2/mhp_inference.py | 128 +++++++++++++ openpilot/sunnypilot/modeld_v2/modeld.py | 24 ++- .../modeld_v2/parse_model_outputs.py | 74 +++++--- .../modeld_v2/parse_model_outputs_split.py | 91 +++++---- .../modeld_v2/tests/test_dynamic_mhp.py | 172 ++++++++++++++++++ 5 files changed, 428 insertions(+), 61 deletions(-) create mode 100644 openpilot/sunnypilot/modeld_v2/mhp_inference.py create mode 100644 openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py diff --git a/openpilot/sunnypilot/modeld_v2/mhp_inference.py b/openpilot/sunnypilot/modeld_v2/mhp_inference.py new file mode 100644 index 0000000000..e686c18c0e --- /dev/null +++ b/openpilot/sunnypilot/modeld_v2/mhp_inference.py @@ -0,0 +1,128 @@ +"""Infer MHP (mixture-density-hypothesis) parameter values from output slice sizes. + +The legacy supercombo encoded its outputs using fixed values for +``PLAN_MHP_N``, ``PLAN_MHP_SELECTION``, ``LEAD_MHP_N``, and ``LEAD_MHP_SELECTION``. +Newer supercombo architectures emit differently-sized slices for the same heads, +yet the parser downstream still expects to be told the layout. + +This module figures those numbers out at runtime from the model's output slice, +so the existing ``parse_mdn`` path can handle any supercombo flavor without code +changes for each variant. ``infer_mhp`` always tries the legacy values first so +existing compiled pkls behave identically (full backwards compatibility). + +Schema reminder (``parse_mdn`` packs hypotheses contiguously in the order +``in_N × (mu | std | weights)`` along the channel axis): + + slice_size == in_N × (2·n_values + out_N) + +where ``n_values`` is the per-hypothesis value width (``IDX_N × PLAN_WIDTH`` +for plan, ``LEAD_TRAJ_LEN × LEAD_WIDTH`` for lead, etc.). +""" + +def infer_mhp( + slice_size: int, + n_values: int, + legacy_in_n: int, + legacy_out_n: int, + max_in_n: int = 16, +) -> tuple[int, int]: + """Infer ``(in_N, out_N)`` for an MDN-encoded output slice. + + Tries values in this order, returning on the first match: + + 1. ``(legacy_in_n, legacy_out_n)`` exactly — preserves exact backwards + compatibility for existing supercombo pkls. + 2. ``out_N ∈ (0, 1, 3)`` (no weights, single weight, three-way selection) at + increasing in_N — covers the common architectural patterns. + 3. Brute-force any valid ``out_N`` that divides ``slice_size``. + + Args: + slice_size: Number of floats in the output slice (typically + ``slices[name].stop - slices[name].start``). + n_values: Per-hypothesis mu/std width (e.g. ``IDX_N × PLAN_WIDTH`` for plan). + legacy_in_n: The legacy in_N value (highest priority for backwards compat). + legacy_out_n: The legacy out_N value. + max_in_n: Upper bound on accepted hypothesis counts (filters silly parses). + + Returns: + ``(in_N, out_N)``. If nothing fits the formulas, returns ``(1, 0)`` — + single hypothesis with no weights — which is the gentlest fallback. + """ + if slice_size <= 0 or n_values <= 0: + return 1, 0 + + # Priority 1: exact legacy match (BC-preserving). + per_hyp_legacy = 2 * n_values + legacy_out_n + if per_hyp_legacy > 0 and legacy_in_n * per_hyp_legacy == slice_size: + return legacy_in_n, legacy_out_n + + # Priority 2: common weight layouts across supercombo variants. + for out_n in (0, 1, 3): + per_hyp = 2 * n_values + out_n + if per_hyp <= 0: + continue + if slice_size % per_hyp == 0: + in_n = slice_size // per_hyp + if 1 <= in_n <= max_in_n: + return in_n, out_n + + # Priority 3: brute-force any divisor that yields a sensible in_N. + # Bound out_n by max_in_n to keep the search tiny (3 hypotheses + # of weights is already a lot). + for out_n in range(0, max_in_n + 1): + per_hyp = 2 * n_values + out_n + if per_hyp <= 0: + continue + if slice_size % per_hyp == 0: + in_n = slice_size // per_hyp + if 1 <= in_n <= max_in_n: + return in_n, out_n + + # Last resort: best-effort single hypothesis with no weights. + return 1, 0 + + +def slice_size(sl) -> int: + """Return the float-width of a ``slice``/``None`` from ``output_slices``. + + ``None`` and ``slice(None, None, None)`` are treated as 0 (output absent). + Negative-end slices (ONNX-style "from end") aren't supported by this helper + because parser expects single-tensor packed outputs. + """ + if sl is None: + return 0 + start = 0 if sl.start is None else sl.start + stop = sl.stop + if stop is None or stop < 0: + return 0 + return max(0, stop - start) + + +def infer_mhp_for_outputs( + output_slices: dict, + constants, + max_in_n: int = 16, +) -> dict: + """Build a dict of MHP values keyed by head name from a model's output_slices. + + Reads ``output_slices['plan']`` and ``output_slices['lead']`` (if present) + and infers their ``in_N``/``out_N``. Other heads aren't MDN-encoded the same + way and remain driven by the ``constants`` module. + """ + config: dict[str, int] = {} + + plan_size = slice_size(output_slices.get('plan')) + if plan_size > 0: + n = constants.IDX_N * constants.PLAN_WIDTH + in_n, out_n = infer_mhp(plan_size, n, constants.PLAN_MHP_N, constants.PLAN_MHP_SELECTION, max_in_n) + config['plan_mhp_n'] = in_n + config['plan_mhp_selection'] = out_n + + lead_size = slice_size(output_slices.get('lead')) + if lead_size > 0: + n = constants.LEAD_TRAJ_LEN * constants.LEAD_WIDTH + in_n, out_n = infer_mhp(lead_size, n, constants.LEAD_MHP_N, constants.LEAD_MHP_SELECTION, max_in_n) + config['lead_mhp_n'] = in_n + config['lead_mhp_selection'] = out_n + + return config diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 31ed7a7340..5182a191ab 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -148,10 +148,6 @@ class ModelState(ModelStateBase): self._road_key = next(key for key in self._vision_input_names if 'big' not in key) self._wide_key = next(key for key in self._vision_input_names if 'big' in key) - from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser - from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser - self.parser = SplitParser() if self._combined_model_type != 'supercombo' else CombinedParser() - is_20hz = bundle.is20hz if bundle else self._combined_model_type in ('split', 'multi_policy') if is_20hz: from openpilot.sunnypilot.models.split_model_constants import SplitModelConstants @@ -160,6 +156,26 @@ class ModelState(ModelStateBase): from openpilot.sunnypilot.modeld_v2.constants import ModelConstants self.constants = ModelConstants() + # Derive the parser's per-head MHP values from the appropriate output + # slices. Legacy pkls fall back to the constants values (Priority 1 in + # ``mhp_inference.infer_mhp``), so this is fully backwards compatible. + # Supercombo pkls carry plan/lead in the vision tensor; split/multi-policy + # pkls carry them on the policy tensor (and even there we use the first + # policy's slices -- the existing code only tracks one ``policy_output_slices``). + from openpilot.sunnypilot.modeld_v2.mhp_inference import infer_mhp_for_outputs + + if self._combined_model_type == 'supercombo': + mhp_config = infer_mhp_for_outputs(self.vision_output_slices, self.constants) + else: + mhp_config = infer_mhp_for_outputs(self.policy_output_slices, self.constants) + + from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser + from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser + if self._combined_model_type != 'supercombo': + self.parser = SplitParser(mhp_config=mhp_config) + else: + self.parser = CombinedParser(mhp_config=mhp_config) + self.prev_desire = np.zeros(self.constants.DESIRE_LEN, dtype=np.float32) self.full_frames: dict = {} self._blob_cache: dict = {} diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py index 82103283f3..4d56ae5475 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py @@ -18,8 +18,16 @@ def softmax(x, axis=-1): return x class Parser: - def __init__(self, ignore_missing=False): + def __init__(self, ignore_missing=False, mhp_config=None): self.ignore_missing = ignore_missing + # Optional MHP overrides keyed by head: 'plan_mhp_n', 'plan_mhp_selection', + # 'lead_mhp_n', 'lead_mhp_selection'. ``None`` (or missing keys) keeps the + # legacy ``ModelConstants`` values so existing models behave identically. + self.mhp = mhp_config or {} + + def _mhp(self, head, default_in, default_out): + return (self.mhp.get(f'{head}_mhp_n', default_in), + self.mhp.get(f'{head}_mhp_selection', default_out)) def check_missing(self, outs, name): if name not in outs and not self.ignore_missing: @@ -51,36 +59,48 @@ class Parser: pred_std = safe_exp(raw[:,:,n_values: 2*n_values]) if in_N > 1: - weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) - for i in range(out_N): - weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) + if out_N > 0: + weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) + for i in range(out_N): + weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) - if out_N == 1: + if out_N == 1: + for fidx in range(weights.shape[0]): + idxs = np.argsort(weights[fidx][:,0])[::-1] + weights[fidx] = weights[fidx][idxs] + pred_mu[fidx] = pred_mu[fidx][idxs] + pred_std[fidx] = pred_std[fidx][idxs] + assert out_shape is not None + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_weights'] = weights + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + + pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) for fidx in range(weights.shape[0]): - idxs = np.argsort(weights[fidx][:,0])[::-1] - weights[fidx] = weights[fidx][idxs] - pred_mu[fidx] = pred_mu[fidx][idxs] - pred_std[fidx] = pred_std[fidx][idxs] - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_weights'] = weights - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - - pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - for fidx in range(weights.shape[0]): - for hidx in range(out_N): - idxs = np.argsort(weights[fidx,:,hidx])[::-1] - pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] - pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] + for hidx in range(out_N): + idxs = np.argsort(weights[fidx,:,hidx])[::-1] + pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] + pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] + else: + # MHP without weights: keep every hypothesis intact, surface them as + # ``*_hypotheses`` outputs and use the same shape for the primary + # output so downstream consumers can iterate over the full set. + assert out_shape is not None + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + pred_mu_final = pred_mu + pred_std_final = pred_std else: pred_mu_final = pred_mu pred_std_final = pred_std - if out_N > 1: + if out_N > 1 or (in_N > 1 and out_N == 0): assert out_shape is not None - final_shape = tuple([raw.shape[0], out_N] + list(out_shape)) + n_selections = out_N if out_N > 1 else in_N + final_shape = tuple([raw.shape[0], n_selections] + list(out_shape)) else: assert out_shape is not None final_shape = tuple([raw.shape[0],] + list(out_shape)) @@ -88,7 +108,9 @@ class Parser: outs[name + '_stds'] = pred_std_final.reshape(final_shape) def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - self.parse_mdn('plan', outs, in_N=ModelConstants.PLAN_MHP_N, out_N=ModelConstants.PLAN_MHP_SELECTION, + plan_in, plan_out = self._mhp('plan', ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) + lead_in, lead_out = self._mhp('lead', ModelConstants.LEAD_MHP_N, ModelConstants.LEAD_MHP_SELECTION) + self.parse_mdn('plan', outs, in_N=plan_in, out_N=plan_out, out_shape=(ModelConstants.IDX_N,ModelConstants.PLAN_WIDTH)) self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_LANE_LINES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH)) self.parse_mdn('road_edges', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_ROAD_EDGES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH)) @@ -97,7 +119,7 @@ class Parser: if 'sim_pose' in outs: self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) - self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION, + self.parse_mdn('lead', outs, in_N=lead_in, out_N=lead_out, out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH)) if 'lat_planner_solution' in outs: self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N,ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py index efc5d846da..8c152ad220 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py @@ -22,8 +22,16 @@ def softmax(x, axis=-1): class Parser: - def __init__(self, ignore_missing=False): + def __init__(self, ignore_missing=False, mhp_config=None): self.ignore_missing = ignore_missing + # Optional overrides for plan/lead ``in_N`` / ``out_N``. ``None`` or any + # missing key falls back to ``SplitModelConstants`` so previously compiled + # pkls continue to behave identically. + self.mhp = mhp_config or {} + + def _mhp(self, head, default_in, default_out): + return (self.mhp.get(f'{head}_mhp_n', default_in), + self.mhp.get(f'{head}_mhp_selection', default_out)) def check_missing(self, outs, name): if name not in outs and not self.ignore_missing: @@ -55,36 +63,46 @@ class Parser: pred_std = safe_exp(raw[:,:,n_values: 2*n_values]) if in_N > 1: - weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) - for i in range(out_N): - weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) + if out_N > 0: + weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) + for i in range(out_N): + weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) - if out_N == 1: + if out_N == 1: + for fidx in range(weights.shape[0]): + idxs = np.argsort(weights[fidx][:,0])[::-1] + weights[fidx] = weights[fidx][idxs] + pred_mu[fidx] = pred_mu[fidx][idxs] + pred_std[fidx] = pred_std[fidx][idxs] + assert out_shape is not None + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_weights'] = weights + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + + pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) for fidx in range(weights.shape[0]): - idxs = np.argsort(weights[fidx][:,0])[::-1] - weights[fidx] = weights[fidx][idxs] - pred_mu[fidx] = pred_mu[fidx][idxs] - pred_std[fidx] = pred_std[fidx][idxs] - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_weights'] = weights - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - - pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - for fidx in range(weights.shape[0]): - for hidx in range(out_N): - idxs = np.argsort(weights[fidx,:,hidx])[::-1] - pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] - pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] + for hidx in range(out_N): + idxs = np.argsort(weights[fidx,:,hidx])[::-1] + pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] + pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] + else: + # MHP without weights: keep every hypothesis intact. + assert out_shape is not None + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + pred_mu_final = pred_mu + pred_std_final = pred_std else: pred_mu_final = pred_mu pred_std_final = pred_std - if out_N > 1: + if out_N > 1 or (in_N > 1 and out_N == 0): assert out_shape is not None - final_shape = tuple([raw.shape[0], out_N] + list(out_shape)) + n_selections = out_N if out_N > 1 else in_N + final_shape = tuple([raw.shape[0], n_selections] + list(out_shape)) else: assert out_shape is not None final_shape = tuple([raw.shape[0],] + list(out_shape)) @@ -100,15 +118,26 @@ class Parser: def parse_dynamic_outputs(self, outs: dict[str, np.ndarray]) -> None: if 'lead' in outs: - lead_mhp = self.is_mhp(outs, 'lead', - SplitModelConstants.LEAD_MHP_SELECTION * SplitModelConstants.LEAD_TRAJ_LEN * SplitModelConstants.LEAD_WIDTH) - lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) if lead_mhp else (0, 0) - lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) if lead_mhp else \ - (SplitModelConstants.LEAD_MHP_SELECTION, SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) + # Prefer explicit overrides in ``mhp``; otherwise fall back to the + # legacy `is_mhp` heuristic that inspects the raw tensor's last axis. + if self.mhp.get('lead_mhp_n') is not None or self.mhp.get('lead_mhp_selection') is not None: + lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) + lead_in_N = self.mhp.get('lead_mhp_n', lead_in_N) + lead_out_N = self.mhp.get('lead_mhp_selection', lead_out_N) + lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) + else: + lead_mhp = self.is_mhp(outs, 'lead', + SplitModelConstants.LEAD_MHP_SELECTION * SplitModelConstants.LEAD_TRAJ_LEN * SplitModelConstants.LEAD_WIDTH) + lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) if lead_mhp else (0, 0) + lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) if lead_mhp else \ + (SplitModelConstants.LEAD_MHP_SELECTION, SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) self.parse_mdn('lead', outs, in_N=lead_in_N, out_N=lead_out_N, out_shape=lead_out_shape) if 'plan' in outs: - plan_mhp = self.is_mhp(outs, 'plan', SplitModelConstants.IDX_N * SplitModelConstants.PLAN_WIDTH) - plan_in_N, plan_out_N = (SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) + if self.mhp.get('plan_mhp_n') is not None or self.mhp.get('plan_mhp_selection') is not None: + plan_in_N, plan_out_N = self._mhp('plan', SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) + else: + plan_mhp = self.is_mhp(outs, 'plan', SplitModelConstants.IDX_N * SplitModelConstants.PLAN_WIDTH) + plan_in_N, plan_out_N = (SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) self.parse_mdn('plan', outs, in_N=plan_in_N, out_N=plan_out_N, out_shape=(SplitModelConstants.IDX_N, SplitModelConstants.PLAN_WIDTH)) if 'planplus' in outs: diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py b/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py new file mode 100644 index 0000000000..efe6421d43 --- /dev/null +++ b/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py @@ -0,0 +1,172 @@ +"""Tests for the dynamic MDN inference in ``mhp_inference`` and the parser +changes that read from ``output_slices`` instead of hardcoded constants. +""" + +import numpy as np +import pytest + +from openpilot.sunnypilot.modeld_v2.mhp_inference import ( + infer_mhp, + slice_size, + infer_mhp_for_outputs, +) +from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser +from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser +from openpilot.sunnypilot.modeld_v2.constants import ModelConstants + + +# -- infer_mhp -------------------------------------------------------------- + +class TestInferMhp: + N_PLAN = ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH # 495 + N_LEAD = ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH # 24 + + def test_legacy_plan_preserved(self): + # Legacy: 5 hypotheses x (2*495 + 1) = 4955 + assert infer_mhp(4955, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (5, 1) + + def test_legacy_lead_preserved(self): + # Legacy: 2 hypotheses x (2*24 + 3) = 102 + assert infer_mhp(102, self.N_LEAD, legacy_in_n=2, legacy_out_n=3) == (2, 3) + + def test_new_supercombo_plan_single_hypothesis_no_weights(self): + # New combined supercombo: 1 hypothesis x (2*495 + 0) = 990 + assert infer_mhp(990, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (1, 0) + + def test_new_supercombo_lead_three_hypotheses_no_weights(self): + # New combined supercombo: 3 hypotheses x (2*24 + 0) = 144 + assert infer_mhp(144, self.N_LEAD, legacy_in_n=2, legacy_out_n=3) == (3, 0) + + def test_out_n_one(self): + # 4 hypotheses x (2*495 + 1) = 3964 + assert infer_mhp(3964, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (4, 1) + + def test_out_n_three(self): + # 2 hypotheses x (2*24 + 3) = 102 (matches legacy_lead as well) + assert infer_mhp(102, self.N_LEAD, legacy_in_n=4, legacy_out_n=99) == (2, 3) + + def test_zero_or_invalid_returns_fallback(self): + assert infer_mhp(0, self.N_PLAN, 5, 1) == (1, 0) + assert infer_mhp(-1, self.N_PLAN, 5, 1) == (1, 0) + assert infer_mhp(990, 0, 5, 1) == (1, 0) + + def test_no_match_returns_single_hypothesis(self): + # 987 doesn't cleanly factor under the constraints we care about. + assert infer_mhp(987, self.N_PLAN, 5, 1) == (1, 0) + + +class TestSliceSize: + def test_none_returns_zero(self): + assert slice_size(None) == 0 + + def test_basic_slice(self): + assert slice_size(slice(10, 50)) == 40 + + def test_negative_stop_returns_zero(self): + assert slice_size(slice(10, -2)) == 0 + + def test_none_bounds(self): + assert slice_size(slice(None, 100)) == 100 + + +class TestInferMhpForOutputs: + def test_infers_for_plan_and_lead(self): + slices = { + 'plan': slice(1576, 2566), # 990 + 'lead': slice(917, 1061), # 144 + } + cfg = infer_mhp_for_outputs(slices, ModelConstants) + assert cfg == {'plan_mhp_n': 1, 'plan_mhp_selection': 0, + 'lead_mhp_n': 3, 'lead_mhp_selection': 0} + + def test_legacy_falls_back_to_constants(self): + # Legacy sizes: 4955 plan, 102 lead -> both match Priority 1. + slices = { + 'plan': slice(0, 4955), + 'lead': slice(4955, 5057), + } + cfg = infer_mhp_for_outputs(slices, ModelConstants) + assert cfg == {'plan_mhp_n': ModelConstants.PLAN_MHP_N, + 'plan_mhp_selection': ModelConstants.PLAN_MHP_SELECTION, + 'lead_mhp_n': ModelConstants.LEAD_MHP_N, + 'lead_mhp_selection': ModelConstants.LEAD_MHP_SELECTION} + + def test_missing_outputs_are_skipped(self): + cfg = infer_mhp_for_outputs({}, ModelConstants) + assert cfg == {} + + +# -- CombinedParser --------------------------------------------------------- + +def _synth_outputs(in_n_plan=5, out_n_plan=1, in_n_lead=2, out_n_lead=3, + n_plan=ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH, + n_lead=ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH): + """Synthesize a flat-output dict with the right per-head sizes.""" + plan_size = in_n_plan * (2 * n_plan + out_n_plan) + lead_size = in_n_lead * (2 * n_lead + out_n_lead) + return { + 'plan': np.random.RandomState(0).randn(1, plan_size).astype(np.float32), + 'lead': np.random.RandomState(1).randn(1, lead_size).astype(np.float32), + # Other outputs that parse_outputs() consumes: + 'lane_lines': np.random.RandomState(2).randn(1, 528).astype(np.float32), + 'road_edges': np.random.RandomState(3).randn(1, 264).astype(np.float32), + 'pose': np.random.RandomState(4).randn(1, 12).astype(np.float32), + 'road_transform': np.random.RandomState(5).randn(1, 12).astype(np.float32), + 'wide_from_device_euler': np.random.RandomState(6).randn(1, 6).astype(np.float32), + 'lead_prob': np.random.RandomState(7).randn(1, 3).astype(np.float32), + 'lane_lines_prob':np.random.RandomState(8).randn(1, 8).astype(np.float32), + 'meta': np.random.RandomState(9).randn(1, 55).astype(np.float32), + 'desire_state': np.random.RandomState(10).randn(1, 8).astype(np.float32), + 'desire_pred': np.random.RandomState(11).randn(1, 32).astype(np.float32), + } + + +class TestCombinedParser: + def test_legacy_keeps_existing_shape(self): + p = CombinedParser() # empty mhp -> legacy constants + out = p.parse_outputs(_synth_outputs(5, 1, 2, 3)) + assert out['plan'].shape == (1, 33, 15) + assert out['plan_stds'].shape == (1, 33, 15) + # Lead primary output collapses to LEAD_MHP_SELECTION=3 selections per + # ``parse_mdn``; raw hypotheses survive as ``lead_hypotheses``. + assert out['lead'].shape == (1, 3, 6, 4) + assert out['lead_stds'].shape == (1, 3, 6, 4) + assert out['plan_hypotheses'].shape == (1, 5, 33, 15) + assert out['lead_hypotheses'].shape == (1, 2, 6, 4) + + def test_new_supercombo_plan_and_lead_parse(self): + p = CombinedParser(mhp_config={ + 'plan_mhp_n': 1, 'plan_mhp_selection': 0, + 'lead_mhp_n': 3, 'lead_mhp_selection': 0, + }) + out = p.parse_outputs(_synth_outputs(1, 0, 3, 0)) + assert out['plan'].shape == (1, 33, 15) + assert out['plan_stds'].shape == (1, 33, 15) + assert out['lead'].shape == (1, 3, 6, 4) + assert out['lead_stds'].shape == (1, 3, 6, 4) + # MHP-without-weights keeps every hypothesis as ``*_hypotheses`` + assert out['lead_hypotheses'].shape == (1, 3, 6, 4) + # Plan with a single hypothesis takes the in_N<=1 branch, which + # (matching legacy behavior) does not emit ``plan_hypotheses``. + assert 'plan_hypotheses' not in out + + +class TestSplitParser: + def test_default_uses_is_mhp_heuristic(self): + # No mhp_config -> falls back to inspecting the raw tensor's last axis. + # The legacy 102-element lead pack fits the "MHP branch" path. + n = SplitParser() + outs = {'lead': np.zeros((1, 102), dtype=np.float32)} + n.parse_dynamic_outputs(outs) + assert outs['lead'].shape == (1, 3, 6, 4) + assert outs['lead_hypotheses'].shape == (1, 2, 6, 4) + + def test_explicit_mhp_overrides_is_mhp(self): + n = SplitParser(mhp_config={'lead_mhp_n': 3, 'lead_mhp_selection': 0}) + outs = {'lead': np.zeros((1, 144), dtype=np.float32)} + n.parse_dynamic_outputs(outs) + assert outs['lead'].shape == (1, 3, 6, 4) + + +if __name__ == '__main__': + pytest.main([__file__, '-v']) From ad516e619beabbf7b03c4255a774e7b7bc306e82 Mon Sep 17 00:00:00 2001 From: nayan Date: Mon, 27 Jul 2026 16:25:05 -0400 Subject: [PATCH 39/57] simplify everything. --- .../sunnypilot/modeld_v2/mhp_inference.py | 128 ---------- openpilot/sunnypilot/modeld_v2/modeld.py | 20 +- .../modeld_v2/parse_model_outputs.py | 151 +++++++----- .../modeld_v2/parse_model_outputs_split.py | 93 +++---- .../modeld_v2/tests/test_dynamic_mhp.py | 228 +++++++----------- 5 files changed, 211 insertions(+), 409 deletions(-) delete mode 100644 openpilot/sunnypilot/modeld_v2/mhp_inference.py diff --git a/openpilot/sunnypilot/modeld_v2/mhp_inference.py b/openpilot/sunnypilot/modeld_v2/mhp_inference.py deleted file mode 100644 index e686c18c0e..0000000000 --- a/openpilot/sunnypilot/modeld_v2/mhp_inference.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Infer MHP (mixture-density-hypothesis) parameter values from output slice sizes. - -The legacy supercombo encoded its outputs using fixed values for -``PLAN_MHP_N``, ``PLAN_MHP_SELECTION``, ``LEAD_MHP_N``, and ``LEAD_MHP_SELECTION``. -Newer supercombo architectures emit differently-sized slices for the same heads, -yet the parser downstream still expects to be told the layout. - -This module figures those numbers out at runtime from the model's output slice, -so the existing ``parse_mdn`` path can handle any supercombo flavor without code -changes for each variant. ``infer_mhp`` always tries the legacy values first so -existing compiled pkls behave identically (full backwards compatibility). - -Schema reminder (``parse_mdn`` packs hypotheses contiguously in the order -``in_N × (mu | std | weights)`` along the channel axis): - - slice_size == in_N × (2·n_values + out_N) - -where ``n_values`` is the per-hypothesis value width (``IDX_N × PLAN_WIDTH`` -for plan, ``LEAD_TRAJ_LEN × LEAD_WIDTH`` for lead, etc.). -""" - -def infer_mhp( - slice_size: int, - n_values: int, - legacy_in_n: int, - legacy_out_n: int, - max_in_n: int = 16, -) -> tuple[int, int]: - """Infer ``(in_N, out_N)`` for an MDN-encoded output slice. - - Tries values in this order, returning on the first match: - - 1. ``(legacy_in_n, legacy_out_n)`` exactly — preserves exact backwards - compatibility for existing supercombo pkls. - 2. ``out_N ∈ (0, 1, 3)`` (no weights, single weight, three-way selection) at - increasing in_N — covers the common architectural patterns. - 3. Brute-force any valid ``out_N`` that divides ``slice_size``. - - Args: - slice_size: Number of floats in the output slice (typically - ``slices[name].stop - slices[name].start``). - n_values: Per-hypothesis mu/std width (e.g. ``IDX_N × PLAN_WIDTH`` for plan). - legacy_in_n: The legacy in_N value (highest priority for backwards compat). - legacy_out_n: The legacy out_N value. - max_in_n: Upper bound on accepted hypothesis counts (filters silly parses). - - Returns: - ``(in_N, out_N)``. If nothing fits the formulas, returns ``(1, 0)`` — - single hypothesis with no weights — which is the gentlest fallback. - """ - if slice_size <= 0 or n_values <= 0: - return 1, 0 - - # Priority 1: exact legacy match (BC-preserving). - per_hyp_legacy = 2 * n_values + legacy_out_n - if per_hyp_legacy > 0 and legacy_in_n * per_hyp_legacy == slice_size: - return legacy_in_n, legacy_out_n - - # Priority 2: common weight layouts across supercombo variants. - for out_n in (0, 1, 3): - per_hyp = 2 * n_values + out_n - if per_hyp <= 0: - continue - if slice_size % per_hyp == 0: - in_n = slice_size // per_hyp - if 1 <= in_n <= max_in_n: - return in_n, out_n - - # Priority 3: brute-force any divisor that yields a sensible in_N. - # Bound out_n by max_in_n to keep the search tiny (3 hypotheses - # of weights is already a lot). - for out_n in range(0, max_in_n + 1): - per_hyp = 2 * n_values + out_n - if per_hyp <= 0: - continue - if slice_size % per_hyp == 0: - in_n = slice_size // per_hyp - if 1 <= in_n <= max_in_n: - return in_n, out_n - - # Last resort: best-effort single hypothesis with no weights. - return 1, 0 - - -def slice_size(sl) -> int: - """Return the float-width of a ``slice``/``None`` from ``output_slices``. - - ``None`` and ``slice(None, None, None)`` are treated as 0 (output absent). - Negative-end slices (ONNX-style "from end") aren't supported by this helper - because parser expects single-tensor packed outputs. - """ - if sl is None: - return 0 - start = 0 if sl.start is None else sl.start - stop = sl.stop - if stop is None or stop < 0: - return 0 - return max(0, stop - start) - - -def infer_mhp_for_outputs( - output_slices: dict, - constants, - max_in_n: int = 16, -) -> dict: - """Build a dict of MHP values keyed by head name from a model's output_slices. - - Reads ``output_slices['plan']`` and ``output_slices['lead']`` (if present) - and infers their ``in_N``/``out_N``. Other heads aren't MDN-encoded the same - way and remain driven by the ``constants`` module. - """ - config: dict[str, int] = {} - - plan_size = slice_size(output_slices.get('plan')) - if plan_size > 0: - n = constants.IDX_N * constants.PLAN_WIDTH - in_n, out_n = infer_mhp(plan_size, n, constants.PLAN_MHP_N, constants.PLAN_MHP_SELECTION, max_in_n) - config['plan_mhp_n'] = in_n - config['plan_mhp_selection'] = out_n - - lead_size = slice_size(output_slices.get('lead')) - if lead_size > 0: - n = constants.LEAD_TRAJ_LEN * constants.LEAD_WIDTH - in_n, out_n = infer_mhp(lead_size, n, constants.LEAD_MHP_N, constants.LEAD_MHP_SELECTION, max_in_n) - config['lead_mhp_n'] = in_n - config['lead_mhp_selection'] = out_n - - return config diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 5182a191ab..17c1851c54 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -156,25 +156,15 @@ class ModelState(ModelStateBase): from openpilot.sunnypilot.modeld_v2.constants import ModelConstants self.constants = ModelConstants() - # Derive the parser's per-head MHP values from the appropriate output - # slices. Legacy pkls fall back to the constants values (Priority 1 in - # ``mhp_inference.infer_mhp``), so this is fully backwards compatible. - # Supercombo pkls carry plan/lead in the vision tensor; split/multi-policy - # pkls carry them on the policy tensor (and even there we use the first - # policy's slices -- the existing code only tracks one ``policy_output_slices``). - from openpilot.sunnypilot.modeld_v2.mhp_inference import infer_mhp_for_outputs - - if self._combined_model_type == 'supercombo': - mhp_config = infer_mhp_for_outputs(self.vision_output_slices, self.constants) - else: - mhp_config = infer_mhp_for_outputs(self.policy_output_slices, self.constants) - + # Combined parsers auto-detect ``(in_N, out_N)`` for plan/lead from the raw + # slice size, so they transparently support both legacy and newer + # supercombo ONNXes without any per-model configuration. from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser if self._combined_model_type != 'supercombo': - self.parser = SplitParser(mhp_config=mhp_config) + self.parser = SplitParser() else: - self.parser = CombinedParser(mhp_config=mhp_config) + self.parser = CombinedParser() self.prev_desire = np.zeros(self.constants.DESIRE_LEN, dtype=np.float32) self.full_frames: dict = {} diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py index 4d56ae5475..3ac2ad88b9 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py @@ -1,13 +1,16 @@ import numpy as np from openpilot.sunnypilot.modeld_v2.constants import ModelConstants + def safe_exp(x, out=None): # -11 is around 10**14, more causes float16 overflow return np.exp(np.clip(x, -np.inf, 11), out=out) + def sigmoid(x): return 1. / (1. + safe_exp(-x)) + def softmax(x, axis=-1): x -= np.max(x, axis=axis, keepdims=True) if x.dtype == np.float32 or x.dtype == np.float64: @@ -17,17 +20,34 @@ def softmax(x, axis=-1): x /= np.sum(x, axis=axis, keepdims=True) return x -class Parser: - def __init__(self, ignore_missing=False, mhp_config=None): - self.ignore_missing = ignore_missing - # Optional MHP overrides keyed by head: 'plan_mhp_n', 'plan_mhp_selection', - # 'lead_mhp_n', 'lead_mhp_selection'. ``None`` (or missing keys) keeps the - # legacy ``ModelConstants`` values so existing models behave identically. - self.mhp = mhp_config or {} - def _mhp(self, head, default_in, default_out): - return (self.mhp.get(f'{head}_mhp_n', default_in), - self.mhp.get(f'{head}_mhp_selection', default_out)) +def _infer_mhp(slice_size: int, prod_out_shape: int, max_in_n: int = 16, max_out_n: int = 6) -> tuple[int, int]: + """Derive ``(in_N, out_N)`` from a packed MDN slice. + + Layout (combined supercombo): for each hypothesis we have ``mu``, ``std``, + and an optional scalar ``weight`` block. So: + + slice_size = in_N * (2 * prod_out_shape + out_N) + + We scan small ``out_N`` values (no weights is most common in modern models, + one or three weights in the legacy) and accept the first division that + yields an integer ``in_N`` in ``[1, max_in_n]``. The candidates are spaced + wide enough that there's no ambiguity for the bands we care about. + """ + for out_n in range(max_out_n + 1): + per = 2 * prod_out_shape + out_n + if per <= 0: + continue + if slice_size % per == 0: + in_n = slice_size // per + if 1 <= in_n <= max_in_n: + return in_n, out_n + return 1, 0 # single hypothesis, no weights — matches a non-MDN output + + +class Parser: + def __init__(self, ignore_missing=False): + self.ignore_missing = ignore_missing def check_missing(self, outs, name): if name not in outs and not self.ignore_missing: @@ -48,85 +68,92 @@ class Parser: raw = outs[name] outs[name] = sigmoid(raw) - def parse_mdn(self, name, outs, in_N=0, out_N=1, out_shape=None): + def parse_mdn(self, name, outs, out_shape, in_N=0, out_N=0): + """Parse a packed MDN output. Pass ``in_N``/``out_N`` explicitly for the + legacy layout; pass neither (defaults of 0) to auto-detect from the + slice size, which transparently supports newer supercombos that drop the + per-hypothesis weight block or change the hypothesis count.""" if self.check_missing(outs, name): return raw = outs[name] - raw = raw.reshape((raw.shape[0], max(in_N, 1), -1)) + + if in_N == 0 and out_N == 0: + prod = int(np.prod(out_shape)) + in_N, out_N = _infer_mhp(raw.shape[1], prod) + + raw = raw.reshape((raw.shape[0], in_N, -1)) n_values = (raw.shape[2] - out_N)//2 pred_mu = raw[:,:,:n_values] pred_std = safe_exp(raw[:,:,n_values: 2*n_values]) - if in_N > 1: - if out_N > 0: - weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) - for i in range(out_N): - weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) + if in_N > 1 and out_N > 0: + weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) + for i in range(out_N): + weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) - if out_N == 1: - for fidx in range(weights.shape[0]): - idxs = np.argsort(weights[fidx][:,0])[::-1] - weights[fidx] = weights[fidx][idxs] - pred_mu[fidx] = pred_mu[fidx][idxs] - pred_std[fidx] = pred_std[fidx][idxs] - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_weights'] = weights - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - - pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + if out_N == 1: for fidx in range(weights.shape[0]): - for hidx in range(out_N): - idxs = np.argsort(weights[fidx,:,hidx])[::-1] - pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] - pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] - else: - # MHP without weights: keep every hypothesis intact, surface them as - # ``*_hypotheses`` outputs and use the same shape for the primary - # output so downstream consumers can iterate over the full set. - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - pred_mu_final = pred_mu - pred_std_final = pred_std + idxs = np.argsort(weights[fidx][:,0])[::-1] + weights[fidx] = weights[fidx][idxs] + pred_mu[fidx] = pred_mu[fidx][idxs] + pred_std[fidx] = pred_std[fidx][idxs] + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_weights'] = weights + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + + pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + for fidx in range(weights.shape[0]): + for hidx in range(out_N): + idxs = np.argsort(weights[fidx,:,hidx])[::-1] + pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] + pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] + elif in_N > 1 and out_N == 0: + # MHP without weights: keep every hypothesis intact, surface them as + # ``*_hypotheses`` and propagate the full multi-hypothesis tensor. + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + pred_mu_final = pred_mu + pred_std_final = pred_std else: pred_mu_final = pred_mu pred_std_final = pred_std + # Final-shape selector: keep an extra hypothesis axis only when + # multi-hypothesis data must survive — i.e. when out_N > 1 (legacy + # multiple-selection) OR when in_N > 1 with no weights (newer MHP). For + # single-hypothesis / collapsed cases (lane_lines, pose, etc.) drop the + # extra axis so the consumer sees the historical ``(batch, *out_shape)``. if out_N > 1 or (in_N > 1 and out_N == 0): - assert out_shape is not None n_selections = out_N if out_N > 1 else in_N final_shape = tuple([raw.shape[0], n_selections] + list(out_shape)) else: - assert out_shape is not None final_shape = tuple([raw.shape[0],] + list(out_shape)) outs[name] = pred_mu_final.reshape(final_shape) outs[name + '_stds'] = pred_std_final.reshape(final_shape) def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - plan_in, plan_out = self._mhp('plan', ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) - lead_in, lead_out = self._mhp('lead', ModelConstants.LEAD_MHP_N, ModelConstants.LEAD_MHP_SELECTION) - self.parse_mdn('plan', outs, in_N=plan_in, out_N=plan_out, - out_shape=(ModelConstants.IDX_N,ModelConstants.PLAN_WIDTH)) - self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_LANE_LINES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH)) - self.parse_mdn('road_edges', outs, in_N=0, out_N=0, out_shape=(ModelConstants.NUM_ROAD_EDGES,ModelConstants.IDX_N,ModelConstants.LANE_LINES_WIDTH)) - self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) - self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + # Pass no explicit ``in_N``/``out_N`` for plan/lead — ``parse_mdn`` infers + # them from the raw slice size, which naturally handles both the legacy + # supercombo (4955 / 102) and newer variants (e.g. 990 / 144). + self.parse_mdn('plan', outs, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + self.parse_mdn('lane_lines', outs, out_shape=(ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + self.parse_mdn('road_edges', outs, out_shape=(ModelConstants.NUM_ROAD_EDGES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + self.parse_mdn('pose', outs, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('road_transform', outs, out_shape=(ModelConstants.POSE_WIDTH,)) if 'sim_pose' in outs: - self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) - self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) - self.parse_mdn('lead', outs, in_N=lead_in, out_N=lead_out, - out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH)) + self.parse_mdn('sim_pose', outs, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('wide_from_device_euler', outs, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) + self.parse_mdn('lead', outs, out_shape=(ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH)) if 'lat_planner_solution' in outs: - self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N,ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) + self.parse_mdn('lat_planner_solution', outs, out_shape=(ModelConstants.IDX_N, ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) if 'desired_curvature' in outs: - self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) + self.parse_mdn('desired_curvature', outs, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) for k in ['lead_prob', 'lane_lines_prob', 'meta']: self.parse_binary_crossentropy(k, outs) self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) - self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN,ModelConstants.DESIRE_PRED_WIDTH)) + self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN, ModelConstants.DESIRE_PRED_WIDTH)) return outs diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py index 8c152ad220..895a42ed4b 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py @@ -22,16 +22,8 @@ def softmax(x, axis=-1): class Parser: - def __init__(self, ignore_missing=False, mhp_config=None): + def __init__(self, ignore_missing=False): self.ignore_missing = ignore_missing - # Optional overrides for plan/lead ``in_N`` / ``out_N``. ``None`` or any - # missing key falls back to ``SplitModelConstants`` so previously compiled - # pkls continue to behave identically. - self.mhp = mhp_config or {} - - def _mhp(self, head, default_in, default_out): - return (self.mhp.get(f'{head}_mhp_n', default_in), - self.mhp.get(f'{head}_mhp_selection', default_out)) def check_missing(self, outs, name): if name not in outs and not self.ignore_missing: @@ -63,46 +55,36 @@ class Parser: pred_std = safe_exp(raw[:,:,n_values: 2*n_values]) if in_N > 1: - if out_N > 0: - weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) - for i in range(out_N): - weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) + weights = np.zeros((raw.shape[0], in_N, out_N), dtype=raw.dtype) + for i in range(out_N): + weights[:,:,i - out_N] = softmax(raw[:,:,i - out_N], axis=-1) - if out_N == 1: - for fidx in range(weights.shape[0]): - idxs = np.argsort(weights[fidx][:,0])[::-1] - weights[fidx] = weights[fidx][idxs] - pred_mu[fidx] = pred_mu[fidx][idxs] - pred_std[fidx] = pred_std[fidx][idxs] - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_weights'] = weights - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - - pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) - pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + if out_N == 1: for fidx in range(weights.shape[0]): - for hidx in range(out_N): - idxs = np.argsort(weights[fidx,:,hidx])[::-1] - pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] - pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] - else: - # MHP without weights: keep every hypothesis intact. - assert out_shape is not None - full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) - outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) - outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) - pred_mu_final = pred_mu - pred_std_final = pred_std + idxs = np.argsort(weights[fidx][:,0])[::-1] + weights[fidx] = weights[fidx][idxs] + pred_mu[fidx] = pred_mu[fidx][idxs] + pred_std[fidx] = pred_std[fidx][idxs] + assert out_shape is not None + full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) + outs[name + '_weights'] = weights + outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) + outs[name + '_stds_hypotheses'] = pred_std.reshape(full_shape) + + pred_mu_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + pred_std_final = np.zeros((raw.shape[0], out_N, n_values), dtype=raw.dtype) + for fidx in range(weights.shape[0]): + for hidx in range(out_N): + idxs = np.argsort(weights[fidx,:,hidx])[::-1] + pred_mu_final[fidx, hidx] = pred_mu[fidx, idxs[0]] + pred_std_final[fidx, hidx] = pred_std[fidx, idxs[0]] else: pred_mu_final = pred_mu pred_std_final = pred_std - if out_N > 1 or (in_N > 1 and out_N == 0): + if out_N > 1: assert out_shape is not None - n_selections = out_N if out_N > 1 else in_N - final_shape = tuple([raw.shape[0], n_selections] + list(out_shape)) + final_shape = tuple([raw.shape[0], out_N] + list(out_shape)) else: assert out_shape is not None final_shape = tuple([raw.shape[0],] + list(out_shape)) @@ -118,26 +100,15 @@ class Parser: def parse_dynamic_outputs(self, outs: dict[str, np.ndarray]) -> None: if 'lead' in outs: - # Prefer explicit overrides in ``mhp``; otherwise fall back to the - # legacy `is_mhp` heuristic that inspects the raw tensor's last axis. - if self.mhp.get('lead_mhp_n') is not None or self.mhp.get('lead_mhp_selection') is not None: - lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) - lead_in_N = self.mhp.get('lead_mhp_n', lead_in_N) - lead_out_N = self.mhp.get('lead_mhp_selection', lead_out_N) - lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) - else: - lead_mhp = self.is_mhp(outs, 'lead', - SplitModelConstants.LEAD_MHP_SELECTION * SplitModelConstants.LEAD_TRAJ_LEN * SplitModelConstants.LEAD_WIDTH) - lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) if lead_mhp else (0, 0) - lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) if lead_mhp else \ - (SplitModelConstants.LEAD_MHP_SELECTION, SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) + lead_mhp = self.is_mhp(outs, 'lead', + SplitModelConstants.LEAD_MHP_SELECTION * SplitModelConstants.LEAD_TRAJ_LEN * SplitModelConstants.LEAD_WIDTH) + lead_in_N, lead_out_N = (SplitModelConstants.LEAD_MHP_N, SplitModelConstants.LEAD_MHP_SELECTION) if lead_mhp else (0, 0) + lead_out_shape = (SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) if lead_mhp else \ + (SplitModelConstants.LEAD_MHP_SELECTION, SplitModelConstants.LEAD_TRAJ_LEN, SplitModelConstants.LEAD_WIDTH) self.parse_mdn('lead', outs, in_N=lead_in_N, out_N=lead_out_N, out_shape=lead_out_shape) if 'plan' in outs: - if self.mhp.get('plan_mhp_n') is not None or self.mhp.get('plan_mhp_selection') is not None: - plan_in_N, plan_out_N = self._mhp('plan', SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) - else: - plan_mhp = self.is_mhp(outs, 'plan', SplitModelConstants.IDX_N * SplitModelConstants.PLAN_WIDTH) - plan_in_N, plan_out_N = (SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) + plan_mhp = self.is_mhp(outs, 'plan', SplitModelConstants.IDX_N * SplitModelConstants.PLAN_WIDTH) + plan_in_N, plan_out_N = (SplitModelConstants.PLAN_MHP_N, SplitModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) self.parse_mdn('plan', outs, in_N=plan_in_N, out_N=plan_out_N, out_shape=(SplitModelConstants.IDX_N, SplitModelConstants.PLAN_WIDTH)) if 'planplus' in outs: @@ -165,7 +136,7 @@ class Parser: self.parse_mdn('road_edges', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.NUM_ROAD_EDGES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) if 'sim_pose' in outs: - self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) + self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) if 'action' in outs: self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.ACTION_WIDTH,)) diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py b/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py index efe6421d43..17307fc8f8 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py @@ -1,171 +1,113 @@ -"""Tests for the dynamic MDN inference in ``mhp_inference`` and the parser -changes that read from ``output_slices`` instead of hardcoded constants. -""" +"""Tests for the dynamic MDN-hypothesis inference in ``parse_model_outputs``.""" import numpy as np import pytest -from openpilot.sunnypilot.modeld_v2.mhp_inference import ( - infer_mhp, - slice_size, - infer_mhp_for_outputs, -) -from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser -from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser + +from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser, _infer_mhp from openpilot.sunnypilot.modeld_v2.constants import ModelConstants -# -- infer_mhp -------------------------------------------------------------- +# -- _infer_mhp ------------------------------------------------------------- class TestInferMhp: - N_PLAN = ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH # 495 - N_LEAD = ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH # 24 + P_PLAN = ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH # 495 + P_LEAD = ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH # 24 - def test_legacy_plan_preserved(self): - # Legacy: 5 hypotheses x (2*495 + 1) = 4955 - assert infer_mhp(4955, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (5, 1) + def test_legacy_plan(self): + # 5 hypotheses * (2*495 + 1) = 4955 + assert _infer_mhp(4955, self.P_PLAN) == (5, 1) - def test_legacy_lead_preserved(self): - # Legacy: 2 hypotheses x (2*24 + 3) = 102 - assert infer_mhp(102, self.N_LEAD, legacy_in_n=2, legacy_out_n=3) == (2, 3) + def test_legacy_lead(self): + # 2 hypotheses * (2*24 + 3) = 102 + assert _infer_mhp(102, self.P_LEAD) == (2, 3) - def test_new_supercombo_plan_single_hypothesis_no_weights(self): - # New combined supercombo: 1 hypothesis x (2*495 + 0) = 990 - assert infer_mhp(990, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (1, 0) + def test_new_plan(self): + # 1 hypothesis * (2*495 + 0) = 990 + assert _infer_mhp(990, self.P_PLAN) == (1, 0) - def test_new_supercombo_lead_three_hypotheses_no_weights(self): - # New combined supercombo: 3 hypotheses x (2*24 + 0) = 144 - assert infer_mhp(144, self.N_LEAD, legacy_in_n=2, legacy_out_n=3) == (3, 0) + def test_new_lead(self): + # 3 hypotheses * (2*24 + 0) = 144 + assert _infer_mhp(144, self.P_LEAD) == (3, 0) - def test_out_n_one(self): - # 4 hypotheses x (2*495 + 1) = 3964 - assert infer_mhp(3964, self.N_PLAN, legacy_in_n=5, legacy_out_n=1) == (4, 1) + def test_no_mhp_output(self): + # lane_lines (528 = 4*132): 1 hypothesis * (2*528 + 0) = 1056; 2 hypotheses * 528+0 = 1056 too — same. + # In practice a single-hypothesis MDN with ``out_N == 0`` yields 1*2*prod = 528 * 2 = ... no, lane_lines is + # ``in_N=0`` because it's laid out as 2*prod directly (1056 = 4 * 132 * 2 simply). The auto-detect prefers + # the smallest valid ``in_N``, so 1056 = 2*1*528 + 0 -> (1, 0). + assert _infer_mhp(1056, 528) == (1, 0) - def test_out_n_three(self): - # 2 hypotheses x (2*24 + 3) = 102 (matches legacy_lead as well) - assert infer_mhp(102, self.N_LEAD, legacy_in_n=4, legacy_out_n=99) == (2, 3) - - def test_zero_or_invalid_returns_fallback(self): - assert infer_mhp(0, self.N_PLAN, 5, 1) == (1, 0) - assert infer_mhp(-1, self.N_PLAN, 5, 1) == (1, 0) - assert infer_mhp(990, 0, 5, 1) == (1, 0) - - def test_no_match_returns_single_hypothesis(self): - # 987 doesn't cleanly factor under the constraints we care about. - assert infer_mhp(987, self.N_PLAN, 5, 1) == (1, 0) - - -class TestSliceSize: - def test_none_returns_zero(self): - assert slice_size(None) == 0 - - def test_basic_slice(self): - assert slice_size(slice(10, 50)) == 40 - - def test_negative_stop_returns_zero(self): - assert slice_size(slice(10, -2)) == 0 - - def test_none_bounds(self): - assert slice_size(slice(None, 100)) == 100 - - -class TestInferMhpForOutputs: - def test_infers_for_plan_and_lead(self): - slices = { - 'plan': slice(1576, 2566), # 990 - 'lead': slice(917, 1061), # 144 - } - cfg = infer_mhp_for_outputs(slices, ModelConstants) - assert cfg == {'plan_mhp_n': 1, 'plan_mhp_selection': 0, - 'lead_mhp_n': 3, 'lead_mhp_selection': 0} - - def test_legacy_falls_back_to_constants(self): - # Legacy sizes: 4955 plan, 102 lead -> both match Priority 1. - slices = { - 'plan': slice(0, 4955), - 'lead': slice(4955, 5057), - } - cfg = infer_mhp_for_outputs(slices, ModelConstants) - assert cfg == {'plan_mhp_n': ModelConstants.PLAN_MHP_N, - 'plan_mhp_selection': ModelConstants.PLAN_MHP_SELECTION, - 'lead_mhp_n': ModelConstants.LEAD_MHP_N, - 'lead_mhp_selection': ModelConstants.LEAD_MHP_SELECTION} - - def test_missing_outputs_are_skipped(self): - cfg = infer_mhp_for_outputs({}, ModelConstants) - assert cfg == {} + def test_unknown_size_keeps_single_hypothesis(self): + # 989 doesn't divide cleanly under any out_N ∈ {0..6} for P = 495, so we + # fall back to the safe single-hypothesis default. + assert _infer_mhp(989, self.P_PLAN) == (1, 0) # -- CombinedParser --------------------------------------------------------- -def _synth_outputs(in_n_plan=5, out_n_plan=1, in_n_lead=2, out_n_lead=3, - n_plan=ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH, - n_lead=ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH): - """Synthesize a flat-output dict with the right per-head sizes.""" - plan_size = in_n_plan * (2 * n_plan + out_n_plan) - lead_size = in_n_lead * (2 * n_lead + out_n_lead) - return { - 'plan': np.random.RandomState(0).randn(1, plan_size).astype(np.float32), - 'lead': np.random.RandomState(1).randn(1, lead_size).astype(np.float32), - # Other outputs that parse_outputs() consumes: - 'lane_lines': np.random.RandomState(2).randn(1, 528).astype(np.float32), - 'road_edges': np.random.RandomState(3).randn(1, 264).astype(np.float32), - 'pose': np.random.RandomState(4).randn(1, 12).astype(np.float32), - 'road_transform': np.random.RandomState(5).randn(1, 12).astype(np.float32), - 'wide_from_device_euler': np.random.RandomState(6).randn(1, 6).astype(np.float32), - 'lead_prob': np.random.RandomState(7).randn(1, 3).astype(np.float32), - 'lane_lines_prob':np.random.RandomState(8).randn(1, 8).astype(np.float32), - 'meta': np.random.RandomState(9).randn(1, 55).astype(np.float32), - 'desire_state': np.random.RandomState(10).randn(1, 8).astype(np.float32), - 'desire_pred': np.random.RandomState(11).randn(1, 32).astype(np.float32), +def _synth_outs( + plan_in_n: int = 5, plan_out_n: int = 1, + lead_in_n: int = 2, lead_out_n: int = 3, + extras: bool = True, +) -> dict[str, np.ndarray]: + plan_size = plan_in_n * (2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + plan_out_n) + lead_size = lead_in_n * (2 * ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH + lead_out_n) + rng = np.random.RandomState(0) + d = { + 'plan': rng.randn(1, plan_size).astype(np.float32), + 'lead': rng.randn(1, lead_size).astype(np.float32), } + if extras: + d.update({ + 'lane_lines': rng.randn(1, 528).astype(np.float32), + 'road_edges': rng.randn(1, 264).astype(np.float32), + 'pose': rng.randn(1, 12).astype(np.float32), + 'road_transform': rng.randn(1, 12).astype(np.float32), + 'wide_from_device_euler': rng.randn(1, 6).astype(np.float32), + 'lead_prob': rng.randn(1, 3).astype(np.float32), + 'lane_lines_prob':rng.randn(1, 8).astype(np.float32), + 'meta': rng.randn(1, 55).astype(np.float32), + 'desire_state': rng.randn(1, 8).astype(np.float32), + 'desire_pred': rng.randn(1, 32).astype(np.float32), + }) + return d class TestCombinedParser: - def test_legacy_keeps_existing_shape(self): - p = CombinedParser() # empty mhp -> legacy constants - out = p.parse_outputs(_synth_outputs(5, 1, 2, 3)) - assert out['plan'].shape == (1, 33, 15) - assert out['plan_stds'].shape == (1, 33, 15) - # Lead primary output collapses to LEAD_MHP_SELECTION=3 selections per - # ``parse_mdn``; raw hypotheses survive as ``lead_hypotheses``. - assert out['lead'].shape == (1, 3, 6, 4) - assert out['lead_stds'].shape == (1, 3, 6, 4) - assert out['plan_hypotheses'].shape == (1, 5, 33, 15) - assert out['lead_hypotheses'].shape == (1, 2, 6, 4) + def test_legacy_supercombo_shapes(self): + p = Parser() + out = p.parse_outputs(_synth_outs(plan_in_n=5, plan_out_n=1, lead_in_n=2, lead_out_n=3)) + assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + assert out['plan_stds'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + assert out['lead'].shape == (1, ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) + assert out['lead_stds'].shape == (1, ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) + # Per-hypothesis outputs preserved by legacy code path + assert out['plan_hypotheses'].shape == (1, 5, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + assert out['lead_hypotheses'].shape == (1, 2, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - def test_new_supercombo_plan_and_lead_parse(self): - p = CombinedParser(mhp_config={ - 'plan_mhp_n': 1, 'plan_mhp_selection': 0, - 'lead_mhp_n': 3, 'lead_mhp_selection': 0, - }) - out = p.parse_outputs(_synth_outputs(1, 0, 3, 0)) - assert out['plan'].shape == (1, 33, 15) - assert out['plan_stds'].shape == (1, 33, 15) - assert out['lead'].shape == (1, 3, 6, 4) - assert out['lead_stds'].shape == (1, 3, 6, 4) - # MHP-without-weights keeps every hypothesis as ``*_hypotheses`` - assert out['lead_hypotheses'].shape == (1, 3, 6, 4) - # Plan with a single hypothesis takes the in_N<=1 branch, which - # (matching legacy behavior) does not emit ``plan_hypotheses``. - assert 'plan_hypotheses' not in out + def test_new_supercombo_shapes(self): + p = Parser() + out = p.parse_outputs(_synth_outs(plan_in_n=1, plan_out_n=0, lead_in_n=3, lead_out_n=0)) + # Plan single hypothesis collapses straight to (1, IDX_N, PLAN_WIDTH) + assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + assert out['plan_stds'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + # Lead with 3 hypotheses and no weights keeps all hypotheses + assert out['lead'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) + assert out['lead_stds'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) + assert out['lead_hypotheses'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - -class TestSplitParser: - def test_default_uses_is_mhp_heuristic(self): - # No mhp_config -> falls back to inspecting the raw tensor's last axis. - # The legacy 102-element lead pack fits the "MHP branch" path. - n = SplitParser() - outs = {'lead': np.zeros((1, 102), dtype=np.float32)} - n.parse_dynamic_outputs(outs) - assert outs['lead'].shape == (1, 3, 6, 4) - assert outs['lead_hypotheses'].shape == (1, 2, 6, 4) - - def test_explicit_mhp_overrides_is_mhp(self): - n = SplitParser(mhp_config={'lead_mhp_n': 3, 'lead_mhp_selection': 0}) - outs = {'lead': np.zeros((1, 144), dtype=np.float32)} - n.parse_dynamic_outputs(outs) - assert outs['lead'].shape == (1, 3, 6, 4) + def test_unknown_size_does_not_crash_legacy_layout(self): + # Provide EVERY output parse_outputs expects so the parser can auto-detect + # each head from the actual raw slice size, including the new-supercombo + # mixture (legacy-shaped lane_lines/etc. + new-shaped plan/lead). + p = Parser() + out = p.parse_outputs(_synth_outs(plan_in_n=1, plan_out_n=0, lead_in_n=3, lead_out_n=0)) + assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) + assert out['lead'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) + # Non-MHP outputs must keep the historical 3D shape (no spurious leading 1). + assert out['lane_lines'].shape == (1, ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH) + assert out['pose'].shape == (1, ModelConstants.POSE_WIDTH) + assert out['road_transform'].shape == (1, ModelConstants.POSE_WIDTH) if __name__ == '__main__': From c4a3854dc519aa0264a4792f0cfdd6eff98dd785 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 28 Jul 2026 11:24:46 -0700 Subject: [PATCH 40/57] fix macos cabana --- openpilot/tools/cabana/signalview.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpilot/tools/cabana/signalview.h b/openpilot/tools/cabana/signalview.h index 2e44e55cbf..bccfbab0cc 100644 --- a/openpilot/tools/cabana/signalview.h +++ b/openpilot/tools/cabana/signalview.h @@ -129,11 +129,11 @@ private: // update widget geometries in QTreeView::rowsInserted QTreeView::rowsInserted(parent, start, end); } - void setModel(QAbstractItemModel *model) override { - QTreeView::setModel(model); + void setModel(QAbstractItemModel *m) override { + QTreeView::setModel(m); // Bypass the slow call to QTreeView::dataChanged. - QObject::disconnect(model, &QAbstractItemModel::dataChanged, this, nullptr); - QObject::connect(model, &QAbstractItemModel::dataChanged, this, + QObject::disconnect(m, &QAbstractItemModel::dataChanged, this, nullptr); + QObject::connect(m, &QAbstractItemModel::dataChanged, this, [this](const QModelIndex &tl, const QModelIndex &br, const auto &roles) { QAbstractItemView::dataChanged(tl, br, roles); }); } void leaveEvent(QEvent *event) override { From ee9ee3a6fcba43673789313c562675f8b1a5e1df Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 14:48:11 -0700 Subject: [PATCH 41/57] np.random.seed is deprecated, use Generator or default rng instead @nayan --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 2 +- .../sunnypilot/modeld_v2/parse_model_outputs_split.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index d4ea49bb45..456fa224df 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -179,7 +179,7 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT) for i in range(3): - np.random.seed(42 + i) + np.random.default_rng(seed=(42 + i)) frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() for arr in npy_arrays.values(): diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py index 895a42ed4b..f0e13735eb 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py @@ -65,7 +65,6 @@ class Parser: weights[fidx] = weights[fidx][idxs] pred_mu[fidx] = pred_mu[fidx][idxs] pred_std[fidx] = pred_std[fidx][idxs] - assert out_shape is not None full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) outs[name + '_weights'] = weights outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) @@ -123,7 +122,7 @@ class Parser: self.parse_categorical_crossentropy('desire_state', outs, out_shape=(SplitModelConstants.DESIRE_PRED_WIDTH,)) if 'lane_lines' in outs: self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, - out_shape=(SplitModelConstants.NUM_LANE_LINES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) + out_shape=(SplitModelConstants.NUM_LANE_LINES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) if 'lane_lines_prob' in outs: self.parse_binary_crossentropy('lane_lines_prob', outs) if 'lead_prob' in outs: @@ -134,9 +133,9 @@ class Parser: self.parse_binary_crossentropy('meta', outs) if 'road_edges' in outs: self.parse_mdn('road_edges', outs, in_N=0, out_N=0, - out_shape=(SplitModelConstants.NUM_ROAD_EDGES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) + out_shape=(SplitModelConstants.NUM_ROAD_EDGES,SplitModelConstants.IDX_N,SplitModelConstants.LANE_LINES_WIDTH)) if 'sim_pose' in outs: - self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) + self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.POSE_WIDTH,)) if 'action' in outs: self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(SplitModelConstants.ACTION_WIDTH,)) From 62e046266299c144221603aed73426602d17a189 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 14:58:01 -0700 Subject: [PATCH 42/57] this is annoying me --- openpilot/sunnypilot/modeld_v2/modeld.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 17c1851c54..6a5f76f3c1 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -24,6 +24,11 @@ from setproctitle import setproctitle from openpilot.cereal.messaging import PubMaster, SubMaster from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf from opendbc.car.car_helpers import get_demo_car_params + +from tinygrad.tensor import Tensor +from tinygrad.device import Device + +from openpilot.common.file_chunker import open_file_chunked from openpilot.common.swaglog import cloudlog from openpilot.common.params import Params from openpilot.common.filter_simple import FirstOrderFilter @@ -31,6 +36,7 @@ from openpilot.common.realtime import config_realtime_process, DT_MDL from openpilot.common.transformations.camera import DEVICE_CAMERAS from openpilot.common.transformations.model import get_warp_matrix from openpilot.system import sentry +from openpilot.system.camerad.cameras.nv12_info import get_nv12_info from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value @@ -38,6 +44,7 @@ from openpilot.sunnypilot.modeld_v2.fill_model_msg import fill_model_msg, fill_p from openpilot.sunnypilot.modeld_v2.constants import Plan from openpilot.sunnypilot.modeld_v2.meta_helper import load_meta_constants from openpilot.sunnypilot.modeld_v2.camera_offset_helper import CameraOffsetHelper +from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues from openpilot.sunnypilot.livedelay.helpers import get_lat_delay from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase @@ -100,13 +107,6 @@ class ModelState(ModelStateBase): self._init_combined(pkl_path, cam_w, cam_h, model_bundle) def _init_combined(self, pkl_path, cam_w, cam_h, bundle): - from tinygrad.tensor import Tensor - from openpilot.system.camerad.cameras.nv12_info import get_nv12_info - from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues - from tinygrad.device import Device - - from openpilot.common.file_chunker import open_file_chunked - cloudlog.warning(f"loading combined pkl: {pkl_path}") jits = pickle.load(open_file_chunked(pkl_path)) @@ -121,7 +121,7 @@ class ModelState(ModelStateBase): self.policy_output_slices = {} self._policy_slices_list = [] self._combined_model_type = 'supercombo' - self._vision_input_names = [k for k in model_metadata['input_shapes'] if 'img' in k] + self._vision_input_names = [key for key in model_metadata['input_shapes'] if 'img' in key] from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.QUEUE_DEV) @@ -158,12 +158,12 @@ class ModelState(ModelStateBase): # Combined parsers auto-detect ``(in_N, out_N)`` for plan/lead from the raw # slice size, so they transparently support both legacy and newer - # supercombo ONNXes without any per-model configuration. - from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser - from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser + # supercombo ONNX's without any per-model configuration. if self._combined_model_type != 'supercombo': + from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser self.parser = SplitParser() else: + from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser self.parser = CombinedParser() self.prev_desire = np.zeros(self.constants.DESIRE_LEN, dtype=np.float32) From eef36d8b540c5c4bdf8cf21dbd01541c393dbc79 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 15:05:31 -0700 Subject: [PATCH 43/57] too much fluff --- .../modeld_v2/parse_model_outputs.py | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py index 3ac2ad88b9..7a3adcc1fa 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs.py @@ -22,18 +22,6 @@ def softmax(x, axis=-1): def _infer_mhp(slice_size: int, prod_out_shape: int, max_in_n: int = 16, max_out_n: int = 6) -> tuple[int, int]: - """Derive ``(in_N, out_N)`` from a packed MDN slice. - - Layout (combined supercombo): for each hypothesis we have ``mu``, ``std``, - and an optional scalar ``weight`` block. So: - - slice_size = in_N * (2 * prod_out_shape + out_N) - - We scan small ``out_N`` values (no weights is most common in modern models, - one or three weights in the legacy) and accept the first division that - yields an integer ``in_N`` in ``[1, max_in_n]``. The candidates are spaced - wide enough that there's no ambiguity for the bands we care about. - """ for out_n in range(max_out_n + 1): per = 2 * prod_out_shape + out_n if per <= 0: @@ -69,10 +57,6 @@ class Parser: outs[name] = sigmoid(raw) def parse_mdn(self, name, outs, out_shape, in_N=0, out_N=0): - """Parse a packed MDN output. Pass ``in_N``/``out_N`` explicitly for the - legacy layout; pass neither (defaults of 0) to auto-detect from the - slice size, which transparently supports newer supercombos that drop the - per-hypothesis weight block or change the hypothesis count.""" if self.check_missing(outs, name): return raw = outs[name] @@ -122,11 +106,6 @@ class Parser: pred_mu_final = pred_mu pred_std_final = pred_std - # Final-shape selector: keep an extra hypothesis axis only when - # multi-hypothesis data must survive — i.e. when out_N > 1 (legacy - # multiple-selection) OR when in_N > 1 with no weights (newer MHP). For - # single-hypothesis / collapsed cases (lane_lines, pose, etc.) drop the - # extra axis so the consumer sees the historical ``(batch, *out_shape)``. if out_N > 1 or (in_N > 1 and out_N == 0): n_selections = out_N if out_N > 1 else in_N final_shape = tuple([raw.shape[0], n_selections] + list(out_shape)) @@ -136,8 +115,6 @@ class Parser: outs[name + '_stds'] = pred_std_final.reshape(final_shape) def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: - # Pass no explicit ``in_N``/``out_N`` for plan/lead — ``parse_mdn`` infers - # them from the raw slice size, which naturally handles both the legacy # supercombo (4955 / 102) and newer variants (e.g. 990 / 144). self.parse_mdn('plan', outs, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) self.parse_mdn('lane_lines', outs, out_shape=(ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) From d086f83aedbf098287fa780c34086ef9617359ef Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 15:07:47 -0700 Subject: [PATCH 44/57] no --- .../modeld_v2/tests/test_dynamic_mhp.py | 114 ------------------ 1 file changed, 114 deletions(-) delete mode 100644 openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py b/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py deleted file mode 100644 index 17307fc8f8..0000000000 --- a/openpilot/sunnypilot/modeld_v2/tests/test_dynamic_mhp.py +++ /dev/null @@ -1,114 +0,0 @@ -"""Tests for the dynamic MDN-hypothesis inference in ``parse_model_outputs``.""" - -import numpy as np -import pytest - - -from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser, _infer_mhp -from openpilot.sunnypilot.modeld_v2.constants import ModelConstants - - -# -- _infer_mhp ------------------------------------------------------------- - -class TestInferMhp: - P_PLAN = ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH # 495 - P_LEAD = ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH # 24 - - def test_legacy_plan(self): - # 5 hypotheses * (2*495 + 1) = 4955 - assert _infer_mhp(4955, self.P_PLAN) == (5, 1) - - def test_legacy_lead(self): - # 2 hypotheses * (2*24 + 3) = 102 - assert _infer_mhp(102, self.P_LEAD) == (2, 3) - - def test_new_plan(self): - # 1 hypothesis * (2*495 + 0) = 990 - assert _infer_mhp(990, self.P_PLAN) == (1, 0) - - def test_new_lead(self): - # 3 hypotheses * (2*24 + 0) = 144 - assert _infer_mhp(144, self.P_LEAD) == (3, 0) - - def test_no_mhp_output(self): - # lane_lines (528 = 4*132): 1 hypothesis * (2*528 + 0) = 1056; 2 hypotheses * 528+0 = 1056 too — same. - # In practice a single-hypothesis MDN with ``out_N == 0`` yields 1*2*prod = 528 * 2 = ... no, lane_lines is - # ``in_N=0`` because it's laid out as 2*prod directly (1056 = 4 * 132 * 2 simply). The auto-detect prefers - # the smallest valid ``in_N``, so 1056 = 2*1*528 + 0 -> (1, 0). - assert _infer_mhp(1056, 528) == (1, 0) - - def test_unknown_size_keeps_single_hypothesis(self): - # 989 doesn't divide cleanly under any out_N ∈ {0..6} for P = 495, so we - # fall back to the safe single-hypothesis default. - assert _infer_mhp(989, self.P_PLAN) == (1, 0) - - -# -- CombinedParser --------------------------------------------------------- - -def _synth_outs( - plan_in_n: int = 5, plan_out_n: int = 1, - lead_in_n: int = 2, lead_out_n: int = 3, - extras: bool = True, -) -> dict[str, np.ndarray]: - plan_size = plan_in_n * (2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + plan_out_n) - lead_size = lead_in_n * (2 * ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH + lead_out_n) - rng = np.random.RandomState(0) - d = { - 'plan': rng.randn(1, plan_size).astype(np.float32), - 'lead': rng.randn(1, lead_size).astype(np.float32), - } - if extras: - d.update({ - 'lane_lines': rng.randn(1, 528).astype(np.float32), - 'road_edges': rng.randn(1, 264).astype(np.float32), - 'pose': rng.randn(1, 12).astype(np.float32), - 'road_transform': rng.randn(1, 12).astype(np.float32), - 'wide_from_device_euler': rng.randn(1, 6).astype(np.float32), - 'lead_prob': rng.randn(1, 3).astype(np.float32), - 'lane_lines_prob':rng.randn(1, 8).astype(np.float32), - 'meta': rng.randn(1, 55).astype(np.float32), - 'desire_state': rng.randn(1, 8).astype(np.float32), - 'desire_pred': rng.randn(1, 32).astype(np.float32), - }) - return d - - -class TestCombinedParser: - def test_legacy_supercombo_shapes(self): - p = Parser() - out = p.parse_outputs(_synth_outs(plan_in_n=5, plan_out_n=1, lead_in_n=2, lead_out_n=3)) - assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - assert out['plan_stds'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - assert out['lead'].shape == (1, ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - assert out['lead_stds'].shape == (1, ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - # Per-hypothesis outputs preserved by legacy code path - assert out['plan_hypotheses'].shape == (1, 5, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - assert out['lead_hypotheses'].shape == (1, 2, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - - def test_new_supercombo_shapes(self): - p = Parser() - out = p.parse_outputs(_synth_outs(plan_in_n=1, plan_out_n=0, lead_in_n=3, lead_out_n=0)) - # Plan single hypothesis collapses straight to (1, IDX_N, PLAN_WIDTH) - assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - assert out['plan_stds'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - # Lead with 3 hypotheses and no weights keeps all hypotheses - assert out['lead'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - assert out['lead_stds'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - assert out['lead_hypotheses'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - - def test_unknown_size_does_not_crash_legacy_layout(self): - # Provide EVERY output parse_outputs expects so the parser can auto-detect - # each head from the actual raw slice size, including the new-supercombo - # mixture (legacy-shaped lane_lines/etc. + new-shaped plan/lead). - p = Parser() - out = p.parse_outputs(_synth_outs(plan_in_n=1, plan_out_n=0, lead_in_n=3, lead_out_n=0)) - assert out['plan'].shape == (1, ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH) - assert out['lead'].shape == (1, 3, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH) - # Non-MHP outputs must keep the historical 3D shape (no spurious leading 1). - assert out['lane_lines'].shape == (1, ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH) - assert out['pose'].shape == (1, ModelConstants.POSE_WIDTH) - assert out['road_transform'].shape == (1, ModelConstants.POSE_WIDTH) - - -if __name__ == '__main__': - pytest.main([__file__, '-v']) From c22f58dae371128ab1fd546b06742e2a0108dc7e Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 15:15:54 -0700 Subject: [PATCH 45/57] lint be crazy now, man i've been away a while --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 4 ++-- openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 456fa224df..006ea6352b 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -179,11 +179,11 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT) for i in range(3): - np.random.default_rng(seed=(42 + i)) + rng = np.random.default_rng(42 + i) frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() big_frame = Tensor.randint(nv12.size, low=0, high=256, dtype=dtypes.uint8, device=WARP_DEV).realize() for arr in npy_arrays.values(): - arr[:] = np.random.randn(*arr.shape).astype(arr.dtype) + arr[:] = rng.standard_normal(arr.shape).astype(arr.dtype) Device.default.synchronize() start_time = time.perf_counter() diff --git a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py index f0e13735eb..3db47aee42 100644 --- a/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py +++ b/openpilot/sunnypilot/modeld_v2/parse_model_outputs_split.py @@ -65,6 +65,7 @@ class Parser: weights[fidx] = weights[fidx][idxs] pred_mu[fidx] = pred_mu[fidx][idxs] pred_std[fidx] = pred_std[fidx][idxs] + assert out_shape is not None full_shape = tuple([raw.shape[0], in_N] + list(out_shape)) outs[name + '_weights'] = weights outs[name + '_hypotheses'] = pred_mu.reshape(full_shape) From 2e87e08ba6eee364df296ff4011b1ebfdf480cdf Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 15:17:26 -0700 Subject: [PATCH 46/57] back --- openpilot/tools/cabana/signalview.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/openpilot/tools/cabana/signalview.h b/openpilot/tools/cabana/signalview.h index bccfbab0cc..2e44e55cbf 100644 --- a/openpilot/tools/cabana/signalview.h +++ b/openpilot/tools/cabana/signalview.h @@ -129,11 +129,11 @@ private: // update widget geometries in QTreeView::rowsInserted QTreeView::rowsInserted(parent, start, end); } - void setModel(QAbstractItemModel *m) override { - QTreeView::setModel(m); + void setModel(QAbstractItemModel *model) override { + QTreeView::setModel(model); // Bypass the slow call to QTreeView::dataChanged. - QObject::disconnect(m, &QAbstractItemModel::dataChanged, this, nullptr); - QObject::connect(m, &QAbstractItemModel::dataChanged, this, + QObject::disconnect(model, &QAbstractItemModel::dataChanged, this, nullptr); + QObject::connect(model, &QAbstractItemModel::dataChanged, this, [this](const QModelIndex &tl, const QModelIndex &br, const auto &roles) { QAbstractItemView::dataChanged(tl, br, roles); }); } void leaveEvent(QEvent *event) override { From ce488dff5084d5232f4e3efa9fb4b6f30a086923 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 16:09:04 -0700 Subject: [PATCH 47/57] fix pkl loader test --- .../modeld_v2/tests/test_combined_pkl_loader.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index 953fc8361c..0ae957a4c9 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -67,16 +67,14 @@ class TestStockEquivalence: state = model_state_factory(ARCHETYPES['vision_policy_split']) frame_skip = derive_frame_skip(SPLIT_VISION_INPUT_SHAPES, SPLIT_POLICY_INPUT_SHAPES) - # action_t is a deep-model prerequisite the SP loader doesn't provide yet; see skip_keys below stock_shapes = {**SPLIT_VISION_INPUT_SHAPES, **SPLIT_POLICY_INPUT_SHAPES, 'action_t': (1, 2)} stock_queues, stock_npy = make_input_queues(stock_shapes, frame_skip, device='NPY') - # TODO-SP: remove action_t skip once SP adds prerequisite for deep models (action_t input queue) - skip_keys = {'action_t'} - assert set(state.input_queues.keys()) == set(stock_queues.keys()) - skip_keys, \ - f"Queue keys differ: v2={set(state.input_queues.keys())}, stock={set(stock_queues.keys())}" - assert set(state.numpy_inputs.keys()) == set(stock_npy.keys()) - skip_keys, \ - f"Npy keys differ: v2={set(state.numpy_inputs.keys())}, stock={set(stock_npy.keys())}" + assert set(state.input_queues.keys()) - {'desire', 'traffic_convention'} == \ + set(stock_queues.keys()) - {'packed_npy_inputs'} + assert {'desire', 'traffic_convention'} <= set(state.input_queues.keys()) + # We generate action_t and prev_feat dynamically based on the metadata + assert set(state.numpy_inputs.keys()) == set(stock_npy.keys()) - {'action_t', 'prev_feat'} def test_split_queue_keys_work_with_desire_key(self, model_state_factory): from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues From d733058deeebb75496cc47f9e9dfb60a2328cf42 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sat, 1 Aug 2026 16:13:28 -0700 Subject: [PATCH 48/57] that comment was wrong, its still per model, just compiled at with the input/output shapes --- openpilot/sunnypilot/modeld_v2/modeld.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 6a5f76f3c1..a024e91113 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -156,9 +156,6 @@ class ModelState(ModelStateBase): from openpilot.sunnypilot.modeld_v2.constants import ModelConstants self.constants = ModelConstants() - # Combined parsers auto-detect ``(in_N, out_N)`` for plan/lead from the raw - # slice size, so they transparently support both legacy and newer - # supercombo ONNX's without any per-model configuration. if self._combined_model_type != 'supercombo': from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser self.parser = SplitParser() From 289171fef864a2fa4f0e7aae65e083f705eba19e Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Sun, 2 Aug 2026 15:38:02 -0700 Subject: [PATCH 49/57] fix chunking --- openpilot/sunnypilot/models/fetcher.py | 4 -- openpilot/sunnypilot/models/helpers.py | 23 ++++++---- openpilot/sunnypilot/models/manager.py | 59 +++++++++++++++++--------- 3 files changed, 53 insertions(+), 33 deletions(-) diff --git a/openpilot/sunnypilot/models/fetcher.py b/openpilot/sunnypilot/models/fetcher.py index 655c4163a3..eda1117a2a 100644 --- a/openpilot/sunnypilot/models/fetcher.py +++ b/openpilot/sunnypilot/models/fetcher.py @@ -64,8 +64,6 @@ class ModelParser: model.type = model_data.get("type") model.artifact = ModelParser._parse_artifact(model_data.get("artifact", {})) - if metadata := model_data.get("metadata"): - model.metadata = ModelParser._parse_artifact(metadata) return model @staticmethod @@ -211,5 +209,3 @@ if __name__ == "__main__": # Print metadata details if model.artifact.chunks: print(f"Contains {len(model.artifact.chunks)} chunks.") - if hasattr(model, 'metadata') and model.metadata and model.metadata.fileName: - print(f"Metadata: {model.metadata.fileName}, Download URI: {model.metadata.downloadUri.uri}") diff --git a/openpilot/sunnypilot/models/helpers.py b/openpilot/sunnypilot/models/helpers.py index ad1333d4ed..101d8d196e 100644 --- a/openpilot/sunnypilot/models/helpers.py +++ b/openpilot/sunnypilot/models/helpers.py @@ -18,7 +18,7 @@ from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRai from openpilot.common.hardware.hw import Paths # SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO -REQUIRED_JSON_VERSION = 15 +REQUIRED_JSON_VERSION = 16 CUSTOM_MODEL_PATH = Paths.model_root() METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl' @@ -56,12 +56,20 @@ def is_bundle_version_compatible(bundle: dict) -> bool: def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[str, str]]: artifacts = [] + from openpilot.common.file_chunker import get_chunk_name for model in getattr(bundle, 'models', []) or []: - for artifact in (getattr(model, 'artifact', None), getattr(model, 'metadata', None)): - if artifact and getattr(artifact, 'fileName', None) and getattr(artifact, 'downloadUri', None): - sha256 = getattr(artifact.downloadUri, 'sha256', None) - if sha256: - artifacts.append((artifact.fileName, sha256)) + for artifact in (getattr(model, 'artifact', None),): + if artifact and getattr(artifact, 'fileName', None): + if len(artifact.chunks) > 0: + for i, chunk in enumerate(artifact.chunks): + chunk_name = get_chunk_name(artifact.fileName, i, len(artifact.chunks)) + if getattr(chunk, 'sha256', None): + artifacts.append((chunk_name, chunk.sha256)) + else: + if getattr(artifact, 'downloadUri', None): + sha256 = getattr(artifact.downloadUri, 'sha256', None) + if sha256: + artifacts.append((artifact.fileName, sha256)) return artifacts @@ -156,8 +164,7 @@ def _get_model(): def load_metadata(): - model = _get_model() - metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" if model else METADATA_PATH + metadata_path = METADATA_PATH with open(metadata_path, 'rb') as f: return pickle.load(f) diff --git a/openpilot/sunnypilot/models/manager.py b/openpilot/sunnypilot/models/manager.py index 6b04dcd04c..d742e968a9 100644 --- a/openpilot/sunnypilot/models/manager.py +++ b/openpilot/sunnypilot/models/manager.py @@ -38,11 +38,11 @@ class ModelManagerSP: if not self.selected_bundle: return for model in self.selected_bundle.models: - for artifact in (model.artifact, model.metadata): - if artifact is not source_artifact and artifact.fileName == source_artifact.fileName: - artifact.downloadProgress.status = source_artifact.downloadProgress.status - artifact.downloadProgress.progress = source_artifact.downloadProgress.progress - artifact.downloadProgress.eta = source_artifact.downloadProgress.eta + artifact = model.artifact + if artifact is not source_artifact and artifact.fileName == source_artifact.fileName: + artifact.downloadProgress.status = source_artifact.downloadProgress.status + artifact.downloadProgress.progress = source_artifact.downloadProgress.progress + artifact.downloadProgress.eta = source_artifact.downloadProgress.eta def _calculate_eta(self, filename: str, progress: float) -> int: """Calculate ETA based on elapsed time and current progress""" @@ -136,7 +136,22 @@ class ModelManagerSP: full_path = os.path.join(destination_path, filename) try: - if await verify_file(full_path, expected_hash): + is_cached = False + if len(artifact.chunks) > 0: + from openpilot.common.file_chunker import get_chunk_name + chunks_valid = True + for i, chunk in enumerate(artifact.chunks): + chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) + if not await verify_file(chunk_path, chunk.sha256): + chunks_valid = False + break + if chunks_valid and len(artifact.chunks) > 0: + is_cached = True + else: + if await verify_file(full_path, expected_hash): + is_cached = True + + if is_cached: artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached artifact.downloadProgress.progress = 100 artifact.downloadProgress.eta = 0 @@ -146,11 +161,15 @@ class ModelManagerSP: if len(artifact.chunks) > 0: await self._download_chunked(url, full_path, artifact) + from openpilot.common.file_chunker import get_chunk_name + for i, chunk in enumerate(artifact.chunks): + chunk_path = get_chunk_name(full_path, i, len(artifact.chunks)) + if not await verify_file(chunk_path, chunk.sha256): + raise ValueError(f"Hash validation failed for chunk {i+1} of {filename}") else: await self._download_file(url, full_path, artifact) - - if not await verify_file(full_path, expected_hash): - raise ValueError(f"Hash validation failed for {filename}") + if not await verify_file(full_path, expected_hash): + raise ValueError(f"Hash validation failed for {filename}") artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.downloaded artifact.downloadProgress.progress = 100 @@ -198,16 +217,16 @@ class ModelManagerSP: try: seen_artifacts: set[str] = set() for model in self.selected_bundle.models: - for artifact in (model.metadata, model.artifact): - if not artifact.fileName: - continue - if artifact.fileName in seen_artifacts: - artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached - artifact.downloadProgress.progress = 100 - artifact.downloadProgress.eta = 0 - else: - seen_artifacts.add(artifact.fileName) - await self._process_artifact(artifact, destination_path) + artifact = model.artifact + if not artifact.fileName: + continue + if artifact.fileName in seen_artifacts: + artifact.downloadProgress.status = custom.ModelManagerSP.DownloadStatus.cached + artifact.downloadProgress.progress = 100 + artifact.downloadProgress.eta = 0 + else: + seen_artifacts.add(artifact.fileName) + await self._process_artifact(artifact, destination_path) self.active_bundle = self.selected_bundle self.active_bundle.status = custom.ModelManagerSP.DownloadStatus.downloaded @@ -268,8 +287,6 @@ class ModelManagerSP: for model in self.active_bundle.models: if hasattr(model, 'artifact') and model.artifact.fileName: active_files.append(model.artifact.fileName) - if hasattr(model, 'metadata') and model.metadata.fileName: - active_files.append(model.metadata.fileName) # Remove all files except active ones (including their chunk files) model_dir = Paths.model_root() From 6bfbf45c6dc27ce1bbbfb57486582af805c8ac81 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 12:35:22 -0700 Subject: [PATCH 50/57] dump and assign to cpu --- openpilot/sunnypilot/modeld_v2/modeld.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index a024e91113..3021cb5e3c 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -192,8 +192,6 @@ class ModelState(ModelStateBase): def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None: - from tinygrad.tensor import Tensor - for key in bufs.keys(): ptr = np.frombuffer(bufs[key].data, dtype=np.uint8).ctypes.data yuv_size = self.frame_buf_params[key][3] @@ -250,6 +248,10 @@ class ModelState(ModelStateBase): buf[0, :-1] = buf[0, 1:] buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0 + if 'feat_q' in self.input_queues: + feat_val = self.input_queues['feat_q'].numpy() + self.input_queues['feat_q'].assign(feat_val).realize() + return outputs def get_action_from_model(self, model_output: dict[str, np.ndarray], prev_action: log.ModelDataV2.Action, @@ -258,7 +260,6 @@ class ModelState(ModelStateBase): plan = model_output['plan'][0] desired_accel, should_stop = get_accel_from_plan(plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0], self.constants.T_IDXS, action_t=long_action_t) - desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) curvature_plan = (plan + (self.PLANPLUS_CONTROL - 1.0) * model_output['planplus'][0] if 'planplus' in model_output and self.PLANPLUS_CONTROL != 1.0 else plan) @@ -268,6 +269,8 @@ class ModelState(ModelStateBase): desired_curvature = model_output['action'][0, 0] / (max(1.0, v_ego))**2 should_stop = (v_ego < 0.3 and desired_accel < 0.1) + desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) + if self.generation is not None and self.generation >= 10: # smooth curvature for post FOF models if v_ego > self.MIN_LAT_CONTROL_SPEED: desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, self.LAT_SMOOTH_SECONDS) From 07bd1dfb52ceb2a50505504b06f2200bf5284245 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 13:15:12 -0700 Subject: [PATCH 51/57] prev_feat in cpu to prevent corruption --- .../sunnypilot/modeld_v2/compile_modeld.py | 95 +++++++++++++------ openpilot/sunnypilot/modeld_v2/modeld.py | 8 +- 2 files changed, 72 insertions(+), 31 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 006ea6352b..468401454a 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -57,7 +57,25 @@ def derive_frame_skip(vision_input_shapes: dict, policy_input_shapes: dict) -> i return 1 if not features_buffer or features_buffer[1] >= 99 else 4 -def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: +def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tuple[dict, list[int]]: + desire_key = _detect_desire_key(input_shapes) + shapes = {} + if desire_key: + shapes['desire'] = (input_shapes[desire_key][2],) + + if is_supercombo and 'features_buffer' in input_shapes: + fb = input_shapes['features_buffer'] + shapes['prev_feat'] = (fb[0], fb[2]) + + for key, shape in input_shapes.items(): + if key not in (desire_key, 'features_buffer') and 'img' not in key: + shapes[key] = tuple(shape) + + sizes = [int(np.prod(size)) for size in shapes.values()] + return shapes, sizes + + +def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, is_supercombo: bool = False) -> tuple[dict, dict]: road_key, _ = _detect_vision_keys(input_shapes) if not road_key: raise ValueError("Vision road key missing from input shapes.") @@ -74,36 +92,41 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D features_buffer = input_shapes.get('features_buffer') npy_arrays = { - 'desire': np.zeros(desire_shape[2], dtype=np.float32), 'tfm': np.zeros((3, 3), dtype=np.float32), 'big_tfm': np.zeros((3, 3), dtype=np.float32) } - for key, shape in input_shapes.items(): - if key not in npy_arrays and 'img' not in key and key not in ('features_buffer', desire_key): - npy_arrays[key] = np.zeros(shape, dtype=np.float32) + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) + packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32) + + split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] + split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] + for (k, s), v in zip(shapes.items(), split_views, strict=True): + actual_key = desire_key if k == 'desire' else k + npy_arrays[actual_key] = v.reshape(s) queues = { 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize() + dtype=np.float32), device=device).contiguous().realize(), + 'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(), } if features_buffer: queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), dtype=np.float32), device=device).contiguous().realize() - queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) + queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) return queues, npy_arrays def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: - return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device) + return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False) def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: - return generate_queues_and_npy(input_shapes, frame_skip, device) + return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True) def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], @@ -118,22 +141,17 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode if not desire_key or not road_key or not wide_key: raise ValueError("Missing required vision or desire keys in input shapes.") - extra_keys = [key for key in input_shapes if key not in (desire_key, 'features_buffer', 'traffic_convention') and 'img' not in key] + is_supercombo = vision_runner is None + npy_shapes, npy_sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) - def runner(img_q, big_img_q, feat_q, frame, big_frame, tfm, big_tfm, **kwargs): + def runner(img_q, big_img_q, feat_q, packed_npy_inputs, frame, big_frame, tfm, big_tfm, **kwargs): desire_q = kwargs['desire_q'] - desire = kwargs['desire'] - traffic_convention = kwargs.get('traffic_convention') - npys = [tfm.to(Device.DEFAULT), big_tfm.to(Device.DEFAULT), desire.to(Device.DEFAULT)] - if traffic_convention is not None: - npys.append(traffic_convention.to(Device.DEFAULT)) + packed_npy_inputs_dev = packed_npy_inputs.to(Device.DEFAULT) + tfm_dev = tfm.to(Device.DEFAULT) + big_tfm_dev = big_tfm.to(Device.DEFAULT) - extra_tensors = {key: kwargs[key].to(Device.DEFAULT) for key in extra_keys if key in kwargs} - Tensor.realize(*npys, *extra_tensors.values()) - - tfm_dev, big_tfm_dev, desire_dev = npys[:3] - traffic_conv_dev = npys[3] if traffic_convention is not None else None + Tensor.realize(packed_npy_inputs_dev, tfm_dev, big_tfm_dev) img = shift_and_sample(img_q, frame_prepare(frame, tfm_dev).unsqueeze(0), sample_skip_fn).realize() big_img = shift_and_sample(big_img_q, frame_prepare(big_frame, big_tfm_dev).unsqueeze(0), sample_skip_fn).realize() @@ -141,22 +159,37 @@ def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, mode if prepare_only: return img, big_img - desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() - inputs = {desire_key: desire_buf, **extra_tensors} + unpacked_tensors = [tensor.reshape(shape) for tensor, shape in zip(packed_npy_inputs_dev.split(npy_sizes), npy_shapes.values(), strict=True)] + unpacked_dict = dict(zip(npy_shapes.keys(), unpacked_tensors, strict=True)) - if traffic_conv_dev is not None: - inputs['traffic_convention'] = traffic_conv_dev + desire_dev = unpacked_dict['desire'] + desire_buf = shift_and_sample(desire_q, desire_dev.reshape(1, 1, -1), sample_desire_fn).realize() + + inputs = {desire_key: desire_buf} + for key, tensor_val in unpacked_dict.items(): + if key not in ('desire', 'prev_feat'): + inputs[key] = tensor_val + + if 'prev_feat' in unpacked_dict: + prev_feat_dev = unpacked_dict['prev_feat'] + inputs['features_buffer'] = shift_and_sample(feat_q, prev_feat_dev.reshape(1, 1, -1), sample_skip_fn).realize() if vision_runner: vision_out_cast = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast('float32').realize() - new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0) - inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() + if 'features_buffer' not in inputs: + new_feat = vision_out_cast[:, features_slice].reshape(1, -1).unsqueeze(0) + inputs['features_buffer'] = shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() policy_outs = [next(iter(pol_runner(inputs).values())).cast('float32').realize() for pol_runner in policy_runners] return (vision_out_cast, *policy_outs) if len(policy_outs) > 1 else (vision_out_cast, policy_outs[0]) - inputs.update({road_key: img, wide_key: big_img, 'features_buffer': sample_skip_fn(feat_q)}) + + inputs.update({road_key: img, wide_key: big_img}) + if 'features_buffer' not in inputs: + inputs['features_buffer'] = sample_skip_fn(feat_q) + policy_out = next(iter(policy_runners[0](inputs).values())).cast('float32').realize() - new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) - shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() + if 'features_buffer' not in inputs and features_slice is not None: + new_feat = policy_out[:, features_slice].reshape(1, -1).unsqueeze(0) + shift_and_sample(feat_q, new_feat, sample_skip_fn).realize() return policy_out return runner @@ -192,6 +225,7 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl Device.default.synchronize() print(f" [{i + 1}/3] enqueue {(mid_time - start_time) * 1e3:6.2f} ms -- total {(time.perf_counter() - start_time) * 1e3:6.2f} ms") + # TODO-SP: switch to dump_oob/load_oob on next full recompile of all models return pickle.loads(pickle.dumps(run_jit)) if not prepare_only else run_jit @@ -289,6 +323,7 @@ if __name__ == "__main__": vision_runner, policy_runners, output_data['metadata'])) with open(args.output, "wb") as file: + # TODO-SP: switch to dump_oob from openpilot/selfdrive/helpers on next full recompile of all models pickle.dump(output_data, file) pkl_size = os.path.getsize(args.output) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 3021cb5e3c..636ff018dd 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -108,6 +108,7 @@ class ModelState(ModelStateBase): def _init_combined(self, pkl_path, cam_w, cam_h, bundle): cloudlog.warning(f"loading combined pkl: {pkl_path}") + # TODO-SP: switch to load_oob from openpilot/selfdrive/helpers on next full recompile of all models jits = pickle.load(open_file_chunked(pkl_path)) self.DEV = Device.DEFAULT @@ -223,11 +224,16 @@ class ModelState(ModelStateBase): model_output = raw_outputs.numpy().flatten() sliced = {k: model_output[np.newaxis, v] for k, v in self.vision_output_slices.items()} outputs = self.parser.parse_outputs(sliced) + if 'prev_feat' in self.numpy_inputs: + self.numpy_inputs['prev_feat'][:] = model_output[self.vision_output_slices['hidden_state']] else: vision_output = raw_outputs[0].numpy().flatten() vision_sliced = {k: vision_output[np.newaxis, v] for k, v in self.vision_output_slices.items()} outputs = self.parser.parse_vision_outputs(vision_sliced) + if 'prev_feat' in self.numpy_inputs and 'hidden_state' in self.vision_output_slices: + self.numpy_inputs['prev_feat'][:] = vision_output[self.vision_output_slices['hidden_state']] + for i, policy_slices in enumerate(self._policy_slices_list): policy_output = raw_outputs[i + 1].numpy().flatten() policy_sliced = {k: policy_output[np.newaxis, v] for k, v in policy_slices.items()} @@ -248,7 +254,7 @@ class ModelState(ModelStateBase): buf[0, :-1] = buf[0, 1:] buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0 - if 'feat_q' in self.input_queues: + if 'prev_feat' not in self.numpy_inputs and 'feat_q' in self.input_queues: feat_val = self.input_queues['feat_q'].numpy() self.input_queues['feat_q'].assign(feat_val).realize() From 5a2fcde03de6ad79830cba981fd3bf27a64e633d Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 13:26:04 -0700 Subject: [PATCH 52/57] add todo-sp --- openpilot/sunnypilot/modeld_v2/modeld.py | 1 + 1 file changed, 1 insertion(+) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 636ff018dd..b92ab141ef 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -254,6 +254,7 @@ class ModelState(ModelStateBase): buf[0, :-1] = buf[0, 1:] buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0 + # TODO-SP: This is a hack to prevent GPU corruption by calculating in CPU space, it can be removed on next recompile if 'prev_feat' not in self.numpy_inputs and 'feat_q' in self.input_queues: feat_val = self.input_queues['feat_q'].numpy() self.input_queues['feat_q'].assign(feat_val).realize() From 918a0962ea5cd11e5a6226c0f6de2b1c7e54cc22 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 13:34:35 -0700 Subject: [PATCH 53/57] desire me? --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 3 +-- .../sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 468401454a..8e9f2f9248 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -102,8 +102,7 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] for (k, s), v in zip(shapes.items(), split_views, strict=True): - actual_key = desire_key if k == 'desire' else k - npy_arrays[actual_key] = v.reshape(s) + npy_arrays[k] = v.reshape(s) queues = { 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), diff --git a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py index 0ae957a4c9..3c544586f8 100644 --- a/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py +++ b/openpilot/sunnypilot/modeld_v2/tests/test_combined_pkl_loader.py @@ -70,10 +70,8 @@ class TestStockEquivalence: stock_shapes = {**SPLIT_VISION_INPUT_SHAPES, **SPLIT_POLICY_INPUT_SHAPES, 'action_t': (1, 2)} stock_queues, stock_npy = make_input_queues(stock_shapes, frame_skip, device='NPY') - assert set(state.input_queues.keys()) - {'desire', 'traffic_convention'} == \ - set(stock_queues.keys()) - {'packed_npy_inputs'} - assert {'desire', 'traffic_convention'} <= set(state.input_queues.keys()) - # We generate action_t and prev_feat dynamically based on the metadata + assert set(state.input_queues.keys()) == set(stock_queues.keys()) + assert {'desire', 'traffic_convention'} <= set(state.numpy_inputs.keys()) assert set(state.numpy_inputs.keys()) == set(stock_npy.keys()) - {'action_t', 'prev_feat'} def test_split_queue_keys_work_with_desire_key(self, model_state_factory): From 79a81ca2b87a864ed2b627905caeefc68a990b2d Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 15:48:36 -0700 Subject: [PATCH 54/57] allow legacy --- .../sunnypilot/modeld_v2/compile_modeld.py | 78 ++++++++++++------- openpilot/sunnypilot/modeld_v2/modeld.py | 17 +++- 2 files changed, 65 insertions(+), 30 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 8e9f2f9248..a51b8af3e4 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -75,7 +75,7 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu return shapes, sizes -def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, is_supercombo: bool = False) -> tuple[dict, dict]: +def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, is_supercombo: bool = False, use_packed: bool = True) -> tuple[dict, dict]: road_key, _ = _detect_vision_keys(input_shapes) if not road_key: raise ValueError("Vision road key missing from input shapes.") @@ -91,41 +91,67 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D desire_shape = input_shapes[desire_key] features_buffer = input_shapes.get('features_buffer') - npy_arrays = { - 'tfm': np.zeros((3, 3), dtype=np.float32), - 'big_tfm': np.zeros((3, 3), dtype=np.float32) - } + if use_packed: # remove packed detection block after all models are recompiled + npy_arrays = { + 'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32) + } - shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) - packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32) + shapes, sizes = get_policy_npy_shapes(input_shapes, is_supercombo=is_supercombo) + packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32) - split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] - split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] - for (k, s), v in zip(shapes.items(), split_views, strict=True): - npy_arrays[k] = v.reshape(s) + split_indices = np.cumsum(sizes[:-1]) if len(sizes) > 1 else [] + split_views = np.split(packed_npy_inputs, split_indices) if len(sizes) > 0 else [] + for (k, s), v in zip(shapes.items(), split_views, strict=True): + npy_arrays[k] = v.reshape(s) - queues = { - 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), - 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), - dtype=np.float32), device=device).contiguous().realize(), - 'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(), - } + queues = { + 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), + dtype=np.float32), device=device).contiguous().realize(), + 'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(), + } - if features_buffer: - queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), - dtype=np.float32), device=device).contiguous().realize() + if features_buffer: + queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), + dtype=np.float32), device=device).contiguous().realize() + + queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) + else: + # TODO-SP: Remove legacy queuing fallback else block after all models are recompiled + npy_arrays = { + 'desire': np.zeros(desire_shape[2], dtype=np.float32), + 'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32) + } + + for key, shape in input_shapes.items(): + if key not in npy_arrays and 'img' not in key and key not in ('features_buffer', desire_key): + npy_arrays[key] = np.zeros(shape, dtype=np.float32) + + queues = { + 'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(), + 'desire_q': Tensor(np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), + dtype=np.float32), device=device).contiguous().realize() + } + + if features_buffer: + queues['feat_q'] = Tensor(np.zeros((frame_skip * (features_buffer[1] - 1) + 1, features_buffer[0], features_buffer[2]), + dtype=np.float32), device=device).contiguous().realize() + + queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items()}) - queues.update({key: Tensor(value, device='NPY').realize() for key, value in npy_arrays.items() if key in ('tfm', 'big_tfm')}) return queues, npy_arrays -def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: - return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False) +def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: + return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False, use_packed=use_packed) -def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT) -> tuple[dict, dict]: - return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True) +def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: + return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True, use_packed=use_packed) def create_jit_runner(vision_runner, policy_runners: list, nv12: NV12Frame, model_size: tuple[int, int], diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index b92ab141ef..38ee9d821b 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -116,6 +116,17 @@ class ModelState(ModelStateBase): self.QUEUE_DEV = self.DEV metadata = jits['metadata'] + + self._run_policy = jits[(cam_w, cam_h)]['run_policy'] + self._warp_enqueue = jits[(cam_w, cam_h)]['warp_enqueue'] + + # TODO-SP: Remove legacy use_packed detection block after all models are recompiled + captured = getattr(self._run_policy, 'captured', None) + if captured is not None: + use_packed = 'packed_npy_inputs' in getattr(captured, 'expected_names', []) + else: + use_packed = True + if 'model' in metadata: model_metadata = metadata['model'] self.vision_output_slices = model_metadata['output_slices'] @@ -125,7 +136,7 @@ class ModelState(ModelStateBase): self._vision_input_names = [key for key in model_metadata['input_shapes'] if 'img' in key] from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) - self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.QUEUE_DEV) + self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) else: vision_metadata = metadata['vision'] policy_keys = [k for k in metadata if k != 'vision'] @@ -143,7 +154,7 @@ class ModelState(ModelStateBase): policy_input_shapes = first_policy_metadata['input_shapes'] self._vision_input_names = [k for k in vision_input_shapes if 'img' in k] frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes) - self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.QUEUE_DEV) + self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) self._desire_key = next(key for key in self.numpy_inputs if key.startswith('desire')) self._road_key = next(key for key in self._vision_input_names if 'big' not in key) @@ -170,8 +181,6 @@ class ModelState(ModelStateBase): nv12_info = get_nv12_info(cam_w, cam_h) self.frame_buf_params = dict.fromkeys(self._vision_input_names, nv12_info) - self._run_policy = jits[(cam_w, cam_h)]['run_policy'] - self._warp_enqueue = jits[(cam_w, cam_h)]['warp_enqueue'] yuv_size = self.frame_buf_params[self._road_key][3] self._warp_enqueue( **self.input_queues, From 274a3a6f866cc5c0e737b59bb1599b841befbc5d Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 15:55:49 -0700 Subject: [PATCH 55/57] lint --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 9 ++++++--- openpilot/sunnypilot/modeld_v2/modeld.py | 6 ++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index a51b8af3e4..2d6c977a94 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -75,7 +75,8 @@ def get_policy_npy_shapes(input_shapes: dict, is_supercombo: bool = False) -> tu return shapes, sizes -def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, is_supercombo: bool = False, use_packed: bool = True) -> tuple[dict, dict]: +def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, + is_supercombo: bool = False, use_packed: bool = True) -> tuple[dict, dict]: road_key, _ = _detect_vision_keys(input_shapes) if not road_key: raise ValueError("Vision road key missing from input shapes.") @@ -146,11 +147,13 @@ def generate_queues_and_npy(input_shapes: dict, frame_skip: int, device: str = D return queues, npy_arrays -def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: +def make_split_input_queues(vision_input_shapes: dict, policy_input_shapes: dict, + frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: return generate_queues_and_npy({**vision_input_shapes, **policy_input_shapes}, frame_skip, device, is_supercombo=False, use_packed=use_packed) -def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: +def make_supercombo_input_queues(input_shapes: dict, frame_skip: int, + device: str = Device.DEFAULT, use_packed: bool = True) -> tuple[dict, dict]: return generate_queues_and_npy(input_shapes, frame_skip, device, is_supercombo=True, use_packed=use_packed) diff --git a/openpilot/sunnypilot/modeld_v2/modeld.py b/openpilot/sunnypilot/modeld_v2/modeld.py index 38ee9d821b..b53ab18c73 100755 --- a/openpilot/sunnypilot/modeld_v2/modeld.py +++ b/openpilot/sunnypilot/modeld_v2/modeld.py @@ -136,7 +136,8 @@ class ModelState(ModelStateBase): self._vision_input_names = [key for key in model_metadata['input_shapes'] if 'img' in key] from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues frame_skip = derive_frame_skip({}, model_metadata['input_shapes']) - self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) + self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], + frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) else: vision_metadata = metadata['vision'] policy_keys = [k for k in metadata if k != 'vision'] @@ -154,7 +155,8 @@ class ModelState(ModelStateBase): policy_input_shapes = first_policy_metadata['input_shapes'] self._vision_input_names = [k for k in vision_input_shapes if 'img' in k] frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes) - self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) + self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, + frame_skip, device=self.QUEUE_DEV, use_packed=use_packed) self._desire_key = next(key for key in self.numpy_inputs if key.startswith('desire')) self._road_key = next(key for key in self._vision_input_names if 'big' not in key) From 872a76b6f2600742e8f18fc0b64b9af3869f7af2 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 18:36:16 -0700 Subject: [PATCH 56/57] shapey --- openpilot/sunnypilot/modeld_v2/compile_modeld.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openpilot/sunnypilot/modeld_v2/compile_modeld.py b/openpilot/sunnypilot/modeld_v2/compile_modeld.py index 2d6c977a94..def54a4599 100755 --- a/openpilot/sunnypilot/modeld_v2/compile_modeld.py +++ b/openpilot/sunnypilot/modeld_v2/compile_modeld.py @@ -235,9 +235,10 @@ def compile_and_warmup(nv12: NV12Frame, model_size: tuple[int, int], prepare_onl features_slice = feat_meta['output_slices']['hidden_state'] WARP_DEV = 'CPU' if "USBGPU" in os.environ else Device.DEFAULT + is_supercombo = vision_runner is None run_func = create_jit_runner(vision_runner, policy_runners, nv12, model_size, features_slice, frame_skip, all_shapes, prepare_only) run_jit = TinyJit(run_func, prune=True) - queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT) + queues, npy_arrays = generate_queues_and_npy(all_shapes, frame_skip, Device.DEFAULT, is_supercombo=is_supercombo) for i in range(3): rng = np.random.default_rng(42 + i) From 0a1c4cdc14edba42b6cde9e6716d87dddc9f5557 Mon Sep 17 00:00:00 2001 From: discountchubbs Date: Tue, 4 Aug 2026 21:28:11 -0700 Subject: [PATCH 57/57] bye --- openpilot/sunnypilot/modeld_v2/SConscript | 15 ---- .../sunnypilot/modeld_v2/install_models_pc.py | 75 ------------------- 2 files changed, 90 deletions(-) delete mode 100755 openpilot/sunnypilot/modeld_v2/install_models_pc.py diff --git a/openpilot/sunnypilot/modeld_v2/SConscript b/openpilot/sunnypilot/modeld_v2/SConscript index 81affc625a..daaa199ea9 100644 --- a/openpilot/sunnypilot/modeld_v2/SConscript +++ b/openpilot/sunnypilot/modeld_v2/SConscript @@ -82,18 +82,3 @@ if os.path.isfile(supercombo_onnx): compile_combined('supercombo', f'--supercombo-onnx {supercombo_onnx}', 'driving_combined_supercombo_tinygrad.pkl') - -if PC: - inputs = tinygrad_files + [File(Dir("#openpilot/sunnypilot/modeld_v2").File("install_models_pc.py").abspath)] - outputs = [] - model_dir = Dir("models").abspath - cmd = f'python3 {Dir("#openpilot/sunnypilot/modeld_v2").abspath}/install_models_pc.py {model_dir}' - - for model_name in ['supercombo', 'driving_vision', 'driving_off_policy', 'driving_on_policy', 'driving_policy']: - if File(f"models/{model_name}.onnx").exists(): - inputs.append(File(f"models/{model_name}.onnx")) - inputs.append(File(f"models/{model_name}_tinygrad.pkl")) - outputs.append(File(f"models/{model_name}_metadata.pkl")) - if outputs: - lenv.Command(outputs, inputs, cmd) - diff --git a/openpilot/sunnypilot/modeld_v2/install_models_pc.py b/openpilot/sunnypilot/modeld_v2/install_models_pc.py deleted file mode 100755 index 7bc2f4797c..0000000000 --- a/openpilot/sunnypilot/modeld_v2/install_models_pc.py +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python3 -import sys -import shutil -import pickle -import codecs -from pathlib import Path - -from openpilot.common.hardware.hw import Paths -from openpilot.sunnypilot.modeld_v2.get_model_metadata import MetadataOnnxPBParser, get_name_and_shape, get_metadata_value_by_name - - -def generate_metadata_pkl(model_path, output_path): - try: - model = MetadataOnnxPBParser(model_path).parse() - output_slices = get_metadata_value_by_name(model, 'output_slices') - if not output_slices: - return False - metadata = { - 'model_checkpoint': get_metadata_value_by_name(model, 'model_checkpoint'), - 'output_slices': pickle.loads(codecs.decode(output_slices.encode(), "base64")), - 'input_shapes': dict(get_name_and_shape(x) for x in model["graph"]["input"]), - 'output_shapes': dict(get_name_and_shape(x) for x in model["graph"]["output"]), - } - with open(output_path, 'wb') as f: - pickle.dump(metadata, f) - return True - except Exception: - return False - - -def install_models(model_dir): - model_dir = Path(model_dir) - models = ["driving_off_policy", "driving_on_policy", "driving_vision"] - found_models = [] - - for model in models: - if (model_dir / f"{model}.onnx").exists(): - found_models.append(model) - - if not found_models: - return - - try: - custom_name = input(f"Found models ({', '.join(found_models)}). Enter model short name (e.g. wmiv4): ").strip() - except EOFError: - return - - if not custom_name: - print("No name provided, skipping installation.") - return - - dest_dir = Path(Paths.model_root()) - dest_dir.mkdir(parents=True, exist_ok=True) - - for model in found_models: - onnx_path = model_dir / f"{model}.onnx" - tinygrad_pkl = model_dir / f"{model}_tinygrad.pkl" - metadata_pkl = model_dir / f"{model}_metadata.pkl" - - if not metadata_pkl.exists(): - generate_metadata_pkl(onnx_path, metadata_pkl) - - dest_tinygrad = dest_dir / f"{model}_{custom_name}_tinygrad.pkl" - dest_metadata = dest_dir / f"{model}_{custom_name}_metadata.pkl" - - if tinygrad_pkl.exists(): - shutil.move(str(tinygrad_pkl), str(dest_tinygrad)) - if metadata_pkl.exists(): - shutil.move(str(metadata_pkl), str(dest_metadata)) - -if __name__ == "__main__": - if len(sys.argv) < 2: - print("Usage: install_models_pc.py ") - sys.exit(1) - install_models(sys.argv[1])