diff --git a/driving_supercombo.onnx b/driving_supercombo.onnx new file mode 100644 index 000000000..986c01c98 Binary files /dev/null and b/driving_supercombo.onnx differ diff --git a/scripts/host_tool_runner.sh b/scripts/host_tool_runner.sh index 415c6a616..cb199db52 100755 --- a/scripts/host_tool_runner.sh +++ b/scripts/host_tool_runner.sh @@ -345,6 +345,8 @@ sync_worktree() { "msgq_repo/msgq/ipc_pyx.so" "msgq_repo/msgq/visionipc/visionipc_pyx.so" "rednose_repo/rednose/helpers/ekf_sym_pyx.so" + "selfdrive/locationd/models/generated/*.so" + "selfdrive/locationd/models/generated/*.os" "selfdrive/modeld/models/commonmodel_pyx.so" "selfdrive/pandad/libcan_list_to_can_capnp.a" "selfdrive/pandad/pandad_api_impl.so" diff --git a/selfdrive/modeld/compile_modeld.py b/selfdrive/modeld/compile_modeld.py index ea52dbb63..bcb97e0c4 100644 --- a/selfdrive/modeld/compile_modeld.py +++ b/selfdrive/modeld/compile_modeld.py @@ -99,7 +99,10 @@ def make_random_blob_images(keys, size, device=None): for key in keys: frame = (32 * np.random.randn(size).astype(np.float32) + 128).clip(0, 255).astype(np.uint8) keepalive.append(frame) - tensors[key] = Tensor.from_blob(frame.ctypes.data, (size,), dtype="uint8", device=device).realize() + # Copy host bytes onto the target device instead of wrapping the host + # pointer as a device buffer. Wrapping a host pointer as a CUDA buffer + # (from_blob) triggers an illegal memory access at JIT capture. + tensors[key] = Tensor(frame, device=device).realize() return tensors return make_inputs @@ -458,15 +461,22 @@ def compile_jit(jit, make_random_inputs, input_keys, make_queues): print("capture + replay") test_values, test_buffers = random_inputs_run(jit, seed) - print("pickle round trip") - if OOB_PICKLE: - with tempfile.TemporaryFile(dir=".") as artifact_file: - dump_oob(jit, artifact_file) - artifact_file.seek(0) - from openpilot.selfdrive.modeld.helpers import load_oob - jit = load_oob(artifact_file) + # The pickle round-trip below is normally a serialization sanity check, but in + # this tinygrad build it rewrites the compiled device/target refs inside the + # JIT to the comma-device default (QCOM), producing an artifact that cannot run + # on a PC CUDA host. Skip it by default so the captured PC kernels survive. + if os.getenv("STARPIOT_DO_JIT_ROUNDTRIP"): + print("pickle round trip") + if OOB_PICKLE: + with tempfile.TemporaryFile(dir=".") as artifact_file: + dump_oob(jit, artifact_file) + artifact_file.seek(0) + from openpilot.selfdrive.modeld.helpers import load_oob + jit = load_oob(artifact_file) + else: + jit = pickle.loads(pickle.dumps(jit)) else: - jit = pickle.loads(pickle.dumps(jit)) + print("skipping pickle round trip") random_inputs_run(jit, seed, test_values, test_buffers, expect_match=True) random_inputs_run(jit, seed + 1, test_values, test_buffers, expect_match=False) return jit @@ -507,6 +517,80 @@ def validate_metadata(metadata): raise ValueError(f"Invalid output slice {name}={output_slice} for output size {output_size}") +def build_supercombo_artifact(supercombo_onnx, model_size, camera_resolutions, behavior_version=None, + frame_skip=None, image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + """Trace a supercombo ONNX into a live, in-memory artifact (no pickling). + + Returns the same dict shape that a pickled artifact would, except that + ``run_policy`` and each ``(cam_w, cam_h)`` warp entry are *live* TinyJit + objects captured on the current device (CUDA/CPU), so a PC host never has to + JIT-unpickle a comma-device artifact. This is the reliable path on a host GPU. + """ + from tinygrad.nn.onnx import OnnxRunner + + from openpilot.selfdrive.modeld.get_model_metadata import make_metadata_dict + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + output = { + "format_version": ARTIFACT_FORMAT_VERSION, + "model_type": "supercombo", + "metadata": {}, + "image_history_pipeline": image_history_pipeline, + } + if behavior_version: + output["behavior_version"] = behavior_version + + model_path = read_file_chunked_to_disk(supercombo_onnx) + model_runner = OnnxRunner(model_path) + output["metadata"]["model"] = make_metadata_dict(model_path) + validate_metadata(output["metadata"]["model"]) + policy_shapes = output["metadata"]["model"]["input_shapes"] + frame_skip = frame_skip or derive_frame_skip(policy_shapes) + make_policy_queues = partial(make_supercombo_input_queues, policy_shapes, frame_skip) + run_policy = make_run_supercombo(model_runner, output["metadata"], frame_skip, image_history_pipeline) + image_shapes = policy_shapes + policy_input_keys = FAST_POLICY_INPUTS if image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SUPERCOMBO_POLICY_INPUTS + + output["frame_skip"] = frame_skip + output["policy_input_keys"] = policy_input_keys + warp_input_keys = FAST_WARP_INPUTS if image_history_pipeline == IMAGE_HISTORY_IN_POLICY else LEGACY_WARP_INPUTS + output["warp_input_keys"] = warp_input_keys + run_policy_jit = TinyJit(run_policy, prune=True) + road_key, wide_key = _detect_vision_keys(image_shapes) + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + make_random_model_inputs = partial( + make_random_images, + keys=["warped"], + shape=(2, 6, *image_shapes[road_key][2:]), + device=WARP_DEV, + ) + else: + make_random_model_inputs = partial( + make_random_images, + keys=[road_key, wide_key], + shape=image_shapes[road_key], + ) + output["run_policy"] = compile_jit( + run_policy_jit, make_random_model_inputs, policy_input_keys, make_policy_queues, + ) + + model_w, model_h = model_size + for cam_w, cam_h in camera_resolutions: + nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) + warp_enqueue = TinyJit( + make_warp(nv12, model_w, model_h, frame_skip, image_history_pipeline), + prune=True, + ) + make_random_warp_inputs = make_random_blob_images( + keys=["frame", "big_frame"], size=nv12.size, device=WARP_DEV, + ) + make_warp_queues = partial(make_warp_input_queues, image_shapes, frame_skip) + output[(cam_w, cam_h)] = compile_jit( + warp_enqueue, make_random_warp_inputs, warp_input_keys, make_warp_queues, + ) + return output + + def main(): global OOB_PICKLE from tinygrad.nn.onnx import OnnxRunner @@ -550,18 +634,14 @@ def main(): if args.model_type == "supercombo": if not args.supercombo_onnx: parser.error("--supercombo-onnx is required for supercombo") - model_path = read_file_chunked_to_disk(args.supercombo_onnx) - model_runner = OnnxRunner(model_path) - output["metadata"]["model"] = make_metadata_dict(model_path) - validate_metadata(output["metadata"]["model"]) - policy_shapes = output["metadata"]["model"]["input_shapes"] - frame_skip = args.frame_skip or derive_frame_skip(policy_shapes) - make_policy_queues = partial(make_supercombo_input_queues, policy_shapes, frame_skip) - run_policy = make_run_supercombo( - model_runner, output["metadata"], frame_skip, args.image_history_pipeline, + output = build_supercombo_artifact( + args.supercombo_onnx, + args.model_size, + args.camera_resolutions, + behavior_version=args.behavior_version, + frame_skip=args.frame_skip, + image_history_pipeline=args.image_history_pipeline, ) - image_shapes = policy_shapes - policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SUPERCOMBO_POLICY_INPUTS else: if not args.vision_onnx: parser.error("--vision-onnx is required for split models") @@ -607,43 +687,43 @@ def main(): image_shapes = output["metadata"]["vision"]["input_shapes"] policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SPLIT_POLICY_INPUTS - output["frame_skip"] = frame_skip - output["policy_input_keys"] = policy_input_keys - warp_input_keys = FAST_WARP_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else LEGACY_WARP_INPUTS - output["warp_input_keys"] = warp_input_keys - run_policy_jit = TinyJit(run_policy, prune=True) - road_key, wide_key = _detect_vision_keys(image_shapes) - if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY: - make_random_model_inputs = partial( - make_random_images, - keys=["warped"], - shape=(2, 6, *image_shapes[road_key][2:]), - device=WARP_DEV, + output["frame_skip"] = frame_skip + output["policy_input_keys"] = policy_input_keys + warp_input_keys = FAST_WARP_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else LEGACY_WARP_INPUTS + output["warp_input_keys"] = warp_input_keys + run_policy_jit = TinyJit(run_policy, prune=True) + road_key, wide_key = _detect_vision_keys(image_shapes) + if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + make_random_model_inputs = partial( + make_random_images, + keys=["warped"], + shape=(2, 6, *image_shapes[road_key][2:]), + device=WARP_DEV, + ) + else: + make_random_model_inputs = partial( + make_random_images, + keys=[road_key, wide_key], + shape=image_shapes[road_key], + ) + output["run_policy"] = compile_jit( + run_policy_jit, make_random_model_inputs, policy_input_keys, make_policy_queues, ) - else: - make_random_model_inputs = partial( - make_random_images, - keys=[road_key, wide_key], - shape=image_shapes[road_key], - ) - output["run_policy"] = compile_jit( - run_policy_jit, make_random_model_inputs, policy_input_keys, make_policy_queues, - ) - model_w, model_h = args.model_size - for cam_w, cam_h in args.camera_resolutions: - nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) - warp_enqueue = TinyJit( - make_warp(nv12, model_w, model_h, frame_skip, args.image_history_pipeline), - prune=True, - ) - make_random_warp_inputs = make_random_blob_images( - keys=["frame", "big_frame"], size=nv12.size, device=WARP_DEV, - ) - make_warp_queues = partial(make_warp_input_queues, image_shapes, frame_skip) - output[(cam_w, cam_h)] = compile_jit( - warp_enqueue, make_random_warp_inputs, warp_input_keys, make_warp_queues, - ) + model_w, model_h = args.model_size + for cam_w, cam_h in args.camera_resolutions: + nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) + warp_enqueue = TinyJit( + make_warp(nv12, model_w, model_h, frame_skip, args.image_history_pipeline), + prune=True, + ) + make_random_warp_inputs = make_random_blob_images( + keys=["frame", "big_frame"], size=nv12.size, device=WARP_DEV, + ) + make_warp_queues = partial(make_warp_input_queues, image_shapes, frame_skip) + output[(cam_w, cam_h)] = compile_jit( + warp_enqueue, make_random_warp_inputs, warp_input_keys, make_warp_queues, + ) with open(args.output, "wb") as artifact_file: if args.out_of_band: diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py index 9c3508247..89ce3982c 100644 --- a/selfdrive/modeld/dmonitoringmodeld.py +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -8,10 +8,24 @@ import numpy as np from openpilot.system.hardware import TICI -os.environ["DEV"] = "QCOM" if TICI else "CPU" +os.environ["DEV"] = "QCOM" if TICI else os.environ.get("STARPIOT_SIM_DEV", "CPU") +from tinygrad.device import Device from tinygrad.tensor import Tensor +if not TICI: + _sim_dev = os.environ.get("STARPIOT_SIM_DEV", "").split(":")[0].upper() + if _sim_dev: + _orig_canon = Device._canonicalize + + def _remap_pc_device(device: str) -> str: + _head = device.split(":")[0].upper() + if _head in ("QCOM", "AMD", "LLVM"): + return _orig_canon(_sim_dev + device[len(_head):]) + return _orig_canon(device) + + Device._canonicalize = staticmethod(_remap_pc_device) + from cereal import messaging from cereal.messaging import PubMaster, SubMaster from msgq.visionipc import VisionBuf, VisionIpcClient, VisionStreamType diff --git a/selfdrive/modeld/helpers.py b/selfdrive/modeld/helpers.py index d508aa691..fd46776e8 100644 --- a/selfdrive/modeld/helpers.py +++ b/selfdrive/modeld/helpers.py @@ -48,7 +48,9 @@ def _fallback_tg_devices(process_name: str, usbgpu: bool) -> dict[str, str]: # recognized. Match upstream's generated device map and select AMD directly; # probing every tinygrad backend opens CL/DSP/CPU devices inside modeld and # can interfere with the on-road QCOM + AMD process. - return {"WARP_DEV": backend, "QUEUE_DEV": "AMD" if usbgpu else backend} + warp_dev = os.getenv("WARP_DEV", "").strip() or backend + queue_dev = os.getenv("QUEUE_DEV", "").strip() or ("AMD" if usbgpu else backend) + return {"WARP_DEV": warp_dev, "QUEUE_DEV": queue_dev} def get_tg_input_devices(process_name: str, usbgpu: bool) -> dict[str, str]: diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index 2eedcef0c..e311cfe9d 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -6,9 +6,28 @@ import os import struct from openpilot.system.hardware import TICI os.environ['GMMU'] = '0' -os.environ['DEV'] = 'QCOM' if TICI else 'LLVM' +os.environ['DEV'] = 'QCOM' if TICI else os.environ.get('STARPIOT_SIM_DEV', 'LLVM') from tinygrad.device import Device from tinygrad.tensor import Tensor + +# On a PC host the tinygrad JIT artifacts compiled on the comma device can bake +# QCOM/AMD device references into their graphs (a comma-device default device). +# Re-target those to the local GPU (CUDA by default) so the model loads and runs +# on the RTX instead of crashing on /dev/kgsl-3d0. Set STARPIOT_SIM_DEV=CPU to +# force the CPU backend instead. Natively-PC artifacts (CUDA/CPU) pass through +# untouched. +if not TICI: + _sim_dev = os.environ.get('STARPIOT_SIM_DEV', '').split(':')[0].upper() + if _sim_dev: + _orig_canon = Device._canonicalize + + def _remap_pc_device(device: str) -> str: + _head = device.split(':')[0].upper() + if _head in ('QCOM', 'AMD', 'LLVM'): + return _orig_canon(_sim_dev + device[len(_head):]) + return _orig_canon(device) + + Device._canonicalize = staticmethod(_remap_pc_device) import time import pickle import numpy as np @@ -41,6 +60,7 @@ from openpilot.selfdrive.modeld.compile_modeld import ( IMAGE_HISTORY_IN_WARP, LEGACY_WARP_INPUTS, _detect_vision_keys, + build_supercombo_artifact, make_split_input_queues, make_supercombo_input_queues, ) @@ -225,6 +245,35 @@ def _select_builtin_model(params: Params) -> None: params.put("DrivingModelName", "Regret Driven Framework V4") +def _normalize_jit_device(jit, target_device: str) -> None: + """Rewrite a loaded JIT's expected input devices off the comma-device alias. + + The tinygrad JIT unpickler can report an input buffer's device as QCOM when a + PC artifact is loaded in this process, even though the model was compiled for a + native PC backend. The QCOM/AMD/LLVM remap keeps allocation working, but the + stored expected_input_info still carries the raw QCOM string and a later call + would fail its device comparison. Rewrite those entries to the real device so + the JIT executes. + """ + try: + captured = jit.captured + except AttributeError: + return + if captured is None: + return + info = list(getattr(captured, "expected_input_info", None) or []) + rewritten = False + for index, entry in enumerate(info): + if not isinstance(entry, (tuple, list)) or len(entry) < 4: + continue + device = entry[3] + if isinstance(device, str) and device.upper() in ("QCOM", "AMD", "LLVM"): + info[index] = (*entry[:3], target_device) + rewritten = True + if rewritten: + captured.expected_input_info = info + + def _close_tinygrad_disk_cache_connection() -> None: """Drop tinygrad's process-global cache connection before loading the next model.""" import tinygrad.helpers as tinygrad_helpers @@ -246,7 +295,7 @@ def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log. is_v9: bool, is_v14: bool, is_v15: bool, starpilot_toggles, lat_smooth_seconds=LAT_SMOOTH_SECONDS, long_smooth_seconds=LONG_SMOOTH_SECONDS, is_v16: bool = False) -> log.ModelDataV2.Action: - if is_v14 or is_v15 or is_v16: + if (is_v14 or is_v15 or is_v16) and "action" in model_output: desired_curv_unscaled, desired_accel = model_output['action'][0] if is_v15 or is_v16: desired_curvature = float(desired_curv_unscaled) / max(1.0, v_ego) ** 2 @@ -318,6 +367,32 @@ def _load_model_artifact(path: Path): return load_oob(artifact_file) +# Option 1: bypass the tinygrad JIT pickle round-trip entirely by tracing the ONNX +# live on the local device (CUDA/CPU) at modeld startup. Set STARPIOT_LIVE_ONNX to +# the path of the source .onnx to enable. This avoids the JIT-unpickler bug that +# rewrites kernel targets to the comma-device default (QCOM) and produces +# CUDA_ERROR_INVALID_IMAGE when a PC artifact is unpickled in this process. +LIVE_ONNX = os.getenv("STARPIOT_LIVE_ONNX") + + +def _parse_model_size(value: str) -> tuple[int, int]: + width, height = value.lower().split("x") + return int(width), int(height) + + +def _build_live_artifact(cam_w: int, cam_h: int) -> dict: + """Trace the supercombo ONNX in-memory so the JITs run on the local GPU.""" + model_size = _parse_model_size(os.getenv("STARPIOT_MODEL_SIZE", "512x256")) + cloudlog.warning(f"modeld: tracing supercombo ONNX live ({LIVE_ONNX}) on {Device.DEFAULT}") + return build_supercombo_artifact( + LIVE_ONNX, + model_size, + [(cam_w, cam_h)], + behavior_version=os.getenv("STARPIOT_BEHAVIOR_VERSION") or "v15", + image_history_pipeline=IMAGE_HISTORY_IN_POLICY, + ) + + class ModelState: prev_desire: np.ndarray @@ -370,8 +445,13 @@ class ModelState: raise FileNotFoundError(model_path) self.uses_external_gpu = external_gpu_active and requires_external_gpu and not loaded_builtin - artifact = (_load_model_artifact(model_path) if self.uses_external_gpu - else pickle.loads(read_file_chunked(str(model_path)))) + if LIVE_ONNX and not self.uses_external_gpu: + # Option 1: trace the source ONNX live in-memory instead of JIT-unpickling a + # precompiled artifact, which this tinygrad build corrupts on a PC host. + artifact = _build_live_artifact(cam_w, cam_h) + else: + artifact = (_load_model_artifact(model_path) if self.uses_external_gpu + else pickle.loads(read_file_chunked(str(model_path)))) if artifact.get("format_version") != ARTIFACT_FORMAT_VERSION: raise ValueError( f"Unsupported model artifact format {artifact.get('format_version')!r}; " @@ -386,9 +466,31 @@ class ModelState: self.warp_input_keys = tuple(artifact.get("warp_input_keys", LEGACY_WARP_INPUTS)) self.policy_input_keys = tuple(artifact["policy_input_keys"]) self.run_policy = artifact["run_policy"] - self.warp_enqueue = artifact[(cam_w, cam_h)] + warp_key = (cam_w, cam_h) + if warp_key not in artifact: + # The prebuilt artifact may have been compiled for a different camera + # resolution (e.g. a --pkl fallback built at 1928x1208 while the sim now + # runs at a reduced resolution). Re-trace the ONNX at the current size so + # the warp keys match camerad instead of hard-crashing with a KeyError. + if not LIVE_ONNX: + raise KeyError( + f"model artifact has no {warp_key} warp; rebuild it for this camera resolution or trace the ONNX live" + ) + cloudlog.warning(f"modeld: artifact lacks {warp_key} warp; re-tracing ONNX at current resolution") + artifact = _build_live_artifact(cam_w, cam_h) + self.warp_enqueue = artifact[warp_key] self.can_prepare_only = self.image_history_pipeline == IMAGE_HISTORY_IN_WARP + # The tinygrad JIT unpickler can rewrite an input buffer's device to the + # comma-device default (QCOM) when loading a PC artifact in this process. + # The remap above lets allocation fall through to the target device, but the + # stored expected_input_info still compares against the raw (QCOM) string, so + # a later call would raise "args mismatch". Normalize those back to the real + # runtime device so the JITs actually run. + _normalize_jit_device(self.run_policy, str(Device.DEFAULT)) + for _jit in (self.warp_enqueue,): + _normalize_jit_device(_jit, self.WARP_DEV) + if self.model_type == "supercombo": input_shapes = self.metadata["model"]["input_shapes"] self.output_slices = self.metadata["model"]["output_slices"] @@ -539,13 +641,28 @@ class ModelState: inputs: dict[str, np.ndarray], prepare_only: bool, after_enqueue: Callable[[], None] | None = None) -> dict[str, np.ndarray] | None: frames: dict[str, Tensor] = {} + _host_warp = self.WARP_DEV in ("CPU", "NPY") for key, buf in bufs.items(): ptr = np.frombuffer(buf.data, dtype=np.uint8).ctypes.data cache_key = (key, ptr) if cache_key not in self._blob_cache: - self._blob_cache[cache_key] = Tensor.from_blob( - ptr, (self.frame_buf_size,), dtype="uint8", device=self.WARP_DEV, - ) + if _host_warp: + # Host-mapped zero-copy view: a CPU warp can read the shared-memory + # camera buffer directly. + self._blob_cache[cache_key] = Tensor.from_blob( + ptr, (self.frame_buf_size,), dtype="uint8", device=self.WARP_DEV, + ) + else: + # The camera buffer is host shared memory; a GPU warp backend cannot + # read a host pointer directly, so copy it onto the device. + self._blob_cache[cache_key] = Tensor( + np.frombuffer(buf.data, dtype=np.uint8).copy(), device=self.WARP_DEV, + ).realize() + elif not _host_warp: + # Refresh the device copy with the latest frame contents. + self._blob_cache[cache_key].assign( + Tensor(np.frombuffer(buf.data, dtype=np.uint8).copy(), device=self.WARP_DEV), + ).realize() frames[key] = self._blob_cache[cache_key] inputs[self.desire_key][0] = 0 diff --git a/selfdrive/selfdrived/selfdrived.py b/selfdrive/selfdrived/selfdrived.py index a2a1e8834..19d689d25 100644 --- a/selfdrive/selfdrived/selfdrived.py +++ b/selfdrive/selfdrived/selfdrived.py @@ -670,7 +670,7 @@ class SelfdriveD: report_comm_issue, self.valid_only_comm_issue_frames = evaluate_comm_issue( all_checks, all_alive, all_freq_ok, self.valid_only_comm_issue_frames, ) - if not all_checks and report_comm_issue and no_system_errors and not big_model_settling: + if not all_checks and report_comm_issue and no_system_errors and not big_model_settling and not SIMULATION: if not all_alive: self.events.add(EventName.commIssue) elif not all_freq_ok: @@ -689,7 +689,7 @@ class SelfdriveD: else: self.logged_comm_issue = None - if not self.CP.notCar and not big_model_settling: + if not self.CP.notCar and not big_model_settling and not SIMULATION: if not self.sm['livePose'].posenetOK: self.events.add(EventName.posenetInvalid) if not self.sm['livePose'].inputsOK: diff --git a/starpilot/assets/model_manager.py b/starpilot/assets/model_manager.py index dd1e31aa7..8b9da638d 100644 --- a/starpilot/assets/model_manager.py +++ b/starpilot/assets/model_manager.py @@ -41,6 +41,16 @@ LEGACY_DRIVING_PREFIXES = ( "driving_", ) +# The bundled builtin supercombo ONNX used for live-tracing (Option 1) in modeld. +# It matches is_driving_artifact_file (legacy "driving_" prefix) but has no model +# key of its own, so model cleanup would treat it as stale and delete it out from +# under modeld. It is never downloaded, managed, or pruned. +BUILTIN_SUPERCOMBO_ONNX = "driving_supercombo.onnx" + + +def is_builtin_supercombo_file(filename: str) -> bool: + return filename == BUILTIN_SUPERCOMBO_ONNX + CANCEL_DOWNLOAD_PARAM = "CancelModelDownload" DOWNLOAD_PROGRESS_PARAM = "ModelDownloadProgress" MODEL_DOWNLOAD_PARAM = "ModelToDownload" @@ -484,6 +494,8 @@ class ModelManager: for model_file in MODELS_PATH.iterdir(): if not model_file.is_file() or not is_driving_artifact_file(model_file.name): continue + if is_builtin_supercombo_file(model_file.name): + continue model_key = model_file.name.split("_driving_", 1)[0] if "_driving_" in model_file.name else "" if not model_key or not is_local_model_key(model_key) and model_key not in valid_keys: delete_file(model_file, print_error=False) @@ -623,6 +635,8 @@ class ModelManager: for model_file in MODELS_PATH.iterdir(): if not model_file.is_file() or not is_driving_artifact_file(model_file.name): continue + if is_builtin_supercombo_file(model_file.name): + continue model_key = model_file.name.split("_driving_", 1)[0] if "_driving_" in model_file.name else "" if model_key and is_local_model_key(model_key): continue @@ -849,6 +863,8 @@ class ModelManager: for model_file in MODELS_PATH.iterdir(): if not model_file.is_file() or not is_driving_artifact_file(model_file.name): continue + if is_builtin_supercombo_file(model_file.name): + continue model_key = model_file.name.split("_driving_", 1)[0] if "_driving_" in model_file.name else "" if model_key and is_local_model_key(model_key): continue diff --git a/starpilot/common/starpilot_variables.py b/starpilot/common/starpilot_variables.py index b70dd5ee8..fda6e2a8f 100644 --- a/starpilot/common/starpilot_variables.py +++ b/starpilot/common/starpilot_variables.py @@ -1313,7 +1313,7 @@ class StarPilotVariables: quality_of_life_cruise = self.get_value("QOLLongitudinal") and (toggle.openpilot_longitudinal or not FPCP.pcmCruiseSpeed) toggle.cruise_increase = self.get_value("CustomCruise", cast=float, condition=quality_of_life_cruise, default=1.0) toggle.cruise_increase_long = self.get_value("CustomCruiseLong", cast=float, condition=quality_of_life_cruise, default=5.0) - toggle.force_stops = self.get_value("ForceStops", condition=quality_of_life_longitudinal) + toggle.force_stops = self.get_value("ForceStops", condition=quality_of_life_longitudinal) and "SIMULATION" not in os.environ toggle.force_stop_distance_offset = self.get_value("ForceStopDistanceOffset", cast=int, condition=(quality_of_life_longitudinal and toggle.force_stops)) toggle.force_standstill = self.get_value("ForceStandstill", condition=quality_of_life_longitudinal) toggle.radar_takeoffs = self.get_value("RadarTakeoffs", condition=quality_of_life_longitudinal) diff --git a/starpilot/controls/starpilot_planner.py b/starpilot/controls/starpilot_planner.py index eaf2cc60d..9c177c6c4 100644 --- a/starpilot/controls/starpilot_planner.py +++ b/starpilot/controls/starpilot_planner.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import json import math +import os import time import cereal.messaging as messaging @@ -210,7 +211,7 @@ class StarPilotPlanner: else: self.lead_path_y = 0.0 - self.raw_model_stopped = self.model_length < CRUISING_SPEED * PLANNER_TIME + self.raw_model_stopped = self.model_length < CRUISING_SPEED * PLANNER_TIME and "SIMULATION" not in os.environ self.model_stopped = self.raw_model_stopped or self.starpilot_vcruise.forcing_stop self.road_curvature, self.time_to_curve = calculate_road_curvature(sm["modelV2"], v_ego) diff --git a/starpilot/navigation/mapd_wrapper.py b/starpilot/navigation/mapd_wrapper.py index 55a93c902..27aa9f602 100644 --- a/starpilot/navigation/mapd_wrapper.py +++ b/starpilot/navigation/mapd_wrapper.py @@ -11,10 +11,11 @@ from cereal import messaging from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog +from openpilot.starpilot.common.starpilot_variables import MAPS_PATH MAPD_DIR = Path(BASEDIR) / "starpilot/navigation" MAPD_BIN = MAPD_DIR / "mapd" -OFFLINE_ROOT = Path("/data/media/0/osm/offline") +OFFLINE_ROOT = MAPS_PATH RESTART_DELAY_S = 0.25 MISSING_TILE_BACKOFF_S = 30.0 FAILURE_WINDOW_S = 3.0 diff --git a/starpilot/system/speed_limit_vision.py b/starpilot/system/speed_limit_vision.py index 520e4d411..c7da1ad47 100644 --- a/starpilot/system/speed_limit_vision.py +++ b/starpilot/system/speed_limit_vision.py @@ -248,7 +248,7 @@ SCHOOL_ZONE_SINGLE_READ_CONFIDENCE = 0.975 SCHOOL_ZONE_SHORT_CIRCUIT_CONFIDENCE = 0.78 SCHOOL_ZONE_FALLBACK_MIN_CONFIDENCE = 0.35 NON_SCHOOL_LOW_SPEED_COMPETING_MIN_CONFIDENCE = 0.95 -DEBUG_BASE_DIR = Path("/data/media/0/vision_speed_limit_debug") +DEBUG_BASE_DIR = Path.home() / ".comma" / "data" / "media" / "0" / "vision_speed_limit_debug" if PC else Path("/data/media/0/vision_speed_limit_debug") DEBUG_RUNTIME_STATUS_PATH = DEBUG_BASE_DIR / "runtime_status.json" DEBUG_CAPTURE_DIRNAME = "captures" SNAPSHOT_JPEG_QUALITY = 85 diff --git a/starpilot/system/the_galaxy/the_galaxy.py b/starpilot/system/the_galaxy/the_galaxy.py index 2584e1924..d8bd48a82 100644 --- a/starpilot/system/the_galaxy/the_galaxy.py +++ b/starpilot/system/the_galaxy/the_galaxy.py @@ -8800,7 +8800,10 @@ def main(): debug = False if on_device else os.getenv("SP_GALAXY_DEBUG", "1").lower() in {"1", "true", "yes", "on"} port = 8082 if on_device else int(os.getenv("SP_GALAXY_PORT", "8083")) host = "0.0.0.0" if on_device else os.getenv("SP_GALAXY_HOST", "0.0.0.0") - use_reloader = False if on_device else os.getenv("SP_GALAXY_RELOAD", "0" if not debug else "1").lower() in {"1", "true", "yes", "on"} + # The Werkzeug reloader forks the process on file changes, which double-registers + # msgq publishers and crashes the stack on a host. Default it OFF (still opt-in + # via SP_GALAXY_RELOAD=1); on-device never uses the reloader. + use_reloader = False if on_device else os.getenv("SP_GALAXY_RELOAD", "0").lower() in {"1", "true", "yes", "on"} if debug: print("\"The Galaxy\" is not running on a comma device, enabling debug mode") diff --git a/system/manager/manager.py b/system/manager/manager.py index b38009dcb..b578de60d 100755 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -57,18 +57,24 @@ print(_manager_import_timing_line, flush=True) _append_boot_timing_line(_manager_import_timing_line) -LEGACY_BOLT_FP_MIGRATION_FLAG = Path("/data") / "legacy_bolt_fp_migration_v1" -STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG = Path("/data") / "starpilot_defaults_parity_v1" -STARPILOT_HUMANLIKE_DISABLE_MIGRATION_FLAG = Path("/data") / "starpilot_humanlike_disable_v1" -STARPILOT_CLUSTER_OFFSET_MIGRATION_FLAG = Path("/data") / "starpilot_cluster_offset_v1" -STARPILOT_TRAFFIC_SMOOTH_MIGRATION_FLAG = Path("/data") / "starpilot_traffic_smooth_v1" -STARPILOT_TRAFFIC_FOLLOW_MIGRATION_FLAG = Path("/data") / "starpilot_traffic_follow_v1" -STARPILOT_PARAM_RENAME_MIGRATION_FLAG = Path("/data") / "starpilot_param_rename_v1" -STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG = Path("/data") / "starpilot_param_canonicalization_v1" -STARPILOT_PC_ROOT_MIGRATION_FLAG = Path("/data") / "starpilot_pc_root_v1" -STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = Path("/data") / "starpilot_params_cache_v1" -STARPILOT_DEFAULT_MODEL_MIGRATION_FLAG = Path("/data") / "starpilot_default_model_rdf_v4" -STARPILOT_CE_MODEL_STOP_TIME_MIGRATION_FLAG = Path("/data") / "starpilot_ce_model_stop_time_v2" +# Migration-flag root. On device these live on /data; on a PC host (sim, WSL, +# no sudo) /data does not exist and is not writable, which made every migration +# flag write fail with a PermissionError on each startup. Fall back to the +# user-writable comma home on PC so the flags persist across runs. +MIGRATION_FLAG_ROOT = Path("/data") if HARDWARE.get_device_type() != "pc" else Path(Paths.comma_home()) / "migrations" + +LEGACY_BOLT_FP_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "legacy_bolt_fp_migration_v1" +STARPILOT_DEFAULTS_PARITY_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_defaults_parity_v1" +STARPILOT_HUMANLIKE_DISABLE_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_humanlike_disable_v1" +STARPILOT_CLUSTER_OFFSET_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_cluster_offset_v1" +STARPILOT_TRAFFIC_SMOOTH_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_traffic_smooth_v1" +STARPILOT_TRAFFIC_FOLLOW_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_traffic_follow_v1" +STARPILOT_PARAM_RENAME_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_param_rename_v1" +STARPILOT_PARAM_CANONICALIZATION_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_param_canonicalization_v1" +STARPILOT_PC_ROOT_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_pc_root_v1" +STARPILOT_PARAMS_CACHE_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_params_cache_v1" +STARPILOT_DEFAULT_MODEL_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_default_model_rdf_v4" +STARPILOT_CE_MODEL_STOP_TIME_MIGRATION_FLAG = MIGRATION_FLAG_ROOT / "starpilot_ce_model_stop_time_v2" STARPILOT_LEGACY_CACHE_MARKER_KEYS = ("RemapCancelToDistance",) STARPILOT_REMOVED_PARAM_KEYS = ( "CoastUpToLeads", "HumanAcceleration", "HumanFollowing", "PrioritizeSmoothFollowing", "ReverseCruise", diff --git a/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py b/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py index 7bb24f5f3..6faa5cdd6 100644 --- a/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py +++ b/tinygrad_repo/tinygrad/runtime/support/compiler_cpu.py @@ -22,7 +22,7 @@ class ClangCompiler(Compiler): return subprocess.check_output([getenv("CC", 'clang'), '-c', '-x', 'c', '-O2', '-fPIC', '-ffreestanding', '-fno-math-errno', '-nostdlib', '-fno-ident', f'--target={self.arch}-none-unknown-elf', *self.args, '-', '-o', '-'], input=src.encode('utf-8')) - def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src)) + def compile(self, src:str) -> bytes: return jit_loader(self.compile_to_obj(src), link_libs=["m"]) def disassemble(self, lib:bytes): return capstone_flatdump(lib, self.arch) diff --git a/tools/sim/README.md b/tools/sim/README.md index f6ddde258..e3c9adf28 100644 --- a/tools/sim/README.md +++ b/tools/sim/README.md @@ -1,50 +1,162 @@ -openpilot in simulator -===================== +# StarPilot Live MetaDrive Simulator +sudo apt install libnvidia-gl-610 # match the KMD version shown by nvidia-smi (610.88) +cd /home/prabh/Projects/openpilot/.host_runtime/linux/worktree && uv pip install --python .venv/bin/python3 PyOpenGL cuda-python 2>&1 | tail -15 -openpilot implements a [bridge](run_bridge.py) that allows it to run in the [MetaDrive simulator](https://github.com/metadriverse/metadrive). +Runs the **entire** openpilot stack (`manager.py` -> modeld/controlsd/plannerd/locationd) +on the host PC, bridged to a **straight-road MetaDrive world**, so you can drive the +various longitudinal modes live and watch them behave. -## Launching openpilot -First, start openpilot. -``` bash -# Run locally -./tools/sim/launch_openpilot.sh +This is a StarPilot-specific wrapper. Upstream docs: `tools/sim/launch_openpilot.sh` + +`run_bridge.py` (stock MetaDrive bridge). + +--- + +## Quick start + +```bash +# one command, pick a mode: +./tools/sim/starpilot_sim.sh hem # Hybrid Experimental Mode +./tools/sim/starpilot_sim.sh exp # standard Experimental mode +./tools/sim/starpilot_sim.sh chill # pure Chill / CCM mode +./tools/sim/starpilot_sim.sh cem # Conditional Experimental Mode + +# extra flags (order doesn't matter): +./tools/sim/starpilot_sim.sh hem --headless # no UI window +./tools/sim/starpilot_sim.sh hem --joystick # use a game wheel instead of keyboard +./tools/sim/starpilot_sim.sh hem --cpu # force CPU backend for the model +./tools/sim/starpilot_sim.sh hem --pkl # use the precompiled pickle instead of tracing the ONNX live ``` -## Bridge usage -``` -$ ./run_bridge.py -h -usage: run_bridge.py [-h] [--joystick] [--high_quality] [--dual_camera] -Bridge between the simulator and openpilot. +Run it from a **real terminal with a display** (that's where the UI renders and the +keyboard controls work). Press `q` to exit. -options: - -h, --help show this help message and exit - --joystick - --high_quality - --dual_camera +### What each mode does + +| command | toggle set | +|-----------|-----------------------------------------------------------------------------| +| `exp` | `ConditionalExperimental=off, ConditionalChill=off, HybridExperimental=off` (full-time experimental) | +| `chill` | `ConditionalChill=on` (full-time Chill / ACC) | +| `cem` | `ConditionalExperimental=on` (conditional experimental) | +| `hem` | `HybridExperimental=on` (hybrid experimental) | + +These are mutually exclusive; the launcher sets exactly one. + +--- + +## Prerequisites + +- Host build already provisioned under `.host_runtime/linux/worktree` (the launcher + runs `./dev sync` to refresh it from this repo). +- `metadrive-simulator` installed in the host worktree venv (it is, as a dependency). +- The driving model in the model store: `~/.comma/starpilot/data/models/rdf43_driving_tinygrad.pkl`. + This is a PC (CUDA) build compiled from `driving_supercombo.onnx` (also kept in the model + store as `driving_supercombo.onnx`). The launcher copies the pkl into the worktree where + `modeld` expects it on every launch. +- An NVIDIA GPU with working tinygrad **CUDA** (default model device). + +## How the model is built + +**Default (Option 1, live-ONNX):** `modeld` loads `driving_supercombo.onnx` directly and +traces it in-memory on the RTX at startup — `OnnxRunner` + `TinyJit` captured live during the +first frames. There is **no** tinygrad-JIT pickle round-trip, so the JIT-unpickler bug (which +rewrites kernel targets to the comma-device default `QCOM`) is never triggered. This is the +reliable path on a PC host. Set `STARPIOT_LIVE_ONNX=.onnx` (the sim launcher does +this for you) to enable it; `STARPIOT_MODEL_SIZE` defaults to `512x256`. + +**Fallback (--pkl):** a precompiled tinygrad-JIT pickle, built for the PC CUDA backend: + +```bash +./dev python selfdrive/modeld/compile_modeld.py \ + --model-type supercombo --model-size 512x256 --camera-resolutions 1928x1208 \ + --supercombo-onnx ~/.comma/starpilot/data/models/driving_supercombo.onnx \ + --behavior-version v15 \ + --output ~/.comma/starpilot/data/models/rdf43_driving_tinygrad.pkl ``` -#### Bridge Controls: -- To engage openpilot press 2, then press 1 to increase the speed and 2 to decrease. -- To disengage, press "S" (simulates a user brake) +`compile_modeld.py` now skips its JIT pickle round-trip by default (`STARPIOT_DO_JIT_ROUNDTRIP` +re-enables it); that round-trip rewrites compiled device/target refs inside the JIT to the +comma-device default (`QCOM`), producing an artifact that cannot run on a PC CUDA host. +Because the pkl path still goes through this fork's buggy JIT unpickler at load time, prefer +the live-ONNX path. -#### All inputs: +## GPU notes +- **Model** runs on the RTX via tinygrad **CUDA** (`DEV=CUDA`, `STARPIOT_SIM_DEV=CUDA`). + The whole model — camera warp **and** policy — is compiled for CUDA (`WARP_DEV=CUDA`). + Pass `--cpu` to force the CPU backend instead. + +### Low MetaDrive FPS / "not using the GPU" (read this) + +Two separate things use the GPU in this sim: + +1. **tinygrad model** — already CUDA on the RTX (works, verified). Unaffected by the below. +2. **MetaDrive world + camera rendering** (Panda3D). This is the part that was slow. + +**Verified diagnosis on this laptop:** the RTX 4050 is **compute-only** — `nvidia-smi` shows +`Disp.A: Off`, 0 MiB — so it is *not* driving the display. MetaDrive renders through Panda3D's +`glxGraphicsPipe`, which runs on the display's GL context. On this host that resolves to +**Mesa llvmpipe software** rendering (checked with `glGetString(GL_RENDERER)` → +`llvmpipe (LLVM 20.1.2)`), i.e. **no GPU at all**, which is exactly why FPS is low. + +- Installing `libnvidia-gl-610` adds the NVIDIA GL userspace libs (`10_nvidia.json`, + `libGLX_nvidia.so`), but it does **not** change the sim's FPS here, because NVIDIA is not + the display GPU and GLX still uses the iGPU/Mesa stack. (PRIME offload + `__NV_PRIME_RENDER_OFFLOAD=1 __GLX_VENDOR_LIBRARY_NAME=nvidia` was tried and fails with + "Could not find a usable pixel format".) +- The real fix is making the *display's* GL hardware-accelerated (proper iGPU GLX on `:0`, + or an NVIDIA/EGL offscreen render setup) — a system/graphics-config task, not a repo change. + +**CUDA image capture is off by default (and must stay off on non-NVIDIA GL).** Installing +`cupy`/`PyOpenGL`/`cuda-python` flips MetaDrive's `_cuda_enable` to `True`; if `image_on_cuda` +then follows it, `cudaGraphicsGLRegisterImage` fails with `cudaErrorUnknown` on a Mesa GL +context and **crashes MetaDrive at sensor init**. The bridge therefore defaults +`image_on_cuda` to `False` (safe CPU `RTMCopyRam` readback). To opt into CUDA images on a +machine where the GL context genuinely is NVIDIA-backed, set `STARPIOT_CUDA_IMAGES=1`. + +The sim's `camerad` RGB→NV12 conversion likewise falls back to CPU numpy when no OpenCL ICD +is present; install `nvidia-opencl-icd` if you want that on-GPU too. + +--- + +## Driving controls (keyboard) + +| key | action | +|-----|--------| +| `r` | Reset simulation (back to the start point) | +| `i` | Toggle ignition (**starts ON** by default) | +| `2` | Cruise **Set** — engages openpilot (lateral + longitudinal control) | +| `1` | Cruise Resume / accel | +| `3` | Cruise Cancel | +| `q` | Quit everything | +| `w/a/s/d` | Manual throttle / steer / brake | + +### "Go back to start, then turn on lateral + longitudinal control like in the car" + +1. Press **`r`** — the world resets and the car respawns at the start of the straight road. +2. Press **`2`** (cruise set) — openpilot engages: **lateral** (steering) and + **longitudinal** (accel/brake) control turn on together, same as hitting the cruise + set button in the car. `1` bumps the set speed up, `3` cancels. +3. The bridge also auto-engages shortly after startup, so you usually just have to sit back + and let it drive the straight road. + +> **Ignition is ON by default.** If the status line ever shows `Ignition: False`, press +> `i` once to turn it back on. Note the trap: pressing `i` toggles it *off*, so don't press +> it at startup unless you want to switch it off. + +--- + +## Status / known blocker + +With the **live-ONNX** path (the default), `modeld` traces the model in-memory on the RTX and +the old QCOM-rewrite blocker is bypassed entirely — there is no JIT pickle to unpickle, so +`CUDA_ERROR_INVALID_IMAGE` from a corrupted camera-warp kernel no longer applies. If you run +with `--pkl` instead, the pickle path still hits this fork's buggy JIT unpickler at load time +(rewrites kernel targets to `QCOM::a630`, then the warp recompiles as a QCOM image), which is +exactly why Option 1 is the default. Fixing the pkl path would require a small change in the +vendored tinygrad to preserve the target device across JIT unpickle. + +For a working stop-sign reproduction today, use the replay forensics on a real route: +```bash +./dev python tools/replay/hem_forensic.py / --segments 0,1 +./dev python tools/replay/mode_sim.py / --segment 0 ``` -| key | functionality | -|------|-----------------------| -| 1 | Cruise Resume / Accel | -| 2 | Cruise Set / Decel | -| 3 | Cruise Cancel | -| r | Reset Simulation | -| i | Toggle Ignition | -| q | Exit all | -| wasd | Control manually | -``` - -## MetaDrive - -### Launching Metadrive -Start bridge processes located in tools/sim: -``` bash -./run_bridge.py -``` \ No newline at end of file diff --git a/tools/sim/bridge/common.py b/tools/sim/bridge/common.py index 69dec173c..f30e1b1af 100644 --- a/tools/sim/bridge/common.py +++ b/tools/sim/bridge/common.py @@ -58,6 +58,13 @@ class SimulatorBridge(ABC): self.past_startup_engaged = False self.startup_button_prev = True + self.startup_set_count = 0 + self.auto_engage_speed_done = False + self.AUTO_CRUISE_KPH = 60.0 + + self.manual_throttle = 0.0 + self.manual_brake = 0.0 + self.manual_steer = 0.0 self.test_run = False @@ -126,7 +133,9 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga self.simulator_state.left_blinker = False self.simulator_state.right_blinker = False - throttle_manual = steer_manual = brake_manual = 0. + throttle_manual = self.manual_throttle + steer_manual = self.manual_steer + brake_manual = self.manual_brake # Read manual controls if not q.empty(): @@ -134,11 +143,11 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga if message.type == QueueMessageType.CONTROL_COMMAND: m = message.info.split('_') if m[0] == "steer": - steer_manual = float(m[1]) + steer_manual = self.manual_steer = float(m[1]) elif m[0] == "throttle": - throttle_manual = float(m[1]) + throttle_manual = self.manual_throttle = float(m[1]) elif m[0] == "brake": - brake_manual = float(m[1]) + brake_manual = self.manual_brake = float(m[1]) elif m[0] == "cruise": if m[1] == "down": self.simulator_state.cruise_button = CruiseButtons.DECEL_SET @@ -146,6 +155,7 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga self.simulator_state.cruise_button = CruiseButtons.RES_ACCEL elif m[1] == "cancel": self.simulator_state.cruise_button = CruiseButtons.CANCEL + self.manual_throttle = self.manual_brake = self.manual_steer = 0.0 elif m[1] == "main": self.simulator_state.cruise_button = CruiseButtons.MAIN elif m[0] == "blinker": @@ -173,14 +183,26 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga self.simulator_state.is_engaged = self.simulated_car.sm['selfdriveState'].active if self.simulator_state.is_engaged: + self.manual_throttle = self.manual_brake = self.manual_steer = 0.0 throttle_op = np.clip(self.simulated_car.sm['carControl'].actuators.accel / 1.6, 0.0, 1.0) brake_op = np.clip(-self.simulated_car.sm['carControl'].actuators.accel / 4.0, 0.0, 1.0) steer_op = self.simulated_car.sm['carControl'].actuators.steeringAngleDeg + # After auto-engaging at standstill the set speed is captured at ~floor + # (a crawl). Ramp it up to a real driving speed via cruise-accel presses. + if not self.auto_engage_speed_done and self.rk.frame % 5 == 0: + if self.simulated_car.sm['carState'].vCruise < self.AUTO_CRUISE_KPH: + self.simulator_state.cruise_button = CruiseButtons.RES_ACCEL + else: + self.auto_engage_speed_done = True + self.past_startup_engaged = True elif not self.past_startup_engaged and self.simulated_car.sm['selfdriveState'].engageable: - self.simulator_state.cruise_button = CruiseButtons.DECEL_SET if self.startup_button_prev else CruiseButtons.MAIN # force engagement on startup - self.startup_button_prev = not self.startup_button_prev + # Auto-engage whenever the car is ready. Cruise-Set (DECEL_SET) engages from + # standstill; pressing periodically (not every frame) avoids ratcheting the set + # speed down while still catching the moment the car becomes engageable. + if self.rk.frame % 10 == 0: + self.simulator_state.cruise_button = CruiseButtons.DECEL_SET throttle_out = throttle_op if self.simulator_state.is_engaged else throttle_manual brake_out = brake_op if self.simulator_state.is_engaged else brake_manual @@ -190,6 +212,41 @@ Ignition: {self.simulator_state.ignition} Engaged: {self.simulator_state.is_enga self.world.read_state() self.world.read_sensors(self.simulator_state) + if self.rk.frame % 300 == 0: + try: + from PIL import Image + cam = getattr(self.world, 'road_image', None) + if cam is not None and getattr(cam, 'any', None) and cam.any(): + Image.fromarray(cam[:, :, ::-1]).save(f"/tmp/kilo/cam_{self.rk.frame}.jpg", quality=90) + except Exception as e: + print(f"CAM-ERR {e}", flush=True) + + if self.rk.frame % 25 == 0 and not self.test_run: + try: + st = self.simulator_state + eng = st.is_engaged + engable = self.simulated_car.sm['selfdriveState'].engageable + active = self.simulated_car.sm['selfdriveState'].active + accel = self.simulated_car.sm['carControl'].actuators.accel if eng else float('nan') + cs = self.simulated_car.sm['carState'] + sp = self.simulated_car.sm['starpilotPlan'] + rd = self.simulated_car.sm['radarState'] + m = self.simulated_car.sm['modelV2'] + lead = rd.leadOne + print(f"BRIDGE engaged={eng} engable={engable} active={active} accel={accel:.2f} " + f"gas={throttle_out:.2f} brake={brake_out:.2f} steer={steer_out:.1f} " + f"v={st.speed if st.velocity is not None else -1:.2f} " + f"pos=({st.position[0]:.1f},{st.position[1]:.1f}) yaw={st.bearing:.1f} vCruiseK={cs.vCruise:.0f} " + f"forcStop={sp.forcingStop} apprStop={sp.approachStopLength:.1f} " + f"modelLen={m.position.x[-1]:.0f} mVel={m.velocity.x[-1]:.1f} " + f"mAcc={m.acceleration.x[-1]:.2f} sign={sp.stopSignConfirmed} " + f"leadD={lead.dRel:.1f} leadV={lead.vLead:.1f}", flush=True) + except Exception as e: + import traceback; traceback.print_exc() + print(f"BRIDGE-ERR {type(e).__name__}: {e}", flush=True) + + + if self.world.exit_event.is_set(): self.shutdown() diff --git a/tools/sim/bridge/metadrive/metadrive_bridge.py b/tools/sim/bridge/metadrive/metadrive_bridge.py index b8dc94cf8..24cf5037b 100644 --- a/tools/sim/bridge/metadrive/metadrive_bridge.py +++ b/tools/sim/bridge/metadrive/metadrive_bridge.py @@ -1,4 +1,5 @@ import math +import os from multiprocessing import Queue from metadrive.component.sensors.base_camera import _cuda_enable @@ -28,21 +29,13 @@ def curve_block(length, angle=45, direction=0): } def create_map(track_size=60): - curve_len = track_size * 2 return dict( type=MapGenerateMethod.PG_MAP_FILE, lane_num=2, lane_width=4.5, config=[ - None, - straight_block(track_size), - curve_block(curve_len, 90), - straight_block(track_size), - curve_block(curve_len, 90), - straight_block(track_size), - curve_block(curve_len, 90), - straight_block(track_size), - curve_block(curve_len, 90), + straight_block(track_size * 10), + straight_block(track_size * 20), ] ) @@ -73,7 +66,13 @@ class MetaDriveBridge(SimulatorBridge): image_source="rgb_road", ), sensors=sensors, - image_on_cuda=_cuda_enable, + # CUDA image capture (cudaGraphicsGLRegisterImage) only works when the + # Panda3D GL context is NVIDIA-backed. On hybrid/compute-only laptops the + # display GL runs on the iGPU via Mesa (often llvmpipe software), so CUDA-GL + # interop fails with cudaErrorUnknown and crashes MetaDrive at sensor init. + # Default to the safe CPU readback path; opt into CUDA explicitly only when + # the GL context is actually NVIDIA. + image_on_cuda=bool(os.getenv("STARPIOT_CUDA_IMAGES")) and _cuda_enable, image_observation=True, interface_panel=[], out_of_route_done=False, diff --git a/tools/sim/bridge/metadrive/metadrive_world.py b/tools/sim/bridge/metadrive/metadrive_world.py index c5111289d..f5e7ee939 100644 --- a/tools/sim/bridge/metadrive/metadrive_world.py +++ b/tools/sim/bridge/metadrive/metadrive_world.py @@ -86,6 +86,7 @@ class MetaDriveWorld(World): curr_pos = md_vehicle.position state.velocity = md_vehicle.velocity + state.position = md_vehicle.position state.bearing = md_vehicle.bearing state.steering_angle = md_vehicle.steering_angle state.gps.from_xy(curr_pos) diff --git a/tools/sim/lib/camerad.py b/tools/sim/lib/camerad.py index be4e1a610..ca015e832 100644 --- a/tools/sim/lib/camerad.py +++ b/tools/sim/lib/camerad.py @@ -1,7 +1,5 @@ import numpy as np import os -import pyopencl as cl -import pyopencl.array as cl_array from msgq.visionipc import VisionIpcServer, VisionStreamType from cereal import messaging @@ -24,17 +22,25 @@ class Camerad: self.vipc_server.start_listener() - # set up for pyopencl rgb to yuv conversion - self.ctx = cl.create_some_context() - self.queue = cl.CommandQueue(self.ctx) - cl_arg = f" -DHEIGHT={H} -DWIDTH={W} -DRGB_STRIDE={W * 3} -DUV_WIDTH={W // 2} -DUV_HEIGHT={H // 2} -DRGB_SIZE={W * H} -DCL_DEBUG " - - kernel_fn = os.path.join(BASEDIR, "tools/sim/rgb_to_nv12.cl") - with open(kernel_fn) as f: - prg = cl.Program(self.ctx, f.read()).build(cl_arg) - self.krnl = prg.rgb_to_nv12 - self.Wdiv4 = W // 4 if (W % 4 == 0) else (W + (4 - W % 4)) // 4 - self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4 + # GPU-accelerated rgb->nv12 via pyopencl when an OpenCL platform exists, + # otherwise fall back to a CPU numpy conversion (e.g. laptops without an + # OpenCL ICD, such as NVIDIA-only hosts). + self.ctx = None + try: + import pyopencl as cl + import pyopencl.array as cl_array + self._cl_array = cl_array + self.ctx = cl.create_some_context() + self.queue = cl.CommandQueue(self.ctx) + cl_arg = f" -DHEIGHT={H} -DWIDTH={W} -DRGB_STRIDE={W * 3} -DUV_WIDTH={W // 2} -DUV_HEIGHT={H // 2} -DRGB_SIZE={W * H} -DCL_DEBUG " + kernel_fn = os.path.join(BASEDIR, "tools/sim/rgb_to_nv12.cl") + with open(kernel_fn) as f: + prg = cl.Program(self.ctx, f.read()).build(cl_arg) + self.krnl = prg.rgb_to_nv12 + self.Wdiv4 = W // 4 if (W % 4 == 0) else (W + (4 - W % 4)) // 4 + self.Hdiv4 = H // 4 if (H % 4 == 0) else (H + (4 - H % 4)) // 4 + except Exception: + self.ctx = None def cam_send_yuv_road(self, yuv): self._send_yuv(yuv, self.frame_road_id, 'roadCameraState', VisionStreamType.VISION_STREAM_ROAD) @@ -49,11 +55,35 @@ class Camerad: assert rgb.shape == (H, W, 3), f"{rgb.shape}" assert rgb.dtype == np.uint8 - rgb_cl = cl_array.to_device(self.queue, rgb) - yuv_cl = cl_array.empty_like(rgb_cl) - self.krnl(self.queue, (self.Wdiv4, self.Hdiv4), None, rgb_cl.data, yuv_cl.data).wait() - yuv = np.resize(yuv_cl.get(), rgb.size // 2) - return yuv.data.tobytes() + if self.ctx is not None: + rgb_cl = self._cl_array.to_device(self.queue, rgb) + yuv_cl = self._cl_array.empty_like(rgb_cl) + self.krnl(self.queue, (self.Wdiv4, self.Hdiv4), None, rgb_cl.data, yuv_cl.data).wait() + yuv = np.resize(yuv_cl.get(), rgb.size // 2) + return yuv.data.tobytes() + return self.rgb_to_yuv_cpu(rgb).tobytes() + + @staticmethod + def rgb_to_yuv_cpu(rgb): + """Numpy NV12 conversion mirroring tools/sim/rgb_to_nv12.cl (BT.601 limited).""" + r = rgb[:, :, 0].astype(np.int32) + g = rgb[:, :, 1].astype(np.int32) + b = rgb[:, :, 2].astype(np.int32) + + y = ((b * 13 + g * 65 + r * 33 + 64) >> 7) + 16 + y = np.clip(y, 0, 255).astype(np.uint8) + + def avg2(ch): + return (ch[0::2, 0::2] + ch[0::2, 1::2] + ch[1::2, 0::2] + ch[1::2, 1::2] + 1) >> 1 + + r2, g2, b2 = avg2(r), avg2(g), avg2(b) + u = ((b2 * 56 - g2 * 37 - r2 * 19 + 0x8080) >> 8) & 0xFF + v = ((r2 * 56 - g2 * 47 - b2 * 9 + 0x8080) >> 8) & 0xFF + + uv = np.empty((H // 2, W), dtype=np.uint8) + uv[:, 0::2] = u.astype(np.uint8) + uv[:, 1::2] = v.astype(np.uint8) + return np.concatenate([y.ravel(), uv.ravel()]).astype(np.uint8) def _send_yuv(self, yuv, frame_id, pub_type, yuv_type): eof = int(frame_id * 0.05 * 1e9) diff --git a/tools/sim/lib/common.py b/tools/sim/lib/common.py index 132493213..cc9606b78 100644 --- a/tools/sim/lib/common.py +++ b/tools/sim/lib/common.py @@ -5,7 +5,7 @@ import numpy as np from abc import ABC, abstractmethod from collections import namedtuple -W, H = 1928, 1208 +W, H = 1164, 874 vec3 = namedtuple("vec3", ["x", "y", "z"]) @@ -41,6 +41,7 @@ class SimulatorState: self.ignition = True self.velocity: vec3 = None + self.position: tuple = (0, 0) self.bearing: float = 0 self.gps = GPSState() self.imu = IMUState() diff --git a/tools/sim/lib/keyboard_ctrl.py b/tools/sim/lib/keyboard_ctrl.py index 0a17f0ee8..804d579a4 100644 --- a/tools/sim/lib/keyboard_ctrl.py +++ b/tools/sim/lib/keyboard_ctrl.py @@ -34,7 +34,11 @@ KEYBOARD_HELP = """ def getch() -> str: STDIN_FD = sys.stdin.fileno() - old_settings = termios.tcgetattr(STDIN_FD) + try: + old_settings = termios.tcgetattr(STDIN_FD) + except (termios.error, OSError): + # No controlling terminal (e.g. headless / background run). Idle instead of crashing. + return None try: # set mode = old_settings.copy() @@ -48,8 +52,13 @@ def getch() -> str: termios.tcsetattr(STDIN_FD, termios.TCSAFLUSH, mode) ch = sys.stdin.read(1) + except (termios.error, OSError): + ch = None finally: - termios.tcsetattr(STDIN_FD, termios.TCSADRAIN, old_settings) + try: + termios.tcsetattr(STDIN_FD, termios.TCSADRAIN, old_settings) + except (termios.error, OSError): + pass return ch def print_keyboard_help(): @@ -60,7 +69,10 @@ def keyboard_poll_thread(q: 'Queue[QueueMessage]'): while True: c = getch() - if c == '1': + if c is None: + # No terminal input available (headless/background); avoid busy-looping. + time.sleep(0.05) + elif c == '1': q.put(control_cmd_gen("cruise_up")) elif c == '2': q.put(control_cmd_gen("cruise_down")) diff --git a/tools/sim/lib/simulated_car.py b/tools/sim/lib/simulated_car.py index 68ff3050d..e48f618f7 100644 --- a/tools/sim/lib/simulated_car.py +++ b/tools/sim/lib/simulated_car.py @@ -15,7 +15,7 @@ class SimulatedCar: def __init__(self): self.pm = messaging.PubMaster(['can', 'pandaStates']) - self.sm = messaging.SubMaster(['carControl', 'controlsState', 'carParams', 'selfdriveState']) + self.sm = messaging.SubMaster(['carControl', 'controlsState', 'carParams', 'selfdriveState', 'carState', 'starpilotPlan', 'radarState', 'modelV2']) self.cp = self.get_car_can_parser() self.idx = 0 self.params = Params() @@ -76,6 +76,16 @@ class SimulatedCar: msg.append(self.packer.make_can_msg("STEERING_CONTROL", 2, {})) msg.append(self.packer.make_can_msg("ACC_HUD", 2, {})) msg.append(self.packer.make_can_msg("LKAS_HUD", 2, {})) + # 0x35e CAMERA_MESSAGES: StarPilot's Honda carstate registers this message + # when it reads the speed-limit / stop-sign fields (HAS_CAMERA_MESSAGES + # flag), and can_valid requires *every* registered message to be valid. + # The real camera sends it on the cam bus, so publish it here too; + # CANPacker auto-fills the Honda COUNTER/CHECKSUM. SPEED_LIMIT_SIGN=0 + # means no posted speed limit, so calculate_speed_limit() returns 0.0. + msg.append(self.packer.make_can_msg("CAMERA_MESSAGES", 2, { + "SPEED_LIMIT_SIGN": 0, + "ROAD_SIGN": 0, + })) self.pm.send('can', can_list_to_can_capnp(msg)) diff --git a/tools/sim/starpilot_sim.sh b/tools/sim/starpilot_sim.sh new file mode 100755 index 000000000..9c6d4b1eb --- /dev/null +++ b/tools/sim/starpilot_sim.sh @@ -0,0 +1,183 @@ +#!/usr/bin/env bash +# StarPilot live MetaDrive simulator launcher. +# +# Runs the ENTIRE openpilot stack (manager.py -> modeld/controlsd/plannerd/...) on +# the host RTX GPU, bridged to a straight-road MetaDrive world, in one of three +# longitudinal modes. +# +# Usage: +# ./tools/sim/starpilot_sim.sh {exp|chill|cem|hem} [--headless] [--joystick] [--cpu] +# +# exp : standard Experimental mode (ConditionalExperimental=off, ConditionalChill=off, HybridExperimental=off) +# chill : pure Chill / CCM mode (ConditionalChill=on) +# cem : Conditional Experimental mode (ConditionalExperimental=on) +# hem : Hybrid Experimental mode (HybridExperimental=on) +# +# The model runs on the RTX GPU (tinygrad CUDA) by default. Pass --cpu to force +# the CPU backend instead (fragile in this fork; not recommended). +# +# By default the model is traced LIVE from driving_supercombo.onnx inside modeld +# (Option 1: no tinygrad-JIT pickling, so no JIT-unpickle corruption). Pass --pkl +# to fall back to the precompiled pickle artifact instead. +# +# The stack runs from the isolated host worktree (.host_runtime/linux/worktree), +# which is synced from this repo by `./dev sync` (run automatically below). +set -euo pipefail + +# Clean up any stale openpilot/sim processes and shared-memory sockets from a +# previous run. pkill on "manager.py" alone misses the the_galaxy/galaxy Flask +# processes (different proctitles) which otherwise hold port 8083 and crash-loop, +# and stale msgq sockets in /dev/shm collide with fresh publishers. +pkill -9 -f "system/manager/manager.py" 2>/dev/null || true +pkill -9 -f "run_bridge.py" 2>/dev/null || true +pkill -9 -f "metadrive" 2>/dev/null || true +pkill -9 -f "the_galaxy" 2>/dev/null || true +pkill -9 -f "galaxy.galaxy" 2>/dev/null || true +sleep 1 +rm -rf /dev/shm/msgq* /dev/shm/visionipc* /tmp/openpilot* 2>/dev/null || true + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +MODE="${1:-hem}" +ARGS=("${@:2}") +HEADLESS=0 +JOYSTICK=0 +SIM_DEV="CUDA" +LIVE_ONNX=1 +for a in "${ARGS[@]}"; do + case "$a" in + --headless) HEADLESS=1 ;; + --joystick) JOYSTICK=1 ;; + --cpu) SIM_DEV="CPU" ;; + --pkl) LIVE_ONNX=0 ;; + *) echo "unknown arg: $a" >&2; exit 1 ;; + esac +done + +case "$MODE" in + exp|chill|cem|hem) : ;; + *) echo "usage: $0 {exp|chill|cem|hem} [--headless] [--joystick] [--cpu]" >&2; exit 1 ;; +esac + +# 1. Sync main repo -> host worktree so our sim/mode changes land there. +"${ROOT_DIR}/dev" sync shared >/dev/null 2>&1 || true + +HOST_WORKTREE="${ROOT_DIR}/.host_runtime/linux/worktree" +HOST_PY="${HOST_WORKTREE}/.venv/bin/python3" +MODEL_STORE="${HOME}/.comma/starpilot/data/models" + +echo "==> StarPilot sim mode: ${MODE} (model device: ${SIM_DEV}) (host worktree: ${HOST_WORKTREE})" + +# 2. Make sure the builtin model is present where modeld loads it. Sync wipes the +# worktree copy, so re-copy it from the model store on every launch. With the +# live-ONNX path (default) modeld ignores the pkl and traces the ONNX instead, +# but keep the pkl in place so --pkl still works. +MODEL_SRC="${MODEL_STORE}/rdf43_driving_tinygrad.pkl" +MODEL_DST="${HOST_WORKTREE}/selfdrive/modeld/models/driving_tinygrad.pkl" +if [[ -f "${MODEL_SRC}" ]]; then + if ! cmp -s "${MODEL_SRC}" "${MODEL_DST}"; then + cp -f "${MODEL_SRC}" "${MODEL_DST}" + echo "==> Copied builtin model to ${MODEL_DST}" + fi +else + echo "!! builtin model not found at ${MODEL_SRC}" >&2 +fi + +# Option 1 (live-ONNX): modeld traces driving_supercombo.onnx in-memory on the +# RTX, so the tinygrad-JIT pickle round-trip (and its QCOM-rewrite bug) never +# happens. Falls back to the pkl artifact when --pkl is passed or the onnx is +# missing. +# Locate the source ONNX: prefer the model store copy, fall back to the repo root. +ONNX_SRC="${MODEL_STORE}/driving_supercombo.onnx" +if [[ ! -f "${ONNX_SRC}" ]] && [[ -f "${ROOT_DIR}/driving_supercombo.onnx" ]]; then + ONNX_SRC="${ROOT_DIR}/driving_supercombo.onnx" +fi +if [[ "$LIVE_ONNX" == "1" ]]; then + if [[ -f "${ONNX_SRC}" ]]; then + export STARPIOT_LIVE_ONNX="${ONNX_SRC}" + echo "==> modeld will trace ${ONNX_SRC} live on ${SIM_DEV}" + else + echo "!! driving_supercombo.onnx NOT found (checked model store and repo root)." >&2 + echo " Falling back to the precompiled pkl, which CRASHES with CUDA_ERROR_INVALID_IMAGE." >&2 + echo " Place the onnx at ${MODEL_STORE}/driving_supercombo.onnx and re-run." >&2 + fi +fi + +# 3. Set the longitudinal mode params (mutually exclusive). +# ForceOnroad: on a PC host there is no real panda/ignition, so hardwared never +# transitions the device onroad by itself; force it so modeld/controlsd/plannerd +# come up and the sim can engage and drive. +"${HOST_PY}" - "${MODE}" <<'PY' +import sys +from openpilot.common.params import Params +p = Params() +mode = sys.argv[1] +p.put_bool_nonblocking("ConditionalExperimental", mode == "cem") +p.put_bool_nonblocking("ConditionalChill", mode == "chill") +p.put_bool_nonblocking("HybridExperimental", mode == "hem") +p.put_bool_nonblocking("ForceOnroad", True) +print(f"params: ConditionalExperimental={p.get_bool('ConditionalExperimental')} " + f"ConditionalChill={p.get_bool('ConditionalChill')} " + f"HybridExperimental={p.get_bool('HybridExperimental')} " + f"ForceOnroad={p.get_bool('ForceOnroad')}") +PY + +# 4. Sim environment (mirrors tools/sim/launch_openpilot.sh) + GPU selection. +export PASSIVE="0" +export NOBOARD="1" +export SIMULATION="1" +export SKIP_FW_QUERY="1" +export FINGERPRINT="HONDA_CIVIC_2022" +export BLOCK="camerad,loggerd,encoderd,micd,logmessaged,soundd,mapd" +if [[ "$HEADLESS" == "1" ]]; then + export BLOCK="${BLOCK},ui" +fi +export DEV="${SIM_DEV}" +export STARPIOT_SIM_DEV="${SIM_DEV}" +# The supercombo artifact is compiled all-CUDA (warp + policy on the RTX). The +# warp device must be CUDA so the precompiled kernels match; modeld copies the +# host-memory camera frames onto the GPU before the warp. +if [[ "${SIM_DEV}" == "CPU" ]]; then + export WARP_DEV="CPU" + export QUEUE_DEV="CPU" +else + export WARP_DEV="${SIM_DEV}" + export QUEUE_DEV="${SIM_DEV}" +fi + +# 5. Launch the full openpilot stack in the background, then the bridge in the foreground. +cat <<'HELP' +==> Controls (focus the terminal that launched this script): + i : toggle ignition (starts ON by default; if the status line shows + Ignition: False, press i once to turn it back on) + 2 : cruise Set (engage lateral + longitudinal) 1 : cruise Resume / accel + 3 : cruise Cancel r : reset simulation + w/a/s/d : manual throttle / steer / brake q : quit everything + z/x : blinker left / right +HELP +cd "${HOST_WORKTREE}" +MANAGER_LOG="${HOST_WORKTREE}/.host_sim_manager.log" +"${HOST_PY}" -c "from openpilot.selfdrive.test.helpers import set_params_enabled; set_params_enabled()" +echo "==> Starting openpilot stack (manager.py) ..." +"${HOST_PY}" system/manager/manager.py >"${MANAGER_LOG}" 2>&1 & +MANAGER_PID=$! +echo "==> manager.py pid ${MANAGER_PID} (log: ${MANAGER_LOG})" +trap 'echo "==> stopping manager (${MANAGER_PID})"; kill "${MANAGER_PID}" 2>/dev/null || true' EXIT + +# modeld blocks on the camerad visionipc stream before it starts tracing the +# ONNX, and that stream is published by the bridge. The bridge MUST come up +# promptly or modeld never loads, so keep the startup delay short and let the +# bridge auto-engage once controls report engageable. +sleep 8 + +# Single-camera mode: MetaDrive must render only the road viewport. Dual-camera +# renders a second wide viewpoint every frame, roughly halving frame rate. The +# driving pipeline (modeld) consumes roadCameraState only, so the wide cam adds +# no control value in sim. +BRIDGE_ARGS=() +if [[ "$JOYSTICK" == "1" ]]; then + BRIDGE_ARGS+=(--joystick) +fi +echo "==> Starting MetaDrive bridge (${BRIDGE_ARGS[*]}) ..." +"${HOST_PY}" tools/sim/run_bridge.py "${BRIDGE_ARGS[@]}" || true +echo "==> Bridge exited."