mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-08-20 07:43:48 +08:00
Fix external GPU model handoff
This commit is contained in:
@@ -143,6 +143,22 @@ def _select_builtin_model(params: Params) -> None:
|
||||
params.put("DrivingModelName", "Regret Driven Framework V4")
|
||||
|
||||
|
||||
def _close_tinygrad_disk_cache_connection() -> None:
|
||||
"""Drop a tinygrad cache connection before handing work to another thread."""
|
||||
import tinygrad.helpers as tinygrad_helpers
|
||||
|
||||
connection = getattr(tinygrad_helpers, "_db_connection", None)
|
||||
if connection is None:
|
||||
return
|
||||
|
||||
try:
|
||||
connection.close()
|
||||
except Exception:
|
||||
cloudlog.exception("failed to close tinygrad disk cache connection")
|
||||
finally:
|
||||
tinygrad_helpers._db_connection = None
|
||||
|
||||
|
||||
def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log.ModelDataV2.Action,
|
||||
lat_action_t: float, long_action_t: float, v_ego: float, mlsim: bool,
|
||||
is_v9: bool, is_v14: bool, is_v15: bool, starpilot_toggles,
|
||||
@@ -593,15 +609,22 @@ def main(demo=False):
|
||||
|
||||
def load_big_model() -> None:
|
||||
nonlocal big_model
|
||||
candidate = None
|
||||
try:
|
||||
wait_usbgpu_link()
|
||||
candidate = ModelState(vipc_client_main.width, vipc_client_main.height, True)
|
||||
if not candidate.uses_external_gpu:
|
||||
raise RuntimeError("external GPU model resolved to the builtin model")
|
||||
candidate.warmup()
|
||||
big_model = candidate
|
||||
except Exception:
|
||||
cloudlog.exception("external GPU model load or warmup failed")
|
||||
candidate = None
|
||||
finally:
|
||||
# tinygrad's global SQLite cache connection is thread-bound. Loading and
|
||||
# warming here can create it in this worker, so close it here before the
|
||||
# model (or native fallback) runs on modeld's main thread.
|
||||
_close_tinygrad_disk_cache_connection()
|
||||
big_model = candidate
|
||||
|
||||
loader = threading.Thread(target=load_big_model, name="big_model_loader", daemon=True)
|
||||
loader.start()
|
||||
|
||||
@@ -19,6 +19,25 @@ def test_external_gpu_uses_a_longer_load_watchdog():
|
||||
assert modeld.BIG_MODEL_RUN_WAIT_TIMEOUT_MS == 3000
|
||||
|
||||
|
||||
def test_tinygrad_disk_cache_connection_is_closed_before_thread_handoff(monkeypatch):
|
||||
import tinygrad.helpers as tinygrad_helpers
|
||||
|
||||
class FakeConnection:
|
||||
def __init__(self):
|
||||
self.closed = False
|
||||
|
||||
def close(self):
|
||||
self.closed = True
|
||||
|
||||
connection = FakeConnection()
|
||||
monkeypatch.setattr(tinygrad_helpers, "_db_connection", connection)
|
||||
|
||||
modeld._close_tinygrad_disk_cache_connection()
|
||||
|
||||
assert connection.closed
|
||||
assert tinygrad_helpers._db_connection is None
|
||||
|
||||
|
||||
def test_out_of_band_artifact_round_trip():
|
||||
artifact = {"weights": np.arange(32, dtype=np.float32), "metadata": {"version": 1}}
|
||||
stream = io.BytesIO()
|
||||
|
||||
Reference in New Issue
Block a user