diff --git a/openpilot/selfdrive/modeld/modeld.py b/openpilot/selfdrive/modeld/modeld.py
index 0d90f1225e..dbb64ec281 100755
--- a/openpilot/selfdrive/modeld/modeld.py
+++ b/openpilot/selfdrive/modeld/modeld.py
@@ -77,6 +77,12 @@ def get_action_from_model(model_output: dict[str, np.ndarray], prev_action: log.
                                 shouldStop=bool(stop))
 
 
+def drop_chestnut() -> None:
+  if "AMD" in Device._opened_devices:
+    for d in Device._opened_devices:
+      Device[d].pending.pop(Device["AMD"], None)
+
+
 class ChestnutGpuState:
   # GPU metrics require modeld's GPU context
   def __init__(self, pm: PubMaster, big: bool):
@@ -281,6 +287,8 @@ def main(demo=False):
     loader.join(BIG_MODEL_TIMEOUT)
     model = big_model
     params.put_bool("ChestnutActive", model is not None)
+    if model is None:
+      drop_chestnut()
 
   small_model = ModelState(vipc_client_main.width, vipc_client_main.height, False) if model is None or CHESTNUT else None
   if model is None:
@@ -413,6 +421,7 @@ def main(demo=False):
         raise
       # fallback to small model
       cloudlog.exception("big model failed, fall back to small")
+      drop_chestnut()
       params.put_bool("ChestnutActive", False)
       model = small_model
       if chestnut_state is not None: