From dd4615850cd8e265ba25c16fa14ec98368599c64 Mon Sep 17 00:00:00 2001
From: Daniel Koepping <elkoled@gmail.com>
Date: Tue, 8 Sep 2026 18:05:03 -0700
Subject: [PATCH] replay chestnut in CI (#38826)

run model replay on chestnut in CI
---
 Jenkinsfile                                             | 3 ++-
 openpilot/selfdrive/test/chestnut.sh                    | 2 +-
 openpilot/selfdrive/test/process_replay/model_replay.py | 9 +++++++--
 3 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index f1450c304b..b6b3eec5e3 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -254,7 +254,8 @@ node {
       },
       'chestnut': {
         deviceStage("chestnut", "mici-chestnut-ci", ["UNSAFE=1", "CHESTNUT=1"], [
-          step("compile big model", "./openpilot/selfdrive/test/chestnut.sh"),
+          step("build", "./openpilot/selfdrive/test/chestnut.sh"),
+          step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py --chestnut"),
         ])
       },
 
diff --git a/openpilot/selfdrive/test/chestnut.sh b/openpilot/selfdrive/test/chestnut.sh
index 747a2b275b..1ffd12c9e4 100755
--- a/openpilot/selfdrive/test/chestnut.sh
+++ b/openpilot/selfdrive/test/chestnut.sh
@@ -3,5 +3,5 @@ set -e
 
 TARGET=openpilot/selfdrive/modeld/models/big_driving_tinygrad.pkl.chunkmanifest
 rm -f "$TARGET"
-scons --cache-disable "$TARGET"
+SCONSFLAGS="-j2 --cache-disable" ./openpilot/system/manager/build.py
 test -s "$TARGET"
diff --git a/openpilot/selfdrive/test/process_replay/model_replay.py b/openpilot/selfdrive/test/process_replay/model_replay.py
index 927c9b38f1..47ca3fb104 100755
--- a/openpilot/selfdrive/test/process_replay/model_replay.py
+++ b/openpilot/selfdrive/test/process_replay/model_replay.py
@@ -25,6 +25,8 @@ SEGMENT = 4
 START_FRAME = 0
 END_FRAME = 60
 
+CHESTNUT = "--chestnut" in sys.argv
+
 SEND_EXTRA_INPUTS = bool(int(os.getenv("SEND_EXTRA_INPUTS", "0")))
 
 DATA_TOKEN = os.getenv("CI_ARTIFACTS_TOKEN","")
@@ -39,7 +41,7 @@ EXEC_TIMINGS = [
 ]
 
 def get_log_fn(test_route, ref="master"):
-  return f"{test_route}_model_tici_{ref}.zst"
+  return f"{test_route}_model_{'chestnut' if CHESTNUT else 'tici'}_{ref}.zst"
 
 def plot(proposed, master, title, tmp):
   proposed = list(proposed)
@@ -170,6 +172,8 @@ def model_replay(lr, frs):
 
   msgs = modeld_msgs + dmonitoringmodeld_msgs
   chestnut = any(m.modelV2.big for m in modeld_msgs if m.which() == "modelV2")
+  if CHESTNUT:
+    assert chestnut and all(m.modelV2.big for m in modeld_msgs if m.which() == "modelV2"), "Chestnut replay must run the big model without fallback"
 
   header = ['model', 'max instant', 'max instant allowed', 'average', 'max average allowed', 'test result']
   rows = []
@@ -285,7 +289,8 @@ if __name__ == "__main__":
       diff_short, diff_long, failed = format_diff(results, log_paths, 'master')
 
       if "CI" in os.environ:
-        comment_replay_report(log_msgs, cmp_log, log_msgs)
+        if not CHESTNUT:
+          comment_replay_report(log_msgs, cmp_log, log_msgs)
         failed = False
         print(diff_long)
       print('-------------\n'*5)