diff --git a/examples/openpilot/compile3.py b/examples/openpilot/compile3.py
index 041c332202..74afadfc14 100644
--- a/examples/openpilot/compile3.py
+++ b/examples/openpilot/compile3.py
@@ -58,9 +58,13 @@ def test(test_val=None):
   Tensor.manual_seed(100)
   new_inputs = {nm:Tensor.randn(*st.shape, dtype=dtype).mul(8).realize() for nm, (st, _, dtype, _) in
                 sorted(zip(run.captured.expected_names, run.captured.expected_st_vars_dtype_device))}
+  new_inputs_numpy = {k:v.numpy() for k,v in new_inputs.items()}
   for _ in range(20):
     st = time.perf_counter()
-    out = run(**new_inputs)
+    # Need to cast non-image inputs from numpy, this is only realistic way to run it
+    inputs = {**{k:v for k,v in new_inputs.items() if 'img' in k},
+              **{k:Tensor(v) for k,v in new_inputs_numpy.items() if 'img' not in k}}
+    out = run(**inputs)
     mt = time.perf_counter()
     val = out['outputs'].numpy()
     et = time.perf_counter()
diff --git a/test/external/external_benchmark_openpilot.py b/test/external/external_benchmark_openpilot.py
index b817725f7a..2811ec891c 100644
--- a/test/external/external_benchmark_openpilot.py
+++ b/test/external/external_benchmark_openpilot.py
@@ -22,6 +22,7 @@ if __name__ == "__main__":
   input_types = {inp.name: tensor_dtype_to_np_dtype(inp.type.tensor_type.elem_type) for inp in onnx_model.graph.input}
   new_inputs = {k:Tensor.randn(*shp, dtype=_from_np_dtype(input_types[k])).mul(8).realize() for k,shp in input_shapes.items()}
   new_inputs_junk = {k:Tensor.randn(*shp, dtype=_from_np_dtype(input_types[k])).mul(8).realize() for k,shp in input_shapes.items()}
+  new_inputs_junk_numpy = {k:v.numpy() for k,v in new_inputs_junk.items()}
 
   # benchmark
   for _ in range(5):
@@ -35,7 +36,10 @@ if __name__ == "__main__":
   for _ in range(20):
     GlobalCounters.reset()
     st = time.perf_counter_ns()
-    ret = next(iter(run_onnx_jit(**new_inputs_junk).values())).cast(dtypes.float32).numpy()
+    # Need to cast non-image inputs from numpy, this is only realistic way to run model
+    inputs = {**{k:v for k,v in new_inputs_junk.items() if 'img' in k},
+              **{k:Tensor(v) for k,v in new_inputs_junk_numpy.items() if 'img' not in k}}
+    ret = next(iter(run_onnx_jit(**inputs).values())).cast(dtypes.float32).numpy()
     print(f"jitted:  {(time.perf_counter_ns() - st)*1e-6:7.4f} ms")
 
   suffix = ""