mirror of
https://gitlvb.teallvbs.xyz/IQ.Lvbs/IQ.Pilot.git
synced 2026-07-16 06:02:07 +08:00
IQ.Lvbs CI [bot]
109 lines
4.1 KiB
Python
109 lines
4.1 KiB
Python
"""Compare a host-compiled (finalized) pkl against a device-compiled reference.
|
|
|
|
Runs on the comma device with DEV=QCOM. Each pkl is executed in its own
|
|
subprocess: loading two near-identical pkls in one process partially merges
|
|
their interned UOp graphs and cross-contaminates results (verified: same-file
|
|
A/B passes, near-identical A/B corrupts policy outputs). Identical seeded
|
|
inputs are fed to both; outputs and timings are compared in the parent.
|
|
"""
|
|
import json, os, subprocess, sys, tempfile
|
|
|
|
import numpy as np
|
|
|
|
|
|
def run_one(pkl_path, npz_path):
|
|
import importlib.util, pickle, time
|
|
from functools import partial
|
|
|
|
spec = importlib.util.spec_from_file_location(
|
|
"cs", "/data/openpilot/iqpilot/selfdrive/iqmodeld/tools/compile_supercombo.py")
|
|
cs = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(cs)
|
|
|
|
from tinygrad.device import Device
|
|
from tinygrad.tensor import Tensor
|
|
from openpilot.system.camerad.cameras.nv12_info import get_nv12_info
|
|
|
|
with open(pkl_path, "rb") as f:
|
|
out = pickle.load(f)
|
|
fs = out["frame_skip"]
|
|
shapes = out["metadata"]["input_shapes"]
|
|
|
|
def run_jit(jit, input_keys, make_queues, make_random_inputs, seed=42, n_timed=20):
|
|
input_queues, npy = make_queues(Device.DEFAULT)
|
|
np.random.seed(seed)
|
|
Tensor.manual_seed(seed)
|
|
for v in npy.values():
|
|
v[:] = np.random.randn(*v.shape).astype(v.dtype)
|
|
random_inputs = make_random_inputs()
|
|
outs = jit(**{k: input_queues[k] for k in input_keys}, **random_inputs)
|
|
Device.default.synchronize()
|
|
vals = [np.copy(o.numpy()) for o in outs]
|
|
bufs = [np.copy(v.numpy()) for v in input_queues.values()]
|
|
times = []
|
|
for _ in range(n_timed):
|
|
st = time.perf_counter()
|
|
jit(**{k: input_queues[k] for k in input_keys}, **random_inputs)
|
|
Device.default.synchronize()
|
|
times.append((time.perf_counter() - st) * 1e3)
|
|
return vals, bufs, float(np.median(times))
|
|
|
|
arrays, times = {}, {}
|
|
vals, bufs, times["run_policy"] = run_jit(
|
|
out["run_policy"], cs.POLICY_INPUTS,
|
|
partial(cs.make_input_queues, shapes, fs),
|
|
partial(cs.make_random_images, keys=["warped"], shape=(2, 6, *shapes["img"][2:])))
|
|
for i, a in enumerate(vals): arrays[f"policy_out_{i}"] = a
|
|
for i, a in enumerate(bufs): arrays[f"policy_q_{i}"] = a
|
|
|
|
for key in out:
|
|
if not (isinstance(key, tuple) and len(key) == 2):
|
|
continue
|
|
cam_w, cam_h = key
|
|
nv12 = cs.NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
|
|
vals, bufs, times[f"warp_{cam_w}x{cam_h}"] = run_jit(
|
|
out[key], cs.WARP_INPUTS,
|
|
partial(cs.make_warp_input_queues, shapes, fs),
|
|
partial(cs.make_random_images, keys=["frame", "big_frame"], shape=nv12.size,
|
|
device=cs.WARP_DEV))
|
|
for i, a in enumerate(vals): arrays[f"warp_{cam_w}x{cam_h}_out_{i}"] = a
|
|
for i, a in enumerate(bufs): arrays[f"warp_{cam_w}x{cam_h}_q_{i}"] = a
|
|
|
|
np.savez(npz_path, **arrays)
|
|
with open(npz_path + ".times.json", "w") as f:
|
|
json.dump(times, f)
|
|
|
|
|
|
if len(sys.argv) == 4 and sys.argv[1] == "--run-one":
|
|
run_one(sys.argv[2], sys.argv[3])
|
|
sys.exit(0)
|
|
|
|
HOST_PKL, DEV_PKL = sys.argv[1], sys.argv[2]
|
|
tmp = tempfile.mkdtemp(prefix="validate_pkls_")
|
|
res = {}
|
|
for tag, pkl in (("host", HOST_PKL), ("ref", DEV_PKL)):
|
|
npz = os.path.join(tmp, f"{tag}.npz")
|
|
subprocess.run([sys.executable, os.path.abspath(__file__), "--run-one", pkl, npz],
|
|
check=True, env=os.environ)
|
|
res[tag] = (np.load(npz), json.load(open(npz + ".times.json")))
|
|
|
|
host, ht = res["host"]
|
|
ref, rt = res["ref"]
|
|
for name in sorted(set(ht) | set(rt)):
|
|
print(f"{name}: host {ht[name]:.2f} ms device-compiled {rt[name]:.2f} ms")
|
|
|
|
all_ok = True
|
|
for name in ref.files:
|
|
x, y = host[name], ref[name]
|
|
if np.array_equal(x, y):
|
|
print(f" {name}: EXACT match ({x.shape})")
|
|
else:
|
|
diff = np.abs(x.astype(np.float64) - y.astype(np.float64))
|
|
rel = diff.max() / (np.abs(y).max() + 1e-9)
|
|
print(f" {name}: max abs diff {diff.max():.6g} max rel {rel:.6g} ({x.shape})")
|
|
if not np.allclose(x, y, rtol=2e-2, atol=2e-2):
|
|
all_ok = False
|
|
|
|
print("VALIDATION", "PASS" if all_ok else "FAIL")
|
|
sys.exit(0 if all_ok else 1)
|