mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-08-21 00:03:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

Revert "Simpler file chunker (#37276)"

Browse Source 
This reverts commit b27fa58444.
This commit is contained in:
Bruce Wayne
2026-02-20 16:43:43 -08:00
parent 09926bf5b5
commit d6af0e6eb5
6 changed files with 72 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
selfdrive/modeld/SConscript
+23 -22
View File
@@ -1,18 +1,14 @@
import os
import glob
from openpilot.common.file_chunker import chunk_file, get_chunk_paths
Import('env', 'arch')
chunker_file = File("#common/file_chunker.py")
lenv = env.Clone()
CHUNK_BYTES = int(os.environ.get("TG_CHUNK_BYTES", str(45 * 1024 * 1024)))
tinygrad_root = env.Dir("#").abspath
tinygrad_files = ["#"+x for x in glob.glob(env.Dir("#tinygrad_repo").relpath + "/**", recursive=True, root_dir=tinygrad_root)
if 'pycache' not in x and os.path.isfile(os.path.join(tinygrad_root, x))]
def estimate_pickle_max_size(onnx_size):
return 1.2 * onnx_size + 10 * 1024 * 1024 # 20% + 10MB is plenty
# Get model metadata
for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
fn = File(f"models/{model_name}").abspath
@@ -30,34 +26,39 @@ image_flag = {
'larch64': 'IMAGE=2',
}.get(arch, 'IMAGE=0')
script_files = [File(Dir("#selfdrive/modeld").File("compile_warp.py").abspath)]
compile_warp_cmd = f'{tg_flags} python3 {Dir("#selfdrive/modeld").abspath}/compile_warp.py '
cmd = f'{tg_flags} python3 {Dir("#selfdrive/modeld").abspath}/compile_warp.py '
from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
warp_targets = []
for cam in [_ar_ox_fisheye, _os_fisheye]:
w, h = cam.width, cam.height
warp_targets += [File(f"models/warp_{w}x{h}_tinygrad.pkl").abspath, File(f"models/dm_warp_{w}x{h}_tinygrad.pkl").abspath]
def chunk_warps(target, source, env):
for t in warp_targets:
chunk_file(t, 1)
chunk_targets = sum([get_chunk_paths(t, estimate_pickle_max_size(0)) for t in warp_targets], [])
lenv.Command(chunk_targets, tinygrad_files + script_files + [chunker_file],
[compile_warp_cmd, chunk_warps])
lenv.Command(warp_targets, tinygrad_files + script_files, cmd)
def tg_compile(flags, model_name):
pythonpath_string = 'PYTHONPATH="${PYTHONPATH}:' + env.Dir("#tinygrad_repo").abspath + '"'
fn = File(f"models/{model_name}").abspath
pkl = fn + "_tinygrad.pkl"
onnx_path = fn + ".onnx"
chunk_targets = get_chunk_paths(pkl, estimate_pickle_max_size(os.path.getsize(onnx_path)))
def do_chunk(target, source, env):
chunk_file(pkl, len(chunk_targets))
return lenv.Command(
chunk_targets,
[onnx_path] + tinygrad_files + [chunker_file],
[f'{pythonpath_string} {flags} {image_flag} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {pkl}',
do_chunk]
out = fn + "_tinygrad.pkl"
full = out + ".full"
parts = out + ".parts"
full_node = lenv.Command(
full,
[fn + ".onnx"] + tinygrad_files,
f'{pythonpath_string} {flags} {image_flag} python3 {Dir("#tinygrad_repo").abspath}/examples/openpilot/compile3.py {fn}.onnx {full}'
)
split_script = File(Dir("#selfdrive/modeld").File("external_pickle.py").abspath)
parts_node = lenv.Command(
parts,
[full_node, split_script, Value(str(CHUNK_BYTES))],
[f'python3 {split_script.abspath} {full} {out} {CHUNK_BYTES}', Delete(full)],
)
lenv.NoCache(parts_node)
lenv.AlwaysBuild(parts_node)
return parts_node
# Compile small models
for model_name in ['driving_vision', 'driving_policy', 'dmonitoring_model']:
tg_compile(tg_flags, model_name)