mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-06-25 12:02:05 +08:00
Armand du Parc Locmaria
52e182611d
* zero ll patched big model * probe in a subprocess so usbgpu lock gets released * compiles * runs * num_jobs gets overwritten, use side effect * poll tg devices * make sure build crashes on missing gpu * fine not to rely on Device.default * seperate tg env for each model runner * comment * Revert "seperate tg env for each model runner" This reverts commit f6470cc4258eaeb3e8e37907ef370871c9af5aa4. * env is shared, gate on flag * no fallback warp dev must be set * build for current device only, unless pc/release * comment * list * listen for plug in * add icon to status bar, read params on every frame (?) * log available devices * try copy out when loading? * Revert "log available devices" This reverts commit e8c52a5d59456d4820ecb13b99a6c46ea1386a20. * Revert "try copy out when loading?" This reverts commit 518f403aa03faeda1950fe3dbce0d9e4c1584455. * don't trigger device probe/caching on modeld prepare * re-export with ll and road edges * dont cache devices in manager process * get USBGPU from params * no usbgpu env * missed one * sconscript don't poll * unconditional env * always explicitely set devices on input tensors * set DEV so amd uses right compiler and iface?? * fix flag * bump tg * rm xdg_cache_home * tg don't bump all the way * missing gmmu=0 at compile time * dm set dev * tg backend * update gitignore * missing import * unused imports * rely on Device.DEFAULT at compile time (already the case bc onnxrunner) * comments * dm warp needs DEV set too * build both smol and big * misc typos * set dev at compile time * don't need * DEV=CPU when getting metadata, ensure we don't grab gpu lock * this would also grab lock * put bool * warp compile always prepare only * missed one * poll ui * missing here * don't force usbgpu at build time * tmp patch fetch_fw * catch all, follow hardwared patterns * simpler * compile make input queues * revert this * group this more readable * rm empty line * make dummy frame using numpy * revert compile make input queues * no compiler at runtime * cleanup * fine to rebuild all on change to device node for now * fix usbgpu_present * fix sconscript * no size in header stream decompress * DEBUG=2 * minimal viable feedback * egpu gray * oops * gotta do this actually * modeld build only depends on modeld devices * don't ship onnx to release? or chunk * don't need * can only set compiler on dev= * none device works, will use default * make linter happy * chunk agnostic onnx input to compile_modeld * chunk big onnx * +x chunker * fix #! * and don't ship chunked onnx to release * firmware now in correct location * better err on missing onnx/chunk * SConscript also need to accept chunked onnx * metadata also need to load maybe chunked * dedupe cmd * this needs to be on cpu * devices are set in the tgflags, we already depend on them * rebuilding on changed order is fine * read file chunked can already load either chunked or not * chunk all big onnx * less confusing * unused import * python device to load onnx bytes * default device for runners, python for metadata * why not * chunked to shm
56 lines
1.9 KiB
Python
Executable File
56 lines
1.9 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
import sys
|
|
import math
|
|
import os
|
|
from pathlib import Path
|
|
|
|
CHUNK_SIZE = 45 * 1024 * 1024 # 45MB, under GitHub's 50MB limit
|
|
|
|
def get_chunk_name(name, idx, num_chunks):
|
|
return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
|
|
|
|
def get_manifest_path(name):
|
|
return f"{name}.chunkmanifest"
|
|
|
|
def _chunk_paths(path, num_chunks):
|
|
return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
|
|
|
|
def get_chunk_targets(path, file_size):
|
|
num_chunks = math.ceil(file_size / CHUNK_SIZE)
|
|
return _chunk_paths(path, num_chunks)
|
|
|
|
def chunk_file(path, targets):
|
|
manifest_path, *chunk_paths = targets
|
|
with open(path, 'rb') as f:
|
|
data = f.read()
|
|
actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
|
|
assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
|
|
for i, chunk_path in enumerate(chunk_paths):
|
|
with open(chunk_path, 'wb') as f:
|
|
f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
|
|
Path(manifest_path).write_text(str(len(chunk_paths)))
|
|
os.remove(path)
|
|
|
|
def get_existing_chunks(path):
|
|
if os.path.isfile(path):
|
|
return [path]
|
|
if os.path.isfile(manifest := get_manifest_path(path)):
|
|
num_chunks = int(Path(manifest).read_text().strip())
|
|
return _chunk_paths(path, num_chunks)
|
|
raise FileNotFoundError(path)
|
|
|
|
def read_file_chunked(path):
|
|
manifest_path = get_manifest_path(path)
|
|
if os.path.isfile(manifest_path):
|
|
num_chunks = int(Path(manifest_path).read_text().strip())
|
|
return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
|
|
if os.path.isfile(path):
|
|
return Path(path).read_bytes()
|
|
raise FileNotFoundError(path)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
path = sys.argv[1]
|
|
chunk_paths = get_chunk_targets(path, os.path.getsize(path))
|
|
chunk_file(path, chunk_paths)
|