mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-09-04 23:23:42 +08:00
Team Noah
This commit is contained in:
firestar5683
+48
-33
@@ -46,7 +46,7 @@ MODEL_CONTEXT_FREQ = 5
|
||||
# GitHub/GitLab advertise a 100 MB per-file limit. Use the decimal limit so
|
||||
# artifacts such as a 104.4 MB PKL are split before they reach the remote.
|
||||
REPOSITORY_FILE_LIMIT = 100_000_000
|
||||
DEFAULT_MULTIPART_SIZE = 95 * 1024 * 1024
|
||||
DEFAULT_CHUNK_SIZE = 45 * 1024 * 1024
|
||||
USBGPU_PROBE_ATTEMPTS = 10
|
||||
DEFAULT_SUPERCOMBO_BEHAVIOR_VERSION = "v16"
|
||||
|
||||
@@ -117,7 +117,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--gpu", "--external-gpu", dest="external_gpu", action="store_true",
|
||||
help="Compile the driving artifact for the USB AMD GPU.")
|
||||
parser.add_argument("--split-artifact", type=Path, help="Split an existing oversized PKL without compiling.")
|
||||
parser.add_argument("--chunk-size-mib", type=int, default=95, help="Multipart size in MiB; must be below 100.")
|
||||
parser.add_argument("--chunk-size-mib", type=int, default=45, help="Native chunk size in MiB; must be below 100.")
|
||||
parser.add_argument("--no-split", action="store_true",
|
||||
help="Keep a single .pkl even if >100 MiB (for local installs, which need one "
|
||||
"file). Auto-enabled for local- model IDs.")
|
||||
@@ -399,6 +399,15 @@ def multipart_output_paths(artifact: Path, output_dir: Path | None = None) -> li
|
||||
]
|
||||
|
||||
|
||||
def chunked_output_paths(artifact: Path, output_dir: Path | None = None) -> list[Path]:
|
||||
output_dir = output_dir or artifact.parent
|
||||
return [
|
||||
*sorted(output_dir.glob(f"{artifact.name}.chunk[0-9][0-9]of[0-9][0-9]")),
|
||||
output_dir / f"{artifact.name}.chunkmanifest",
|
||||
output_dir / f"{artifact.name}.sha256",
|
||||
]
|
||||
|
||||
|
||||
def _update_local_artifact_metadata(model_key: str, external_gpu: bool) -> None:
|
||||
metadata_path = MODELS_PATH / ".model_artifacts.json"
|
||||
try:
|
||||
@@ -461,7 +470,7 @@ def install_local_artifact(artifact: Path, model_key: str, version: str, externa
|
||||
def split_oversized_artifact(
|
||||
artifact: Path,
|
||||
output_dir: Path | None = None,
|
||||
chunk_size: int = DEFAULT_MULTIPART_SIZE,
|
||||
chunk_size: int = DEFAULT_CHUNK_SIZE,
|
||||
force: bool = False,
|
||||
) -> list[Path]:
|
||||
artifact = artifact.resolve()
|
||||
@@ -469,46 +478,52 @@ def split_oversized_artifact(
|
||||
if not artifact.is_file():
|
||||
raise FileNotFoundError(artifact)
|
||||
if chunk_size <= 0 or chunk_size >= REPOSITORY_FILE_LIMIT:
|
||||
raise ValueError("Multipart chunk size must be between 1 byte and 100 MiB.")
|
||||
raise ValueError("Chunk size must be between 1 byte and 100 MiB.")
|
||||
|
||||
remove_paths(multipart_output_paths(artifact, output_dir))
|
||||
remove_paths([*multipart_output_paths(artifact, output_dir), *chunked_output_paths(artifact, output_dir)])
|
||||
if artifact.stat().st_size <= REPOSITORY_FILE_LIMIT and not force:
|
||||
return []
|
||||
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
digest = hashlib.sha256()
|
||||
part_paths: list[Path] = []
|
||||
chunk_paths: list[Path] = []
|
||||
artifact_size = artifact.stat().st_size
|
||||
chunk_count = (artifact_size + chunk_size - 1) // chunk_size
|
||||
if chunk_count > 99:
|
||||
raise ValueError(f"Artifact requires {chunk_count} chunks; two-digit chunk names support at most 99.")
|
||||
with open(artifact, "rb") as source:
|
||||
for index in range(100):
|
||||
part_path = output_dir / f"{artifact.name}.p{index:02d}"
|
||||
part_size = 0
|
||||
with open(part_path, "wb") as part_file:
|
||||
while part_size < chunk_size:
|
||||
chunk = source.read(min(1024 * 1024, chunk_size - part_size))
|
||||
for index in range(chunk_count):
|
||||
chunk_path = output_dir / f"{artifact.name}.chunk{index + 1:02d}of{chunk_count:02d}"
|
||||
written = 0
|
||||
with open(chunk_path, "wb") as chunk_file:
|
||||
while written < chunk_size:
|
||||
chunk = source.read(min(1024 * 1024, chunk_size - written))
|
||||
if not chunk:
|
||||
break
|
||||
part_file.write(chunk)
|
||||
chunk_file.write(chunk)
|
||||
digest.update(chunk)
|
||||
part_size += len(chunk)
|
||||
if part_size == 0:
|
||||
part_path.unlink()
|
||||
break
|
||||
part_paths.append(part_path)
|
||||
if not part_paths:
|
||||
written += len(chunk)
|
||||
if written == 0:
|
||||
chunk_path.unlink()
|
||||
raise RuntimeError("Unexpected end of artifact while chunking.")
|
||||
chunk_paths.append(chunk_path)
|
||||
if not chunk_paths:
|
||||
raise ValueError(f"Artifact is empty: {artifact}")
|
||||
|
||||
manifest_path = output_dir / f"{artifact.name}.chunkmanifest"
|
||||
manifest_path.write_text(str(chunk_count))
|
||||
checksum_path = output_dir / f"{artifact.name}.sha256"
|
||||
checksum_path.write_text(f"{digest.hexdigest()} {artifact.name}\n")
|
||||
|
||||
verify_digest = hashlib.sha256()
|
||||
for part_path in part_paths:
|
||||
with open(part_path, "rb") as part_file:
|
||||
for chunk in iter(lambda: part_file.read(1024 * 1024), b""):
|
||||
for chunk_path in chunk_paths:
|
||||
with open(chunk_path, "rb") as chunk_file:
|
||||
for chunk in iter(lambda: chunk_file.read(1024 * 1024), b""):
|
||||
verify_digest.update(chunk)
|
||||
if verify_digest.hexdigest() != digest.hexdigest():
|
||||
remove_paths([*part_paths, checksum_path])
|
||||
raise RuntimeError("Split artifact failed checksum verification.")
|
||||
return [*part_paths, checksum_path]
|
||||
remove_paths([*chunk_paths, manifest_path, checksum_path])
|
||||
raise RuntimeError("Chunked artifact failed checksum verification.")
|
||||
return [manifest_path, *chunk_paths, checksum_path]
|
||||
|
||||
|
||||
def compile_driving(
|
||||
@@ -528,6 +543,7 @@ def compile_driving(
|
||||
removed = remove_paths(sorted({
|
||||
output_path,
|
||||
*multipart_output_paths(output_path, output_dir),
|
||||
*chunked_output_paths(output_path, output_dir),
|
||||
*output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl"),
|
||||
*output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.p[0-9][0-9]"),
|
||||
*output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl.sha256"),
|
||||
@@ -552,6 +568,7 @@ def compile_driving(
|
||||
str(frame_skip),
|
||||
"--image-history-pipeline",
|
||||
image_history_pipeline,
|
||||
"--out-of-band",
|
||||
*source_args,
|
||||
]
|
||||
if version:
|
||||
@@ -572,7 +589,6 @@ def compile_driving(
|
||||
"GMMU": "0",
|
||||
"TC_OPT": "2",
|
||||
})
|
||||
command.append("--out-of-band")
|
||||
wait_for_external_gpu()
|
||||
command = external_gpu_compile_command(command)
|
||||
subprocess.run(command, cwd=REPO_ROOT, env=compile_env, check=True)
|
||||
@@ -695,13 +711,12 @@ def main() -> int:
|
||||
if is_local and not args.no_install:
|
||||
install_local_artifact(output, model_key, version, args.external_gpu)
|
||||
else:
|
||||
multipart_outputs = split_oversized_artifact(output)
|
||||
if multipart_outputs:
|
||||
print(" artifact exceeds 100 MB; created repository-safe multipart files:")
|
||||
for multipart_output in multipart_outputs:
|
||||
print(f" {multipart_output.name} ({multipart_output.stat().st_size} bytes)")
|
||||
output.unlink()
|
||||
print(f" removed oversized source artifact {output.name}")
|
||||
chunked_outputs = split_oversized_artifact(output, force=True)
|
||||
print(" created repository-safe native chunks:")
|
||||
for chunked_output in chunked_outputs:
|
||||
print(f" {chunked_output.name} ({chunked_output.stat().st_size} bytes)")
|
||||
output.unlink()
|
||||
print(f" removed source artifact {output.name}")
|
||||
print("Done.")
|
||||
return 0
|
||||
|
||||
|
||||
@@ -17,10 +17,10 @@ if str(REPO_ROOT / "scripts") not in sys.path:
|
||||
from model_compiler import REPOSITORY_FILE_LIMIT, split_oversized_artifact
|
||||
|
||||
DEFAULT_OPENPILOT = Path.home() / "openpilot"
|
||||
DEFAULT_WORKSPACE = Path("/Volumes/T5/StarPilot-Model-Rebuild-2026-06-22")
|
||||
DEFAULT_SOURCE_MAP = REPO_ROOT / "scripts/model_source_map_v22.json"
|
||||
DEFAULT_MANIFEST = DEFAULT_WORKSPACE / "manifests/model_names_v22.json"
|
||||
REMOTE = os.environ.get("STAR_PILOT_MODEL_REMOTE", "comma@192.168.3.109")
|
||||
DEFAULT_WORKSPACE = Path("/Volumes/T5/StarPilot-Model-Rebuild")
|
||||
DEFAULT_SOURCE_MAP = REPO_ROOT / "scripts/model_source_map_v25.json"
|
||||
DEFAULT_MANIFEST = DEFAULT_WORKSPACE / "manifests/model_names_v25.json"
|
||||
REMOTE = os.environ.get("STAR_PILOT_MODEL_REMOTE", "comma@192.168.3.110")
|
||||
REMOTE_ROOT = Path("/data/openpilot")
|
||||
SSH_OPTIONS = ("-o", "ConnectTimeout=10", "-o", "ConnectionAttempts=1")
|
||||
|
||||
@@ -149,9 +149,9 @@ def extract_git_file(repo: Path, ref: str, git_path: str, destination: Path) ->
|
||||
temporary.replace(destination)
|
||||
|
||||
|
||||
def find_model_paths(repo: Path, ref: str, input_format: str) -> list[str]:
|
||||
def find_model_paths(repo: Path, ref: str, input_format: str, uses_external_gpu: bool = False) -> list[str]:
|
||||
requested = (
|
||||
("driving_supercombo.onnx",)
|
||||
(("big_driving_supercombo.onnx",) if uses_external_gpu else ("driving_supercombo.onnx",))
|
||||
if input_format == "supercombo"
|
||||
else (
|
||||
"driving_vision.onnx",
|
||||
@@ -186,7 +186,7 @@ def extract_model(model_id: str, source: dict, repo: Path, workspace: Path) -> d
|
||||
output_dir = workspace / "onnx" / model_id
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
extracted = []
|
||||
for git_path in find_model_paths(repo, ref, input_format):
|
||||
for git_path in find_model_paths(repo, ref, input_format, bool(source.get("uses_external_gpu"))):
|
||||
filename = Path(git_path).name
|
||||
destination = output_dir / f"{model_id}_{filename}"
|
||||
extract_git_file(repo, ref, git_path, destination)
|
||||
@@ -222,34 +222,37 @@ def remote(command: str, *, capture: bool = False):
|
||||
|
||||
def stage_ready_artifact(artifact: Path, workspace: Path) -> None:
|
||||
ready_path = workspace / "ready-for-resources" / artifact.name
|
||||
multipart_outputs = split_oversized_artifact(artifact, ready_path.parent)
|
||||
if multipart_outputs:
|
||||
ready_path.unlink(missing_ok=True)
|
||||
for multipart_output in multipart_outputs:
|
||||
multipart_output.chmod(0o644)
|
||||
else:
|
||||
shutil.copyfile(artifact, ready_path)
|
||||
ready_path.chmod(0o644)
|
||||
chunked_outputs = split_oversized_artifact(artifact, ready_path.parent, force=True)
|
||||
ready_path.unlink(missing_ok=True)
|
||||
for chunked_output in chunked_outputs:
|
||||
chunked_output.chmod(0o644)
|
||||
|
||||
|
||||
def pull_remote_artifact(remote_output: str, local_output: Path) -> None:
|
||||
"""Pull either a single artifact or the compiler's repository-safe parts."""
|
||||
"""Pull a single artifact or native chunks and reconstruct the T5 archive copy."""
|
||||
local_output.unlink(missing_ok=True)
|
||||
for stale in local_output.parent.glob(f"{local_output.name}.p[0-9][0-9]"):
|
||||
stale.unlink()
|
||||
for stale in local_output.parent.glob(f"{local_output.name}.chunk[0-9][0-9]of[0-9][0-9]"):
|
||||
stale.unlink()
|
||||
local_output.parent.joinpath(f"{local_output.name}.chunkmanifest").unlink(missing_ok=True)
|
||||
local_output.parent.joinpath(f"{local_output.name}.sha256").unlink(missing_ok=True)
|
||||
local_output.parent.mkdir(parents=True, exist_ok=True)
|
||||
run(["rsync", "-az", "-e", "ssh -o ConnectTimeout=10 -o ConnectionAttempts=1", f"{REMOTE}:{remote_output}*", f"{local_output.parent}/"])
|
||||
|
||||
parts = sorted(local_output.parent.glob(f"{local_output.name}.p[0-9][0-9]"))
|
||||
parts = sorted(local_output.parent.glob(f"{local_output.name}.chunk[0-9][0-9]of[0-9][0-9]"))
|
||||
if local_output.is_file():
|
||||
for part in parts:
|
||||
part.unlink()
|
||||
local_output.parent.joinpath(f"{local_output.name}.chunkmanifest").unlink(missing_ok=True)
|
||||
local_output.parent.joinpath(f"{local_output.name}.sha256").unlink(missing_ok=True)
|
||||
return
|
||||
checksum_path = local_output.parent / f"{local_output.name}.sha256"
|
||||
if not parts or not checksum_path.is_file():
|
||||
raise FileNotFoundError(f"Remote compiler returned neither {local_output.name} nor verified parts")
|
||||
chunk_manifest = local_output.parent / f"{local_output.name}.chunkmanifest"
|
||||
if not parts or not checksum_path.is_file() or not chunk_manifest.is_file():
|
||||
raise FileNotFoundError(f"Remote compiler returned neither {local_output.name} nor verified native chunks")
|
||||
if int(chunk_manifest.read_text().strip()) != len(parts):
|
||||
raise ValueError(f"Native chunk count mismatch for {local_output.name}")
|
||||
with open(local_output, "wb") as destination:
|
||||
for part in parts:
|
||||
with open(part, "rb") as source:
|
||||
@@ -261,6 +264,7 @@ def pull_remote_artifact(remote_output: str, local_output: Path) -> None:
|
||||
raise ValueError(f"Reassembled {local_output.name} checksum mismatch: {actual} != {expected}")
|
||||
for part in parts:
|
||||
part.unlink()
|
||||
chunk_manifest.unlink()
|
||||
checksum_path.unlink()
|
||||
|
||||
|
||||
@@ -307,7 +311,7 @@ def artifact_result(model_id: str, path: Path, status: str) -> dict:
|
||||
"path": str(path),
|
||||
"size": path.stat().st_size,
|
||||
"sha256": sha256_file(path),
|
||||
"multipart": path.stat().st_size > REPOSITORY_FILE_LIMIT,
|
||||
"chunk_count": len(list((path.parent.parent / "ready-for-resources").glob(f"{path.name}.chunk[0-9][0-9]of[0-9][0-9]"))),
|
||||
}
|
||||
|
||||
|
||||
@@ -338,16 +342,7 @@ def validate_model(model_id: str, version: str, workspace: Path) -> dict:
|
||||
def update_manifest(base_manifest: Path, workspace: Path, source_map: dict) -> dict:
|
||||
payload = load_json(base_manifest)
|
||||
models = payload["models"] if isinstance(payload, dict) else payload
|
||||
if not any(model.get("id") == "deeprl3v2" for model in models):
|
||||
models.append({
|
||||
"id": "deeprl3v2",
|
||||
"name": "Deep RL 3 V2 👀📡",
|
||||
"version": "v15",
|
||||
"series": "OP Series",
|
||||
"released": "2026-06-17",
|
||||
"community_favorite": False,
|
||||
})
|
||||
multipart_handoff = []
|
||||
build_results = []
|
||||
for model in models:
|
||||
source = source_map.get(model["id"], {})
|
||||
if "uses_external_gpu" in source:
|
||||
@@ -357,25 +352,30 @@ def update_manifest(base_manifest: Path, workspace: Path, source_map: dict) -> d
|
||||
model.pop("artifact_size", None)
|
||||
model.pop("artifact_sha256", None)
|
||||
model.pop("artifact_urls", None)
|
||||
if not artifact.is_file() or artifact.stat().st_size <= REPOSITORY_FILE_LIMIT:
|
||||
model.pop("artifact_chunk_count", None)
|
||||
if not artifact.is_file():
|
||||
model.pop("artifact_url", None)
|
||||
else:
|
||||
multipart_handoff.append({
|
||||
continue
|
||||
chunks = sorted((workspace / "ready-for-resources").glob(f"{artifact.name}.chunk[0-9][0-9]of[0-9][0-9]"))
|
||||
model.update({
|
||||
"artifact_format": "tinygrad_single_v1",
|
||||
"artifact_size": artifact.stat().st_size,
|
||||
"artifact_sha256": sha256_file(artifact),
|
||||
"artifact_chunk_count": len(chunks),
|
||||
})
|
||||
build_results.append({
|
||||
"id": model["id"],
|
||||
"filename": artifact.name,
|
||||
"size": artifact.stat().st_size,
|
||||
"sha256": sha256_file(artifact),
|
||||
"parts": [
|
||||
path.name
|
||||
for path in sorted((workspace / "ready-for-resources").glob(f"{artifact.name}.p[0-9][0-9]"))
|
||||
],
|
||||
"chunks": [path.name for path in chunks],
|
||||
})
|
||||
output = {"models": models}
|
||||
output_path = workspace / "manifests/model_names_v22.json"
|
||||
output_path = workspace / "manifests/model_names_v25.json"
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_path.write_text(json.dumps(output, indent=2, ensure_ascii=False) + "\n")
|
||||
(workspace / "ready-for-resources" / "multipart.json").write_text(
|
||||
json.dumps(multipart_handoff, indent=2) + "\n",
|
||||
(workspace / "ready-for-resources" / "artifacts.json").write_text(
|
||||
json.dumps(build_results, indent=2) + "\n",
|
||||
)
|
||||
return output
|
||||
|
||||
|
||||
+26
-17
@@ -31,12 +31,12 @@ OPENPILOT_REPO = "commaai/openpilot"
|
||||
RESOURCES_REPO = os.environ.get("STARPILOT_RESOURCES_REPO", "firestar5683/StarPilot-Resources")
|
||||
HF_BUCKET = os.environ.get("STARPILOT_HF_BUCKET", "StarPilot-Driving/StarPilot-Resources")
|
||||
RESOURCE_BRANCH = "Models"
|
||||
MANIFEST_VERSION = "v24"
|
||||
MANIFEST_VERSION = "v25"
|
||||
DEFAULT_BEHAVIOR_VERSION = "v16"
|
||||
DEVICE_ROOT = "/data/openpilot"
|
||||
REPOSITORY_FILE_LIMIT = 100_000_000
|
||||
LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1"
|
||||
CHUNK_SUFFIX_RE = re.compile(r"\.p\d{2}$")
|
||||
CHUNK_SUFFIX_RE = re.compile(r"\.chunk\d{2}of\d{2}$")
|
||||
SHA_RE = re.compile(r"(?<![0-9a-f])([0-9a-f]{40})(?![0-9a-f])", re.IGNORECASE)
|
||||
DATE_RE = re.compile(r"([A-Za-z]+\s+\d{1,2},\s+\d{4})")
|
||||
MODEL_ID_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$")
|
||||
@@ -531,12 +531,15 @@ def remote_compile(info: ReleaseInfo, source: Path, ip: str, workspace: Path, ke
|
||||
for filename in remote_files:
|
||||
run(scp_base(ip) + [f"comma@{ip}:{output_dir}/{filename}", str(artifact_dir / filename)])
|
||||
|
||||
parts = sorted(artifact_dir.glob(f"{artifact_prefix}.p[0-9][0-9]"))
|
||||
parts = sorted(artifact_dir.glob(f"{artifact_prefix}.chunk[0-9][0-9]of[0-9][0-9]"))
|
||||
full_artifact = artifact_dir / artifact_prefix
|
||||
chunk_manifest_path = artifact_dir / f"{artifact_prefix}.chunkmanifest"
|
||||
checksum_path = artifact_dir / f"{artifact_prefix}.sha256"
|
||||
if parts:
|
||||
if full_artifact.exists() or not checksum_path.is_file():
|
||||
raise ReleaseError("Device returned invalid multipart output")
|
||||
if full_artifact.exists() or not checksum_path.is_file() or not chunk_manifest_path.is_file():
|
||||
raise ReleaseError("Device returned invalid native chunk output")
|
||||
if int(chunk_manifest_path.read_text(encoding="utf-8").strip()) != len(parts):
|
||||
raise ReleaseError("Device chunk manifest does not match the returned chunks")
|
||||
expected = checksum_path.read_text(encoding="utf-8").split()[0].lower()
|
||||
digest = hashlib.sha256()
|
||||
size = 0
|
||||
@@ -547,8 +550,8 @@ def remote_compile(info: ReleaseInfo, source: Path, ip: str, workspace: Path, ke
|
||||
size += len(chunk)
|
||||
actual = digest.hexdigest()
|
||||
if actual != expected:
|
||||
raise ReleaseError(f"Multipart checksum mismatch: {actual} != {expected}")
|
||||
artifact_files = [*parts, checksum_path]
|
||||
raise ReleaseError(f"Native chunk checksum mismatch: {actual} != {expected}")
|
||||
artifact_files = [chunk_manifest_path, *parts, checksum_path]
|
||||
elif full_artifact.is_file():
|
||||
size = full_artifact.stat().st_size
|
||||
actual = sha256_file(full_artifact)
|
||||
@@ -564,7 +567,7 @@ def remote_compile(info: ReleaseInfo, source: Path, ip: str, workspace: Path, ke
|
||||
"status": "compiled",
|
||||
"size": size,
|
||||
"sha256": expected,
|
||||
"multipart": bool(parts),
|
||||
"chunk_count": len(parts),
|
||||
"files": [path.name for path in artifact_files],
|
||||
"path": str(artifact_dir),
|
||||
}
|
||||
@@ -587,6 +590,7 @@ def manifest_entry(info: ReleaseInfo, result: dict) -> dict:
|
||||
"artifact_format": "tinygrad_single_v1",
|
||||
"artifact_size": result["size"],
|
||||
"artifact_sha256": result["sha256"],
|
||||
"artifact_chunk_count": result["chunk_count"],
|
||||
"uses_external_gpu": info.uses_external_gpu,
|
||||
}
|
||||
|
||||
@@ -629,14 +633,15 @@ def hf_copy(source: Path, bucket: str, remote_path: str) -> None:
|
||||
run([hf, "buckets", "cp", str(source), destination, "--format", "quiet"])
|
||||
|
||||
|
||||
def upload_huggingface(info: ReleaseInfo, result: dict, workspace: Path, bucket: str, manifest: Path, upload_onnx: bool, source: Path) -> None:
|
||||
def upload_huggingface(info: ReleaseInfo, result: dict, workspace: Path, bucket: str, manifest_version: str,
|
||||
manifest: Path, upload_onnx: bool, source: Path) -> None:
|
||||
artifact_dir = Path(result["path"])
|
||||
for filename in result["files"]:
|
||||
hf_copy(artifact_dir / filename, bucket, f"models/{info.model_id}/{filename}")
|
||||
hf_copy(artifact_dir / filename, bucket, f"models/{manifest_version}/{info.model_id}/{filename}")
|
||||
if upload_onnx:
|
||||
hf_copy(source, bucket, f"onnx/{info.model_id}/{source.name}")
|
||||
hf_copy(manifest, bucket, f"manifests/{manifest.name}")
|
||||
print(f"Hugging Face upload complete: {bucket}/models/{info.model_id}/")
|
||||
print(f"Hugging Face upload complete: {bucket}/models/{manifest_version}/{info.model_id}/")
|
||||
|
||||
|
||||
def git_output(repo: Path, args: list[str]) -> str:
|
||||
@@ -658,13 +663,16 @@ def check_resources_repo(repo: Path, branch: str) -> None:
|
||||
raise ReleaseError("Resources checkout has unpushed or missing remote commits; sync it before releasing")
|
||||
|
||||
|
||||
def push_github(info: ReleaseInfo, result: dict, resources_repo: Path, manifest: Path, branch: str, force: bool) -> None:
|
||||
def push_github(info: ReleaseInfo, result: dict, resources_repo: Path, manifest_version: str,
|
||||
manifest: Path, branch: str, force: bool) -> None:
|
||||
artifact_dir = Path(result["path"])
|
||||
artifact_names = list(result["files"])
|
||||
release_dir = resources_repo / manifest_version / info.model_id
|
||||
release_dir.mkdir(parents=True, exist_ok=True)
|
||||
destination_paths = []
|
||||
stale_relative: list[str] = []
|
||||
for filename in artifact_names:
|
||||
destination = resources_repo / filename
|
||||
destination = release_dir / filename
|
||||
if destination.exists() and not force:
|
||||
raise ReleaseError(f"Artifact already exists in GitHub checkout: {destination}; use --force to replace it")
|
||||
shutil.copy2(artifact_dir / filename, destination)
|
||||
@@ -673,7 +681,7 @@ def push_github(info: ReleaseInfo, result: dict, resources_repo: Path, manifest:
|
||||
prefix = f"{info.model_id}_driving_tinygrad.pkl"
|
||||
if force:
|
||||
allowed = {path.name for path in destination_paths}
|
||||
for stale in resources_repo.glob(f"{prefix}*"):
|
||||
for stale in release_dir.glob(f"{prefix}*"):
|
||||
if stale.name not in allowed and stale.is_file():
|
||||
stale.unlink()
|
||||
stale_relative.append(str(stale.relative_to(resources_repo)))
|
||||
@@ -770,11 +778,12 @@ def main() -> int:
|
||||
resources_repo = args.resources_repo.expanduser().resolve()
|
||||
check_resources_repo(resources_repo, args.resources_branch)
|
||||
manifest = update_manifest(resources_repo, info, result, args.manifest_version)
|
||||
upload_huggingface(info, result, workspace, args.hf_bucket, manifest, not args.no_onnx_upload, source)
|
||||
push_github(info, result, resources_repo, manifest, args.resources_branch, args.force)
|
||||
upload_huggingface(info, result, workspace, args.hf_bucket, args.manifest_version,
|
||||
manifest, not args.no_onnx_upload, source)
|
||||
push_github(info, result, resources_repo, args.manifest_version, manifest, args.resources_branch, args.force)
|
||||
print("\nRelease complete.")
|
||||
print(f" local artifact: {result['path']}")
|
||||
print(f" Hugging Face: {args.hf_bucket}/models/{info.model_id}/")
|
||||
print(f" Hugging Face: {args.hf_bucket}/models/{args.manifest_version}/{info.model_id}/")
|
||||
print(f" GitHub: {RESOURCES_REPO}/{args.resources_branch}")
|
||||
return 0
|
||||
except (ReleaseError, subprocess.CalledProcessError) as error:
|
||||
|
||||
@@ -0,0 +1,429 @@
|
||||
{
|
||||
"tr14223": {
|
||||
"ref": "64fd3f986081ba6cb488d02e799845367add29e2",
|
||||
"input_format": "split",
|
||||
"source_id": "tr1422"
|
||||
},
|
||||
"tr15223": {
|
||||
"ref": "ed38ca8cc998f0a2b47cde29da11b782e4551e8b",
|
||||
"input_format": "split",
|
||||
"source_id": "tr1522"
|
||||
},
|
||||
"tr16223": {
|
||||
"ref": "e4b6a2736580dc427473bcf8a6e3b08eb352edd4",
|
||||
"input_format": "split",
|
||||
"source_id": "tr16"
|
||||
},
|
||||
"letr223": {
|
||||
"ref": "c2fa07b82c2e09964e4b6dccb3e4465a2e4ad138",
|
||||
"input_format": "split",
|
||||
"source_id": "letr22"
|
||||
},
|
||||
"spacelab22223": {
|
||||
"ref": "d3a0378ae601f61307a53d1173c251ee36c56317",
|
||||
"input_format": "split",
|
||||
"source_id": "spacelab2222"
|
||||
},
|
||||
"vikander223": {
|
||||
"ref": "d134fdd7d51ba84848866c0265e37508c38f9366",
|
||||
"input_format": "split",
|
||||
"source_id": "vikander22"
|
||||
},
|
||||
"fof223": {
|
||||
"ref": "d807b5c476658dba7760d30949baa619074c4a4e",
|
||||
"input_format": "split",
|
||||
"source_id": "fof22"
|
||||
},
|
||||
"vfof223": {
|
||||
"ref": "c790d3f58ad32cf15f384e6018d942b2348f0c5b",
|
||||
"input_format": "split",
|
||||
"source_id": "vfof22"
|
||||
},
|
||||
"dtr223": {
|
||||
"ref": "5b405920ec9511e146432de08171d6f93295248b",
|
||||
"input_format": "split",
|
||||
"source_id": "dtr22"
|
||||
},
|
||||
"dtr6223": {
|
||||
"ref": "a6cf39c07a67dbc346af03010093fe4788c402d0",
|
||||
"input_format": "split",
|
||||
"source_id": "dtr622"
|
||||
},
|
||||
"uvdtr6223": {
|
||||
"ref": "079d0461604632719e73dc5b6a5eb9897c624da2",
|
||||
"input_format": "split",
|
||||
"source_id": "uvdtr622"
|
||||
},
|
||||
"sp2223": {
|
||||
"ref": "f1b8f510b2c554191ca15cebe35f2c5b304cf23c",
|
||||
"input_format": "split",
|
||||
"source_id": "sp222"
|
||||
},
|
||||
"fp223": {
|
||||
"ref": "95aff72e351e935221ec965aa0384dfd502d383d",
|
||||
"input_format": "split",
|
||||
"source_id": "fp22"
|
||||
},
|
||||
"kv223": {
|
||||
"ref": "16e87d4c72e5efec18032f6b4a5f6ec35d0df4e4",
|
||||
"input_format": "split",
|
||||
"source_id": "kv22"
|
||||
},
|
||||
"gwm3223": {
|
||||
"ref": "93b26a91a62b69b809162ad2bf65e36be2158dc1",
|
||||
"input_format": "split",
|
||||
"source_id": "gwm322"
|
||||
},
|
||||
"gwm5223": {
|
||||
"ref": "e3ee440f7d5ac12c50db6381e292a55a69b745a7",
|
||||
"input_format": "split",
|
||||
"source_id": "gwm522"
|
||||
},
|
||||
"gwm6223": {
|
||||
"ref": "91a1cea814f76d605d5b15de9634a3c8f79df509",
|
||||
"input_format": "split",
|
||||
"source_id": "gwm622"
|
||||
},
|
||||
"gwm8223": {
|
||||
"ref": "96e7b310b164b55fb73abdd505b66fb36630718e",
|
||||
"input_format": "split",
|
||||
"source_id": "gwm822"
|
||||
},
|
||||
"gwm9223": {
|
||||
"ref": "f2413040a8560c0c17c18353a3c75124a2f09d17",
|
||||
"input_format": "split",
|
||||
"source_id": "gwm922"
|
||||
},
|
||||
"cgwm223": {
|
||||
"ref": "94dd3bd9107cd413ed2418f9e6c0cf805a9ca495",
|
||||
"input_format": "split",
|
||||
"source_id": "cgwm22"
|
||||
},
|
||||
"bd223": {
|
||||
"ref": "b74f5189a74446015c0cf78a4a9f0134a347ae3b",
|
||||
"input_format": "split",
|
||||
"source_id": "bd22"
|
||||
},
|
||||
"nr223": {
|
||||
"ref": "266b642180f0e9278e1db46ccf8c1a2a8dae4bb3",
|
||||
"input_format": "split",
|
||||
"source_id": "nr22"
|
||||
},
|
||||
"tcp2223": {
|
||||
"ref": "19084132cd3956413a0fdfcdc18971a55df5e0ff",
|
||||
"input_format": "split",
|
||||
"source_id": "tcp222"
|
||||
},
|
||||
"tcp3223": {
|
||||
"ref": "5eb912c025a2207c7d428deee74ded49e5905527",
|
||||
"input_format": "split",
|
||||
"source_id": "tcp322"
|
||||
},
|
||||
"fbw223": {
|
||||
"ref": "c4488e3411285f6fc3d008bca885e05731fd75c2",
|
||||
"input_format": "split",
|
||||
"source_id": "fbw22"
|
||||
},
|
||||
"nevada223": {
|
||||
"ref": "3193eac5e385aa010694a8ac192ff38ffe000193",
|
||||
"input_format": "split",
|
||||
"source_id": "nevada22"
|
||||
},
|
||||
"wmi4223": {
|
||||
"ref": "2d8e596a0208dafe4ed2d945b07d6a7d72f70fb8",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi422"
|
||||
},
|
||||
"wmi523": {
|
||||
"ref": "545e0ed13f85e4c744fa1f69095863ea9dbe6d6b",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi52"
|
||||
},
|
||||
"wmi623": {
|
||||
"ref": "54c6c5776a37dc56fde626314f53ec3316d88a48",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi62"
|
||||
},
|
||||
"wmi723": {
|
||||
"ref": "aa3d90c92a7bd7a4e7db1b3fdee3d3e522358823",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi72"
|
||||
},
|
||||
"wmi7223": {
|
||||
"ref": "a2ace1ed6b84d7c7a2dbf990c1b566f9a9df167f",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi722"
|
||||
},
|
||||
"wmi823": {
|
||||
"ref": "4c438f59e7dcf5fddc769032882b62d0cb805e9b",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi82"
|
||||
},
|
||||
"wmi923": {
|
||||
"ref": "8950897d7e4d2ba2426b2b31cf29f25e87a3d4ba",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi92"
|
||||
},
|
||||
"wmi1023": {
|
||||
"ref": "855f5e4ddefd69a20cc4e9da004eb53f3e00d950",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi102"
|
||||
},
|
||||
"wmi1123": {
|
||||
"ref": "7e3fd7a63c5a09c3fabe108b1b62bba3cf684878",
|
||||
"input_format": "split",
|
||||
"source_id": "wmi112"
|
||||
},
|
||||
"cd21023": {
|
||||
"ref": "55f66e2246359c6593605399a0199d94d13ad90d",
|
||||
"input_format": "split",
|
||||
"source_id": "cd2102"
|
||||
},
|
||||
"op223": {
|
||||
"ref": "ae34a24c59d85df7efad5fde2b760fb6d0b9dd6e",
|
||||
"input_format": "split",
|
||||
"source_id": "op22"
|
||||
},
|
||||
"op323": {
|
||||
"ref": "7e548dd765873bed301c0a19cfe10c3ca6be2bbe",
|
||||
"input_format": "split",
|
||||
"source_id": "op32"
|
||||
},
|
||||
"op423": {
|
||||
"ref": "25abcc49fce2f6268d8cabf759e31c626edbf584",
|
||||
"input_format": "split",
|
||||
"source_id": "op42"
|
||||
},
|
||||
"op523": {
|
||||
"ref": "b390b98c5a359c293fdc2501af79b19867fcdac4",
|
||||
"input_format": "split",
|
||||
"source_id": "op52"
|
||||
},
|
||||
"op623": {
|
||||
"ref": "831f13da61d937bb40ea0f50590cdc3b5380f313",
|
||||
"input_format": "split",
|
||||
"source_id": "op62"
|
||||
},
|
||||
"opv73": {
|
||||
"ref": "cb327933002bc1a00bf60a3c20af2eb7a5f653e5",
|
||||
"input_format": "split",
|
||||
"source_id": "opv7"
|
||||
},
|
||||
"opv83": {
|
||||
"ref": "052692b25d63c5ddda276b5c2271383b6aff129f",
|
||||
"input_format": "split",
|
||||
"source_id": "opv8"
|
||||
},
|
||||
"opv93": {
|
||||
"ref": "eae2a73e0ac600d7f0342fc4397568e0733fe6e2",
|
||||
"input_format": "split",
|
||||
"source_id": "opv9"
|
||||
},
|
||||
"opv103": {
|
||||
"ref": "5faf14e04e4038ec4673d119c950b46051630205",
|
||||
"input_format": "split",
|
||||
"source_id": "opv10"
|
||||
},
|
||||
"opv113": {
|
||||
"ref": "bbde5ddb5a0ee35c98a962029355f9a95403a825",
|
||||
"input_format": "split",
|
||||
"source_id": "opv11"
|
||||
},
|
||||
"opv123": {
|
||||
"ref": "7d4c295c27bc3f72727c49ec73b98e45745b588d",
|
||||
"input_format": "split",
|
||||
"source_id": "opv12"
|
||||
},
|
||||
"opv133": {
|
||||
"ref": "faeeaad3a5841694b4c3bd6d37e41e3db4b4873e",
|
||||
"input_format": "split",
|
||||
"source_id": "opv13"
|
||||
},
|
||||
"op163": {
|
||||
"ref": "72101c6e50ea594f24e75d3b605abf8b5cab1d6b",
|
||||
"input_format": "split",
|
||||
"source_id": "op16"
|
||||
},
|
||||
"op16d3": {
|
||||
"ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4",
|
||||
"input_format": "split",
|
||||
"source_id": "op16d"
|
||||
},
|
||||
"op16dv23": {
|
||||
"ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4",
|
||||
"input_format": "split",
|
||||
"source_id": "op16dv2"
|
||||
},
|
||||
"rlv1dl3": {
|
||||
"ref": "38c0e3c95de5b63b56ecbbe507abc49b9b0091a3",
|
||||
"input_format": "split",
|
||||
"source_id": "rlv1dl"
|
||||
},
|
||||
"deeprl33": {
|
||||
"ref": "a1060427b9b3ac1d11cbad05380929ea8235840a",
|
||||
"input_format": "split",
|
||||
"source_id": "deeprl3"
|
||||
},
|
||||
"deeprl333": {
|
||||
"ref": "4604c98f21c9a22230e417d17d03c402b9b5f869",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "deeprl33"
|
||||
},
|
||||
"rl343": {
|
||||
"ref": "4e6b072404110056ccf9bcc232c718ac71d45478",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "rl34"
|
||||
},
|
||||
"drl3": {
|
||||
"ref": "a881acfbd59cb9ed18c54535b835721e2bc96def",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "drl"
|
||||
},
|
||||
"nopp3": {
|
||||
"ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "nopp"
|
||||
},
|
||||
"tobyrl3": {
|
||||
"ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "tobyrl"
|
||||
},
|
||||
"michael-rl3": {
|
||||
"ref": "968fd3c989b993cf03187aa0e05cd2088e275551",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "michael-rl"
|
||||
},
|
||||
"michael-rl23": {
|
||||
"ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "michael-rl2"
|
||||
},
|
||||
"karnbir3": {
|
||||
"ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "karnbir"
|
||||
},
|
||||
"karnbir23": {
|
||||
"ref": "682d33d0414d7cef5a7286c24298003503ea13f4",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "karnbir2"
|
||||
},
|
||||
"ms23": {
|
||||
"ref": "d70b13931793bc0e4d8efd36128dd66f227a3f81",
|
||||
"input_format": "split",
|
||||
"source_id": "ms2"
|
||||
},
|
||||
"pp2223": {
|
||||
"ref": "50c78a9dd670a305ca898b96245cadbc2ecdc1a6",
|
||||
"input_format": "split",
|
||||
"source_id": "pp222"
|
||||
},
|
||||
"ds2223": {
|
||||
"ref": "5ff0e48f90d662d8c9d5061adc63e90c198c9b81",
|
||||
"input_format": "split",
|
||||
"source_id": "ds222"
|
||||
},
|
||||
"nn2223": {
|
||||
"ref": "83b81b83cdf76a577946a1567f4644e1a018443c",
|
||||
"input_format": "split",
|
||||
"source_id": "nn222"
|
||||
},
|
||||
"sc23": {
|
||||
"ref": "e4a4b4b1adf2d19fedab4d195faa382b061fa754",
|
||||
"input_format": "split",
|
||||
"source_id": "sc2"
|
||||
},
|
||||
"pop23": {
|
||||
"ref": "6f71783a8a8faa07ddaeef5bbb6809b4f4f44a15",
|
||||
"input_format": "split",
|
||||
"source_id": "pop2"
|
||||
},
|
||||
"pop223": {
|
||||
"ref": "62bf6fb072880905a4c490f0f4f4a6b3c23346ec",
|
||||
"input_format": "split",
|
||||
"source_id": "pop22"
|
||||
},
|
||||
"nid223": {
|
||||
"ref": "13e79e9fad60c19e751e4f9ab0538d39f1bb54dd",
|
||||
"input_format": "split",
|
||||
"source_id": "nid22"
|
||||
},
|
||||
"kerrygold223": {
|
||||
"ref": "dc7a92ea630e4f6053082fc25eca83938927b91e",
|
||||
"input_format": "split",
|
||||
"source_id": "kerrygold22"
|
||||
},
|
||||
"deeprl3v23": {
|
||||
"ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "deeprl3v2"
|
||||
},
|
||||
"rh3": {
|
||||
"ref": "93f5aa469a72b7621aef7da7901c100e0113e4d9",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "rh"
|
||||
},
|
||||
"gyhu3": {
|
||||
"ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "gyhu"
|
||||
},
|
||||
"rdfv23": {
|
||||
"ref": "a95e2c25cae5fbf1afba7628bfb7acc4af59e0cc",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "rdf"
|
||||
},
|
||||
"rdf23": {
|
||||
"ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "rdf2"
|
||||
},
|
||||
"tsf": {
|
||||
"ref": "4d911346cde4e0d2978a625f31679808284cc19d",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "tsf"
|
||||
},
|
||||
"tsfdo": {
|
||||
"ref": "0394d7284cfc229f772ee4fa73ff724ba8245110",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "tsfdo"
|
||||
},
|
||||
"lebowski3": {
|
||||
"ref": "fa0c6876d3cf070e91e25e5353ceadc68a5b3285",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "lebowski",
|
||||
"uses_external_gpu": true
|
||||
},
|
||||
"bmrlnapv2": {
|
||||
"ref": "ba37c02c7c64225b8aec79411d1a8c2f5c1aa343",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "bmrlnapv2",
|
||||
"uses_external_gpu": true
|
||||
},
|
||||
"happybirthday": {
|
||||
"ref": "db06f5f36caa379e112bafd38423e37b66fa4149",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "happybirthday",
|
||||
"uses_external_gpu": true
|
||||
},
|
||||
"berightthere": {
|
||||
"ref": "bf74ce5448306088b4e9bc843472c39402930bc7",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "berightthere",
|
||||
"uses_external_gpu": true
|
||||
},
|
||||
"sad": {
|
||||
"ref": "30de303d5ffb63957f8acbdb256fd4c7d360455a",
|
||||
"input_format": "supercombo",
|
||||
"source_id": "sad",
|
||||
"uses_external_gpu": true
|
||||
},
|
||||
"bmrlnapv6": {
|
||||
"ref": "9d683c06518c0358fb402f38468a7030700c38ac",
|
||||
"input_format": "supercombo",
|
||||
"uses_external_gpu": true,
|
||||
"source_id": "bmrlnapv6"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,18 +79,19 @@ def test_runtime_scan_excludes_model_weights_but_flags_runtime_code():
|
||||
|
||||
|
||||
def test_update_manifest_replaces_one_entry(tmp_path: Path):
|
||||
manifest = tmp_path / "model_names_v24.json"
|
||||
manifest = tmp_path / "model_names_v25.json"
|
||||
manifest.write_text(json.dumps({"models": [{"id": "old"}]}) + "\n")
|
||||
info = parse_pasted_release(RELEASE_TEXT, "bmrlnapv4", "v16")
|
||||
path = update_manifest(
|
||||
tmp_path,
|
||||
info,
|
||||
{"size": 123, "sha256": "a" * 64},
|
||||
"v24",
|
||||
{"size": 123, "sha256": "a" * 64, "chunk_count": 2},
|
||||
"v25",
|
||||
)
|
||||
payload = json.loads(path.read_text())
|
||||
assert len(payload["models"]) == 2
|
||||
entry = payload["models"][1]
|
||||
assert entry["id"] == "bmrlnapv4"
|
||||
assert entry["artifact_size"] == 123
|
||||
assert entry["artifact_chunk_count"] == 2
|
||||
assert entry["uses_external_gpu"]
|
||||
|
||||
Reference in New Issue
Block a user