mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-09-18 13:33:53 +08:00
Code Issues Packages Projects Releases Wiki Activity

cilantro lime

Browse Source
This commit is contained in:
firestar5683
2026-09-09 17:46:41 -05:00
parent 962d8b6719
commit 22707891bd
15 changed files with 449 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/model_release.py
+96 -4
View File
@@ -19,6 +19,7 @@ import shlex
import shutil
import subprocess
import sys
import tempfile
import urllib.error
import urllib.parse
import urllib.request
@@ -619,6 +620,70 @@ def update_manifest(repo: Path, info: ReleaseInfo, result: dict, manifest_versio
return path
def manifest_models(payload: object) -> list[dict]:
models = payload.get("models") if isinstance(payload, dict) else payload
if not isinstance(models, list) or not models:
raise ReleaseError("Unsupported or empty Hugging Face manifest")
if any(not isinstance(model, dict) or not str(model.get("id") or "").strip() for model in models):
raise ReleaseError("Hugging Face manifest contains an invalid model entry")
model_ids = [str(model["id"]).strip() for model in models]
if len(model_ids) != len(set(model_ids)):
raise ReleaseError("Hugging Face manifest contains duplicate model IDs")
return models
def accelerator_artifact_map(payload: object) -> dict[tuple[str, str], dict]:
artifacts: dict[tuple[str, str], dict] = {}
for model in manifest_models(payload):
model_id = str(model.get("id") or "").strip()
model_artifacts = model.get("accelerator_artifacts")
if not model_id or not isinstance(model_artifacts, dict):
continue
for accelerator, metadata in model_artifacts.items():
if isinstance(metadata, dict):
artifacts[(model_id, str(accelerator))] = metadata
return artifacts
def validate_manifest_update(before: object, after: object, replacing_model_id: str) -> None:
before_by_id = {str(model["id"]).strip(): model for model in manifest_models(before)}
after_by_id = {str(model["id"]).strip(): model for model in manifest_models(after)}
before_ids = set(before_by_id)
after_ids = set(after_by_id)
expected_ids = before_ids | {replacing_model_id}
if after_ids != expected_ids:
missing = sorted(expected_ids - after_ids)
unexpected = sorted(after_ids - expected_ids)
raise ReleaseError(
"Refusing to publish a manifest with an unexpected model set"
+ (f"; missing: {', '.join(missing)}" if missing else "")
+ (f"; unexpected: {', '.join(unexpected)}" if unexpected else "")
)
before_artifacts = accelerator_artifact_map(before)
after_artifacts = accelerator_artifact_map(after)
regressions = [
f"{model_id}:{accelerator}"
for (model_id, accelerator), metadata in before_artifacts.items()
if model_id != replacing_model_id and after_artifacts.get((model_id, accelerator)) != metadata
]
if regressions:
raise ReleaseError(
"Refusing to publish a manifest that removes or changes existing accelerator metadata for: "
+ ", ".join(sorted(regressions))
)
unrelated_changes = sorted(
model_id for model_id, model in before_by_id.items()
if model_id != replacing_model_id and after_by_id[model_id] != model
)
if unrelated_changes:
raise ReleaseError(
"Refusing to publish a manifest that changes unrelated model entries: "
+ ", ".join(unrelated_changes)
)
def find_hf() -> str:
candidates = [shutil.which("hf"), str(Path.home() / ".local/bin/hf")]
for candidate in candidates:
@@ -633,15 +698,42 @@ def hf_copy(source: Path, bucket: str, remote_path: str) -> None:
run([hf, "buckets", "cp", str(source), destination, "--format", "quiet"])
def refresh_huggingface_manifest(manifest: Path, bucket: str) -> dict:
remote_path = f"manifests/{manifest.name}"
source = f"hf://buckets/{bucket}/{remote_path}"
manifest.parent.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(prefix=".model-release-", dir=manifest.parent) as temporary_dir:
candidate = Path(temporary_dir) / manifest.name
run([find_hf(), "buckets", "cp", source, str(candidate), "--format", "quiet"])
try:
payload = json.loads(candidate.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError) as error:
raise ReleaseError(f"Invalid live Hugging Face manifest: {error}") from error
accelerator_artifact_map(payload)
candidate.replace(manifest)
return payload
def prepare_huggingface_manifest(manifest: Path, info: ReleaseInfo, result: dict,
bucket: str, manifest_version: str) -> Path:
live_payload = refresh_huggingface_manifest(manifest, bucket)
updated_manifest = update_manifest(manifest.parent, info, result, manifest_version)
updated_payload = json.loads(updated_manifest.read_text(encoding="utf-8"))
validate_manifest_update(live_payload, updated_payload, info.model_id)
return updated_manifest
def upload_huggingface(info: ReleaseInfo, result: dict, workspace: Path, bucket: str, manifest_version: str,
manifest: Path, upload_onnx: bool, source: Path) -> None:
manifest: Path, upload_onnx: bool, source: Path) -> Path:
artifact_dir = Path(result["path"])
for filename in result["files"]:
hf_copy(artifact_dir / filename, bucket, f"models/{manifest_version}/{info.model_id}/{filename}")
if upload_onnx:
hf_copy(source, bucket, f"onnx/{info.model_id}/{source.name}")
manifest = prepare_huggingface_manifest(manifest, info, result, bucket, manifest_version)
hf_copy(manifest, bucket, f"manifests/{manifest.name}")
print(f"Hugging Face upload complete: {bucket}/models/{manifest_version}/{info.model_id}/")
return manifest
def git_output(repo: Path, args: list[str]) -> str:
@@ -777,9 +869,9 @@ def main() -> int:
result = remote_compile(info, source, ip, workspace, args.keep_device_files)
resources_repo = args.resources_repo.expanduser().resolve()
check_resources_repo(resources_repo, args.resources_branch)
manifest = update_manifest(resources_repo, info, result, args.manifest_version)
upload_huggingface(info, result, workspace, args.hf_bucket, args.manifest_version,
manifest, not args.no_onnx_upload, source)
manifest = resources_repo / f"model_names_{args.manifest_version}.json"
manifest = upload_huggingface(info, result, workspace, args.hf_bucket, args.manifest_version,
manifest, not args.no_onnx_upload, source)
push_github(info, result, resources_repo, args.manifest_version, manifest, args.resources_branch, args.force)
print("\nRelease complete.")
print(f" local artifact: {result['path']}")
scripts/tests/test_model_release.py
+148 -1
View File
@@ -3,8 +3,20 @@ from __future__ import annotations
import json
from pathlib import Path
import pytest
from scripts import model_release
from scripts.model_release import parse_lfs_pointer, parse_pasted_release, runtime_file, update_manifest
from scripts.model_release import (
ReleaseError,
parse_lfs_pointer,
parse_pasted_release,
prepare_huggingface_manifest,
refresh_huggingface_manifest,
runtime_file,
update_manifest,
upload_huggingface,
validate_manifest_update,
)
RELEASE_TEXT = """
@@ -95,3 +107,138 @@ def test_update_manifest_replaces_one_entry(tmp_path: Path):
assert entry["artifact_size"] == 123
assert entry["artifact_chunk_count"] == 2
assert entry["uses_external_gpu"]
def test_prepare_manifest_refreshes_live_copy_before_adding_model(tmp_path: Path, monkeypatch):
manifest = tmp_path / "model_names_v25.json"
manifest.write_text(json.dumps({"models": [{"id": "stale"}]}) + "\n")
live_payload = {
"models": [{
"id": "small-model",
"model_lab_eligible": True,
"accelerator_artifacts": {
"chestnut": {
"artifact_filename": "small-model_driving_chestnut_tinygrad.pkl",
"artifact_sha256": "b" * 64,
"execution_device": "AMD",
},
},
}],
}
def fake_refresh(path, bucket):
assert bucket == "StarPilot-Driving/StarPilot-Resources"
path.write_text(json.dumps(live_payload) + "\n")
return live_payload
monkeypatch.setattr(model_release, "refresh_huggingface_manifest", fake_refresh)
info = parse_pasted_release(RELEASE_TEXT, "bmrlnapv4", "v16")
prepared = prepare_huggingface_manifest(
manifest,
info,
{"size": 123, "sha256": "a" * 64, "chunk_count": 2},
"StarPilot-Driving/StarPilot-Resources",
"v25",
)
models = {entry["id"]: entry for entry in json.loads(prepared.read_text())["models"]}
assert set(models) == {"small-model", "bmrlnapv4"}
assert models["small-model"] == live_payload["models"][0]
assert models["bmrlnapv4"]["artifact_sha256"] == "a" * 64
def test_manifest_guard_rejects_unrelated_accelerator_metadata_regression():
chestnut = {"execution_device": "AMD", "artifact_sha256": "a" * 64}
before = {"models": [
{"id": "keep", "accelerator_artifacts": {"chestnut": chestnut}},
{"id": "replace", "accelerator_artifacts": {"chestnut": chestnut}},
]}
after = {"models": [{"id": "keep"}, {"id": "replace"}]}
with pytest.raises(ReleaseError, match="keep:chestnut"):
validate_manifest_update(before, after, "replace")
validate_manifest_update(
{"models": [{"id": "replace", "accelerator_artifacts": {"chestnut": chestnut}}]},
{"models": [{"id": "replace"}]},
"replace",
)
with pytest.raises(ReleaseError, match="unrelated model entries: keep"):
validate_manifest_update(
{"models": [{"id": "keep", "model_lab_eligible": True}]},
{"models": [{"id": "keep", "model_lab_eligible": False}, {"id": "replace"}]},
"replace",
)
def test_refresh_manifest_is_atomic_and_uses_hugging_face_bucket(tmp_path: Path, monkeypatch):
manifest = tmp_path / "model_names_v25.json"
manifest.write_text(json.dumps({"models": [{"id": "stale"}]}) + "\n")
live_payload = {"models": [{"id": "live"}]}
commands = []
monkeypatch.setattr(model_release, "find_hf", lambda: "/usr/bin/hf")
def fake_run(command, **kwargs):
commands.append(command)
Path(command[4]).write_text(json.dumps(live_payload) + "\n")
monkeypatch.setattr(model_release, "run", fake_run)
assert refresh_huggingface_manifest(manifest, "owner/resources") == live_payload
assert json.loads(manifest.read_text()) == live_payload
assert commands[0][0:4] == [
"/usr/bin/hf",
"buckets",
"cp",
"hf://buckets/owner/resources/manifests/model_names_v25.json",
]
def test_refresh_manifest_keeps_local_copy_when_live_json_is_invalid(tmp_path: Path, monkeypatch):
manifest = tmp_path / "model_names_v25.json"
original = {"models": [{"id": "safe"}]}
manifest.write_text(json.dumps(original) + "\n")
monkeypatch.setattr(model_release, "find_hf", lambda: "/usr/bin/hf")
monkeypatch.setattr(model_release, "run", lambda command, **kwargs: Path(command[4]).write_text("{"))
with pytest.raises(ReleaseError, match="Invalid live Hugging Face manifest"):
refresh_huggingface_manifest(manifest, "owner/resources")
assert json.loads(manifest.read_text()) == original
def test_upload_refreshes_manifest_after_artifacts(tmp_path: Path, monkeypatch):
artifact_dir = tmp_path / "artifacts"
artifact_dir.mkdir()
manifest = tmp_path / "resources" / "model_names_v25.json"
manifest.parent.mkdir()
manifest.write_text(json.dumps({"models": [{"id": "old"}]}) + "\n")
source = tmp_path / "model.onnx"
calls = []
info = parse_pasted_release(RELEASE_TEXT, "bmrlnapv4", "v16")
result = {
"path": str(artifact_dir),
"files": ["bmrlnapv4_driving_tinygrad.pkl"],
"size": 123,
"sha256": "a" * 64,
"chunk_count": 0,
}
monkeypatch.setattr(model_release, "hf_copy", lambda source, bucket, remote: calls.append(("copy", remote)))
def fake_prepare(path, release_info, release_result, bucket, version):
calls.append(("prepare", path.name))
return path
monkeypatch.setattr(model_release, "prepare_huggingface_manifest", fake_prepare)
returned = upload_huggingface(
info, result, tmp_path, "owner/resources", "v25", manifest, True, source,
)
assert returned == manifest
assert calls == [
("copy", "models/v25/bmrlnapv4/bmrlnapv4_driving_tinygrad.pkl"),
("copy", "onnx/bmrlnapv4/model.onnx"),
("prepare", "model_names_v25.json"),
("copy", "manifests/model_names_v25.json"),
]