From 6ee00d3f6073a820c6cf415f4921371770cd04e7 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Tue, 23 Jun 2026 00:17:15 -0500 Subject: [PATCH] Support multipart model artifacts --- docs/MODEL_REBUILD.md | 36 ++++-- scripts/model_compiler.py | 95 ++++++++++++++ scripts/model_rebuild_pipeline.py | 41 ++++-- starpilot/assets/download_functions.py | 111 +++++++++++++++-- starpilot/assets/model_manager.py | 24 +++- starpilot/assets/tests/test_model_pipeline.py | 117 ++++++++++++++++++ 6 files changed, 390 insertions(+), 34 deletions(-) diff --git a/docs/MODEL_REBUILD.md b/docs/MODEL_REBUILD.md index 30650a282..18257a18d 100644 --- a/docs/MODEL_REBUILD.md +++ b/docs/MODEL_REBUILD.md @@ -23,7 +23,7 @@ Important directories: - `compiled/`: completed unified driving PKLs. - `driver-monitoring/`: DM ONNX, model PKL, metadata, and camera warps. - `ready-for-resources/`: flat repository-upload handoff. -- `external-upload/`: artifacts over 100 MiB plus `handoff.json`. +- Oversized models are represented by repository-safe `.p00`, `.p01`, and `.sha256` files in `ready-for-resources/`. - `logs/`: one remote compilation log per model. - `results/`: source and artifact checksum records. - `manifests/`: generated `model_names_v22.json`. @@ -84,6 +84,28 @@ The lower-level device compiler also supports direct use: `--version` records behavioral semantics only. It does not change artifact layout. +If the compiled PKL exceeds 100 MiB, `./models` automatically keeps the full +local PKL and creates 95 MiB upload parts beside it: + +```text +deeprl3v2_driving_tinygrad.pkl +deeprl3v2_driving_tinygrad.pkl.p00 +deeprl3v2_driving_tinygrad.pkl.p01 +deeprl3v2_driving_tinygrad.pkl.sha256 +``` + +To split an already compiled artifact: + +```bash +./models --split-artifact /path/to/deeprl3v2_driving_tinygrad.pkl \ + --output-dir /path/to/upload-ready +``` + +Upload only the numbered parts and checksum when the full PKL exceeds the +repository limit. The downloader reassembles into a temporary file, verifies +the companion SHA-256, and atomically installs the final PKL. No manifest field +is required for multipart artifacts. + ## Driver Monitoring Stage the current DM ONNX in `uncompiledmodels`, then run: @@ -112,14 +134,10 @@ python3 scripts/model_rebuild_pipeline.py manifest \ --base-manifest /path/to/model_names_v21.json ``` -The generator preserves existing IDs and behavioral metadata, adds `deeprl3v2`, and writes: - -- `artifact_format` -- `artifact_size` -- `artifact_sha256` -- optional `artifact_url` - -Files above 100 MiB are listed in `external-upload/handoff.json`. Upload those files to Dropbox, use a direct-download URL, add it as `artifact_url`, and regenerate or edit the final manifest without changing its size or SHA-256 fields. +The generator preserves existing IDs and behavioral metadata and adds +`deeprl3v2`. Manifest v22 implies the unified single-PKL runtime layout. +Repository-hosted multipart files are discovered by naming convention, so no +size, hash, format, or part-count metadata is required. ## Runtime Verification diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py index 2493355c8..a5927fca5 100644 --- a/scripts/model_compiler.py +++ b/scripts/model_compiler.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse import codecs +import hashlib import json import os import pickle @@ -36,6 +37,8 @@ MEDMODEL_INPUT_SIZE = (512, 256) DM_INPUT_SIZE = (1440, 960) MODEL_RUN_FREQ = 20 MODEL_CONTEXT_FREQ = 5 +REPOSITORY_FILE_LIMIT = 100 * 1024 * 1024 +DEFAULT_MULTIPART_SIZE = 95 * 1024 * 1024 def build_compile_env() -> dict[str, str]: @@ -77,6 +80,8 @@ def parse_args() -> argparse.Namespace: ) parser.add_argument("--list", action="store_true", help="List staged models and exit.") parser.add_argument("--force", action="store_true", help="Accepted for compatibility; selected outputs are always replaced.") + parser.add_argument("--split-artifact", type=Path, help="Split an existing oversized PKL without compiling.") + parser.add_argument("--chunk-size-mib", type=int, default=95, help="Multipart size in MiB; must be below 100.") args, unknown = parser.parse_known_args() dynamic_flags = [value[2:] for value in unknown if value.startswith("--")] @@ -93,6 +98,10 @@ def parse_args() -> argparse.Namespace: args.model = None if args.dm and args.model: parser.error("Use either --dm or a driving model ID.") + if args.split_artifact and (args.dm or args.model): + parser.error("--split-artifact cannot be combined with --dm or a model ID.") + if not 1 <= args.chunk_size_mib < 100: + parser.error("--chunk-size-mib must be between 1 and 99.") return args @@ -264,11 +273,81 @@ def remove_paths(paths: list[Path]) -> int: return count +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as artifact_file: + for chunk in iter(lambda: artifact_file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def multipart_output_paths(artifact: Path, output_dir: Path | None = None) -> list[Path]: + output_dir = output_dir or artifact.parent + return [ + *sorted(output_dir.glob(f"{artifact.name}.p[0-9][0-9]")), + output_dir / f"{artifact.name}.sha256", + ] + + +def split_oversized_artifact( + artifact: Path, + output_dir: Path | None = None, + chunk_size: int = DEFAULT_MULTIPART_SIZE, + force: bool = False, +) -> list[Path]: + artifact = artifact.resolve() + output_dir = (output_dir or artifact.parent).resolve() + if not artifact.is_file(): + raise FileNotFoundError(artifact) + if chunk_size <= 0 or chunk_size >= REPOSITORY_FILE_LIMIT: + raise ValueError("Multipart chunk size must be between 1 byte and 100 MiB.") + + remove_paths(multipart_output_paths(artifact, output_dir)) + if artifact.stat().st_size <= REPOSITORY_FILE_LIMIT and not force: + return [] + + output_dir.mkdir(parents=True, exist_ok=True) + digest = hashlib.sha256() + part_paths: list[Path] = [] + with open(artifact, "rb") as source: + for index in range(100): + part_path = output_dir / f"{artifact.name}.p{index:02d}" + part_size = 0 + with open(part_path, "wb") as part_file: + while part_size < chunk_size: + chunk = source.read(min(1024 * 1024, chunk_size - part_size)) + if not chunk: + break + part_file.write(chunk) + digest.update(chunk) + part_size += len(chunk) + if part_size == 0: + part_path.unlink() + break + part_paths.append(part_path) + if not part_paths: + raise ValueError(f"Artifact is empty: {artifact}") + + checksum_path = output_dir / f"{artifact.name}.sha256" + checksum_path.write_text(f"{digest.hexdigest()} {artifact.name}\n") + + verify_digest = hashlib.sha256() + for part_path in part_paths: + with open(part_path, "rb") as part_file: + for chunk in iter(lambda: part_file.read(1024 * 1024), b""): + verify_digest.update(chunk) + if verify_digest.hexdigest() != digest.hexdigest(): + remove_paths([*part_paths, checksum_path]) + raise RuntimeError("Split artifact failed checksum verification.") + return [*part_paths, checksum_path] + + def compile_driving(model_key: str, files: dict[str, Path], input_format: str, version: str, output_dir: Path) -> Path: model_type, source_args = driving_compile_args(files, input_format) output_path = output_dir / f"{model_key}_driving_tinygrad.pkl" removed = remove_paths([ output_path, + *multipart_output_paths(output_path, output_dir), *output_dir.glob(f"{model_key}_driving_*_tinygrad.pkl"), *output_dir.glob(f"{model_key}_driving_*_metadata.pkl"), ]) @@ -349,6 +428,17 @@ def list_models(staged: dict[str, dict[str, Path]], input_root: Path) -> int: def main() -> int: args = parse_args() + if args.split_artifact: + outputs = split_oversized_artifact( + args.split_artifact, + args.output_dir, + args.chunk_size_mib * 1024 * 1024, + force=True, + ) + for output in outputs: + print(f" saved {output.name} ({output.stat().st_size} bytes)") + return 0 + staged = find_staged_models(args.input_dir) if args.list: return list_models(staged, args.input_dir) @@ -379,6 +469,11 @@ def main() -> int: print(f"Compiling {model_key} ({input_format}, {version_label}) from {args.input_dir} -> {args.output_dir}") output = compile_driving(model_key, files, input_format, version, args.output_dir) print(f" saved {output.name}") + multipart_outputs = split_oversized_artifact(output) + if multipart_outputs: + print(" artifact exceeds 100 MiB; created repository-safe multipart files:") + for multipart_output in multipart_outputs: + print(f" {multipart_output.name} ({multipart_output.stat().st_size} bytes)") print("Done.") return 0 diff --git a/scripts/model_rebuild_pipeline.py b/scripts/model_rebuild_pipeline.py index 717f07b2c..3f33a1331 100644 --- a/scripts/model_rebuild_pipeline.py +++ b/scripts/model_rebuild_pipeline.py @@ -11,6 +11,11 @@ import sys from pathlib import Path REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT / "scripts") not in sys.path: + sys.path.insert(0, str(REPO_ROOT / "scripts")) + +from model_compiler import split_oversized_artifact + DEFAULT_OPENPILOT = Path.home() / "openpilot" DEFAULT_WORKSPACE = Path("/Volumes/T5/StarPilot-Model-Rebuild-2026-06-22") DEFAULT_SOURCE_MAP = REPO_ROOT / "scripts/model_source_map_v22.json" @@ -204,9 +209,22 @@ def remote(command: str, *, capture: bool = False): return result +def stage_ready_artifact(artifact: Path, workspace: Path) -> None: + ready_path = workspace / "ready-for-resources" / artifact.name + multipart_outputs = split_oversized_artifact(artifact, ready_path.parent) + if multipart_outputs: + ready_path.unlink(missing_ok=True) + for multipart_output in multipart_outputs: + multipart_output.chmod(0o644) + else: + shutil.copyfile(artifact, ready_path) + ready_path.chmod(0o644) + + def compile_model(model_id: str, source: dict, version: str, workspace: Path, force: bool) -> dict: local_output = workspace / "compiled" / f"{model_id}_driving_tinygrad.pkl" if local_output.is_file() and not force: + stage_ready_artifact(local_output, workspace) return artifact_result(model_id, local_output, "skipped") source_dir = workspace / "onnx" / model_id @@ -230,13 +248,7 @@ def compile_model(model_id: str, source: dict, version: str, workspace: Path, fo run(["rsync", "-az", f"{REMOTE}:{remote_output}", str(local_output)]) local_output.chmod(0o644) - ready_path = workspace / "ready-for-resources" / local_output.name - shutil.copyfile(local_output, ready_path) - ready_path.chmod(0o644) - if local_output.stat().st_size > 100 * 1024 * 1024: - external_path = workspace / "external-upload" / local_output.name - shutil.copyfile(local_output, external_path) - external_path.chmod(0o644) + stage_ready_artifact(local_output, workspace) result = artifact_result(model_id, local_output, "compiled") (workspace / "results" / f"{model_id}_artifact.json").write_text(json.dumps(result, indent=2) + "\n") return result @@ -249,7 +261,7 @@ def artifact_result(model_id: str, path: Path, status: str) -> dict: "path": str(path), "size": path.stat().st_size, "sha256": sha256_file(path), - "external_upload": path.stat().st_size > 100 * 1024 * 1024, + "multipart": path.stat().st_size > 100 * 1024 * 1024, } @@ -288,7 +300,7 @@ def update_manifest(base_manifest: Path, workspace: Path) -> dict: "released": "2026-06-17", "community_favorite": False, }) - external_handoff = [] + multipart_handoff = [] for model in models: artifact = workspace / "compiled" / f"{model['id']}_driving_tinygrad.pkl" model.pop("artifact_format", None) @@ -298,19 +310,22 @@ def update_manifest(base_manifest: Path, workspace: Path) -> dict: if not artifact.is_file() or artifact.stat().st_size <= 100 * 1024 * 1024: model.pop("artifact_url", None) else: - external_handoff.append({ + multipart_handoff.append({ "id": model["id"], "filename": artifact.name, "size": artifact.stat().st_size, "sha256": sha256_file(artifact), - "artifact_url": model.get("artifact_url", ""), + "parts": [ + path.name + for path in sorted((workspace / "ready-for-resources").glob(f"{artifact.name}.p[0-9][0-9]")) + ], }) output = {"models": models} output_path = workspace / "manifests/model_names_v22.json" output_path.parent.mkdir(parents=True, exist_ok=True) output_path.write_text(json.dumps(output, indent=2, ensure_ascii=False) + "\n") - (workspace / "external-upload" / "handoff.json").write_text( - json.dumps(external_handoff, indent=2) + "\n", + (workspace / "ready-for-resources" / "multipart.json").write_text( + json.dumps(multipart_handoff, indent=2) + "\n", ) return output diff --git a/starpilot/assets/download_functions.py b/starpilot/assets/download_functions.py index 814b8261f..07f34a4fd 100644 --- a/starpilot/assets/download_functions.py +++ b/starpilot/assets/download_functions.py @@ -11,8 +11,11 @@ from pathlib import Path from openpilot.starpilot.common.starpilot_utilities import delete_file, is_url_pingable RESOURCES_REPO = os.getenv("STARPILOT_RESOURCES_REPO", "firestar5683/StarPilot-Resources") +GITLAB_RESOURCES_REPO = os.getenv("STARPILOT_GITLAB_RESOURCES_REPO", "firestar5683/FrogPilot-Resources") GITHUB_URL = f"https://raw.githubusercontent.com/{RESOURCES_REPO}" -GITLAB_URL = f"https://gitlab.com/{RESOURCES_REPO}/-/raw" +GITLAB_URL = f"https://gitlab.com/{GITLAB_RESOURCES_REPO}/-/raw" +LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1\n" +MAX_MULTIPART_FILES = 100 def normalize_download_url(url: str) -> str: @@ -25,6 +28,37 @@ def normalize_download_url(url: str) -> str: return urllib.parse.urlunsplit(parsed._replace(query=urllib.parse.urlencode(query))) +def append_url_suffix(url: str, suffix: str) -> str: + parsed = urllib.parse.urlsplit(str(url or "").strip()) + return urllib.parse.urlunsplit(parsed._replace(path=f"{parsed.path}{suffix}")) + + +def is_git_lfs_pointer(path: Path) -> bool: + if not path.is_file() or path.stat().st_size > 1024: + return False + with open(path, "rb") as artifact_file: + return artifact_file.read(len(LFS_POINTER_PREFIX)) == LFS_POINTER_PREFIX + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with open(path, "rb") as artifact_file: + for chunk in iter(lambda: artifact_file.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def get_remote_text(url: str, suppress_errors=False) -> str: + try: + response = requests.get(normalize_download_url(url), timeout=10) + response.raise_for_status() + return response.text.strip() + except Exception as error: + if not suppress_errors: + handle_request_error(error, None, None, None, None) + return "" + + def check_github_rate_limit(): try: response = requests.get("https://api.github.com/rate_limit") @@ -88,12 +122,76 @@ def download_file(cancel_param, destination, progress_param, url, download_param temp_file_path.rename(destination) print(f"Download complete: {destination.name}") + return True except Exception as error: if suppress_errors: - return + return False print(f"Download request error: {error}") handle_request_error(error, destination, download_param, progress_param, params_memory) + return False + + +def download_multipart_file(cancel_param, destination, progress_param, url, download_param, params_memory): + del download_param + checksum_text = get_remote_text(append_url_suffix(url, ".sha256"), suppress_errors=True) + expected_sha256 = checksum_text.split(maxsplit=1)[0].lower() if checksum_text else "" + if len(expected_sha256) != 64 or any(char not in "0123456789abcdef" for char in expected_sha256): + return False + + parts: list[tuple[str, int]] = [] + for index in range(MAX_MULTIPART_FILES): + part_url = append_url_suffix(url, f".p{index:02d}") + part_size = get_remote_file_size(part_url, suppress_errors=True) + if part_size <= 0: + break + parts.append((part_url, part_size)) + if not parts: + return False + + destination.parent.mkdir(parents=True, exist_ok=True) + total_size = sum(part_size for _, part_size in parts) + downloaded_size = 0 + digest = hashlib.sha256() + temp_path = None + + try: + with tempfile.NamedTemporaryFile(dir=destination.parent, delete=False) as output_file: + temp_path = Path(output_file.name) + for part_number, (part_url, expected_part_size) in enumerate(parts, start=1): + print(f"Downloading {destination.name} part {part_number}/{len(parts)} ({expected_part_size} bytes)") + part_size = 0 + with requests.get(normalize_download_url(part_url), stream=True, timeout=(10, 60)) as response: + response.raise_for_status() + for chunk in response.iter_content(chunk_size=1024 * 1024): + if params_memory.get_bool(cancel_param): + return False + if not chunk: + continue + output_file.write(chunk) + digest.update(chunk) + part_size += len(chunk) + downloaded_size += len(chunk) + params_memory.put(progress_param, f"{(downloaded_size / total_size) * 100:.0f}%") + if part_size != expected_part_size: + print(f"Part size mismatch for {part_url}: {part_size} != {expected_part_size}") + return False + + params_memory.put(progress_param, "Verifying authenticity...") + if digest.hexdigest() != expected_sha256: + print(f"SHA-256 mismatch for reassembled {destination.name}") + return False + + temp_path.replace(destination) + temp_path = None + print(f"Reassembled and verified {destination.name}") + return True + except Exception as error: + print(f"Multipart download failed for {destination.name}: {error}") + return False + finally: + if temp_path is not None: + temp_path.unlink(missing_ok=True) def get_remote_file_size(url, suppress_errors=False): try: @@ -143,6 +241,9 @@ def verify_download(file_path, url, allow_unknown_size=False, expected_size=None if not file_path.is_file(): print(f"File not found: {file_path}") return False + if is_git_lfs_pointer(file_path): + print(f"Git LFS pointer received instead of artifact: {file_path}") + return False actual_size = file_path.stat().st_size if expected_size and actual_size != expected_size: @@ -150,11 +251,7 @@ def verify_download(file_path, url, allow_unknown_size=False, expected_size=None return False if expected_sha256: - digest = hashlib.sha256() - with open(file_path, "rb") as artifact_file: - for chunk in iter(lambda: artifact_file.read(1024 * 1024), b""): - digest.update(chunk) - if digest.hexdigest().lower() != expected_sha256: + if sha256_file(file_path).lower() != expected_sha256: print(f"SHA-256 mismatch for {file_path}") return False diff --git a/starpilot/assets/model_manager.py b/starpilot/assets/model_manager.py index bfcdf28fc..2266b474c 100644 --- a/starpilot/assets/model_manager.py +++ b/starpilot/assets/model_manager.py @@ -9,6 +9,7 @@ from pathlib import Path from openpilot.starpilot.assets.download_functions import ( GITLAB_URL, download_file, + download_multipart_file, get_repository_url, handle_error, handle_request_error, @@ -584,21 +585,21 @@ class ModelManager: for filename in required_files: file_path = MODELS_PATH / filename - candidate_urls: list[tuple[str, bool]] = [] + candidate_urls: list[tuple[str, bool, bool]] = [] custom_url = artifact_urls.get(filename, "").strip() if custom_url: - candidate_urls.append((custom_url, True)) + candidate_urls.append((custom_url, True, False)) file_url = f"{repo_url}/Models/{filename}" - candidate_urls.append((file_url, False)) + candidate_urls.append((file_url, False, True)) fallback_url = f"{GITLAB_URL}/Models/{filename}" if fallback_url != file_url: - candidate_urls.append((fallback_url, False)) + candidate_urls.append((fallback_url, False, True)) download_succeeded = False - for candidate_url, allow_unknown_size in candidate_urls: + for candidate_url, allow_unknown_size, allow_multipart in candidate_urls: download_file( CANCEL_DOWNLOAD_PARAM, file_path, @@ -607,6 +608,7 @@ class ModelManager: MODEL_DOWNLOAD_PARAM, self.params_memory, allow_unknown_size=allow_unknown_size, + suppress_errors=True, ) if self.params_memory.get_bool(CANCEL_DOWNLOAD_PARAM): handle_error(None, "Download cancelled...", "Download cancelled...", MODEL_DOWNLOAD_PARAM, DOWNLOAD_PROGRESS_PARAM, self.params_memory) @@ -622,6 +624,18 @@ class ModelManager: ): download_succeeded = True break + delete_file(file_path, print_error=False) + + if allow_multipart and download_multipart_file( + CANCEL_DOWNLOAD_PARAM, + file_path, + DOWNLOAD_PROGRESS_PARAM, + candidate_url, + MODEL_DOWNLOAD_PARAM, + self.params_memory, + ): + download_succeeded = True + break if not download_succeeded: handle_error(file_path, "Verification failed...", f"Verification failed for {filename}", MODEL_DOWNLOAD_PARAM, DOWNLOAD_PROGRESS_PARAM, self.params_memory) diff --git a/starpilot/assets/tests/test_model_pipeline.py b/starpilot/assets/tests/test_model_pipeline.py index 9eea87a43..859ff0337 100644 --- a/starpilot/assets/tests/test_model_pipeline.py +++ b/starpilot/assets/tests/test_model_pipeline.py @@ -1,3 +1,6 @@ +import hashlib + +from scripts.model_compiler import split_oversized_artifact from openpilot.starpilot.assets import download_functions from openpilot.starpilot.assets.model_manager import MANIFEST_CANDIDATES, ModelManager from openpilot.starpilot.common.model_versions import UNIFIED_ARTIFACT_FORMAT @@ -44,3 +47,117 @@ def test_download_verification_uses_manifest_size_and_sha(tmp_path, monkeypatch) allow_unknown_size=True, expected_size=artifact.stat().st_size + 1, ) + + +def test_lfs_pointer_is_not_accepted_as_model(tmp_path, monkeypatch): + artifact = tmp_path / "model.pkl" + artifact.write_text( + "version https://git-lfs.github.com/spec/v1\n" + f"oid sha256:{'0' * 64}\n" + "size 123456789\n", + ) + monkeypatch.setattr(download_functions, "get_remote_file_size", lambda *args, **kwargs: artifact.stat().st_size) + assert not download_functions.verify_download(artifact, "https://example.com/model.pkl") + + +def test_multipart_download_is_atomic_and_checksum_verified(tmp_path, monkeypatch): + payload = b"first part" + b"second part" + expected_sha = hashlib.sha256(payload).hexdigest() + part_payloads = { + "https://example.com/model.pkl.p00": b"first part", + "https://example.com/model.pkl.p01": b"second part", + } + + class FakeResponse: + def __init__(self, data): + self.data = data + self.text = data.decode() + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def raise_for_status(self): + pass + + def iter_content(self, chunk_size): + del chunk_size + yield self.data + + def fake_get(url, **kwargs): + del kwargs + if url.endswith(".sha256"): + return FakeResponse(f"{expected_sha} model.pkl\n".encode()) + return FakeResponse(part_payloads[url]) + + def fake_size(url, **kwargs): + del kwargs + return len(part_payloads.get(url, b"")) + + class FakeParams: + def get_bool(self, key): + del key + return False + + def put(self, key, value): + del key, value + + monkeypatch.setattr(download_functions.requests, "get", fake_get) + monkeypatch.setattr(download_functions, "get_remote_file_size", fake_size) + destination = tmp_path / "model.pkl" + + assert download_functions.download_multipart_file( + "cancel", destination, "progress", "https://example.com/model.pkl", "download", FakeParams(), + ) + assert destination.read_bytes() == payload + + +def test_multipart_checksum_failure_leaves_no_model(tmp_path, monkeypatch): + class FakeResponse: + text = f"{'0' * 64} model.pkl" + + def __enter__(self): + return self + + def __exit__(self, *args): + pass + + def raise_for_status(self): + pass + + def iter_content(self, chunk_size): + del chunk_size + yield b"corrupt" + + monkeypatch.setattr(download_functions.requests, "get", lambda *args, **kwargs: FakeResponse()) + monkeypatch.setattr( + download_functions, + "get_remote_file_size", + lambda url, **kwargs: len(b"corrupt") if url.endswith(".p00") else 0, + ) + + class FakeParams: + def get_bool(self, key): + del key + return False + + def put(self, key, value): + del key, value + + destination = tmp_path / "model.pkl" + assert not download_functions.download_multipart_file( + "cancel", destination, "progress", "https://example.com/model.pkl", "download", FakeParams(), + ) + assert not destination.exists() + + +def test_oversized_artifact_split_round_trip(tmp_path): + artifact = tmp_path / "model.pkl" + artifact.write_bytes(b"multipart artifact") + + outputs = split_oversized_artifact(artifact, chunk_size=10, force=True) + part_paths = [path for path in outputs if path.suffix != ".sha256"] + assert len(part_paths) == 2 + assert b"".join(path.read_bytes() for path in part_paths) == artifact.read_bytes()