Compare commits

..

1 Commits

Author SHA1 Message Date
github-actions[bot] 8928e1df16 Update Python packages 2026-11-16 23:17:56 +00:00
5 changed files with 505 additions and 695 deletions
@@ -249,11 +249,7 @@ class ModelsLayout(Widget):
return {"name": name, "downloading": True, "progress": progress, "status_text": tr("verifying")}
if ds.downloading in statuses:
speed = max((p.speed for p in progresses), default=0.0)
speed_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
if any(p.resuming for p in progresses):
status_text = tr("resuming...") + f" {speed_text}"
else:
status_text = speed_text
status_text = f"{speed / 1024 ** 2:.1f} MB/s" if speed > 0 else "--.- MB/s"
return {"name": name, "downloading": True, "progress": progress, "status_text": status_text}
if statuses <= {ds.downloaded, ds.cached}:
return {"name": name, "text_color": ON_COLOR, "icon": "icons/checkmark.png"}
@@ -267,8 +267,8 @@ class ModelsLayoutMici(NavScroller):
self.current_model_info.info_text.set_text(info_text)
if manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.failed:
self.current_model_info.info_header.set_text(tr("error"))
self.current_model_info.info_text.set_text(f"{manager.selectedBundle.internalName.lower()} | {tr('download failed')}")
self.current_model_info.info_header.set_text(tr("error") + self._download_progress)
self.current_model_info.info_text.set_text(tr("download failed"))
elif manager.selectedBundle and manager.selectedBundle.status == custom.ModelManagerSP.DownloadStatus.downloading:
self.cancel_download_btn.set_visible(ui_state.params.get("ModelManager_DownloadRef") is not None)
@@ -277,7 +277,6 @@ class ModelsLayoutMici(NavScroller):
count = 0
verifying = False
canceling = False
resuming = False
speed = 0.0
for model in manager.selectedBundle.models:
count += 1
@@ -287,13 +286,12 @@ class ModelsLayoutMici(NavScroller):
progress += p.progress
verifying = verifying or p.status == custom.ModelManagerSP.DownloadStatus.verifying
canceling = canceling or p.canceling
resuming = resuming or p.resuming
speed = max(speed, p.speed) # only one file downloads at a time, so the non-zero speed is the live one
elif p.status in (custom.ModelManagerSP.DownloadStatus.downloaded,
custom.ModelManagerSP.DownloadStatus.cached):
progress += 100.0
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("resuming...") if resuming else tr("downloading")
header = tr("canceling...") if canceling else tr("verifying") if verifying else tr("downloading")
self.current_model_info.current_model_header.set_text(header)
self.cancel_download_btn.set_text(tr("cancel verification") if verifying else tr("cancel download"))
self.current_model_info.current_model_header._shimmer = True
+29 -120
View File
@@ -6,7 +6,6 @@ See the LICENSE.md file in the root directory for more details.
"""
import asyncio
import json
import os
import re
import threading
@@ -29,11 +28,8 @@ DOWNLOAD_TIMEOUT = (30, 30)
# HuggingFace throttles each connection to ~1-2 MB/s but not the connection count; 12 saturates the device link
MAX_CONCURRENT_PIECES = 12
PIECE_SIZE = 8 * 1024 * 1024
PIECE_ATTEMPTS = 3 # per piece, before the transfer as a whole is retried
PIECE_ATTEMPTS = 3
PIECE_BACKOFF = 1.0 # seconds, growing linearly: waits 1 s then 2 s between a piece's three attempts
DOWNLOAD_ATTEMPTS = 4 # whole-transfer attempts, each resuming from the sidecar
RETRY_BACKOFF = 2.0 # seconds, doubling: waits 2, 4 then 8 s between the four transfer attempts
RETRY_POLL = 0.25 # how often a backoff checks for a cancel
TRANSIENT_HTTP_STATUS = frozenset({408, 429, 500, 502, 503, 504})
REPORT_INTERVAL = 0.5 # seconds between progress publications
# byte ranges count encoded bytes but iter_content yields decoded ones, so ask for an unencoded body
@@ -53,69 +49,26 @@ def _is_transient(e: BaseException) -> bool:
class _Progress:
def __init__(self, done: set[int], done_bytes: int):
self._bytes = done_bytes
self._done = set(done)
def __init__(self):
self._bytes = 0
self._lock = threading.Lock()
def add_bytes(self, n: int) -> None:
with self._lock:
self._bytes += n
def mark_done(self, index: int) -> None:
def snapshot(self) -> int:
with self._lock:
self._done.add(index)
def snapshot(self) -> tuple[int, set[int]]:
with self._lock:
return self._bytes, set(self._done)
return self._bytes
def _piece_ranges(total: int, piece_size: int) -> list[tuple[int, int]]:
return [(start, min(start + piece_size, total)) for start in range(0, total, piece_size)]
def _sidecar_path(path: str) -> str:
return f"{path}.download"
def _download_in_progress(path: str) -> bool:
return os.path.isfile(_sidecar_path(path))
def _save_resume_state(path: str, layout: dict, done: set[int]) -> None:
tmp = _sidecar_path(path) + ".tmp"
with open(tmp, "w") as f:
json.dump({**layout, "done": sorted(done)}, f)
f.flush()
os.fsync(f.fileno())
os.replace(tmp, _sidecar_path(path)) # a power loss leaves the old sidecar or the new one, never a torn file
dir_fd = os.open(os.path.dirname(path) or ".", os.O_RDONLY)
try:
os.fsync(dir_fd)
finally:
os.close(dir_fd)
def _remove_download_state(path: str) -> None:
for stale in (_sidecar_path(path), _sidecar_path(path) + ".tmp"):
if os.path.isfile(stale):
os.remove(stale)
def _prepare_target(path: str, layout: dict) -> set[int]:
total = layout["total"]
try:
with open(_sidecar_path(path)) as f:
state = json.load(f)
if isinstance(state, dict) and all(state.get(k) == v for k, v in layout.items()) and os.path.getsize(path) == total:
return {i for i in state.get("done", []) if isinstance(i, int)}
except (OSError, ValueError, TypeError): # missing, unreadable or malformed sidecar: start over
pass
_save_resume_state(path, layout, set())
def _prepare_target(path: str, total: int) -> None:
with open(path, "wb") as f:
f.truncate(total)
return set()
def _content_range_total(response: requests.Response) -> int | None:
@@ -157,9 +110,6 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
f.truncate() # unknown length: the body defines the file size
if end is not None and written != end - start:
raise requests.exceptions.ConnectionError(f"short read: {written} of {end - start} bytes")
f.flush()
os.fsync(f.fileno()) # before it is recorded, so a power loss never leaves the sidecar naming unsynced bytes
progress.mark_done(index)
return
except requests.RequestException as e:
progress.add_bytes(-written) # the piece restarts from scratch
@@ -170,11 +120,6 @@ def _fetch_piece(url: str, path: str, index: int, start: int, end: int | None, p
raise DownloadCancelled("Download cancelled") from None
def _remove_file(path: str) -> None:
if os.path.isfile(path):
os.remove(path)
class ModelManagerSP:
"""Manages model downloads and status reporting"""
@@ -241,12 +186,11 @@ class ModelManagerSP:
else:
eta = self._calculate_eta(artifact.fileName, progress)
self._set_progress(artifact, custom.ModelManagerSP.DownloadStatus.downloading, progress, eta, speed)
artifact.downloadProgress.resuming = False # shown only for the first report, when the bar jumps to the resumed point
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, path: str, layout: dict,
started: tuple[int, set[int], float]) -> None:
async def _report_until_done(self, tasks: list[asyncio.Future], artifact, progress: _Progress, total: int,
started: tuple[int, float]) -> None:
pending = set(tasks)
last_bytes, saved, last_time = started
last_bytes, last_time = started
speed = 0.0
while pending:
done, pending = await asyncio.wait(pending, timeout=REPORT_INTERVAL)
@@ -255,17 +199,14 @@ class ModelManagerSP:
if self._download_interrupted():
raise DownloadCancelled("Download cancelled")
now = time.monotonic()
done_bytes, done_pieces = progress.snapshot()
if done_pieces != saved:
_save_resume_state(path, layout, done_pieces)
saved = done_pieces
done_bytes = progress.snapshot()
if (dt := now - last_time) > 0:
instant = max(0.0, (done_bytes - last_bytes) / dt) # bytes/s this tick; clamped since retried pieces can lower the byte count
speed = instant if speed == 0 else SPEED_SMOOTHING * speed + (1 - SPEED_SMOOTHING) * instant
if speed < SPEED_FLOOR: # a stall decays toward 0 without reaching it, and a near-0 speed overflows the eta
speed = 0.0
last_time, last_bytes = now, done_bytes
self._publish_progress(artifact, done_bytes, layout["total"], speed)
self._publish_progress(artifact, done_bytes, total, speed)
async def _download_file(self, url: str, path: str, artifact) -> None:
self._download_start_times[artifact.fileName] = time.monotonic()
@@ -273,18 +214,16 @@ class ModelManagerSP:
total, ranged = await loop.run_in_executor(None, _probe_size, url)
pieces: list[tuple[int, int | None]] = list(_piece_ranges(total, PIECE_SIZE)) if ranged else [(0, None)]
layout = {"total": total, "piece_size": PIECE_SIZE, "ranged": ranged, "sha256": artifact.downloadUri.sha256}
done = await loop.run_in_executor(None, _prepare_target, path, layout)
resumed_bytes = sum(end - start for i, (start, end) in enumerate(pieces) if i in done and end is not None)
progress = _Progress(done, resumed_bytes) # the bar and eta start from the resumed pieces
await loop.run_in_executor(None, _prepare_target, path, total)
progress = _Progress()
cancel = threading.Event()
started = *progress.snapshot(), time.monotonic() # before any worker can land bytes
started = progress.snapshot(), time.monotonic() # before any worker can land bytes
with ThreadPoolExecutor(max_workers=MAX_CONCURRENT_PIECES) as pool:
tasks = [loop.run_in_executor(pool, _fetch_piece, url, path, i, start, end, progress, cancel, self._block_size)
for i, (start, end) in enumerate(pieces) if i not in done]
for i, (start, end) in enumerate(pieces)]
try:
await self._report_until_done(tasks, artifact, progress, path, layout, started)
await self._report_until_done(tasks, artifact, progress, total, started)
except BaseException as e:
cancel.set()
if isinstance(e, DownloadCancelled):
@@ -293,32 +232,9 @@ class ModelManagerSP:
for task in tasks:
task.cancel() # drops queued pieces; running ones see the event at their next block
await asyncio.gather(*tasks, return_exceptions=True)
await loop.run_in_executor(None, pool.shutdown) # let running pieces stop so none lands after the save
_save_resume_state(path, layout, progress.snapshot()[1])
raise
_remove_download_state(path)
artifact.downloadProgress.resuming = False
del self._download_start_times[artifact.fileName]
async def _download_with_retries(self, url: str, path: str, artifact) -> None:
for attempt in range(DOWNLOAD_ATTEMPTS):
try:
await self._download_file(url, path, artifact)
return
except Exception as e:
if not _is_transient(e) or attempt == DOWNLOAD_ATTEMPTS - 1:
raise
delay = RETRY_BACKOFF * 2 ** attempt
cloudlog.warning(f"Retrying {artifact.fileName} in {delay:g}s after {type(e).__name__}: {e}")
progress = artifact.downloadProgress
self._set_progress(artifact, progress.status, progress.progress, progress.eta, 0.0) # stalled: no speed
deadline = time.monotonic() + delay
while (remaining := deadline - time.monotonic()) > 0:
if self._download_interrupted():
raise DownloadCancelled("Download cancelled") from e
await asyncio.sleep(min(RETRY_POLL, remaining))
async def _process_artifact(self, artifact, destination_path: str) -> None:
if not artifact.downloadUri.uri:
return None
@@ -330,39 +246,37 @@ class ModelManagerSP:
filename = artifact.fileName
full_path = os.path.join(destination_path, filename)
status = custom.ModelManagerSP.DownloadStatus
artifact.downloadProgress.resuming = False
artifact.downloadProgress.canceling = False
try:
if _download_in_progress(full_path):
cloudlog.info(f"Resuming interrupted download of {filename}")
artifact.downloadProgress.resuming = True
else:
self._set_progress(artifact, status.verifying, 0)
if await verify_file(full_path, expected_hash):
self._set_progress(artifact, status.cached, 100)
return
self._set_progress(artifact, status.verifying, 0)
if await verify_file(full_path, expected_hash):
self._set_progress(artifact, status.cached, 100)
return
self._set_progress(artifact, status.downloading, 0)
await self._download_with_retries(url, full_path, artifact)
await self._download_file(url, full_path, artifact)
self._set_progress(artifact, status.verifying, 99)
if not await verify_file(full_path, expected_hash):
_remove_file(full_path)
_remove_download_state(full_path)
raise ValueError(f"Hash validation failed for {filename}")
self._set_progress(artifact, status.downloaded, 100)
except DownloadCancelled: # not a failure: the request handler clears the row
except DownloadCancelled:
self._download_start_times.pop(artifact.fileName, None)
artifact.downloadProgress.status = status.failed
artifact.downloadProgress.eta = 0
artifact.downloadProgress.speed = 0
self._sync_artifact_progress(artifact)
if self.selected_bundle:
self.selected_bundle.status = status.failed
self._report_status()
raise
except Exception as e:
cloudlog.error(f"Error downloading {filename}: {str(e)}")
if os.path.isfile(full_path): # noqa: ASYNC240
os.remove(full_path)
artifact.downloadProgress.status = status.failed
artifact.downloadProgress.eta = 0
artifact.downloadProgress.speed = 0
@@ -419,8 +333,6 @@ class ModelManagerSP:
self.params.put(ACTIVE_BUNDLE_KEYS[source], model_bundle.to_dict(), block=True)
self.active_bundle = get_active_bundle(self.params, chestnut=self.chestnut_present)
except DownloadCancelled:
raise # not a failure
except Exception:
if self.selected_bundle is not None:
self.selected_bundle.status = custom.ModelManagerSP.DownloadStatus.failed
@@ -448,14 +360,11 @@ class ModelManagerSP:
self._download_ref = ref_to_download
try:
self.download(model_to_download, Paths.model_root(), source)
except DownloadCancelled:
self.selected_bundle = None
except Exception as e:
cloudlog.exception(e) # a failure stays on the row until the next request
else:
self.selected_bundle = None
cloudlog.exception(e)
finally:
self._release_download_ref()
self.selected_bundle = None
def main_thread(self) -> None:
"""Main thread for model management"""
@@ -11,13 +11,11 @@ import contextlib
import dataclasses
import hashlib
import http.server
import json
import math
import os
import random
import re
import requests
import stat
import tempfile
import threading
import time
@@ -57,8 +55,6 @@ class ServerState:
fail_ranges: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> HTTP status, every time
fail_times: dict[tuple[int, int], int] = dataclasses.field(default_factory=dict) # range -> how many more requests 503
short_once: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges whose first answer is cut short
fail_after_others: bool = False # hold each failure until every other piece has been sent
sent: set[tuple[int, int]] = dataclasses.field(default_factory=set) # ranges sent in full
stall_ranges: set[tuple[int, int]] = dataclasses.field(default_factory=set)
stall_event: threading.Event | None = None
stall_released_by_event: bool | None = None
@@ -92,10 +88,6 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
state.fail_times[rng] -= 1
status = 503
if status is not None:
others = {piece_range(i) for i in range(NUM_PIECES)} - {rng}
deadline = time.monotonic() + 5
while state.fail_after_others and not others <= state.sent and time.monotonic() < deadline:
time.sleep(0.01)
self.send_response(status)
self.end_headers()
return
@@ -123,8 +115,6 @@ class DownloadHandler(http.server.BaseHTTPRequestHandler):
self.wfile.write(payload[100:])
else:
self.wfile.write(payload)
if rng is not None:
state.sent.add(rng)
@contextlib.contextmanager
@@ -143,10 +133,7 @@ def threaded_server(handler):
class InlineExecutor(concurrent.futures.Executor):
def submit(self, fn, /, *args, **kwargs):
future: concurrent.futures.Future = concurrent.futures.Future()
try:
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
except BaseException as e:
future.set_exception(e)
future.set_result(fn(*args, **kwargs)) # the piece lands before the reporter starts
return future
@@ -160,12 +147,11 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
self.addCleanup(self._tmp.cleanup)
self.dest = self._tmp.name
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100), ('RETRY_BACKOFF', 0.01)):
for name, value in (('PIECE_SIZE', PIECE_SIZE), ('REPORT_INTERVAL', 0.05), ('PIECE_BACKOFF', 0.01), ('SPEED_FLOOR', 100)):
self.enterContext(mock.patch.object(manager_module, name, value))
self.reported: list[float] = []
self.reported_statuses: list[int] = []
self.reported_resuming: list[bool] = []
self.manager = ModelManagerSP.__new__(ModelManagerSP)
self.params = self.manager.params = mock.MagicMock()
@@ -186,7 +172,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
self.reported.append(float(artifact.downloadProgress.progress))
status = artifact.downloadProgress.status
self.reported_statuses.append(getattr(status, 'raw', status)) # .raw: _DynamicEnum equals its int but does not hash like it
self.reported_resuming.append(bool(artifact.downloadProgress.resuming))
def make_artifact(self):
bundle = custom.ModelManagerSP.ModelBundle.new_message()
@@ -203,24 +188,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
def path(self) -> str:
return os.path.join(self.dest, FILE_NAME)
@property
def sidecar(self) -> str:
return manager_module._sidecar_path(self.path)
def resume_state(self) -> dict:
with open(self.sidecar) as f:
return json.load(f)
def write_resume_state(self, done: list[int], total: int = len(WHOLE_BODY), file_size: int | None = None,
file_sha256: str = sha256(WHOLE_BODY)) -> None:
with open(self.path, 'wb') as f:
f.truncate(len(WHOLE_BODY) if file_size is None else file_size)
for i in done:
f.seek(i * PIECE_SIZE)
f.write(WHOLE_BODY[i * PIECE_SIZE:(i + 1) * PIECE_SIZE])
with open(self.sidecar, 'w') as f:
json.dump({"total": total, "piece_size": PIECE_SIZE, "ranged": True, "sha256": file_sha256, "done": done}, f)
def piece_requests(self) -> list[tuple[int, int]]:
return [r for r in self.server.request_ranges if r is not None and r != (0, 0)]
@@ -242,17 +209,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
def download_bundle(self, source: str):
asyncio.run(self.manager._download_bundle(self._bundle, self.dest, source))
def request_download(self) -> dict:
self.make_artifact()
self._bundle.ref = "ref"
self._bundle.minimumSelectorVersion = helpers.REQUIRED_JSON_VERSION
self.manager.source_models = {"qcom": [self._bundle], "chestnut": []}
self.manager.params, store = self._make_params_with_store()
store["ModelManager_DownloadRef"] = "ref"
with mock.patch.object(manager_module.Paths, 'model_root', return_value=self.dest):
self.manager._process_download_requests()
return store
def read_path(self) -> bytes:
with open(self.path, 'rb') as f:
return f.read()
@@ -269,7 +225,6 @@ class ManagerDownloadTestBase(OpenpilotTestCase):
params.get.side_effect = get
params.put.side_effect = put
params.remove.side_effect = lambda key: put(key, None)
return params, store
@@ -283,7 +238,6 @@ class TestManagerDownload(ManagerDownloadTestBase):
assert max(self.reported) <= 99.0, "progress holds below 100 until the hash check"
assert artifact.fileName not in self.manager._download_start_times
assert artifact.downloadProgress.speed > 0, "a running download shows its speed"
assert not os.path.exists(self.sidecar), "the resume state goes once every piece has landed"
def test_speed_counts_bytes_landed_before_the_first_report(self):
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
@@ -342,42 +296,9 @@ class TestManagerDownload(ManagerDownloadTestBase):
def test_permanent_failure_is_not_retried(self):
self.server.fail_ranges = {piece_range(1): 404}
self.enterContext(mock.patch.object(manager_module, 'ThreadPoolExecutor', lambda max_workers: InlineExecutor()))
with self.assertRaises(requests.exceptions.HTTPError):
self.process_artifact()
self.download_file()
assert self.piece_requests().count(piece_range(1)) == 1
assert os.path.isfile(self.path)
assert self.resume_state()["done"] == [i for i in range(NUM_PIECES) if i != 1]
def test_failed_transfer_resumes_on_retry(self):
self.server.fail_times = {piece_range(1): manager_module.PIECE_ATTEMPTS} # the first transfer gives up on piece 1
self.server.fail_after_others = True # by then every other piece is saved
self.process_artifact()
assert self.read_path() == WHOLE_BODY
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS + 1, "the retry fetched piece 1"
others = [i for i in range(NUM_PIECES) if i != 1]
assert all(self.piece_requests().count(piece_range(i)) == 1 for i in others), "the retry skipped every saved piece"
def test_lasting_transient_failure_gives_up(self):
self.server.fail_ranges = {piece_range(1): 503}
with self.assertRaises(requests.exceptions.HTTPError):
self.process_artifact()
assert self.piece_requests().count(piece_range(1)) == manager_module.PIECE_ATTEMPTS * manager_module.DOWNLOAD_ATTEMPTS
def test_cancel_ends_retry_backoff(self):
self.enterContext(mock.patch.object(manager_module, 'RETRY_BACKOFF', 30.0))
self.manager._download_ref = b"ref"
def fail_then_cancel(*args):
self.params.get.return_value = None # the user cancels while the manager waits to retry
raise requests.exceptions.ConnectionError("dropped")
transfer = self.enterContext(mock.patch.object(self.manager, '_download_file', side_effect=fail_then_cancel))
started = time.monotonic()
with self.assertRaises(DownloadCancelled):
self.process_artifact()
assert time.monotonic() - started < 5, "the wait must end at the cancel, not after 30 s"
assert transfer.call_count == 1, "no second transfer after a cancel"
assert self.artifact.downloadProgress.status != custom.ModelManagerSP.DownloadStatus.failed
def test_removing_download_ref_cancels(self):
stall_event = self.stall(3)
@@ -395,51 +316,6 @@ class TestManagerDownload(ManagerDownloadTestBase):
assert self.server.stall_released_by_event is True, "the cancel must have caught the piece mid-transfer"
end = piece_range(3)[1] + 1
assert self.read_path()[end - 100:end] == bytes(100), "the cancelled piece must stop before its last bytes land"
done = self.resume_state()["done"]
assert done and 3 not in done, "a cancel records the finished pieces, and only those, for resume"
def test_resume_skips_complete_pieces(self):
self.write_resume_state(done=[0, 4])
self.download_file()
assert piece_range(0) not in self.piece_requests() and piece_range(4) not in self.piece_requests()
assert self.read_path() == WHOLE_BODY
assert min(self.reported) >= 2 * PIECE_SIZE / len(WHOLE_BODY) * 100 - 1, "progress must not restart below the resumed share"
def test_unusable_resume_state_starts_over(self):
def malformed(content: bytes):
self.write_resume_state(done=[0])
with open(self.sidecar, 'wb') as f:
f.write(content)
layout = {"total": len(WHOLE_BODY), "piece_size": PIECE_SIZE, "ranged": True, "sha256": sha256(WHOLE_BODY)}
cases = {
"other size": lambda: self.write_resume_state(done=[0], total=len(WHOLE_BODY) + 1),
"other bytes": lambda: self.write_resume_state(done=[0], file_sha256="0" * 64),
"truncated file": lambda: self.write_resume_state(done=[0], file_size=PIECE_SIZE),
"not an object": lambda: malformed(b'[]'),
"cut-off json": lambda: malformed(b'{"total": '),
"done not a list": lambda: malformed(json.dumps({**layout, "done": 3}).encode()),
"done not ints": lambda: malformed(json.dumps({**layout, "done": ["0", None]}).encode()),
}
for name, write_state in cases.items():
with self.subTest(name):
self.server.request_ranges = []
write_state()
self.download_file()
assert sorted(self.piece_requests()) == [piece_range(i) for i in range(NUM_PIECES)], "every piece is fetched again"
assert self.read_path() == WHOLE_BODY
def test_pieces_and_resume_state_are_synced_to_disk(self):
real_fsync, synced = os.fsync, []
def record_sync(fd):
st = os.fstat(fd)
synced.append("dir" if stat.S_ISDIR(st.st_mode) else "model" if os.path.exists(self.path) and st.st_ino == os.stat(self.path).st_ino else "sidecar")
real_fsync(fd)
with mock.patch.object(manager_module.os, 'fsync', side_effect=record_sync):
self.download_file()
assert synced.count("model") == NUM_PIECES, f"expected one fsync per piece, got {synced.count('model')}"
assert "sidecar" in synced and "dir" in synced, "sidecar file and directory must be fsync'd"
def test_replaced_download_ref_queues_instead_of_canceling(self):
self.params.get.return_value = b"other-ref"
@@ -469,26 +345,9 @@ class TestProcessArtifact(ManagerDownloadTestBase):
self.server.corrupt = True
with self.assertRaises(ValueError):
self.process_artifact()
assert not os.path.isfile(self.path) and not os.path.exists(self.sidecar), "a corrupt file must not be resumed from"
assert not os.path.isfile(self.path)
assert self.artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.failed
def test_interrupted_download_resumes_without_reverifying(self):
self.write_resume_state(done=[0, 1, 2])
artifact = self.process_artifact()
ds = custom.ModelManagerSP.DownloadStatus
assert self.reported_statuses[0] == ds.downloading, "an in-progress download is not verified first"
assert piece_range(0) not in self.piece_requests()
assert artifact.downloadProgress.status == ds.downloaded
assert any(self.reported_resuming), "a resumed download is flagged resuming until its first progress report"
assert not artifact.downloadProgress.resuming
def test_resume_with_every_piece_done_clears_resuming(self):
self.write_resume_state(done=list(range(NUM_PIECES)))
artifact = self.process_artifact()
assert self.piece_requests() == []
assert artifact.downloadProgress.status == custom.ModelManagerSP.DownloadStatus.downloaded
assert not artifact.downloadProgress.resuming
def test_probe_failure_moves_past_verifying(self):
self.server.fail_ranges = {(0, 0): 404} # the size probe
with self.assertRaises(requests.exceptions.HTTPError):
@@ -515,23 +374,6 @@ class TestProcessArtifact(ManagerDownloadTestBase):
self.download_bundle("qcom")
assert ACTIVE_BUNDLE_KEYS["qcom"] not in store
assert os.path.isfile(self.path), "cancel must not delete a cached model"
assert self.manager.selected_bundle.status != custom.ModelManagerSP.DownloadStatus.failed, "a cancel is not a failure"
def test_failed_download_stays_on_the_row(self):
self.server.fail_ranges = {piece_range(1): 404}
store = self.request_download()
selected = self.manager.selected_bundle
assert selected is not None and selected.status == custom.ModelManagerSP.DownloadStatus.failed, "a failure stays on the row"
assert store["ModelManager_DownloadRef"] is None, "the failed request is released"
def test_finished_download_clears_the_row(self):
self.request_download()
assert self.manager.selected_bundle is None
def test_cancelled_download_clears_the_row(self):
self.pm.send.side_effect = lambda *args: self.manager.params.remove("ModelManager_DownloadRef") # cancel at the first report
self.request_download()
assert self.manager.selected_bundle is None
def test_download_ref_is_released_only_if_unchanged(self):
for current, removed in ((b"new-ref", False), (b"ref", True)):
Generated
+468 -403
View File
File diff suppressed because it is too large Load Diff