diff --git a/jetlink_repo/LICENSE b/jetlink_repo/LICENSE new file mode 100644 index 0000000000..2186c11202 --- /dev/null +++ b/jetlink_repo/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Zeph Leggett + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/jetlink_repo/README.md b/jetlink_repo/README.md new file mode 100644 index 0000000000..7a2c31704e --- /dev/null +++ b/jetlink_repo/README.md @@ -0,0 +1,20 @@ +# Jetlink runtime for StarPilot + +This subtree supplies the Jetlink remote-model client used by **StarPilot**. +For installing, connecting, selecting, and running it with StarPilot, follow +[the StarPilot JetLink guide](../openpilot/starpilot/models/JETLINK.md). + +The comma runs StarPilot; the connected host runs the upstream Jetlink application. +Host apps, installers, development references, and original history are maintained +in [ZoomPilot / Jetlink](https://github.com/zoompilot/jetlink/tree/c17cd5c). +This runtime subtree does not contain the upstream host applications or docs; +their source attribution does not require installing ZoomPilot on the comma. + +Jetlink is experimental. Use separate power and USB 3, set up parked, and verify +StarPilot's active model status. See the local guide for recovery and limitations. + +## Attribution + +Jetlink was developed by Zeph Leggett / ZoomPilot. Copyright (c) 2026 Zeph Leggett. +The original [MIT license](LICENSE) is preserved without modification. +[Upstream README and credits](https://github.com/zoompilot/jetlink/blob/c17cd5c/README.md). diff --git a/jetlink_repo/jetlink/__init__.py b/jetlink_repo/jetlink/__init__.py new file mode 100644 index 0000000000..d61cb1f2de --- /dev/null +++ b/jetlink_repo/jetlink/__init__.py @@ -0,0 +1,13 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +jetlink: run openpilot's large driving models on an attached Jetson. +""" +# The one place the version is written: pyproject.toml reads it from here, and +# so does the release check (macos/scripts/check-version.sh). The comma runs +# jetlink from a checkout, not an install, so this is also all it has. The Swift +# reads it from Pinned.swift: run JetlinkKit/Scripts/make_pins.py after a bump. +__version__ = '0.8.3' diff --git a/jetlink_repo/jetlink/client.py b/jetlink_repo/jetlink/client.py new file mode 100644 index 0000000000..c692f997d8 --- /dev/null +++ b/jetlink_repo/jetlink/client.py @@ -0,0 +1,604 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The comma side of the link. + +Knows nothing about openpilot: warped frame and packed scalars in, model output +out. The glue lives in the fork (sunnypilot/jetlink/), so any fork can import +this. + +Every error is a LinkError, which modeld already catches and answers with the +small model. A frame blocks until the answer arrives, as it does on a chestnut; +only a stall past FRAME_TIMEOUT is a failure. +""" +from __future__ import annotations + +import json +import logging +import os +import secrets +import sys +import time +from collections import deque +from collections.abc import Callable +from pathlib import Path +from typing import NamedTuple + +import numpy as np + +from jetlink import protocol as P +from jetlink.spec import CHUNK, DEFAULT_FRAME_SKIP, ModelSpec +from jetlink.transport.base import LinkError, LinkTimeout, Message, Transport + + +def _name(enum_cls, value) -> str: + """Enum name for a value off the wire, which may be anything.""" + try: + return enum_cls(value).name + except ValueError: + return f'unknown({value})' + +log = logging.getLogger('jetlink.client') + +ProgressFn = Callable[[str, float, str], None] + +# Not a frame budget: a long frame is a dropped camera frame, which modeld +# tolerates as it does on a chestnut. This is chestnut's HCQDEV_WAIT_TIMEOUT_MS +# (3000) - past it the far end is gone, not slow. +FRAME_TIMEOUT = 3.0 + +StopFn = Callable[[], bool] + +# openpilot logs the whole output vector as modelV2.rawPredictions when this is +# set, from what infer_end returns. The server keeps the hidden state, so ask for +# it on those runs, or the log would carry zeros where it goes. +RAW_PRED_ENV = 'SEND_RAW_PRED' + + +class Sent(NamedTuple): + """A frame sent and not yet answered.""" + seq: int + frame_id: int + flags: int + sent_at: float + + +class EngineMissing(LinkError): + """The server has no engine and this caller has no ONNX to upload. + + Raised in modeld, which never carries the file: the Jetson's cache was pruned, + re-flashed or swapped since a provisioning run recorded it ready. + """ + + +def _default_name() -> str: + """Something the Jetson's journal can tell one comma process from another by.""" + return (Path(sys.argv[0]).stem if sys.argv else '') or 'python' + + +class JetlinkClient: + def __init__(self, transport: Transport, deadline: float = FRAME_TIMEOUT, + name: str | None = None, want_hidden: bool | None = None): + self.t = transport + self.deadline = deadline + # Flag.WANT_HIDDEN on every frame; see RAW_PRED_ENV + self.want_hidden = bool(os.environ.get(RAW_PRED_ENV)) if want_hidden is None else want_hidden + # Who the server logs this connection as. The nonce is per client object, + # so two processes taking turns on one gadget, or one process reopening + # the link, are separable in a journal whose clock is wrong anyway. + self.name = name or _default_name() + self.nonce = secrets.token_hex(4) + self.seq = 0 + self.spec: ModelSpec | None = None + self.progress_cb: ProgressFn | None = None + self._engine_state: dict | None = None + self._should_stop: StopFn = lambda: False + self.dead = False + self.last_timings = (0, 0, 0) # gpu_us, queue_us, total_us, server-side + self.last_state: dict | None = None # most recent piggybacked telemetry + # the newest model output read off the link, whichever call read it: what + # a frame whose own reply is late publishes again (JetlinkModelState.end) + self.last_output: np.ndarray | None = None + # frames sent and not yet answered, oldest first. More than one while + # frames go out without being waited for (drain), or after a reply a + # frame gave up on (infer_end with hold) + self._in_flight: deque[Sent] = deque() + # seqs sent with no reply wanted (leave): an ERROR to one is not this link's + self._no_reply_wanted: set[int] = set() + + # -- construction --------------------------------------------------------- + + @classmethod + def open_ffs(cls, mount: str = '/dev/ffs-jetlink', gadget: str | None = None, + udc: str | None = None, **kw) -> JetlinkClient: + """This end is the USB gadget (FunctionFS). + + The roles follow what the two kernels support, not who is the client: AGNOS + has F_FS built in, and a host needs no driver at all. See docs/transport.md. + """ + from jetlink.transport.ffs import FfsTransport + return cls(FfsTransport(mount, gadget=gadget, udc=udc), **kw) + + @classmethod + def open_borrowed_ffs(cls, mount: str, udc: str, bounce=None, owner_gadget: str | None = None, + **kw) -> JetlinkClient: + """This end is the USB gadget, over a gadget another process owns. + + The owner holds ep0 and the UDC bind for as long as the link is enabled, + so a drive starting or ending is no longer an unplug the host has to + recover from. See FfsTransport.borrowed. + """ + from jetlink.transport.ffs import FfsTransport + return cls(FfsTransport.borrowed(mount, udc, bounce=bounce, owner_gadget=owner_gadget), **kw) + + @classmethod + def open_tcp(cls, host: str, port: int = 5599, **kw) -> JetlinkClient: + from jetlink.transport.tcp import TcpTransport + return cls(TcpTransport.connect(host, port), **kw) + + @classmethod + def open_listen(cls, address: str, timeout: float | None = None, **kw) -> JetlinkClient: + """Wait for one peer to dial `address`, '[HOST:]PORT' (every interface by + default), and run the session over that connection, as the comma takes a + phone's dial (docs/transport.md). For the bench and parity scripts.""" + from jetlink.transport.tcp import TcpTransport + host, _, port = address.rpartition(':') + if not port.isdigit(): + raise ValueError(f'listen on [HOST:]PORT, not {address!r}') + transport, peer = TcpTransport.listen_once(host or '0.0.0.0', int(port), timeout) + log.info('peer dialed in from %s:%d', peer[0], peer[1]) + return cls(transport, **kw) + + @classmethod + def open_socket(cls, sock, **kw) -> JetlinkClient: + """Over a socket somebody else connected: a phone that dialed the comma + over the cable's network interface, accepted by whoever holds the loan + (jetlink.openpilot.link.connect). The same session as open_tcp from here on.""" + from jetlink.transport.tcp import TcpTransport + return cls(TcpTransport(sock), **kw) + + # -- plumbing ------------------------------------------------------------- + + def _next_seq(self) -> int: + self.seq = (self.seq + 1) & 0xFFFFFFFF + return self.seq + + def _dispatch(self, msg: Message) -> None: + """Handle the messages the server may send at any time.""" + if msg.msg_type == P.Msg.PROGRESS: + p = json.loads(bytes(msg.payload)) + if self.progress_cb: + self.progress_cb(p.get('stage', ''), float(p.get('frac', 0.0)), p.get('msg', '')) + elif msg.msg_type == P.Msg.ENGINE_RESP: + state = json.loads(bytes(msg.payload)) + # A build that finished for a model this client has moved off would + # otherwise read as ready, and every frame after it as NOT_READY. + wanted = (self._engine_state or {}).get('sha256') + if wanted is not None and state.get('sha256') not in (None, wanted): + log.warning("ignoring engine state for %s (we want %s)", + str(state.get('sha256'))[:16], wanted[:16]) + else: + self._engine_state = state + elif msg.msg_type == P.Msg.ERROR: + e = json.loads(bytes(msg.payload)) + if msg.seq in self._no_reply_wanted: + # an older server answering a message it does not know; see leave() + self._no_reply_wanted.discard(msg.seq) + log.info("the server did not take seq %d (%s: %s)", msg.seq, e.get('error'), e.get('detail')) + return + raise LinkError(f"server error: {e.get('error')}: {e.get('detail')}") + + def _expect(self, msg_type: int, seq: int, timeout: float | None) -> Message: + """Wait for one specific reply, servicing anything unsolicited on the way. + A timeout of 0 still takes a reply that has already arrived: one read + that does not wait, as the transport does it.""" + end = None if timeout is None else time.monotonic() + timeout + read_once_late = False + while True: + remaining = None if end is None else end - time.monotonic() + if remaining is not None and remaining <= 0: + if read_once_late: + raise LinkTimeout(f'timed out waiting for {_name(P.Msg, msg_type)} seq={seq}') + read_once_late, remaining = True, 0.0 + msg = self.t.recv(timeout=remaining) + if msg.msg_type == msg_type and msg.seq == seq: + return msg + if msg.msg_type in (P.Msg.PROGRESS, P.Msg.ENGINE_RESP, P.Msg.ERROR): + self._dispatch(msg) + continue + if msg.msg_type == P.Msg.INFER_RESP and any(f.seq == msg.seq for f in self._in_flight): + # the answer to a frame nobody waited for, or one a frame gave up + # waiting for: read like any other, then on to what is wanted + self._take_reply(msg) + continue + # A reply a caller gave up on. Drop it, or every frame reads one behind. + log.warning("discarding stale %s seq=%d (waiting for %s seq=%d)", + _name(P.Msg, msg.msg_type), msg.seq, _name(P.Msg, msg_type), seq) + + # -- handshake ------------------------------------------------------------ + + def hello(self, timeout: float = 5.0) -> dict: + """Introduce this client. The server starts its session over on a hello, + so this is also how a new owner of the gadget takes over one the server + never saw end; see Session.greet in JetlinkServer.""" + seq = self._next_seq() + client = {'nonce': self.nonce, 'name': self.name} + try: + link = self.t.link_info() + except Exception: + link = {} # never worth a failed hello + if link: + # Which link this is and how fast: the server shows it (USB 3, USB 2, + # TCP), and only this end always knows. See Transport.link_info. + client['link'] = link + # a new session: frames of the old one still unanswered will not be, and + # its last output is nothing the new model may publish + self._in_flight.clear() + self.last_output = None + self.t.send_json(P.Msg.HELLO_REQ, seq, {'client': client}) + return json.loads(bytes(self._expect(P.Msg.HELLO_RESP, seq, timeout).payload)) + + def state(self, timeout: float = 2.0) -> dict: + seq = self._next_seq() + self.t.send_json(P.Msg.STATE_REQ, seq, {}) + return json.loads(bytes(self._expect(P.Msg.STATE_RESP, seq, timeout).payload)) + + def ping(self, timeout: float = 1.0) -> float: + seq = self._next_seq() + t0 = time.perf_counter() + self.t.send(P.Msg.PING, seq) + self._expect(P.Msg.PONG, seq, timeout) + return time.perf_counter() - t0 + + def shutdown(self, reason: str = '', timeout: float = 5.0) -> dict: + """Ask the Jetson to power off, not sleep: only a DC cycle or the button + brings it back, so this is for the comma's low-battery shutdown alone. The + reply comes before the box goes down.""" + seq = self._next_seq() + self.t.send_json(P.Msg.SHUTDOWN_REQ, seq, {'reason': reason}) + return json.loads(bytes(self._expect(P.Msg.SHUTDOWN_RESP, seq, timeout).payload)) + + def leave(self, reason: str, timeout: float = 0.5, **measured) -> None: + """Say why this client stops using the link (protocol.LEAVE_REASONS), and + what it measured over it (JetlinkModelState.trips): the server logs both + next to its own numbers, which are the half the comma cannot see, and the + other way round. No reply is read, and the connection stays: a later hello + starts a new session over it. An older server answers ERROR + unknown_message, which _dispatch discards by this seq. Never raises, and + says nothing over a dead link; a send that fails leaves it dead.""" + if self.dead: + return + seq = self._next_seq() + self._no_reply_wanted.add(seq) + try: + self.t.send_json(P.Msg.LEAVE, seq, {'reason': reason, **measured}, timeout=timeout) + except LinkError as e: + self.dead = True + log.warning("could not say why the link is left (%s)", e) + + # -- model provisioning --------------------------------------------------- + + def ensure_engine(self, sha256: str, nbytes: int, onnx_path: str | Path | None = None, + frame_skip: int = DEFAULT_FRAME_SKIP, + progress: ProgressFn | None = None, + build_timeout: float = 900.0, + should_stop: StopFn | None = None) -> ModelSpec: + """Make the server ready to run this model, uploading and building if needed. + + Blocks until ready or the build fails, and returns the spec the server + derived from the ONNX. Only the server parses the model, so a device whose + parser rejects an export (tinygrad and the org.tinygrad domain) still runs. + + `onnx_path` None means the caller cannot upload (modeld) and a missing + engine is EngineMissing. `progress(stage, frac, msg)` has stage in + upload/patch/parse/build/load. `should_stop` is polled during long waits. + """ + self.progress_cb = progress + self._should_stop = should_stop or (lambda: False) + self.spec = None + self._engine_state = None + + seq = self._next_seq() + self.t.send_json(P.Msg.ENGINE_REQ, seq, {'sha256': sha256, 'nbytes': nbytes, 'frame_skip': frame_skip}) + resp = json.loads(bytes(self._expect(P.Msg.ENGINE_RESP, seq, 60.0).payload)) + self._engine_state = resp + log.info("server engine state: %s (%s)", resp['state'], resp.get('detail', '')) + + if resp['state'] == 'need_upload': + if onnx_path is None: + raise EngineMissing(f"server has no engine for {sha256[:16]} ({resp.get('detail', '')})") + self._upload(Path(onnx_path), nbytes, int(resp.get('chunk') or CHUNK)) + + try: + self._await_ready(build_timeout) + except EngineMissing: + # need_upload came later, once the job holding the device (a preload + # of another model) was done. A caller with the file sends it now. + if onnx_path is None: + raise + self._upload(Path(onnx_path), nbytes, int(self._engine_state.get('chunk') or CHUNK)) + self._await_ready(build_timeout) + spec = ModelSpec.from_dict(self._engine_state['spec']) + if spec.sha256 != sha256: + raise LinkError(f"server answered for {spec.sha256[:16]}, we asked for {sha256[:16]}") + self.spec = spec + return spec + + def _stopped(self) -> None: + if self._should_stop(): + raise LinkError("stopped while waiting for the engine") + + def _upload(self, onnx_path: Path, nbytes: int, chunk: int) -> None: + log.info("uploading %s (%d MB)", onnx_path.name, nbytes >> 20) + t0 = time.time() + sent = 0 + with open(onnx_path, 'rb') as f: + while data := f.read(chunk): + self._stopped() + self.t.send(P.Msg.UPLOAD_CHUNK, self._next_seq(), + (sent.to_bytes(8, 'little'), data)) + sent += len(data) + if self.progress_cb: + self.progress_cb('upload', sent / nbytes, f'{sent >> 20}/{nbytes >> 20} MB') + seq = self._next_seq() + self.t.send_json(P.Msg.UPLOAD_DONE, seq, {}) + resp = json.loads(bytes(self._expect(P.Msg.ENGINE_RESP, seq, 300.0).payload)) + self._engine_state = resp + rate = sent / max(1e-6, time.time() - t0) / 1e6 + log.info("uploaded %d MB at %.1f MB/s -> %s", sent >> 20, rate, resp['state']) + if resp['state'] == 'failed': + raise LinkError(f"upload rejected: {resp.get('detail')}") + + def _await_ready(self, timeout: float) -> None: + end = time.monotonic() + timeout + while True: + st = (self._engine_state or {}).get('state') + if st == 'ready': + if 'spec' not in (self._engine_state or {}): + raise LinkError("server reported ready without a model spec") + return + if st == 'failed': + raise LinkError(f"engine build failed: {self._engine_state.get('detail')}") + if st == 'need_upload': + # pushed when the device came free: without this the wait ran to + # build_timeout, a whole drive for provisioning's 1800 s + sha = str(self._engine_state.get('sha256')) + raise EngineMissing(f"server has no engine for {sha[:16]} ({self._engine_state.get('detail', '')})") + if time.monotonic() > end: + raise LinkTimeout(f"engine not ready after {timeout:.0f}s (state={st})") + self._stopped() + try: + self._dispatch(self.t.recv(timeout=min(1.0, max(0.1, end - time.monotonic())))) + except LinkTimeout: + continue + + # -- inference ------------------------------------------------------------ + + def infer_begin(self, warped: np.ndarray, packed: np.ndarray, frame_id: int = 0, + reset: bool = False, want_state: bool = False, deadline: float | None = None, + silence: float | None = None) -> int: + """Send a frame and return immediately with its sequence number. + + Split from infer_end so the caller can work while the Jetson is busy; + openpilot publishes chestnutState in that window. A frame nobody calls + infer_end for is read by the next drain or infer_end. A host that has + answered none of them for `silence` (the client's deadline by default) + is gone, not slow, and the link is done here, before another frame goes + out to it. `deadline` bounds this frame's send. + """ + if self.spec is None: + raise LinkError("ensure_engine() first") + if self.dead: + raise LinkError("link previously failed") + silence = self.deadline if silence is None else silence + if self.waiting_for() > silence: + self.dead = True + raise LinkError(f"no answer to {len(self._in_flight)} frames in {silence:.1f}s; link abandoned") + warped = _as_bytes(warped, self.spec.warped_nbytes, 'warped') + packed = _as_bytes(packed, self.spec.packed_nbytes, 'packed') + seq = self._next_seq() + flags = ((P.Flag.RESET_QUEUES if reset else 0) | (P.Flag.WANT_STATE if want_state else 0) + | (P.Flag.WANT_HIDDEN if self.want_hidden else 0)) + try: + self._in_flight.append(Sent(seq, frame_id, flags, time.monotonic())) + self.t.send(P.Msg.INFER_REQ, seq, (P.pack_infer_req(frame_id, flags), warped, packed), + timeout=self.deadline if deadline is None else deadline) + except LinkError: + self.dead = True + raise + return seq + + def infer_end(self, seq: int, deadline: float | None = None, hold: float | None = None) -> np.ndarray | None: + """Block for the frame's output, as modeld blocks on a chestnut. + + A long frame is a dropped camera frame, which modeld tolerates. Only a stall + past `deadline` (the client's by default) is a failure, and then the link + is done. With `hold`, wait that long and no longer: None comes back when + the reply has not, the link is as good as it was, and the reply is read + off the stream by whatever reads it next. Another frame can go out + meanwhile, so the camera frame is not dropped waiting for this one. + """ + sent = next((f for f in self._in_flight if f.seq == seq), None) + if sent is None: + raise LinkError(f'frame seq {seq} was not sent, or has been read already') + failing = hold is None + if failing: + hold = (self.deadline if deadline is None else deadline) - (time.monotonic() - sent.sent_at) + try: + if failing and hold <= 0: + raise LinkTimeout('frame deadline elapsed during send') + msg = self._expect(P.Msg.INFER_RESP, seq, hold) + except LinkTimeout as e: + if not failing: + return None + self.dead = True + raise LinkError(f"no answer for frame in {self.deadline:.1f}s; link abandoned") from e + except LinkError: + self.dead = True + raise + return self._take_reply(msg) + + def drain(self) -> int: + """Read every reply that has already arrived, without waiting for one that + has not. How many frames were answered. What they carried is kept as + last_output, last_timings and last_state, as infer_end keeps it; a frame + the server failed is this link's failure, as it would have been there.""" + answered = 0 + while self._in_flight: # with nothing out there is nothing to read for + try: + msg = self.t.recv(timeout=0) + except LinkTimeout: + break + except LinkError: + self.dead = True + raise + if msg.msg_type == P.Msg.INFER_RESP: + self._take_reply(msg) + answered += 1 + else: + self._dispatch(msg) + return answered + + @property + def unanswered(self) -> int: + """Frames sent and not yet answered.""" + return len(self._in_flight) + + def waiting_for(self) -> float: + """How long the oldest frame still unanswered has been out, in seconds; + 0 with none. A host that has gone quiet shows here before any frame + waits on it.""" + return time.monotonic() - self._in_flight[0].sent_at if self._in_flight else 0.0 + + def _take_reply(self, msg: Message) -> np.ndarray: + """One INFER_RESP off the stream: checked against the frame it answers, + its timings and telemetry kept, the frame and any older one taken out of + flight. Raises, with the link dead, on anything but a good answer.""" + while self._in_flight and self._in_flight[0].seq != msg.seq: + # an older frame the server never answered: it answers in order, and a + # frame that fell out of its session (a hello between) is not coming + skipped = self._in_flight.popleft() + log.warning('frame %d (seq %d) was never answered', skipped.frame_id, skipped.seq) + if not self._in_flight: + self.dead = True + raise LinkError(f'inference response seq {msg.seq} answers no frame sent') + _, frame_id, flags, _ = self._in_flight.popleft() + if msg.payload.nbytes < P.INFER_RESP_SIZE: + self.dead = True + raise LinkError('inference response is missing its header') + fid, status, gpu_us, queue_us, total_us = P.unpack_infer_resp(msg.payload) + self.last_timings = (gpu_us, queue_us, total_us) + if status != P.Status.OK: + self.dead = True + raise LinkError(f"inference failed: {_name(P.Status, status)} (frame {fid})") + if fid != frame_id: + self.dead = True + raise LinkError(f'inference response frame {fid}, expected {frame_id}') + whole = bool(flags & P.Flag.WANT_HIDDEN) + n = self.spec.output_nelem if whole else self.spec.reply_nelem + end = P.INFER_RESP_SIZE + n * 4 + if msg.payload.nbytes < end: + self.dead = True + raise LinkError('inference response is missing model outputs') + if msg.payload.nbytes > end: + # piggybacked telemetry, when asked for; anything else is outputs laid + # out some other way (hidden_state left in by a server that says + # protocol 3, or a slice the two ends resolve differently): every float + # after it would be misread + state = _telemetry(msg.payload[end:]) if flags & P.Flag.WANT_STATE else None + if state is None: + self.dead = True + raise LinkError(f'inference response is {msg.payload.nbytes} bytes, expected {end}') + self.last_state = state + receive = getattr(self.t, 'last_receive', None) + if receive is not None and total_us > 50_000: + log.warning('frame %d receive maxima: prepare %.1f read_wait %.1f handoff %.1f ms; ' + 'server gpu %.1f queue %.1f total %.1f ms', fid, + receive['prepare'] * 1e3, receive['read_wait'] * 1e3, receive['handoff'] * 1e3, + gpu_us / 1e3, queue_us / 1e3, total_us / 1e3) + self.last_output = _whole_output(self.spec, np.frombuffer(msg.payload, np.float32, n, P.INFER_RESP_SIZE), whole) + return self.last_output + + def infer(self, warped: np.ndarray, packed: np.ndarray, frame_id: int = 0, + reset: bool = False, deadline: float | None = None, + want_state: bool = False) -> np.ndarray: + """One frame. Returns the model output as float32, shaped (n,), laid out + as the spec's output_slices say; hidden_state reads as zeros unless + `want_hidden`, since it stays on the server. + + `warped` is (2, 6, H, W) uint8 off openpilot's warp, `packed` the float32 + packed_npy_inputs. Either may be any buffer, so a tinygrad Tensor.data() + memoryview reaches the wire with no numpy round trip. + """ + return self.infer_end(self.infer_begin(warped, packed, frame_id, reset, want_state, deadline), deadline) + + @property + def lendable(self) -> bool: + """Could another process take the IO over from this link? See + FfsTransport.lendable.""" + return self.t.lendable + + def rebind(self) -> bool: + """Make the peer see the link arrive again, where that is a thing this + transport can do. See FfsTransport.rebind; False everywhere else.""" + return self.t.rebind() + + def release_endpoints(self) -> bool: + """Give the endpoints up without giving the gadget up. See + FfsTransport.release_endpoints; False everywhere else.""" + return self.t.release_endpoints() + + def close(self) -> None: + # a closed client is as dead as a failed one: whoever still holds it must + # open a new one rather than reuse a closed socket or endpoint file + self.dead = True + self.t.close() + + +def _telemetry(tail) -> dict | None: + """A WANT_STATE reply's telemetry: the JSON object after the outputs. None + for anything else, floats included, which is a reply laid out another way.""" + raw = bytes(tail) + if raw[:1] != b'{': + return None + try: + state = json.loads(raw) + except ValueError: + return None + return state if isinstance(state, dict) else None + + +def _whole_output(spec: ModelSpec, reply: np.ndarray, whole: bool) -> np.ndarray: + """The output vector, from a reply that may leave hidden_state out. + + Always a new array: `reply` views the transport's reusable receive buffer. + The full length keeps the spec's output_slices valid, so the caller slices + as it did when the whole vector crossed the link. + """ + hidden = spec.hidden_range + if whole or hidden is None: + return reply.copy() + start, stop = hidden + out = np.empty(spec.output_nelem, np.float32) + out[:start] = reply[:start] + out[start:stop] = 0 + out[stop:] = reply[start:] + return out + + +def _as_bytes(buf, expect: int, name: str) -> memoryview: + """Byte view over a numpy array or any buffer, size-checked. + + Bytes rather than shape, so the caller can pass whatever it has and a + mismatch fails here instead of misparsing on the far end. + """ + mv = memoryview(buf) + if not mv.contiguous: + raise LinkError(f"{name} must be contiguous") + mv = mv.cast('B') + if mv.nbytes != expect: + raise LinkError(f"{name} is {mv.nbytes} bytes, expected {expect}") + return mv diff --git a/jetlink_repo/jetlink/comma/__init__.py b/jetlink_repo/jetlink/comma/__init__.py new file mode 100644 index 0000000000..db13cd73d9 --- /dev/null +++ b/jetlink_repo/jetlink/comma/__init__.py @@ -0,0 +1,17 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +What runs on the comma itself, beside openpilot: the USB gadget (gadget), the +process that holds it for the whole time the link is on (owner), the lease +other processes borrow its endpoints on (lending), the USB-C port (port), and +the one root script all of it goes through (root), the VM tuning included. For +the comma four and the comma 3X. + +The standard library and jetlink's transport only: the owner that imports it +stays resident at about 10 MB. openpilot starts the owner through +jetlink.openpilot.owner, with what the fork's adapter names: the settings, the +chestnut's USB ids and the provisioning worker. +""" diff --git a/jetlink_repo/jetlink/comma/gadget.py b/jetlink_repo/jetlink/comma/gadget.py new file mode 100644 index 0000000000..c9d7ccebba --- /dev/null +++ b/jetlink_repo/jetlink/comma/gadget.py @@ -0,0 +1,463 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The comma's USB gadget, and how to look at it, using nothing but the +standard library and jetlink's own transport. + +Kept apart from openpilot so the process that owns the gadget can be small. +Holding ep0 needs sysfs and a unix socket; `openpilot.common.swaglog` costs +28 MB because it drags numpy, capnp and zmq in to publish a log line, and +`openpilot.common.params` imports swaglog, so a module that touches either +prices the owner out of being minimal. Measured on the comma: python plus this +plus the FunctionFS transport is 10.4 MB against 47.5 MB for the daemon that +imported the world. + +Nothing in jetlink.comma may import openpilot or knows a param: the settings +come from jetlink.openpilot.settings, over the directory and the keys the +fork's adapter names. tests/test_comma_gadget.py holds the line. + +jetlink.openpilot's heavy processes import it too, and bind() points `log` at +the fork's cloudlog (set_logger), so their lines still reach the drive's log +while the owner's go to a file. +""" +from __future__ import annotations + +import json +import logging +import os +import time +from pathlib import Path + +from jetlink.comma import root +from jetlink.transport import ffs +from jetlink.transport.base import UDC_SYSFS, udc_speed +from jetlink.transport.tcp import CABLE_ADDRESS, DEFAULT_PORT + +# -- logging -------------------------------------------------------------- +# a module-level indirection rather than an import: the owner has no swaglog +# and must not grow one, and every other process wants its lines in the drive. +log = logging.getLogger('jetlink.comma.gadget') + + +def set_logger(logger) -> None: + """Send this module's lines somewhere else, and the root script's failures + with them: jetlink.openpilot.bind points it at the fork's cloudlog.""" + global log + log = logger + root.log = logger + + +# -- what carries the link ------------------------------------------------ +# The Jetlink setting names the host: USB (a Jetson or a Mac on the +# FunctionFS vendor interface) or iOS (an iPhone, which gives apps no USB +# access). For iOS the gadget is composite, with a CDC-NCM network interface +# whose comma end is 192.168.60.1 (jetlink-root.sh gadget --ios, and net, which +# also runs the DHCP server), and the phone dials CABLE_ADDR whenever it holds +# a lease on that network. +# Whoever holds the loan listens for the dial, the owner while nobody does; +# nothing writes to the endpoint files, which a phone never reads. The setting, +# not a guess, says which: a hello over FunctionFS to a phone blocks 15 s and +# bounces the gadget, and waiting to see whether a phone dials cost every +# Jetson reconnect 5 to 10 s. +LINK = Path("/dev/shm/jetlink-link") # "cable " while a phone is dialed in +NET_STATUS = Path("/dev/shm/jetlink-net") # jetlink-root.sh: "ok 192.168.60.1 ", "error: ...", "net: off" (USB) +CABLE_ADDR = (CABLE_ADDRESS, DEFAULT_PORT) + + +def read(path: Path) -> str: + """A record's or a sysfs file's text, stripped; '' when it is missing or + unreadable.""" + try: + return path.read_text().strip() + except OSError: + return '' + + +def _write(path: Path, text: str | None, what: str) -> bool: + """Write a record, or remove it with None. False, and logged, when that fails.""" + try: + if text is None: + path.unlink(missing_ok=True) + else: + path.write_text(text) + return True + except OSError: + log.exception("jetlink: could not %s", what) + return False + + +def link_state() -> tuple[str | None, str | None]: + """The link record in one read: the gadget the owner built, 'cable' for + iOS (the phone's network interface on the gadget) or 'usb', and on the + cable the phone's address while a link is up. (None, None) before the + owner has said.""" + record = read(LINK).split() + if record[:1] not in (['cable'], ['usb']): + return None, None + return record[0], (record[1] if record[0] == 'cable' and len(record) > 1 else None) + + +def link_kind(mode: str | None = None) -> str: + """link_state's kind, or a stand-in: `mode`, the Jetlink setting as + the caller read it, stands in only until the owner has said: it may have + moved and be waiting for the car to park. Without either, 'usb'.""" + kind, _ = link_state() + return kind or ('cable' if mode == 'ios' else 'usb') + + +def link_peer() -> str | None: + """The phone's address, while a cable link is up.""" + return link_state()[1] + + +def note_link(kind: str, peer: str | None = None) -> None: + """The owner's record of the gadget it built, 'usb' or 'cable', and on the + cable the phone that dialed in; see link_kind and link_peer.""" + _write(LINK, f"{kind} {peer}".strip() if peer else kind, "record the link") + + +def clear_link() -> None: + _write(LINK, None, "clear the link record") + + +def net_status() -> str | None: + """What jetlink-root.sh said about the gadget's network interface, if it ran.""" + return read(NET_STATUS) or None + + +def usb_speed() -> str | None: + """The bus speed the host enumerated us at: 458 KB a frame is ~1 ms on + super-speed and ~11 ms on high-speed, so this is the first thing to read + when the link is slow. None while unbound or where the UDC does not say.""" + udc = bound_udc() + return None if udc is None else udc_speed(udc, str(UDC_PATH)) + + +# -- where the gadget lives ----------------------------------------------- +# the comma is the USB gadget and the Jetson the host, decided by the kernels: +# AGNOS has CONFIG_USB_F_FS built in, L4T images are often stripped of the +# gadget modules. See docs/transport.md +GADGET_PATH = Path(ffs.GADGET) +FFS_MOUNT = Path(ffs.MOUNT) +UDC_PATH = Path(UDC_SYSFS) +# the network function jetlink-root.sh gadget --ios adds after ffs.jetlink. Its +# netdev is not usb0, which the modem holds, but whatever the kernel names it +NET_FUNCTION = 'ncm.usb0' +# written by scripts/comma/jetlink-root.sh gadget, which the owner runs when the +# link is on and there is no gadget: "ok", or "error: " +GADGET_STATUS = Path("/dev/shm/jetlink-gadget") +# the owner's own: "error: the lender could not listen: " while nothing +# can borrow the gadget from it, absent otherwise. Not GADGET_STATUS, which +# root owns and every build rewrites +LENDER_STATUS = Path("/dev/shm/jetlink-lender") +CC_ORIENTATION = Path('/sys/class/power_supply/usb/typec_cc_orientation') +# the owner's pid while it has released the gadget on purpose so the Jetson can +# sleep. Presence comes from this, not the UDC; a marker whose writer is dead is +# a leftover from a kill +DORMANT = Path("/dev/shm/jetlink-dormant") +# hardwared's request to power the Jetson off; see backend.shutdown +SHUTDOWN_REQUEST = Path("/dev/shm/jetlink-shutdown") +# the owner's status record: everything the readers used to take from the +# files above one by one, rewritten whole every step, which makes it the +# owner's heartbeat too. A clean stop removes it +STATUS = Path("/dev/shm/jetlink/status.json") +# how old that record may be before a reader takes its owner for gone: six of +# its 0.5 s steps. A wait inside a step renews it (Owner.beat) +HEARTBEAT_TIMEOUT = 3.0 +# the owner's start times, for its crash-loop backoff (see the owner) +STARTS = Path("/dev/shm/jetlink/starts.json") +# held with flock by the owner for its whole life, so there is only ever one +OWNER_LOCK = Path("/dev/shm/jetlink/owner.lock") +# what the server's last hello said (lending.SERVER_FIELDS), kept apart from +# the status record so a clean stop leaves it for the next owner +SERVER = Path("/dev/shm/jetlink/server.json") +# how long a host that stopped reading configured still counts as there. The +# owner holds the gadget for as long as the link is enabled, so presence no +# longer blinks at every handover; what is left to bridge is a USB3 link +# recovery passing through "addressed", and a bounce made on purpose when a +# host will not enumerate (wait_for_host) +PRESENCE_HOLD = 5.0 + + +def write_record(path: Path, record) -> None: + """Replace a JSON record whole: a reader gets the old one or the new one, + never half of either. Raises OSError, for the writer to say once; the + temporary goes with a failure (a SIGKILL between the two leaves it for + the next owner to clear, clear_leftovers).""" + path.parent.mkdir(parents=True, exist_ok=True) + tmp = path.with_name(f".{path.name}.{os.getpid()}") + try: + tmp.write_text(json.dumps(record)) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise + + +def clear_leftovers(directory: Path) -> None: + """The temporaries a writer killed between its write and its rename left + in `directory`. For the owner's start, when nothing else writes there.""" + try: + for leftover in directory.glob('.*.json.*'): + leftover.unlink(missing_ok=True) + except OSError: + pass + + +def owner_status() -> dict | None: + """The owner's status record, or None without one: no owner has run since + boot, the last one stopped cleanly, or it is one too old to write it.""" + try: + record = json.loads(read(STATUS)) + except ValueError: + return None + return record if isinstance(record, dict) else None + + +def owner_alive(record: dict) -> bool: + """Was this record written within HEARTBEAT_TIMEOUT? time.monotonic is the + system's CLOCK_MONOTONIC, the same in every process on the device, and + unlike the wall clock it does not jump when the comma sets its time.""" + at = record.get('at') + return isinstance(at, (int, float)) and time.monotonic() - at < HEARTBEAT_TIMEOUT + + +def _status_error(path: Path) -> str | None: + """The reason in an "ok" or "error: " file. A missing file is not an + error: whatever writes it has not run.""" + reason = read(path) + if not reason or reason == 'ok': + return None + return reason.removeprefix('error:').strip() or None + + +def build_error() -> str | None: + """Why the gadget could not be built, if it could not.""" + return _status_error(GADGET_STATUS) + + +def gadget_error() -> str | None: + """Why the link is unavailable, if it is: the gadget could not be built, or + it was and nothing can borrow it from the owner. + + Both happen in the owner, nowhere a user would look, so this is how the + reason reaches the panels. + """ + return build_error() or _status_error(LENDER_STATUS) + + +def note_lender_error(reason: str | None) -> None: + """The owner's record of a lender that cannot listen, or None once it can.""" + _write(LENDER_STATUS, None if reason is None else f"error: the lender could not listen: {reason}\n", + "record the lender's state") + + +def bound_udc() -> str | None: + """The device controller our gadget is attached to, if it is attached.""" + return read(GADGET_PATH / "UDC") or None + + +def other_gadget() -> str | None: + """Another gadget bound to a device controller, by name: on a comma, AGNOS's + ADB gadget (g1) while ADB is on. jetlink-root.sh refuses to take the + controller from it.""" + for udc in GADGET_PATH.parent.glob("*/UDC"): + if udc.parent != GADGET_PATH and read(udc): + return udc.parent.name + return None + + +def udc_state() -> str | None: + """What the device controller says about the bus, or None if we are unbound. + + "configured" is a host that has us; "default" and "addressed" are one that + reset the bus and stopped part way, which is what a Jetson that took the + bind as a wake and did not finish waking looks like. + """ + udc = bound_udc() + return None if udc is None else read(UDC_PATH / udc / "state") or None + + +def host_attached() -> bool: + """Has a host (the Jetson) enumerated and configured us?""" + return udc_state() == "configured" + + +def cc_orientation() -> int | None: + """What the USB-C port controller reads on the CC pin, so electrically true + whether or not anything enumerated: 0 is a port with nothing on it, 1 or 2 a + cable with a live host behind it (it cannot say what kind). A legacy A-to-C + cable's pull-up rides on the host's VBUS and reads the same. None where the + kernel does not say.""" + try: + return int(read(CC_ORIENTATION)) + except ValueError: + return None + + +def port_has_host() -> bool: + """Does the USB-C port controller see a host on the cable?""" + return bool(cc_orientation()) + + +# how long the UDC may sit half enumerated with a host on the cable before the +# gadget is bounced. A real enumeration is milliseconds; this only fires for a +# host that answered the bind with a bus reset and then stopped, which is what +# an unarmed hub does to a box asleep. See FfsTransport.rebind +STALLED_ENUMERATION = 20.0 +STALLED_STATES = ('default', 'addressed') +HOST_POLL = 0.5 + + +def wait_for_host(timeout: float, bounce=None, should_stop=None, report=None, mode: str | None = None) -> bool: + """Wait for the Jetson to enumerate us, bouncing a bus that stalled. + + The gadget stays bound throughout. An unbind is an unplug as the far end sees + it, and while one boots it takes ~50 s a cycle: doing that on a timer landed + an unplug on a box that was seconds from enumerating. + + The one case that needs an edge is a host that took the bind as a wake, reset + the bus and stopped. The UDC then sits in default or addressed with the CC + pin still showing a host, and only another connect moves it: that is what + `bounce` is for, and it is spent once. + + On the cable there is nothing to wait for: the connect that made the client + already reached the phone. The UDC is configured too, but by the phone, and + it is the dial that proved it is there. `mode` is the link setting, for + before the owner has recorded which gadget it built (link_kind). + """ + if link_kind(mode) == 'cable': + return True + deadline = time.monotonic() + timeout + stalled_since = None + bounced = False + reported = False + while True: + state = udc_state() + if state == "configured": + return True + now = time.monotonic() + if now >= deadline or (should_stop is not None and should_stop()): + return False + if report is not None and not reported: + reported = True + report() + if state in STALLED_STATES and port_has_host(): + stalled_since = now if stalled_since is None else stalled_since + if not bounced and bounce is not None and now - stalled_since > STALLED_ENUMERATION: + bounced = True + log.warning("jetlink: the bus has been half enumerated for %.0f s, bouncing the gadget", + STALLED_ENUMERATION) + try: + bounce() + except Exception: + log.exception("jetlink: could not bounce the gadget") + else: + stalled_since = None + time.sleep(HOST_POLL) + + +def setup_gadget(ios: bool) -> bool: + """Create the gadget, for USB or iOS: there is none yet, or the setting + moved between the two. The script records "ok" or the reason itself, in + GADGET_STATUS, so a failure here reaches the offroad alert. Off AGNOS + there is nothing to create it with, and this is a False.""" + if not root.run('gadget', *(['--ios'] if ios else [])): + return False + log.warning("jetlink: gadget set up for %s", 'iOS' if ios else 'USB') + return link_configured() + + +def built_for_ios() -> bool: + """Does the gadget carry the network function, as jetlink-root.sh gadget --ios builds it?""" + return os.path.lexists(GADGET_PATH / 'configs' / 'c.1' / NET_FUNCTION) + + +def net_up() -> bool: + """Bring the gadget's network interface up, for a phone to dial over. + + The netdev does not exist until the first UDC bind (f_ncm registers it in + its bind), and the gadget subcommand never binds, so the owner runs this + after it binds. `net` is idempotent: nmcli unmanaged, 192.168.60.1/24, the + DHCP server, and NET_STATUS written as "ok 192.168.60.1 " or + "error: ". + """ + if not root.run('net'): + return False + status = net_status() or '' + log.warning("jetlink: gadget network: %s", status or 'no status written') + return status.startswith('ok') + + +def link_configured() -> bool: + """Can we even attempt a link? The gadget exists. + + Not host_attached(): the UDC only binds when something opens ep0, and nothing + opens ep0 unless the link looks usable. Waiting for a host deadlocks. + + The cable too: a phone is on the gadget's own network interface, which + exists only while ep0 is held. + """ + if build_error() is not None: + return False + try: + return (FFS_MOUNT / "ep0").exists() + except OSError: + # a root-only mount raises PermissionError from stat; unusable either way + return False + + +def set_dormant(on: bool) -> None: + _write(DORMANT, str(os.getpid()) if on else None, "update the dormant marker") + + +def dormant() -> bool: + """Has a live owner released the gadget on purpose?""" + try: + pid = int(read(DORMANT)) + except ValueError: + return False + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + pass # alive, just not ours to signal + return True + + +def request_shutdown(reason: str) -> bool: + return _write(SHUTDOWN_REQUEST, json.dumps({'reason': reason}), "write the shutdown request") + + +def pending_shutdown() -> str | None: + """The reason in a shutdown request that has not been dealt with, if any. + + Read twice a second for the life of the process and almost never there, so + the miss is a stat rather than an open that raises. + """ + if not SHUTDOWN_REQUEST.exists(): + return None + try: + return str(json.loads(read(SHUTDOWN_REQUEST)).get('reason', '')) + except ValueError: + return None + + +def await_shutdown(timeout: float, poll: float = 0.25) -> bool: + """Wait for the owner's run to take a shutdown request. False if nobody did + within `timeout`, and then the request is withdrawn.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if not SHUTDOWN_REQUEST.exists(): + return True + time.sleep(poll) + finish_shutdown() + return False + + +def finish_shutdown() -> None: + _write(SHUTDOWN_REQUEST, None, "remove the shutdown request") diff --git a/jetlink_repo/jetlink/comma/lending.py b/jetlink_repo/jetlink/comma/lending.py new file mode 100644 index 0000000000..4e7bcc5da9 --- /dev/null +++ b/jetlink_repo/jetlink/comma/lending.py @@ -0,0 +1,624 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Who may do endpoint IO on the gadget, while one process owns it throughout. + +The owner (owner.py) holds ep0 for as long as the link is on, and nothing else +ever does: when two processes took turns at it, every change of owner was an +unplug, a fresh libusb open and a fresh server session as the Jetson saw it. + +What still has to change hands is the right to read the endpoint files. +FunctionFS keeps a queued read queued until something completes it, so a second +reader would sit behind the first and take its reply. This is the handshake for +that: modeld borrows for the length of a drive, the owner stays off the +endpoints while it does, and the connection is the lease, so a modeld that is +killed returns it by dying. There is no "give it back" message: the socket +closing is the only signal, because it is the only one a killed process sends. + +An iPhone cannot take the endpoint files: it is on the gadget's network +interface and dials the comma at CABLE_ADDR. Whoever holds the loan listens +for that dial: the owner while nobody does (CableListener), so a parked phone +shows connected and a dial is news that starts a provisioning run, and the +borrower while it holds the loan (Loan.accept), for the drive or the run. The +handshake is the handover: the owner closes its listener and the dial it +holds before it says yes, and listens again once the loan ends. No socket +changes hands, and nothing shares one. +""" +from __future__ import annotations + +import errno +import json +import os +import select +import socket +import threading +import time +from collections.abc import Callable +from pathlib import Path + +from jetlink.comma import gadget + +SOCKET = Path('/dev/shm/jetlink-lend.sock') +# how long a borrower waits for the owner to put the gadget down. It only has +# something to put down if it was mid-provision at ignition, and then it is one +# re-enumeration; the usual answer is immediate +BORROW_TIMEOUT = 8.0 +# a stuck write is already 15 s old by the time this is asked for +BOUNCE_TIMEOUT = 10.0 +# the owner records a note at once; this only bounds one that is wedged +NOTE_TIMEOUT = 5.0 +RETRY = 0.25 +POLL = 0.5 +# what a borrower passes on of the server's hello (Loan.note_server): the owner +# never speaks the protocol, and sleep_after is how it knows whether letting +# go of the gadget lets the Jetson sleep. The rest is for its status record +SERVER_FIELDS = ('protocol', 'device', 'backend', 'runtime_version', 'trt_version', 'sleep_after') + + +def _send(conn: socket.socket, msg: dict) -> None: + conn.sendall(json.dumps(msg).encode() + b'\n') + + +def _recv_line(conn: socket.socket, buf: bytearray, deadline: float) -> dict | None: + """One json message off the socket, or None if the deadline passes first. + + Both ends speak newline-delimited json over a stream, so a message can arrive + in pieces or with the next one behind it, and the recv timeout is a poll + rather than a refusal: the lender answers one borrower at a time and can be a + moment late while it lets go of the one before. + + The peer going away raises, because that is the one thing neither end may + read as "nothing yet": for the lender it is the whole lease ending. + """ + while time.monotonic() < deadline: + if b'\n' in buf: + line, _, rest = bytes(buf).partition(b'\n') + buf[:] = rest + return json.loads(line) + try: + chunk = conn.recv(4096) + except TimeoutError: + continue + if not chunk: + raise ConnectionResetError('the peer closed the link socket') + buf.extend(chunk) + return None + + +class Loan: + """The right to do endpoint IO on a gadget the owner holds, or on the + cable the right to listen for the phone's dial (accept). + + Held for the length of a drive: modeld is stopped at every ignition-off and + SIGKILLed if it lingers, so the socket closing is how the link is handed + back, and a modeld that crashed hands it back the same way. + """ + + def __init__(self, conn: socket.socket, buf: bytearray, mount: str, udc: str, name: str = 'modeld'): + self.conn = conn + self.mount = mount + self.udc = udc + self.name = name + # a phone is the host: the link is its dial, taken by accept, and the + # endpoint files are left alone. The owner's answer says (_take) + self.cable = False + # the cable listener, bound on the first accept and kept for the loan + self._srv: socket.socket | None = None + self.bound: tuple | None = None + self._buf = buf + self._lock = threading.Lock() + self._closed = False + + @property + def closed(self) -> bool: + return self._closed + + def bounce(self) -> bool: + """Ask the owner to take the gadget down and put it back up. + + The only thing that dequeues a FunctionFS write nobody is reading is the + unbind, and the unbind belongs to whoever holds ep0. Called from the write + watchdog on a link that is already 15 s stuck, so the re-enumeration it + costs is not the expensive part. + """ + with self._lock: + if self._closed: + return False + try: + _send(self.conn, {'op': 'bounce'}) + reply = _recv_line(self.conn, self._buf, time.monotonic() + BOUNCE_TIMEOUT) + except OSError: + gadget.log.exception("jetlink: could not ask for a gadget bounce") + return False + return bool(reply and reply.get('ok')) + + def note_server(self, hello: dict) -> bool: + """Tell the owner what the server said in its hello, which the owner + cannot ask for itself. Every borrower sends it after every hello, so + modeld's join refreshes it each drive and a Jetson moved to another + power supply is known by the next one. False when the owner did not + take it: an older owner answers "unknown op", which is not an error. + + An answer that does not come within NOTE_TIMEOUT, or is not one, closes + the loan. The exchange has no ids, so an answer that came later would be + read as the answer to the next request on it, a renewal's or a bounce's, + and every one after that would be one behind.""" + fields = {k: hello[k] for k in SERVER_FIELDS if k in hello} + with self._lock: + if self._closed: + return False + try: + _send(self.conn, {'op': 'server', **fields}) + reply = _recv_line(self.conn, self._buf, time.monotonic() + NOTE_TIMEOUT) + except (OSError, ValueError) as e: + reply, why = None, str(e) or type(e).__name__ + else: + why = 'no answer' if reply is None else f'the answer {reply!r}' + if not isinstance(reply, dict): + gadget.log.warning("jetlink: the owner did not take what the server said (%s), letting the loan go", why) + self._closed = True + _shut(self.conn) + return False + return bool(reply.get('ok')) + + def accept(self, timeout: float) -> socket.socket: + """The phone's next dial, within `timeout` s: the link on the cable. + + The listener is bound on the first call and kept for the loan's life, so a + phone that dials again after a lost link is taken here, with nobody else + involved. The newest dial wins, as on the owner's listener: the app may + have restarted behind an older one. Raises TimeoutError with no dial in + time, and OSError when the address could not be bound for the whole wait: + the owner lets go of it before it says yes, but a run before us may still + be exiting. + """ + if not self.cable: + raise RuntimeError('this loan is the endpoint files, not the cable') + deadline = time.monotonic() + timeout + while self._srv is None: + try: + srv = _listen() + except OSError: + if time.monotonic() >= deadline: + raise + time.sleep(RETRY) + continue + self._srv, self.bound = srv, srv.getsockname() + while True: + if select.select([self._srv], [], [], max(0.0, deadline - time.monotonic()))[0]: + conn = newest_dial(self._srv) + if conn is not None: + return conn + if time.monotonic() >= deadline: + raise TimeoutError(f'no phone dialed in {timeout:.0f} s') + + def _take(self, timeout: float) -> dict | None: + """Ask the owner for the link until it lends one or `timeout` passes; the + reply that lent it. The loan is closed when the owner is gone.""" + try: + reply = _ask(self.conn, self._buf, self.name, time.monotonic() + timeout) + except (OSError, ValueError, KeyError): + if not self._closed: # a close() from another thread wakes the ask this way + gadget.log.exception("jetlink: could not ask the owner for the link") + self._closed = True + _close(self.conn) + return None + if reply is None: + return None + self.mount, self.udc = str(reply['mount']), str(reply['udc']) + self.cable = bool(reply.get('cable')) + return reply + + def close(self) -> None: + # not behind the lock: the shutdown is what wakes an ask on another thread + self._closed = True + _shut(self.conn) + with self._lock: + _close(self.conn) + srv, self._srv, self.bound = self._srv, None, None + _close(srv) + + +def borrow(name: str = 'modeld', timeout: float = BORROW_TIMEOUT, path: Path | None = None) -> Loan | None: + """Ask the owner for the link, or None if there is nobody to ask: the link + was only just turned on, or the owner died or cannot listen, which it says + in gadget.gadget_error(). The caller asks again later; only the owner ever + holds ep0. `path` defaults to SOCKET as it is when called, so a test that + points SOCKET elsewhere is never lent the real owner's link. + """ + try: + conn = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + conn.settimeout(POLL) + conn.connect(str(SOCKET if path is None else path)) + except OSError: + return None # no owner listening + loan = Loan(conn, bytearray(), '', '', name=name) + reply = loan._take(timeout) + if reply is None: + loan.close() + return None + gadget.log.warning("jetlink: borrowed %s from the owner", _what_was_lent(reply)) + return loan + + +def _ask(conn: socket.socket, buf: bytearray, name: str, deadline: float) -> dict | None: + """Ask for the link until the owner lends it or `deadline` passes: the reply + that lent it. None when out of time or refused.""" + while time.monotonic() < deadline: + _send(conn, {'op': 'borrow', 'name': name}) + reply = _recv_line(conn, buf, deadline) + if reply is None: + return None # out of time + if reply.get('ok'): + return reply + if not reply.get('retry'): + gadget.log.warning("jetlink: the owner would not lend the gadget (%s)", reply.get('detail')) + return None + time.sleep(RETRY) + return None + + +def _what_was_lent(reply: dict) -> str: + if reply.get('cable'): + return "the cable link" + return f"the gadget (udc {reply.get('udc')})" + + +# between attempts to bind the cable address: usb0 has no address until +# jetlink-root.sh net has run, and a bind that keeps failing is not worth a log +# line twice a second +CABLE_BIND_BACKOFF = 5.0 + + +def _listen() -> socket.socket: + """A listener on CABLE_ADDR, non-blocking, or OSError: the address is + usb0's, which may not exist yet, or another process still has it.""" + from jetlink.transport.tcp import TcpTransport + srv = TcpTransport.listen(*gadget.CABLE_ADDR, backlog=2) + srv.setblocking(False) + return srv + + +def newest_dial(srv: socket.socket) -> socket.socket | None: + """Every dial waiting on the non-blocking listener `srv`, the newest kept + and the others closed: the app may have restarted behind an older one. + None with nobody waiting. The dial comes back blocking, as a transport + wants it.""" + conn = None + while True: + try: + newer, _ = srv.accept() + except (BlockingIOError, InterruptedError): + break + _close(conn) + conn = newer + if conn is not None: + conn.setblocking(True) + return conn + + +class CableListener: + """The owner's ear for a phone: one accept socket on CABLE_ADDR, open while + the gadget is presented and nobody holds the loan, holding at most one dial + at a time. + + A dial is accepted from the owner's step, never read: what it proves is that + a phone is on the cable, and whether its app is running. A newer dial + replaces an older one, so a phone whose app restarted is not stuck behind + its own dead connection. A borrower takes the port over (vacate) and + listens for the phone itself. + """ + + def __init__(self): + self._srv: socket.socket | None = None + self._sock: socket.socket | None = None + self.peer: str | None = None + self.bound: tuple | None = None + self.next_open = 0.0 + # we hung up on the phone because its borrower finished, so its next dial + # is the same phone coming back, not news; `news` says which the last + # accepted dial was + self.redial_expected = False + self.news = False + self._last_error = '' + self._lock = threading.Lock() + + @property + def listening(self) -> bool: + return self._srv is not None + + @property + def held(self) -> bool: + """Is a phone's dial in hand?""" + return self._sock is not None + + def open(self) -> bool: + """Bind, or say why not. Never raises: the address is usb0's, which may + not exist yet, and the owner's loop must carry on without it.""" + if self._srv is not None: + return True + now = time.monotonic() + if now < self.next_open: + return False + self.next_open = now + CABLE_BIND_BACKOFF + try: + srv = _listen() + except OSError as e: + why = ('usb0 has no address yet' if e.errno == errno.EADDRNOTAVAIL else str(e)) + if why != self._last_error: + self._last_error = why + gadget.log.warning("jetlink: cannot listen for a phone on %s:%d (%s)", *gadget.CABLE_ADDR, why) + return False + self._srv = srv + self.bound = srv.getsockname() + self._last_error = '' + gadget.log.warning("jetlink: listening for a phone on %s:%d", *self.bound[:2]) + return True + + def poll(self) -> str | None: + """Take a dial if one is waiting. The peer's address when one arrived, + else None. A held dial the phone has since dropped is let go here too, + so a borrower is never handed a dead socket.""" + if self._srv is None: + return None + self._drop_if_dead() + try: + conn = newest_dial(self._srv) + except OSError: + gadget.log.exception("jetlink: the cable listener failed") + return None + if conn is None: + return None + with self._lock: + old, self._sock = self._sock, conn + self.peer = str(conn.getpeername()[0]) + self.news, self.redial_expected = not self.redial_expected, False + _close(old) + return self.peer + + def _drop_if_dead(self) -> None: + """A phone that closed its end. It sends nothing until a hello.""" + with self._lock: + sock = self._sock + if sock is None or not hung_up(sock): + return + self._sock, self.peer = None, None + gadget.log.warning("jetlink: the phone hung up") + _close(sock) + + def vacate(self) -> None: + """Stop listening and let the held dial go, for a borrower that listens + itself: the port is free when the owner's yes lands, and the phone dials + again in a moment, to the borrower. Its dial after the loan is the same + phone coming back, not news. From the lender's thread.""" + self.release(expect_redial=True) + self._stop_listening() + self.next_open = 0.0 # listen again on the first step after the loan ends + + def release(self, expect_redial: bool = False) -> None: + """Close the held dial. The phone dials again and the next accept + replaces it: a borrower that has finished, or died, must not leave the + phone talking to nobody. `expect_redial` marks that next dial as ours to + expect rather than a phone turning up.""" + with self._lock: + sock, self._sock = self._sock, None + self.peer = None + self.redial_expected = expect_redial and sock is not None + _close(sock) + + def close(self) -> None: + self.release() + self._stop_listening() + + def _stop_listening(self) -> None: + with self._lock: + srv, self._srv = self._srv, None + self.bound = None + _close(srv) + + +def _close(sock: socket.socket | None) -> None: + if sock is not None: + try: + sock.close() + except OSError: + pass + + +def _shut(sock: socket.socket) -> None: + """End the connection for both ends, so the owner sees the lease end.""" + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def hung_up(sock: socket.socket, wait: float = 0.0) -> bool: + """Whether the far end closed `sock`, waiting up to `wait` s to see. Bytes + waiting read as alive, and the peek leaves them for whoever shares the socket; + a peer that sends nothing unasked is readable only at EOF.""" + try: + # select first: a socket with a timeout would wait that long in recv + if not select.select([sock], [], [], wait)[0]: + return False + return sock.recv(1, socket.MSG_PEEK | socket.MSG_DONTWAIT) == b'' + except (BlockingIOError, InterruptedError, TimeoutError): + return False # a sharer took the bytes between the two + except OSError: + return True + + +class Lender: + """The owner's side: one borrower at a time, for as long as it stays connected. + + `lendable` says whether the gadget is in the state a borrower can take over + from, bound with no endpoint file open here; while it is not, a borrow is + answered "retry" and the daemon's own loop puts it there. `cable` says the + host is a phone (Jetlink iOS): the loan is then the right to + listen for its dial, never the endpoint files, which a phone does not read, + and `vacate` frees the port first (CableListener.vacate). `server` takes + what a borrower passes on of the server's hello (Loan.note_server), with + the borrower's name, on this thread. + """ + + def __init__(self, lendable: Callable[[], bool], bounce: Callable[[], bool], + path: Path | None = None, cable: Callable[[], bool] | None = None, + vacate: Callable[[], None] | None = None, server: Callable[[str, dict], None] | None = None): + self._lendable = lendable + self._bounce = bounce + self._cable = cable or (lambda: False) + self._vacate = vacate or (lambda: None) + self._server = server + # what this borrower was last told it has, so each change is logged once + self._told = '' + # as it is when made, for the same reason as borrow's + self.path = SOCKET if path is None else path + self.borrower = '' + self._sock: socket.socket | None = None + self._thread: threading.Thread | None = None + self._stop = threading.Event() + self._lent = threading.Event() + # why the last start could not listen, for the owner to report + self.error: str | None = None + + @property + def listening(self) -> bool: + """Can anybody ask us for the endpoints? If not, nobody can use the link, + and the owner says so and starts this again; see Owner.ensure_lender.""" + return self._sock is not None + + @property + def lent(self) -> bool: + """Is somebody using the endpoints? True from the first ask, not the first + successful one: the daemon has to get off them before it can say yes.""" + return self._lent.is_set() + + def start(self) -> bool: + """Listen for borrowers. False, with the reason in `error`, on a read-only + /dev/shm or a path somebody else owns; the caller logs it and tries again, + so this does not.""" + if self._thread is not None: + return True + sock = None + try: + self._clear_stale() + sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + sock.bind(str(self.path)) + sock.listen(1) + sock.settimeout(POLL) + except OSError as e: + if sock is not None: + sock.close() + self.error = str(e) or type(e).__name__ + return False + self.error = None + self._sock = sock + self._thread = threading.Thread(target=self._serve, name='jetlink_lend', daemon=True) + self._thread.start() + return True + + def stop(self) -> None: + self._stop.set() + if self._thread is not None: + self._thread.join(2.0) + self._thread = None + if self._sock is None: + return # never listened: the path, if any, is somebody else's + self._sock.close() + self._sock = None + try: + self.path.unlink(missing_ok=True) + except OSError: + pass + + def _clear_stale(self) -> None: + """A socket file a dead daemon left behind. Proven dead by a connect that + is refused, so a second owner cannot take the link from a live one.""" + if not self.path.exists(): + return + probe = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + try: + probe.settimeout(0.5) + probe.connect(str(self.path)) + except OSError: + os.unlink(self.path) + finally: + probe.close() + + def _serve(self) -> None: + while not self._stop.is_set(): + try: + conn, _ = self._sock.accept() + except (TimeoutError, OSError): + continue + try: + self._handle(conn) + except Exception: + gadget.log.exception("jetlink: the borrower's connection failed") + finally: + conn.close() + if self._lent.is_set(): + gadget.log.warning("jetlink: %s handed the %s back", self.borrower or 'the borrower', + 'cable link' if self._told == 'cable' else 'gadget') + # the owner's step listens for the phone again, now that the port is free + self._told = '' + self._lent.clear() + self.borrower = '' + + def _tell(self, what: str, *msg) -> None: + """Log a lend when it is not what this borrower already had: a renewal + that changes nothing is not news.""" + if self._told != what: + self._told = what + gadget.log.warning(*msg) + + def _handle(self, conn: socket.socket) -> None: + conn.settimeout(POLL) + buf = bytearray() + while not self._stop.is_set(): + try: + msg = _recv_line(conn, buf, time.monotonic() + POLL) + except (OSError, ValueError): + return # the borrower exited, was killed mid-drive, or is not one + # None is only the poll coming round again; the lease ends on EOF, which + # is what a modeld that manager stopped or SIGKILLed sends + if msg is not None: + self._answer(conn, msg) + + def _answer(self, conn: socket.socket, msg: dict) -> None: + op = msg.get('op') + if op == 'borrow': + self.borrower = str(msg.get('name') or 'a borrower') + self._lent.set() + if self._cable(): + # a phone: the link is its dial, which the borrower listens for + # itself; the owner's listener and the dial it holds go first, so the + # port is free when this answer lands. The endpoint files stay put + self._vacate() + self._tell('cable', "jetlink: lending the cable link to %s", self.borrower) + _send(conn, {'ok': True, 'cable': True, 'udc': gadget.bound_udc() or '', 'mount': str(gadget.FFS_MOUNT)}) + return + udc = gadget.bound_udc() + if not (udc and self._lendable()): + # the daemon is mid-exchange, or has not bound yet. It sees `lent` on + # its next cycle and puts the endpoints down for us + _send(conn, {'ok': False, 'retry': True, 'detail': 'the gadget is still in use here'}) + return + self._tell('gadget', "jetlink: lending the gadget to %s, udc %s", self.borrower, udc) + _send(conn, {'ok': True, 'udc': udc, 'mount': str(gadget.FFS_MOUNT)}) + elif op == 'bounce': + gadget.log.warning("jetlink: %s asked for a gadget bounce", self.borrower) + _send(conn, {'ok': bool(self._bounce())}) + elif op == 'server' and self._server is not None: + try: + self._server(self.borrower or 'a borrower', {k: msg[k] for k in SERVER_FIELDS if k in msg}) + except Exception: + # a note is never worth the lease: a raise here would end the loan + # under a borrower that is using the endpoints + gadget.log.exception("jetlink: could not record what the server said") + _send(conn, {'ok': True}) + else: + _send(conn, {'ok': False, 'detail': f'unknown op {op!r}'}) diff --git a/jetlink_repo/jetlink/comma/owner.py b/jetlink_repo/jetlink/comma/owner.py new file mode 100644 index 0000000000..b82e8f2cc4 --- /dev/null +++ b/jetlink_repo/jetlink/comma/owner.py @@ -0,0 +1,957 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Holds the USB gadget, and nothing else. + +The comma is the USB device: the link exists only while some process holds ep0 +with the UDC bound. This is that process, for as long as the link is enabled, +onroad and offroad alike, and no other process ever holds ep0. Whoever wants +to move bytes borrows the endpoint files over a unix socket (lending.py) and +the gadget never leaves the bus. + +The Jetlink setting names the host. For USB (a Jetson or a Mac) the +gadget is FunctionFS alone and the endpoint files are lent at once. For iOS +the gadget is composite: the phone is on its network interface and dials this +process (lending.CableListener), and the loan carries its socket; the endpoint +files are never lent, and the gadget is never released, settled or bounced, +since every unbind drops the phone's network interface with it. + +It is deliberately small. Everything heavy jetlink does is episodic, so none of +it lives here: a download, an upload and a TensorRT build all belong to the +provisioning run (jetlink.openpilot.provision), which this spawns when there is +something to do and which exits when there is not. That keeps a parked car and +a drive alike at one resident jetlink process of about 13 MB rather than +47.5 MB, and it is why nothing in this module may import swaglog, Params, +numpy, capnp or zmq; see gadget.py and tests/test_comma_gadget.py. The caller +names the worker and hands over the settings: jetlink.openpilot.owner makes +and runs the Owner from what the fork's adapter says (OwnerConfig), so nothing +here knows an openpilot module or a param name. + +What it knows, it writes down whole every step in its status record +(gadget.STATUS): the gadget, the host, presence with its hold, the errors. The +UI and hardwared read that one file rather than the gadget's own files one by +one, and its timestamp is the heartbeat that tells them the owner is gone. + +manager stops this on shutdown with SIGINT and SIGKILLs it 5 s later, so every +long wait polls `stop`: a FunctionFS owner killed mid-transfer leaves the +gadget in a state only a reboot clears. +""" +from __future__ import annotations + +import errno +import fcntl +import json +import logging +import os +import subprocess +import time +from collections.abc import Mapping, Sequence +from pathlib import Path + +from jetlink.comma import gadget, lending, port, root + +POLL = 0.5 +# how long the gadget is held after the last thing that wanted it. The server +# sleeps 120 s after the gadget goes; a stop inside the hold rejoins at once, +# one outside it costs the ~8 s wake. It also covers a jetson that is still +# enumerating when the run that woke it has already finished +DORMANT_HOLD = 60.0 +# how long settle() waits for its own re-enumeration before carrying on +SETTLE_TIMEOUT = 10.0 +# after a borrower lets go. The server has just lost its client and is closing +# the gadget and reopening it, two seconds at a time +LEASE_SETTLE = 4.0 +RECONNECT_BACKOFF = 5.0 +GADGET_SETUP_BACKOFF = 60.0 +# a build refused because another gadget holds the controller: the host turns +# that one off itself (openpilot's ADB toggle with the link on), within seconds +GADGET_HELD_RETRY = 2.0 +# between attempts to bring the gadget's network interface up after a bind +NET_BACKOFF = 5.0 +# between attempts to listen for borrowers after one failed +LENDER_BACKOFF = 30.0 +# between runs of the worker that found nothing to do. It costs a couple of +# seconds of imports, so it is spawned on a change and not on a timer +WORKER_BACKOFF = 300.0 +SHUTDOWN_RETRY = 2.0 # a shutdown run that exited with the request still there +WORKER_GRACE = 10.0 +# between opens of an ep0 that answered EBUSY: a borrower from before this +# owner started still has the endpoint files, and the open succeeds the moment +# it lets go. A refused open changes nothing in the kernel, so it is safe to +# repeat; only a read of an endpoint no host has enabled is not (_ensure_epfiles) +EP0_BUSY_RETRY = 1.0 + +# A crash loop re-enumerates the Jetson at every start, since the gadget goes +# with the owner that held it. Each start is written down and a clean stop +# takes its own back out, so what is left is owners that died or were killed. +# With CRASH_FREE of those inside CRASH_WINDOW the next start waits, holding +# nothing, CRASH_BACKOFF doubling up to CRASH_BACKOFF_MAX +CRASH_WINDOW = 600.0 +CRASH_FREE = 3 +CRASH_BACKOFF = 10.0 +CRASH_BACKOFF_MAX = 300.0 + +# the log is rotated: the loop below logs a traceback per cycle if something stays broken +LOG_BYTES = 1 << 20 + + +def logger(path: Path) -> logging.Logger: + """The owner's log, to stderr and `path`: swaglog costs 28 MB, so the owner + keeps its own. The worker's lines go to the drive; these are for a bench + session, which wants them after the drive.""" + log = logging.getLogger('jetlink.owner') + log.setLevel(logging.INFO) + handlers: list[logging.Handler] = [logging.StreamHandler()] + try: + from logging.handlers import RotatingFileHandler + handlers.append(RotatingFileHandler(path, maxBytes=LOG_BYTES, backupCount=1)) + except OSError: + pass # a read-only or missing /data/log; stderr is enough + for handler in handlers: + handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)-7s %(message)s')) + log.addHandler(handler) + return log + + +def server_sleeps(server: dict | None) -> bool: + """Does the far end suspend when the gadget goes, by its hello's + sleep_after? No hello yet, or a server too old to say, is a yes: letting go + of one that stays up only costs a rebind, and holding on to one that sleeps + keeps it awake all night.""" + try: + return float((server or {}).get('sleep_after', 1.0)) > 0 + except (TypeError, ValueError): + return True + + +def _starts() -> list[float]: + try: + starts = json.loads(gadget.STARTS.read_text()) + except (OSError, ValueError): + return [] + return [t for t in starts if isinstance(t, (int, float))] if isinstance(starts, list) else [] + + +def _write_starts(starts: list[float]) -> None: + try: + gadget.write_record(gadget.STARTS, starts) + except OSError: + gadget.log.exception("jetlink: could not record the owner's starts") + + +def note_start(now: float) -> tuple[float, int]: + """Write this start down. How long it should wait before taking the + gadget, and how many owners inside CRASH_WINDOW never stopped cleanly.""" + starts = [t for t in _starts() if 0.0 <= now - t < CRASH_WINDOW] + _write_starts([*starts, now]) + died = len(starts) + if died < CRASH_FREE: + return 0.0, died + return min(CRASH_BACKOFF_MAX, CRASH_BACKOFF * 2 ** (died - CRASH_FREE)), died + + +def forget_start(started: float) -> None: + """A clean stop: this start no longer counts against the next one.""" + _write_starts([t for t in _starts() if t != started]) + + +class Owner: + def __init__(self, worker: Sequence[str], cwd: str | None = None, env: Mapping[str, str] | None = None, *, + settings, chestnut_ids): + # the provisioning run: its argv, its working directory, and what it gets + # over this process's environment + self.worker_argv = list(worker) + self.worker_cwd = cwd + self.worker_env = dict(env or {}) + # mode(), offroad() and marks(): the link setting, whether the car is + # parked, and when the pick and the built model last changed + self.settings = settings + self.transport = None + self.stop = False + self.dormant = False + # the link mode the root tuning was applied for, None while stock + self.tuned: str | None = None + # when there was last something to do. The hold runs from here, not from + # process start: this daemon is not restarted at ignition any more, so a + # hold measured from its birth expires once and never applies again, and + # every wake released the gadget a second later with the jetson still + # coming up + self.idle_since = time.monotonic() + self.next_attempt = 0.0 + self.next_gadget_attempt = 0.0 + self.failed_ios: bool | None = None # the mode whose last build failed, until one works + self.next_worker = 0.0 + self.shutting_down = False # the run in flight is the one asking the jetson to power off + self.next_shutdown_run = 0.0 + self.lease_settled = 0.0 + self.attached = False + self.configured = False # attached, as of the last step: for the edges + self.dialed = False # a phone dialed in since the last run + self.built_ios: bool | None = None # what the gadget was built for; None until looked at + self._peer: str | None = None # the phone published as dialed in + self.net_ready = False # usb0 configured for this bind + self.next_net_attempt = 0.0 + self.lender_failed = False # said once, until it listens again + self.next_lender = 0.0 + self.worker: subprocess.Popen | None = None + self.seen: dict[str, int] = {} # watched param -> mtime when last looked + self.had_host = False + # the status record every other process reads (gadget.STATUS): the setting + # the last step acted on, when a host last had us configured (presence's + # hold), when the record was last written, and a failure writing it, said once + self.mode: str | None = None + self.last_configured = 0.0 + self.published = 0.0 + self.status_error: str | None = None + # what this process never speaks the protocol to learn: the fields of the + # server's last hello, which every borrower passes on over its loan + # (lending.SERVER_FIELDS), and whether the last run left work undone, from + # its exit status + self.server: dict | None = None + self.unfinished = False + self.ep0_busy = False # said once, until an open works + self.switch_waiting = False # a rebuild held off by a borrower of a dead owner, said once + # why a crash loop's backoff holds this start back, for the status record + self.backing_off: str | None = None + self.lock_fd: int | None = None # gadget.OWNER_LOCK, held for the life of run() + # this owner's entry in gadget.STARTS, until a stop that was asked for takes it back + self.born: float | None = None + self.port = port.Port(chestnut_ids) + self.cable = lending.CableListener() + self.lender = lending.Lender(self.lendable, self.bounce_gadget, cable=self.cable_mode, vacate=self.cable.vacate, + server=self.note_server) + + # -- the gadget ----------------------------------------------------------- + + def close_link(self) -> None: + """Always go through this: a FunctionFS owner that exits without closing + can wedge the driver until a reboot.""" + transport, self.transport = self.transport, None + # the phone's network interface goes with the gadget, and its dial with it + self.cable.close() + self.net_ready = False + self.publish() + if transport is not None: + try: + transport.close() + except Exception: + gadget.log.exception("jetlink: error closing the link") + + def lendable(self) -> bool: + """Is the gadget in a state a borrower can take the endpoints over from?""" + return self.transport is not None and self.transport.lendable + + def cable_mode(self) -> bool: + """Is the link a phone's dial rather than the endpoint files? On a gadget + built for iOS, always: the host is a phone, which never reads them, so a + borrower listens for its dial instead (lending.Loan.accept). The build, + not the setting: a setting moved while somebody borrowed waits for them.""" + return bool(self.built_ios) + + def publish(self, peer: str | None = None) -> None: + """What the gadget is, for every other process's link_kind, and on the + cable which phone dialed in.""" + self._peer = peer + if self.built_ios is None: + gadget.clear_link() + else: + gadget.note_link('cable' if self.built_ios else 'usb', peer) + + def bounce_gadget(self) -> bool: + """One unplug and replug, for a borrower whose write has no reader. + + Unbinding is the only thing that makes FunctionFS dequeue a write the host + is not draining, and the unbind belongs to whoever holds ep0. See + FfsTransport._abort_write. Only a borrower on FunctionFS asks: for iOS + nothing is ever lent the endpoint files. + """ + if self.transport is None: + return False + try: + return bool(self.transport.rebind()) + except Exception: + gadget.log.exception("jetlink: could not bounce the gadget for the borrower") + return False + finally: + # the unbind took usb0 with it; the rebind made a bare one + self.net_ready = False + self.next_net_attempt = 0.0 + + def open_link(self) -> bool: + """Present the gadget so a Jetson can enumerate whenever it powers on. + + The transport, not a client: this end never speaks the protocol, and + jetlink.client would bring numpy with it. + """ + if self.transport is not None: + return True + try: + from jetlink.transport.ffs import FfsTransport + self.transport = FfsTransport(str(gadget.FFS_MOUNT), gadget=str(gadget.GADGET_PATH)) + gadget.log.warning("jetlink: gadget presented, waiting for a jetson") + self.net_ready = False + self.next_net_attempt = 0.0 + self.ep0_busy = False + return True + except OSError as e: + if e.errno == errno.EBUSY and os.path.basename(os.fsdecode(e.filename or '')) == 'ep0': + # an owner started after one that died: its borrower (modeld mid-drive, + # or the dead owner's own run) still has ep1 and ep2, and FunctionFS + # refuses ep0 until they are closed. Nothing was opened, and the UDC + # was never touched, so the borrower's link carries on meanwhile + if not self.ep0_busy: + gadget.log.warning("jetlink: ep0 is busy, something still has the gadget's endpoint files open " + "(most likely a borrower from before this owner); presenting it once they are closed") + self.ep0_busy = True + self.next_attempt = time.monotonic() + EP0_BUSY_RETRY + return False + return self._open_failed() + except Exception: + return self._open_failed() + + def _open_failed(self) -> bool: + gadget.log.exception("jetlink: could not present the gadget") + self.next_attempt = time.monotonic() + RECONNECT_BACKOFF + return False + + def ensure_gadget(self, ios: bool) -> bool: + """Is there a gadget to present? Create it, for iOS or USB, if there is + none: nothing sets it up at boot, so the owner's first step does, parked + or onroad, and a link turned on later gets one at once.""" + if gadget.link_configured(): + return True + return self.build(ios) + + def build(self, ios: bool) -> bool: + """Set the gadget up for USB or iOS. A failure backs off, since the script + is sudo, configfs and for iOS the network; a success does not, or a switch + just after a build would wait out the backoff. The backoff is the failed + mode's: a half-built iOS gadget never delays the way back to USB, and a + controller another gadget holds is tried again in seconds.""" + now = time.monotonic() + if now < self.next_gadget_attempt and ios == self.failed_ios: + return False + if not gadget.setup_gadget(ios): + self.failed_ios = ios + held = gadget.other_gadget() is not None + self.next_gadget_attempt = now + (GADGET_HELD_RETRY if held else GADGET_SETUP_BACKOFF) + return False + self.failed_ios = None + self.built_ios = ios + self.publish() + return True + + def ensure_net(self) -> None: + """usb0 exists only once the UDC is bound, so the network comes up here, + after open_link, once per bind and again after a bounce. Retried on a + backoff: the script is sudo, nmcli and dnsmasq, not twice a second.""" + if self.transport is None or self.net_ready: + return + now = time.monotonic() + if now < self.next_net_attempt: + return + self.next_net_attempt = now + NET_BACKOFF + self.net_ready = gadget.net_up() + + def settle(self) -> None: + """Put the gadget back to bound with nothing open on it. + + ep0 and the descriptors stay here throughout; only the endpoint files go. + See FfsTransport.release_endpoints. Not over the cable: nothing of ours + is open on the endpoints, and the re-enumeration it may cost would drop + the phone's network interface. + """ + if self.transport is None or self.lendable() or self.built_ios: + return + gadget.log.warning("jetlink: putting the endpoints down, keeping the gadget bound") + if not self.transport.release_endpoints(): + return + gadget.wait_for_host(SETTLE_TIMEOUT, bounce=self.bounce_gadget, should_stop=self.waiting) + + def hold(self, ios: bool) -> None: + """Everything this process does once the car is moving, or once somebody + has the endpoints. + + Keep the gadget on the bus and stay off it. One sysfs read a cycle: a + process that wakes up to do work on modeld's core is a dropped frame. + """ + self.wake() + if self.transport is not None: + return self.settle() + # a failed open waits its backoff here too: a busy ep0 is asked once a + # second, not every step, and a broken one is not a traceback a step + if time.monotonic() >= self.next_attempt and self.ensure_gadget(ios): + self.open_link() + + # -- the parked car ------------------------------------------------------- + + def go_dormant(self) -> None: + """Release the gadget so the Jetson can sleep. The marker goes first so + present() never blinks. Never on the cable: see step().""" + gadget.log.warning("jetlink: nothing left to do, releasing the gadget so the jetson can sleep") + gadget.set_dormant(True) + self.close_link() + self.dormant = True + self.had_host = False + + def wake(self) -> None: + """Present the gadget again. If the Jetson is asleep, the bind wakes it.""" + if not self.dormant: + return + gadget.log.warning("jetlink: presenting the gadget again") + gadget.set_dormant(False) + self.dormant = False + self.idle_since = time.monotonic() + + def note_server(self, borrower: str, fields: dict) -> None: + """What a borrower passed on of the server's hello (lending.Loan.note_server), + on the lender's thread. Said when it changes, which is how a server moved + to --sleep-after 0 or back shows in this log, and kept in gadget.SERVER, + which outlives this owner: a run with nothing to do never says hello, so + the next owner would otherwise take an always-on Jetson for one that + sleeps and let the gadget go until the next drive.""" + known, self.server = self.server or {}, dict(fields) + if any(known.get(k) != fields.get(k) for k in ('device', 'sleep_after')) or not known: + gadget.log.warning("jetlink: %s says the server on %s %s when the gadget goes (sleep_after %s)", borrower, + fields.get('device') or 'the far end', 'sleeps' if server_sleeps(fields) else 'stays up', + fields.get('sleep_after')) + if fields != known: + try: + gadget.write_record(gadget.SERVER, self.server) + except OSError as e: + gadget.log.warning("jetlink: could not keep what the server said (%s)", e) + + def far_end_sleeps(self) -> bool: + """Does the far end suspend when the gadget goes, as its last hello said?""" + return server_sleeps(self.server) + + # -- the worker ----------------------------------------------------------- + + def worker_running(self) -> bool: + if self.worker is None: + return False + if self.worker.poll() is None: + return True + gadget.log.warning("jetlink: the provisioning run finished (%s)", self.worker.returncode) + # 0 is a run with nothing left to do; anything else, a crash included, is + # tried again once WORKER_BACKOFF has passed + self.unfinished = self.worker.returncode != 0 + self.worker = None + self.shutting_down = False + # the far end may still be waking; give it the hold before letting go + self.idle_since = time.monotonic() + # after the run, not before it: a run writes JetlinkSpec itself, so a mark + # taken at spawn always differs by the time it exits and every successful + # provision started a second one + self.note_marks() + return False + + def note_marks(self) -> None: + self.seen = self.settings.marks() + self.had_host = gadget.host_attached() + + def wanted(self) -> str | None: + """Why the worker should run, or None. Everything that decides whether + there is work needs the catalog, the spec and the Jetson, so the worker + decides; this only notices the things that could have changed the answer.""" + if not self.seen: + return 'nothing has been checked since boot' + changed = [k for k, v in self.settings.marks().items() if self.seen.get(k) != v] + if changed: + return f"{', '.join(changed)} changed" + if self.dialed: + return 'a phone dialed in' + if self.attached and not self.had_host: + return 'a jetson turned up' + if time.monotonic() >= self.next_worker and self.unfinished: + return 'the last run left something to do' + return None + + def spawn_worker(self, why: str) -> None: + gadget.log.warning("jetlink: starting a provisioning run: %s", why) + self.dialed = False + self.note_marks() + self.next_worker = time.monotonic() + WORKER_BACKOFF + try: + self.worker = subprocess.Popen(self.worker_argv, cwd=self.worker_cwd, + env={**os.environ, **self.worker_env}) + except Exception: + gadget.log.exception("jetlink: could not start the provisioning run") + self.worker = None + + def stop_worker(self) -> None: + if self.worker is None: + return + self.worker.terminate() + deadline = time.monotonic() + WORKER_GRACE + while True: + try: + # a run that finished its round on the way out says so like any other + self.unfinished = self.worker.wait(POLL) != 0 + break + except subprocess.TimeoutExpired: + if time.monotonic() >= deadline: + self.worker.kill() + self.unfinished = True + break + self.beat() # a run inside a hello can take the whole grace + self.worker = None + self.shutting_down = False + + # -- the status record ------------------------------------------------------ + + def status_record(self) -> dict: + """What every other process reads instead of the gadget's files one by one + (jetlink.openpilot.status), presence and its hold worked out once, here.""" + now = time.monotonic() + state = gadget.udc_state() + configured = state == 'configured' + if configured: + self.last_configured = now + if self.dormant: + # no enumeration during suspend; the CC line still tells a sleeping host from an unplugged one + present = gadget.port_has_host() + else: + present = configured or now - self.last_configured < gadget.PRESENCE_HOLD + return { + 'pid': os.getpid(), + 'at': now, + 'mode': self.mode, + 'link': None if self.built_ios is None else ('cable' if self.built_ios else 'usb'), + 'peer': self._peer, + # root's jetlink-gadget and the lender's error, as gadget_error() puts + # them together, or why a crash loop's backoff is holding this start + 'error': self.backing_off or gadget.gadget_error(), + 'net': gadget.net_status(), + 'dormant': self.dormant, + 'udc': state, + 'speed': gadget.usb_speed() if configured else None, + 'present': present, + 'server': self.server, + 'worker': self.worker is not None and self.worker.poll() is None, + 'unfinished': self.unfinished, + # asked to stop: a record a SIGKILL during the teardown leaves behind is + # an owner that was stopped, not one that died, and readers take it for none + 'stopping': self.stop, + } + + def publish_status(self) -> None: + """Rewrite the status record, which is the heartbeat as well. Never + raises; a failure is logged once, until a write works again.""" + self.stopping_cleanly() + try: + gadget.write_record(gadget.STATUS, self.status_record()) + except Exception as e: + # the last record would go stale under a live owner and read as + # "service stopped"; without one the readers go back to + # the gadget's files, which are still right + self.forget_status() + error = f"{type(e).__name__}: {e}" + if error != self.status_error: + self.status_error = error + gadget.log.error("jetlink: could not write the status record (%s); readers read the gadget's files", error) + return + self.status_error = None + self.published = time.monotonic() + + def beat(self) -> None: + """Renew the record from inside a wait that can outlast the readers' + HEARTBEAT_TIMEOUT, at most once a POLL.""" + if time.monotonic() - self.published >= POLL: + self.publish_status() + + def waiting(self) -> bool: + """should_stop for a wait inside a step: keeps the heartbeat going.""" + self.beat() + return self.stop + + def forget_status(self) -> None: + """A clean stop leaves no record, so only an owner that died leaves a + heartbeat behind to go stale; the readers go back to the files.""" + try: + gadget.STATUS.unlink(missing_ok=True) + except OSError: + pass + + # -- the loop ------------------------------------------------------------- + + def request_stop(self, *_) -> None: + self.stop = True + + def stopping_cleanly(self) -> None: + """A stop that was asked for is clean however long its teardown takes: + manager SIGKILLs an owner still in stop_worker's 10 s grace 5 s after its + SIGINT, and a start left in gadget.STARTS would count as a death, three of + them a crash loop. So the start is taken back as soon as the main thread + sees the request, at the next record (within 0.5 s), not in the finally + a SIGKILL never reaches. Not in the signal handler: it can land inside + a write of the same file.""" + if self.stop and self.born is not None: + forget_start(self.born) + self.born = None + + def step(self) -> None: + # each read is a file; take them once and pass them down + mode = self.mode = self.settings.mode() + if mode == 'off': + if self.transport is not None: + gadget.log.warning("jetlink: disabled, releasing the link") + self.close_link() + self.stop_worker() + self.wake() + if self.tuned: + root.run('vm', 'restore') + root.run('udc', 'restore') + if self.tuned == 'ios': + root.run('draw', 'on', timeout=root.PORT_TIMEOUT) + self.tuned = None + self.port.off() + return + self.link_step(mode == 'ios') + if self.tuned != mode: + # jetlink-root.sh vm: the recording VM tuning the gadget's reads need; + # udc: the USB device side kept on for a host that powers the port + # (port.py); draw, iOS only: no current drawn from the port, which an + # iPhone cannot supply. While the link is on. After the step, so the + # first gadget does not wait on them. Put back only when the link is + # turned off or leaves iOS, never on exit: manager stops this at + # ignition, just as the contention starts, and a phone plugged in before + # it is back should still come up. A start assumes the draw stock, so a + # cut a dead iOS owner left lasts until the reboot that clears it + if self.tuned is None: + root.run('vm', 'apply') + root.run('udc', 'apply') + if mode == 'ios': + if root.run('draw', 'off', timeout=root.PORT_TIMEOUT): + gadget.log.info("jetlink: drawing no current from the USB-C port, for an iPhone") + elif self.tuned == 'ios': + root.run('draw', 'on', timeout=root.PORT_TIMEOUT) + self.tuned = mode + + def ensure_lender(self) -> None: + """Listen for borrowers, or say why not and try again later. + + Only this process ever holds ep0, so without the lender nothing can use + the link: the gadget stays held all the same, since letting it go would + be an unplug that helps nobody, and the reason goes where + gadget.gadget_error() reads it, which the panels show. + """ + if self.lender.listening: + return + now = time.monotonic() + if now < self.next_lender: + return + if self.lender.start(): + if self.lender_failed: + gadget.log.warning("jetlink: listening for borrowers again") + self.lender_failed = False + gadget.note_lender_error(None) + return + self.next_lender = now + LENDER_BACKOFF + if not self.lender_failed: + gadget.log.error("jetlink: nothing can borrow the gadget, the lender could not listen on %s (%s); " + "trying again every %.0f s", self.lender.path, self.lender.error, LENDER_BACKOFF) + self.lender_failed = True + gadget.note_lender_error(self.lender.error) + + def link_step(self, ios: bool) -> None: + """A step with the link on, for an iPhone or not.""" + self.ensure_lender() + # before anything is presented: a C-to-C host has to find a device here. + # The charging param is read every step; turned off, a phone already + # charging keeps charging until the next plug + self.port.update(configured=self.configured, charge=ios and self.settings.charge_phone()) + + offroad = self.settings.offroad() + if self.switch_mode(offroad, ios): + return + self.attached = gadget.host_attached() + self.watch_the_port() + + # before the worker gate: hardwared waits 25 s for this and a build in + # flight takes minutes, so a shutdown request cannot queue behind one + reason = gadget.pending_shutdown() + if reason is not None and not self.lender.lent: + if self.worker_running(): + if self.shutting_down: + # the run asking the jetson. Stopping it here restarted it every + # step, half a second, less than it takes to start: on the bench + # nothing ever asked, and hardwared gave up after its 25 s + return + self.stop_worker() + if time.monotonic() < self.next_shutdown_run: + return + self.wake() + if self.open_link(): + self.next_shutdown_run = time.monotonic() + SHUTDOWN_RETRY + self.spawn_worker(f'the jetson has to be shut down: {reason}') + self.shutting_down = True + return + + if not offroad: + # the drive has started and the endpoints belong to modeld. A run of ours + # holding the lease would keep it out for the whole drive; the server's + # build carries on and modeld picks the engine up over its loan + self.stop_worker() + + if self.lender.lent or not offroad: + if self.lender.lent: + # the window starts when the last borrower lets go. Its own deadline, + # because a host arriving clears the worker backoff and a borrower + # letting go looks like one arriving + self.lease_settled = time.monotonic() + LEASE_SETTLE + self.idle_since = time.monotonic() + return self.hold(ios) + + if self.worker_running(): + return self.hold(ios) + + if time.monotonic() < max(self.next_attempt, self.lease_settled): + return + + why = self.wanted() + if why is not None: + if not self.ensure_gadget(ios): + return + self.wake() + if not self.open_link(): + return + return self.spawn_worker(why) + + # a phone never sleeps, and letting go takes the network interface it + # dials over: for iOS the gadget stays up whatever the record says + sleeps = self.far_end_sleeps() and not self.built_ios + if self.transport is None: + # nothing to do and nothing presented: only worth a bind if the far end + # stays awake for it + if not sleeps and self.ensure_gadget(ios): + self.open_link() + return + if sleeps and time.monotonic() - self.idle_since >= DORMANT_HOLD: + self.go_dormant() + else: + self.settle() + + def watch_the_port(self) -> None: + """The edges of the USB link, and for iOS the phone's dial. + + For iOS the gadget's network comes up after each bind and the owner + listens for the phone while nobody holds the loan; a borrower listens + for it itself, and the owner listens again once the loan ends. The host + going away takes the dial with it, so the next one is looked at afresh. + """ + if self.attached and not self.configured: + gadget.log.warning("jetlink: a host configured us at %s", gadget.usb_speed() or 'an unknown speed') + elif self.configured and not self.attached and self.cable.held: + gadget.log.warning("jetlink: the host went away, the cable link with it") + self.cable.release() + self.dialed = False + self.publish() + self.configured = self.attached + if not self.built_ios: + return + self.ensure_net() + if self.lender.lent: + return # the borrower listens for the phone; the port is its + if self.transport is not None and self.net_ready and not self.cable.listening: + self.cable.open() # not before: the bind to 192.168.60.1 fails without usb0 + peer = self.cable.poll() + if peer is not None: + self.publish(peer) + gadget.log.warning("jetlink: cable link from %s", peer) + self.had_host = True + self.idle_since = time.monotonic() + # a reason for a run, unless it is the phone coming back after we hung + # up on it, or a run is already going and its borrow will take this dial + running = self.worker is not None and self.worker.poll() is None + if self.cable.news and not running: + self.dialed = True + elif not self.cable.held and self._peer is not None: + self.publish() # the phone hung up, or its borrower finished + + def switch_mode(self, offroad: bool, ios: bool) -> bool: + """Rebuild the gadget when the setting moved between USB and iOS: they are + different devices, and the rebuild is an unplug. True when this step went + on it. + + A switch waits for the car to park: the panels lock the setting while + driving, and one written some other way waits too. Building the mode that + is set is never held back, onroad included. Parked, leaving iOS is not held + back either: an iOS gadget lends nobody the endpoint files (cable_mode), so + nothing of a borrower's is under the rebuild, and a borrower waiting for a + phone that never dials, or a run stuck with a silent one, kept the comma on + iOS until a power cycle. Entering iOS unplugs a USB host whose endpoint + files may be lent, so it waits for nobody to be on the link, except a run + of ours, which is stopped. + + What is built is learned once, onroad too: an iOS gadget taken for USB + would lend a phone the endpoint files.""" + if self.built_ios is None: + self.built_ios = gadget.built_for_ios() if gadget.link_configured() else ios + self.publish() + # a failed build of the other mode left the gadget half built: build the + # set mode again. A failure of the set mode itself is build()'s to retry + half_built = self.failed_ios is not None and self.failed_ios != ios + if ios == self.built_ios and not half_built: + return False + if ios != self.built_ios: + # the way out of iOS whose USB build failed is retried onroad too: held + # for the drive, the comma would sit on iOS waiting for a phone + if not offroad and not (not ios and self.failed_ios is False): + return False + if ios: + if self.worker_running(): + if self.shutting_down: + return False # the run asking the jetson to power off finishes first + self.stop_worker() # its loan ends as it exits + return True + if self.lender.lent: + return False + if self.transport is None and gadget.bound_udc(): + # bound, and not by us: the borrower of an owner that died (its run) + # is still on the endpoints. The rebuild's unbind would pull the + # gadget from under it, so it waits for the borrower to let go, when + # the kernel unbinds + if not self.switch_waiting: + gadget.log.warning("jetlink: the setting is now iOS; the gadget is still bound for a borrower " + "from before this owner, rebuilding it once that lets go") + self.switch_waiting = True + return False + self.switch_waiting = False + if time.monotonic() < self.next_gadget_attempt and ios == self.failed_ios: + return True # this mode's last build failed; build() says when to try again + self.stop_worker() # bounded: WORKER_GRACE, then SIGKILL + gadget.log.warning("jetlink: the setting is now %s, rebuilding the gadget", 'iOS' if ios else 'USB') + self.close_link() + self.dialed = False + self.build(ios) + return True + + def adopt(self) -> None: + """Start from whatever an owner before this one left, dead or alive. + + The configfs gadget is used as it is: ensure_gadget and switch_mode look + at it before building anything, and the first bind presents it again. + What the last owner heard of the server still holds (gadget.SERVER, which + a clean stop leaves); without it an owner started again while parked took + an always-on Jetson for one that sleeps, and let the gadget go until the + next drive's hello. The last owner's own + records do not hold: the link it published, its dormant marker and its + lender's error. The lender clears the socket it left once a connect + proves it dead, and the port's first update lets go of its hold. + """ + gadget.clear_link() + gadget.set_dormant(False) + gadget.note_lender_error(None) + gadget.clear_leftovers(gadget.STATUS.parent) + if self.server is None: + try: + heard = json.loads(gadget.SERVER.read_text()) + except (OSError, ValueError): + heard = None + self.server = heard if isinstance(heard, dict) else None + + def sit_out(self, until: float, died: int) -> None: + """A crash loop's backoff: hold nothing, not the gadget, the lender or + the port, until `until` or a stop. The status record says why, and that + is the offroad alert while the link is on. + + A power-off request ends it: hardwared waits 25 s for the owner to take + one, and the Jetson on its own supply stays on otherwise.""" + self.backing_off = (f"the service keeps stopping ({died} times in {CRASH_WINDOW / 60:.0f} min), " + f"waiting {until - time.monotonic():.0f} s before starting it again") + gadget.log.warning("jetlink: %s", self.backing_off) + self.publish_status() + try: + while not self.stop and time.monotonic() < until: + if gadget.SHUTDOWN_REQUEST.exists(): + gadget.log.warning("jetlink: a power-off request ends the wait") + break + time.sleep(min(POLL, max(0.0, until - time.monotonic()))) + self.beat() + finally: + self.backing_off = None + + def take_lock(self) -> bool: + """One owner at a time. A second owner beside a live one (a manager that + was SIGKILLed without its cleanup starts jetlinkd again while the orphan + still holds the gadget) would clear that owner's records, overwrite its + status record and release its port hold. The kernel lets an flock go with + the process however it dies, and the fd is not inherited by a run. + Without a /dev/shm to lock in, the owner runs as it did before.""" + try: + gadget.OWNER_LOCK.parent.mkdir(parents=True, exist_ok=True) + fd = os.open(gadget.OWNER_LOCK, os.O_RDWR | os.O_CREAT, 0o644) + except OSError as e: + gadget.log.error("jetlink: could not open the owner's lock (%s), running without it", e) + return True + try: + fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError: + try: + holder = os.pread(fd, 32, 0).decode(errors='replace').strip() or 'unknown' + except OSError: + holder = 'unknown' + os.close(fd) + gadget.log.error("jetlink: another owner (pid %s) holds the gadget, exiting", holder) + return False + os.ftruncate(fd, 0) + os.pwrite(fd, str(os.getpid()).encode(), 0) + self.lock_fd = fd + return True + + def run(self) -> None: + """Hold the gadget until stopped. Returns at once, touching nothing, when + another owner is alive.""" + if not self.take_lock(): + return + try: + self.hold_the_gadget() + finally: + fd, self.lock_fd = self.lock_fd, None + if fd is not None: + os.close(fd) + + def hold_the_gadget(self) -> None: + born = self.born = time.monotonic() + wait, died = note_start(born) + self.adopt() + # the heartbeat from the start: the first step builds the gadget + self.publish_status() + try: + if wait: + self.sit_out(born + wait, died) + while not self.stop: + started = time.monotonic() + try: + self.step() + except Exception: + # nothing may escape: restarting in a loop is worse than sitting out a cycle + gadget.log.exception("jetlink: unhandled error") + self.close_link() + self.next_attempt = time.monotonic() + RECONNECT_BACKOFF + self.publish_status() + time.sleep(max(0.0, POLL - (time.monotonic() - started))) + finally: + # first, inside manager's 5 s: it stops this when a chestnut turns up, + # and the comma has to host that. The sysctls stay: a stop here is where + # a drive begins + self.port.off() + self.lender.stop() + # about this process's lender; with no owner there is nothing to borrow + # anyway, and a chestnut may be why it stopped + gadget.note_lender_error(None) + self.stop_worker() + self.close_link() + gadget.set_dormant(False) + self.forget_status() + # an owner killed before it got this far without being asked to stop + # counts as one that died; one that was asked took it back already + if self.born is not None: + forget_start(born) + self.born = None + gadget.log.warning("jetlink: stopped") + diff --git a/jetlink_repo/jetlink/comma/port.py b/jetlink_repo/jetlink/comma/port.py new file mode 100644 index 0000000000..1d31febca5 --- /dev/null +++ b/jetlink_repo/jetlink/comma/port.py @@ -0,0 +1,405 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The comma's USB-C port, kept the device end of a USB link. + +The port is dual role. It hosts a chestnut, and it is the device for a jetlink +host. An A-to-C cable settles which by construction: the A end only pulls CC +up, so the comma can only be the sink and the device. A C-to-C cable does not. +Both ends are dual role, and the comma can come out the source and the host: +facing a Mac, nothing enumerates; facing an iPhone or a Jetson's own USB-C +port, the far end enumerates as a device. Carrot's Jetson on C-to-C read +"Powered cable w/ sink" and enumerated as 0955:7020. + +While the link is on, a chestnut is the only thing the comma should host on +this port. So once the comma has been the source for a few seconds with no +chestnut on the port, whatever is on the other end is a host that lost the +toss, and the port is held at sink until that cable comes out. Nothing happens +anywhere else: the comma as the device (every USB-A host, a C-to-C host that +won), a power supply and a chestnut all leave the port as AGNOS boots it. +Holding for the whole session would be simpler and would hide a chestnut +plugged in while the link is on, since chestnut_present() needs the comma to +host it before jetlink stands aside. + +The lever is the charger's DISABLE_POWER_ROLE_SWITCH voter, forced from +debugfs. It is the one that holds. The charger puts the port back to dual role +on every unplug and refuses a role written through the power supply once +nothing is attached, so the policy engine's rev3_sink_only and dual_role/mode +last one plug at most; a forced voter gates all of those writes. + +The voter only moves the power role, and a USB PD power role swap leaves the +data roles where they were. The comma's policy engine, as the source, accepts +a PR_Swap at any time, and its sink startup keeps the host role it had. So a +far end that came out the device and then asked for the source role, as a +phone or tablet that powers what it plugs into may, leaves the comma the sink +and still the host: no hold fires, and none would help. For that pair the +comma asks over USB PD for a data role swap (the dual_role class's data_role, +which sends a DR_Swap and moves nothing else), a few times, then once for a +hard reset, after which a sink is the device by the spec. Neither is ever sent +to a far end the comma is the device of. + +USB PD is otherwise left alone; a host that comes back negotiates as over any +cable. The hold does not survive a reboot. + +The right roles are not enough on their own: the comma also has to turn its +device side on. As a sink the policy engine does that only when the charger +detection read a USB port or floating data lines, and dwc3 turns it off again +when the lines read floating and no host has enumerated it in 10 s. Behind a +hub the lines are a USB port's. On a direct cable an Apple port connects them +only once USB PD is done, and a comma that boots with the phone plugged in has +no gadget bound for far longer than 10 s. So while the link is on both are +told to keep the device (jetlink-root.sh udc apply, which the owner runs with +the VM tuning), a sink that is the device with the device side still off a few +seconds into a plug has it turned on, and an empty port with it still on has it +turned off. + +An iPhone on a direct cable ends up powering the comma: after a hold it is the +source and the host. It can be the host and charge from the comma instead, and +it negotiates USB PD, so with the link set to iOS and phone charging turned on +the comma asks it once a plug for the source role (a PR_Swap, jetlink-root.sh +port source), once the gadget is configured. The swap leaves the data roles +and the link as they are. The comma's own sink capabilities never invite the +phone to offer it. Being the source and still the device is left alone: only +the host role is judged. + +Phone charging is off unless the fork's param turns it on (Keys.charge_phone): +some iPhones lose the link once the comma powers them. The link comes first, +so a link that goes down while the comma charges the phone has the source role +handed back (port sink), and the phone powers the port again. Charging the +phone adds up to 7.5 W through the comma's boost converter, so a comma that +runs hot waits to ask until it has cooled. A phone that takes the source role +back, or is handed it, is not asked again until the next plug. + +Every change of the port's roles is a line in the owner's log, so a plug that +did not connect can be read afterwards. +""" +from __future__ import annotations + +import os +import time +from pathlib import Path + +from jetlink.comma import gadget, root + +USBPD = Path('/sys/class/usbpd/usbpd0') +POWER_ROLE = USBPD / 'current_pr' +DATA_ROLE = USBPD / 'current_dr' +CONTRACT = USBPD / 'contract' +USB_PSY = Path('/sys/class/power_supply/usb') +TYPEC_MODE = USB_PSY / 'typec_mode' +CHARGER = USB_PSY / 'real_type' +# dwc3's glue for usb0: 'peripheral' while the device side is on +UDC_MODE = Path('/sys/devices/platform/soc/a600000.ssusb/mode') +USB_DEVICES = Path('/sys/bus/usb/devices') +THERMAL = Path('/sys/devices/virtual/thermal') +# the zones hardwared takes the comma's temperature from (its ThermalConfig's +# cpu, gpu, memory and pmic), the same names on the four and the 3X +HOT_ZONES = frozenset([f'cpu{i}-{c}-usr' for c in ('silver', 'gold') for i in range(4)] + + ['gpu0-usr', 'gpu1-usr', 'ddr-usr', 'pm8998_tz', 'pm8005_tz']) +# a comma this hot, the hottest zone smoothed over a few seconds, does not +# start charging a phone. Above the fan's setpoints (75 C on the four, 80 C on +# the 3X) and below openpilot's overheated bands (100 C and 96 C) +HOT_C = 90.0 +HOT_SMOOTHING = 0.2 # of each half-second reading +# how long the comma hosts the far end before judging it. A chestnut enumerates +# well inside this, and a far end that powers us and wants the host role has +# asked for it by then +SWAP_AFTER = 3.0 +# DR_Swap requests to a far end that powers us and came out the device, a plug, +# this far apart; then one hard reset +SWAP_TRIES = 3 +RETRY_AFTER = 2.0 +# a device side turned on that went off again is turned on no sooner than +# this: dwc3 gives floating lines 10 s +RESTART_AFTER = 10.0 +# how long the port reads empty before a hold is let go. The hold itself is an +# unplug and replug, so this has to outlast the replug +RELEASE_AFTER = 5.0 +# how long the port reads empty before a plug counts as gone. A device let go +# by a hold reattaches in a DRP toggle and a CC debounce, well under this +UNPLUGGED = 1.5 +ATTACHED = ('source', 'sink') + + +def power_role() -> str | None: + """'source', 'sink' or 'none' from the policy engine, or None without one.""" + return gadget.read(POWER_ROLE) or None + + +def data_role() -> str | None: + """'dfp' (the comma is the host), 'ufp' (the device) or 'none'.""" + return gadget.read(DATA_ROLE) or None + + +def thermal_zones() -> list[Path]: + """The temp files of HOT_ZONES; none off a comma.""" + try: + names = os.listdir(THERMAL) + except OSError: + return [] + zones = [] + for name in names: + if name.startswith('thermal_zone') and gadget.read(THERMAL / name / 'type') in HOT_ZONES: + zones.append(THERMAL / name / 'temp') + return zones + + +def hottest(zones: list[Path]) -> float | None: + """The hottest of `zones` in degrees C, or None when none reads.""" + temps = [] + for zone in zones: + try: + temps.append(int(gadget.read(zone)) / 1000) + except ValueError: + continue + return max(temps, default=None) + + +def chestnut_attached(chestnut_ids: frozenset[tuple[int, int]]) -> bool: + """Is a chestnut enumerated, running or in its ROM? It can only be on this port.""" + try: + names = os.listdir(USB_DEVICES) + except OSError: + return False + for name in names: + if ':' in name: + continue # an interface + try: + ids = tuple(int((USB_DEVICES / name / f).read_text(), 16) for f in ('idVendor', 'idProduct')) + except (OSError, ValueError): + continue # a device going away + if ids in chestnut_ids: + return True + return False + + +def run_script(command: str) -> bool: + """jetlink-root.sh port hold|off|device|reset|source|sink. Both commas have + the levers, so a False is a failure, and root.run has logged why. Its timeout + is short because the off in the owner's finally comes before the FunctionFS + close, inside manager's 5 s; source and sink wait for the far end's PR_Swap.""" + return root.run('port', command, timeout=root.SWAP_TIMEOUT if command in ('source', 'sink') else root.PORT_TIMEOUT) + + +def run_udc(command: str) -> bool: + """jetlink-root.sh udc start|stop, on the same short timeout.""" + return root.run('udc', command, timeout=root.PORT_TIMEOUT) + + +class Port: + """Called once a cycle by the owner. A cycle reads the power role, the data + role while something is attached, and the device controller's mode while a + host that powers us has not configured the gadget; sudo only runs on a + change.""" + + def __init__(self, chestnut_ids): + # what a chestnut enumerates as, running or in its ROM: openpilot's + # CHESTNUT_USB_IDS and CHESTNUT_ROM_USB_IDS, as the fork's adapter hands + # them over (OwnerConfig.chestnut_ids), so a chestnut being flashed is + # never taken for a host + self.chestnut_ids = frozenset(chestnut_ids) + self.zones: list[Path] | None = None # thermal_zones(), found on first use + self._reset() + + def _reset(self) -> None: + self.cleared = False # the port put back as AGNOS boots it, once a session + self.deferred = False # that waits for a host an owner before this one had + self.held = False # the voter is forced to sink + # this plug has been judged, so leave it until it comes out. Also set for + # the length of a hold until a host comes back + self.settled = False + self.role: str | None = None + self.role_since = 0.0 + self.roles: tuple[str | None, str | None] = (None, None) # power and data + self.roles_since = 0.0 + # something is on the port, or was until less than UNPLUGGED ago. Set at + # the start, so the first empty port is looked at too + self.plugged = True + self.asks = 0 # this plug's: SWAP_TRIES DR_Swaps, then a hard reset + self.next_ask = 0.0 + self.next_start = 0.0 + # this plug's phone charging: asked for (once a plug), the swap went + # through, said once that the comma is too hot to ask yet, and the + # smoothed temperature while waiting to ask + self.charge_asked = False + self.charging = False + self.charge_since = 0.0 + self.said_hot = False + self.temp: float | None = None + + def update(self, now: float | None = None, configured: bool = False, charge: bool = False) -> None: + """configured: a host had the gadget configured at the last cycle, so the + device side is on. charge: the link is set to iOS with phone charging on, + so a host that powers the comma is asked to charge from it instead.""" + if not self.cleared: + # whatever an owner killed mid-hold left behind. Not under a live link: + # an owner started after one that died can find a borrower still on the + # gadget the dead one presented, through a hold it made, and back at + # dual role the port may toss the roles again. Once no host has us + # configured the link has gone anyway, and the hold goes then + if gadget.host_attached(): + if not self.deferred: + gadget.log.warning("jetlink: a host is still on the gadget; leaving the USB-C port as it is until it goes") + self.deferred = True + else: + run_script('off') + self.cleared = True + now = time.monotonic() if now is None else now + role = power_role() + data = data_role() if role in ATTACHED else None + if (role, data) != self.roles: + if role is not None: + self._note(role, data) + if self.charging and (role, data) == ('sink', 'ufp'): + gadget.log.warning("jetlink: the iPhone took the source role back; not asking again until the next plug") + self.charging = False + self.roles, self.roles_since = (role, data), now + if role != self.role: + self.role, self.role_since = role, now + lasted = now - self.role_since + if role in ATTACHED: + self.plugged = True + elif self.plugged and lasted >= UNPLUGGED: + self._unplugged() + if role == 'sink' and data == 'ufp': + if not configured: + self._keep_the_device_on(now) + elif charge and not self.charge_asked and now - self.roles_since >= SWAP_AFTER: + self._charge_the_phone(now) + if self.charging and not configured and role == 'source': + self._stop_charging(now) + if self.held: + if role == 'sink': + self.settled = False # a host came back + elif role != 'source' and lasted >= RELEASE_AFTER: + self._release() + # the accessory reattaches in a moment; time the gap afresh + self.role_since, self.plugged = now, True + elif data == 'dfp' or (role == 'source' and data != 'ufp'): + # the comma is the host, which only a chestnut should make it. The + # source is held at sink; a sink is asked over USB PD. A source that is + # the device got there by a swap, and is what charging a phone looks like + hosted = now - self.roles_since + if not self.settled and hosted >= SWAP_AFTER and now >= self.next_ask: + if self.asks == 0 and chestnut_attached(self.chestnut_ids): + self.settled = True + elif role == 'source': + self._hold(hosted) + else: + self._ask_for_a_host(now) + elif role == 'sink' or not self.plugged: + self.settled = False # a host, or the plug is gone + + def _note(self, role: str, data: str | None) -> None: + gadget.log.info("jetlink: USB-C port %s, %s; USB PD contract %s; Type-C %s; charger detection %s", + role, {'dfp': 'host', 'ufp': 'device'}.get(data, 'no data role'), + gadget.read(CONTRACT) or 'unknown', gadget.read(TYPEC_MODE) or 'unknown', + gadget.read(CHARGER) or 'unknown') + + def _unplugged(self) -> None: + self.plugged = False + self.asks = 0 # the next plug is asked afresh + self.charge_asked = self.charging = self.said_hot = False + self.temp = None + # the policy engine turns off only a device side it turned on itself, so + # one turned on here, by this owner or one before it, is turned off here + if gadget.read(UDC_MODE) == 'peripheral': + run_udc('stop') + + def _keep_the_device_on(self, now: float) -> None: + """A host powers the port and the comma is its device, so the device side + should be on; see the module's docstring for when it is not.""" + if now - self.roles_since < SWAP_AFTER or now < self.next_start: + return + mode = gadget.read(UDC_MODE) + if mode in ('', 'peripheral'): + return + gadget.log.warning("jetlink: a host powers the USB-C port but the USB device controller is %s " + "(charger detection %s); turning its device side on", mode, gadget.read(CHARGER) or 'unknown') + self.next_start = now + RESTART_AFTER + run_udc('start') + + def _charge_the_phone(self, now: float) -> None: + """The phone powers the comma and hosts it over a USB PD contract: ask it + for the source role, unless the comma is hot. See the module's docstring.""" + if gadget.read(CONTRACT) != 'explicit': + return # a PR_Swap needs USB PD; a phone without it is left powering us + if self.zones is None: + self.zones = thermal_zones() + temp = hottest(self.zones) + if temp is not None: + self.temp = temp if self.temp is None else self.temp + (temp - self.temp) * HOT_SMOOTHING + if self.temp >= HOT_C: + if not self.said_hot: + gadget.log.warning("jetlink: the comma is at %.0f C; charging the iPhone waits until it is below %.0f C", + self.temp, HOT_C) + self.said_hot = True + return + self.charge_asked = True + if self.held: + # the voter gates every power role change + self.held = False + run_script('off') + gadget.log.warning("jetlink: the iPhone powers the comma; asking it over USB PD to charge from it instead") + if run_script('source'): + gadget.log.warning("jetlink: charging the iPhone") + self.charging, self.charge_since = True, now + else: + gadget.log.warning("jetlink: the iPhone kept the source role; not asking again until the next plug") + + def _stop_charging(self, now: float) -> None: + """The link went down with the comma charging the phone: what a swap must + never cost. Hand the phone the source role back, so it powers the port as + it does with charging off, and leave this plug alone from then on.""" + self.charging = False # before the swap, whose sink+ufp would read as the phone taking it back + gadget.log.warning("jetlink: the link went down %.0f s into charging the iPhone; giving it the source role back", + now - self.charge_since) + if run_script('sink'): + gadget.log.warning("jetlink: the iPhone powers the comma again; not charging it until the next plug") + else: + gadget.log.warning("jetlink: the iPhone kept charging from the comma; unplug the cable to start over") + + def _ask_for_a_host(self, now: float) -> None: + """The far end powers the comma and is still its device; see the module's + docstring for how that comes about.""" + self.asks += 1 + self.next_ask = now + RETRY_AFTER + if self.asks <= SWAP_TRIES: + gadget.log.warning("jetlink: the far end powers the USB-C port but came out the device; asking it over USB PD " + "to take the host role (%d of %d)", self.asks, SWAP_TRIES) + run_script('device') + elif self.asks == SWAP_TRIES + 1: + gadget.log.warning("jetlink: the far end did not take the host role; resetting USB PD") + run_script('reset') + else: + gadget.log.warning("jetlink: the far end still powers the USB-C port as the device; leaving it until the next " + "plug. A USB-C hub, or a USB-C to USB-A adapter and an A-to-C cable, settles the roles") + self.settled = True + + def _hold(self, lasted: float) -> None: + gadget.log.warning(f"jetlink: hosting something that is not a chestnut for {lasted:.0f} s on the USB-C port; holding it as a device") + # settled stays set if the hold did not happen, so this plug is not tried every cycle + self.held = run_script('hold') + self.settled = True + + def _release(self) -> None: + if self.settled: + # a sink-only accessory: it comes back as a sink on dual role, and + # holding again would only cycle it + gadget.log.warning("jetlink: no host came back on the USB-C port; leaving it dual role until the next plug") + else: + gadget.log.warning("jetlink: the USB-C port is empty; back to dual role") + self.held = False + run_script('off') + + def off(self) -> None: + """Undo a hold. Outside one the port is already as AGNOS boots it. A + device side turned on here is left: this runs as a chestnut turns up, and + turning it off would take the host role from under the chestnut; the next + owner turns it off once the port is empty.""" + if self.held: + run_script('off') + self._reset() diff --git a/jetlink_repo/jetlink/comma/root.py b/jetlink_repo/jetlink/comma/root.py new file mode 100644 index 0000000000..ee0d6f5cd3 --- /dev/null +++ b/jetlink_repo/jetlink/comma/root.py @@ -0,0 +1,63 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Root on the comma: scripts/comma/jetlink-root.sh under sudo -n, and nothing +else. The script leaves its own record of the gadget and its network in +/dev/shm (jetlink-gadget, jetlink-net), so a False here only says the call +failed; the reason is in the log line and, for the gadget, in those files. + + root.run('gadget', '--ios') + root.run('port', 'hold', timeout=root.PORT_TIMEOUT) + root.run('vm', 'apply') + +Only AGNOS has the gadget stack, the port's lever and the sysctls, so +everywhere else run() is a quiet False and nothing is spawned. +""" +from __future__ import annotations + +import logging +import os +import subprocess +from pathlib import Path + +log = logging.getLogger('jetlink.comma') + +AGNOS = os.path.isfile('/AGNOS') + +# Resolved, so that /jetlink, a symlink into jetlink_repo, still +# leads to the checkout's scripts. +SCRIPT = Path(__file__).resolve().parents[2] / 'scripts' / 'comma' / 'jetlink-root.sh' + +# sudo, configfs, and for iOS nmcli and dnsmasq +TIMEOUT = 30.0 +# sudo and two echos take tens of ms. Short, because the owner lets the port go +# before it closes FunctionFS, inside manager's 5 s +PORT_TIMEOUT = 2.0 +# port source: the kernel waits up to 2 s for the far end's PR_Swap +SWAP_TIMEOUT = 4.0 + + +def run(*args: str, timeout: float = TIMEOUT) -> bool: + """sudo -n bash SCRIPT *args, on AGNOS. True when it exits 0; otherwise one + log line, with the last thing the script wrote to stderr, and False. Off + AGNOS a False with no log line. Never raises.""" + if not AGNOS: + return False + what = ' '.join((SCRIPT.name, *args)) + try: + result = subprocess.run(['sudo', '-n', 'bash', str(SCRIPT), *args], stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, timeout=timeout, text=True, errors='replace') + except subprocess.TimeoutExpired: + log.error("jetlink: %s timed out after %.0f s", what, timeout) + return False + except Exception as e: + log.error("jetlink: %s did not run: %s", what, e) + return False + if result.returncode == 0: + return True + lines = (result.stderr or '').strip().splitlines() + log.error("jetlink: %s failed (exit %d): %s", what, result.returncode, lines[-1] if lines else 'no output') + return False diff --git a/jetlink_repo/jetlink/onnx_meta.py b/jetlink_repo/jetlink/onnx_meta.py new file mode 100644 index 0000000000..15c96ccc50 --- /dev/null +++ b/jetlink_repo/jetlink/onnx_meta.py @@ -0,0 +1,97 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +A driving model's graph metadata: input and output names and shapes, plus +openpilot's output_slices and model_checkpoint metadata_props. + +Adapter over whichever parser the host has, neither a hard dependency. +tinygrad's OnnxPBParser is preferred and is what openpilot's +get_model_metadata.py uses: it walks the protobuf without materialising 766 MB +of weights, and a comma has it already. The onnx package is the fallback +everywhere else. +""" +from __future__ import annotations + +import codecs +import pickle +from dataclasses import dataclass, field + + +@dataclass +class OnnxMeta: + inputs: dict[str, tuple[int, ...]] = field(default_factory=dict) + outputs: dict[str, tuple[int, ...]] = field(default_factory=dict) + props: dict[str, str] = field(default_factory=dict) + + @property + def output_slices(self) -> dict[str, slice]: + """openpilot stashes a base64 pickle of the output slice map in metadata_props.""" + raw = self.props.get('output_slices') + if raw is None: + raise KeyError("output_slices not in model metadata_props") + return pickle.loads(codecs.decode(raw.encode(), 'base64')) + + @property + def model_checkpoint(self) -> str | None: + return self.props.get('model_checkpoint') + + +def _parse_tinygrad(path: str) -> OnnxMeta: + """Same approach as openpilot's get_model_metadata.py, without importing it. + + ModelProto is narrowed to two fields, so the 766 MB of initializers are + skipped rather than built. + """ + from tinygrad.nn.onnx import OnnxPBParser + + class _MetaParser(OnnxPBParser): + def _parse_ModelProto(self) -> dict: + obj: dict = {"graph": {"input": [], "output": []}, "metadata_props": []} + for fid, wire_type in self._parse_message(self.reader.len): + if fid == 7: + obj["graph"] = self._parse_GraphProto() + elif fid == 14: + obj["metadata_props"].append(self._parse_StringStringEntryProto()) + else: + self.reader.skip_field(wire_type) + return obj + + model = _MetaParser(path).parse() + meta = OnnxMeta() + for key, dest in (('input', meta.inputs), ('output', meta.outputs)): + for vi in model['graph'][key]: + dest[vi['name']] = tuple(int(d) if isinstance(d, int) else 0 for d in vi['parsed_type'].shape) + for prop in model['metadata_props']: + meta.props[prop['key']] = prop['value'] + return meta + + +def _parse_onnx(path: str) -> OnnxMeta: + import onnx + + model = onnx.load(path, load_external_data=False) + meta = OnnxMeta() + for vis, dest in ((model.graph.input, meta.inputs), (model.graph.output, meta.outputs)): + for vi in vis: + dest[vi.name] = tuple(d.dim_value for d in vi.type.tensor_type.shape.dim) + for p in model.metadata_props: + meta.props[p.key] = p.value + return meta + + +def parse_file(path: str) -> OnnxMeta: + errors = [] + for name, fn in (('tinygrad', _parse_tinygrad), ('onnx', _parse_onnx)): + try: + return fn(path) + except Exception as e: + # not just ImportError: a parser that chokes on a newer layout falls + # through to the other one rather than aborting + errors.append(f'{name}: {type(e).__name__}: {e}') + raise RuntimeError( + "could not read model metadata; need tinygrad or the onnx package. Tried:\n " + + "\n ".join(errors)) + diff --git a/jetlink_repo/jetlink/openpilot/__init__.py b/jetlink_repo/jetlink/openpilot/__init__.py new file mode 100644 index 0000000000..ce21058571 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/__init__.py @@ -0,0 +1,255 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +jetlink on an openpilot device: everything between the fork and the comma's +device layer (jetlink.comma). + +The fork implements Openpilot (interface.py) once, in an adapter module that +holds every openpilot import, binds it here (bind) and calls the methods of +the Jetlink that comes back. That is the whole contract between the two repos: +the names in __all__, the Jetlink methods and Status, which +tests/openpilot/test_api.py and test_interface.py pin. API names its version: +the adapter checks it exactly and treats any other value as jetlink being +absent, an offroad alert and no link rather than a crash. Everything else in +this package is jetlink's own and changes without notice. + +API changes only for a breaking change to the contract. A new Status field, a +new Jetlink method, or a new interface member that jetlink reads with getattr +and a fallback, is additive and keeps it. + +Imported by the resident gadget owner, so nothing heavy at module level: the +parts a heavy process uses are imported when it first uses them. +""" +from __future__ import annotations + +import threading +import time + +from jetlink.comma import gadget +from jetlink.openpilot.interface import MODES, STATES, Keys, ModelFace, Openpilot, OwnerConfig, conformance +from jetlink.openpilot.parts import Parts, for_this_process +from jetlink.openpilot.status import Status + +API = 1 + +__all__ = ['API', 'MODES', 'STATES', 'Jetlink', 'Keys', 'ModelFace', 'Openpilot', 'OwnerConfig', 'Status', 'bind', + 'conformance'] + +# how long a caller waits for the owner's run to shut the Jetson down. Wake +# from suspend is ~8 s to a server +SHUTDOWN_TIMEOUT = 25.0 + + +def bind(op) -> Jetlink: + """jetlink for this process, over the fork's adapter. One per process: it + keeps the process's caches (parts.for_this_process).""" + return Jetlink(for_this_process(op)) + + +class Jetlink: + """What the fork calls: one per process, from bind(). Its public methods are + the API; the parts behind it are not.""" + + def __init__(self, parts: Parts): + self._parts = parts + self._log = parts.log + # prepare() said yes in this process, so attach() may join modeld + self._prepared = False + # what -> the last failure logged reading it; cleared by a read that works + self._failures: dict[str, str] = {} + # when this process asked for the Jetson to power off, until it was taken + self._shutdown_asked: float | None = None + + def _read(self, what: str, read, fallback): + """read(), or fallback(error) when it raises. The readers never raise: + they are asked several times a second, from threads that have nothing to + do with jetlink. A failure is logged once per distinct error, and again + after a read that worked.""" + try: + value = read() + except Exception as e: + error = f"{type(e).__name__}: {e}" + if self._failures.get(what) != error: + self._failures[what] = error + self._log.exception("jetlink: could not read %s", what) + return fallback(error) + self._failures.pop(what, None) + return value + + def _mode(self) -> str: + """The link setting, or 'off' when it cannot be read.""" + return self._read('the link setting', lambda: self._parts.settings.mode(), lambda error: 'off') + + def enabled(self) -> bool: + """Has the user turned the link on, with no chestnut fitted? The setting, + never link state or readiness. A chestnut runs the big model natively and + the link stays off beside it, so jetlinkd never takes the USB controller + from it. manager's should_run for jetlinkd, on every device.""" + mode = self._mode() + return self._read('whether the link is on', lambda: self._parts.enabled(mode), lambda error: False) + + def status(self) -> Status: + """One snapshot for the UI and the panels.""" + from jetlink.openpilot import status + mode = self._mode() + return self._read('the status', lambda: status.read(self._parts, mode), lambda error: status.failed(error, mode)) + + def reason(self) -> str | None: + """Why the link the user asked for cannot run: hardwared's offroad alert. + None with the link off. The files the gadget and the build leave, nothing + else: hardwared asks twice a second on every device.""" + from jetlink.openpilot import status + mode = self._mode() + return self._read('why the link cannot run', lambda: status.reason(self._parts, mode), + lambda error: status.failure(error, mode)) + + def prepare(self) -> bool: + """Will the link join this modeld? modeld only, before it goes realtime: + enabled(), then the process-wide setup that has to happen before then, + which is also a last veto.""" + from jetlink.openpilot.status import NO_WARP + from jetlink.openpilot.warp import init_device + self._prepared = False + if not self.enabled(): + return False + # the link is not worth waiting for: attach() joins in the background. + # enabled() is the setting alone, so this is where a device that cannot + # present a gadget at all says so; nothing here would ever reach a Jetson + if not gadget.link_configured(): + self._log.warning("jetlink: no usable gadget (%s), staying on the small model", + gadget.gadget_error() or 'not set up') + return False + # the warp is a build product and nothing compiles one at runtime, so one + # missing now stays missing, and saying no keeps modeld on the plain small + # model; the offroad alert has said why + if not self._parts.warps.built(): + self._log.warning("jetlink: %s, staying on the small model", NO_WARP) + return False + # the last hook before modeld goes SCHED_FIFO on core 7, and the GPU's init + # spawns a thread that would inherit that. See warp.init_device + init_device(self._log) + self._prepared = True + return True + + def attach(self, small, cam_w: int, cam_h: int): + """Join the link to modeld, once the camera is up and `small` is built: + the joining model, which drives as `small` until the Jetson is there. + + None unless prepare() said yes in this process: without it the GPU's + thread would start on modeld's realtime core. If the joining model cannot + be built, `small`, and the failure is logged. + """ + if not self._prepared: + return None + from jetlink.openpilot.joining import join + try: + return join(self._parts, cam_w, cam_h, small) + except Exception: + self._log.exception("jetlink load failed") + return small + + def request_shutdown(self, reason: str = '') -> bool: + """The device is powering off for good: ask for the Jetson to go down + with it, and return at once. True when the request now waits for the + owner, which shutdown_pending() follows; False when there was nothing to + ask, with the link off, beside a chestnut, or with no Jetson known to be + there. Never raises. + + hardwared calls this once and goes on publishing deviceState, putting + DoShutdown once shutdown_pending() clears or SHUTDOWN_TIMEOUT has passed. + The owner wakes a sleeping Jetson and starts a run that asks it; the + wake and one round trip take ~10 s. + """ + try: + return self._ask_for_power_off(reason) + except Exception: + self._log.exception("jetlink: shutdown request failed") + return False + + def shutdown_pending(self) -> bool: + """Has the owner still to take the power-off request? A stat, for a + caller's loop. Never raises. + + Once SHUTDOWN_TIMEOUT has passed since this process asked, the request is + withdrawn and False: a comma that outlives its DoShutdown must not have + the Jetson powered off later, when nobody wants it off, and the log says + the request went unanswered.""" + try: + pending = gadget.SHUTDOWN_REQUEST.exists() + except Exception: + return False + asked, now = self._shutdown_asked, time.monotonic() + if asked is None: + return pending + if not pending: + # the run removes it whether or not the Jetson answered; its log says which + self._log.warning("jetlink: the owner took the shutdown request after %.1f s", now - asked) + elif now - asked >= SHUTDOWN_TIMEOUT: + self._log.warning("jetlink: nobody took the shutdown request within %.0f s", SHUTDOWN_TIMEOUT) + gadget.finish_shutdown() + pending = False + if not pending: + self._shutdown_asked = None + return pending + + def shutdown(self, reason: str = '', timeout: float = SHUTDOWN_TIMEOUT) -> None: + """request_shutdown(), then wait up to `timeout` for the owner to take it, + on a thread abandoned at the deadline: for a caller with nothing to do + meanwhile. hardwared keeps publishing, and uses the two above.""" + if not self.enabled(): + return + + def request(): + try: + self._request_shutdown(reason, timeout) + except Exception: + self._log.exception("jetlink: shutdown request failed") + + t = threading.Thread(target=request, name='accelerator-shutdown', daemon=True) + t.start() + t.join(timeout) + if t.is_alive(): + self._log.warning("jetlink: shutdown request still pending after %.0f s, going on without it", timeout) + + def _request_shutdown(self, reason: str, timeout: float) -> None: + if not self._ask_for_power_off(reason): + return + if gadget.await_shutdown(timeout): + self._log.warning("jetlink: shutdown request handed to the jetson") + else: + self._log.warning("jetlink: nobody took the shutdown request within %.0f s", timeout) + + def _ask_for_power_off(self, reason: str) -> bool: + """Hand the request to the owner: nothing else can touch the link, and + the request is a file (gadget.SHUTDOWN_REQUEST) the owner's step looks + for before anything else. Skipped when no Jetson is known to be there + (dormant counts as there). A run busy in a long provision is stopped for + it; the caller's timeout covers a Jetson that never answers.""" + from jetlink.openpilot.status import Presence + if not self.enabled(): + return False + # a presence of its own, off the files: one this process's readers keep + # refreshed, or the owner's with its hold, would count a Jetson seen + # seconds before the power-off, and the run would wait 20 s for a host + # that has gone + if not Presence().present(): + return False + self._log.warning("jetlink: asking the jetson to power off: %s", reason) + if not gadget.request_shutdown(reason): + return False + self._shutdown_asked = time.monotonic() + return True + + def should_extend_catalog(self) -> bool: + """Should the model manager's big-model catalog carry the newer catalogs' + models? With no chestnut fitted. Hardware, not the link setting: the model + manager drops a pick its catalog does not list.""" + return not self._parts.chestnut_fitted() + + def extend_catalog(self, catalog: dict) -> dict: + """The big-model catalog the model manager fetched, with the models newer + catalogs list folded in. Never raises.""" + return self._parts.models.big_catalog(catalog) diff --git a/jetlink_repo/jetlink/openpilot/interface.py b/jetlink_repo/jetlink/openpilot/interface.py new file mode 100644 index 0000000000..2e14831df5 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/interface.py @@ -0,0 +1,221 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +What jetlink needs from openpilot, as one interface the fork implements. + +jetlink never imports openpilot. The fork has one adapter module that holds +every openpilot import jetlink needs; it implements `Openpilot` below and +hands the object in, and jetlink.openpilot reaches openpilot through it and +nothing else. So jetlink is tested against a fake, and an openpilot sync that +moves something jetlink relies on fails the adapter's tests in the fork, where +the fix belongs. + +The interface is split by process. The resident gadget owner gets data only +(OwnerConfig), no callbacks, so it stays at the standard library and about +10 MB. The heavy processes get the adapter object and each uses its side: +StatusSide in manager, the UI, hardwared and the model manager, WorkerSide in +the provisioning run, ModelSide in modeld, BuildSide in the warp build. + +The standard library only: the owner imports this module. +""" +from __future__ import annotations + +import importlib +import inspect +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +# the Jetlink setting: off; a Jetson, a Linux PC or a Mac on USB; an +# iPhone on the cable. The INT param holds the index +MODES = ('off', 'usb', 'ios') +# what a joining model reports as big_model_state, by the names of the fork's +# modelDataV2SP.acceleratorState enum +STATES = ('none', 'joining', 'retrying', 'ready', 'running', 'unavailable') + + +@dataclass(frozen=True) +class Keys: + """The params the fork declares for jetlink, by name: jetlink itself knows no param name.""" + link: str # INT, an index into MODES: the Jetlink setting + offroad: str # BOOL: is the car parked (manager's IsOffroad) + progress: str # JSON {stage, frac, msg, drops}: provisioning and join progress + spec: str # JSON: the built model's spec and whether its engine is built + pointers: str # JSON: catalog ref -> {oid, size} + # sunnypilot's model manager: the big-model pick, JSON {ref, displayName}, + # and the big-model catalog, JSON {bundles}. Without one both are None, and + # jetlink runs its own default big model + big_model: str | None = None + catalog: str | None = None + # BOOL: an iPhone on a direct cable is asked to charge from the comma + # (comma.port). Off when unset, and without the key + charge_phone: str | None = None + + +@dataclass(frozen=True) +class OwnerConfig: + """What the resident gadget owner needs, built without importing anything heavy.""" + params_dir: Path # the params store's directory, by params.cc's rule + keys: Keys # the settings it reads and the pick it watches + chestnut_ids: frozenset[tuple[int, int]] # (vid, pid) of comma's chestnut, running or in its ROM: never held as a host + adapter: str # the adapter module, which the provisioning run is started with + cwd: Path # where the run starts: the checkout + env: Mapping[str, str] # over the owner's environment, for the run + log_file: Path # the owner's rotating log + + +@dataclass(frozen=True) +class ModelFace: + """What openpilot's modeld reads off a ModelState, supplied for comma's large model.""" + parser: Callable[[], Any] # a new Parser: .parse_outputs(dict[str, ndarray]) -> dict + frame_size: Callable[[int, int], int] # a camera frame buffer's size for (w, h): get_nv12_info(w, h)[3] + desire_len: int # ModelConstants.DESIRE_LEN + constants: Any # modeld_v2's ModelConstants, which modeld_tinygrad reads off the model + lat_smooth_seconds: float # modeld's LAT_SMOOTH_SECONDS + long_smooth_seconds: float # modeld's LONG_SMOOTH_SECONDS + get_action_from_model: Callable[..., Any] # modeld's action function + + +# The sides are typing Protocols so a type checker can hold the adapter to +# them; conformance() is the check both repos run. A parameter jetlink passes +# by keyword is keyword-only here, and only those names must match. + +class StatusSide(Protocol): + """Every reader: manager, the UI, hardwared, the model manager.""" + keys: Keys + log: Any # logging.Logger-like (cloudlog): debug, info, warning, error, exception + catalog_selector: int # the model manager's REQUIRED_JSON_VERSION; 0 without a model manager + + def params_dir(self) -> Path: + """The params store's directory, by params.cc's rule, worked out on every + call so it follows OPENPILOT_PREFIX as Params does. jetlink reads the link + setting and IsOffroad there as files, for every setting a heavy process + reads (manager's should_run among them), so it is cheap and never raises.""" + + def get(self, key: str) -> Any: + """A param's decoded value. None when unset or unknown to this build; never raises.""" + + def chestnut_present(self) -> bool: + """Is comma's chestnut fitted? A USB walk; jetlink caches the answer.""" + + def camera(self) -> tuple[int, int, int, int]: + """This device's (cam_w, cam_h, model_w, model_h): the warp modeld asks for.""" + + def warp_path(self, cam_w: int, cam_h: int, model_w: int, model_h: int) -> Path: + """Where the build puts the warp for this geometry.""" + + +class WorkerSide(StatusSide, Protocol): + """The provisioning run: writes, files, the network.""" + basedir: Path # the checkout, whose .lfsconfig names the nearest LFS server + + def put(self, key: str, value: Any, *, block: bool = False) -> None: + """Write a param. May raise, as Params does for a key this build does not declare.""" + + def remove(self, key: str) -> None: + """Clear a param.""" + + def model_root(self) -> Path: + """The model manager's model root (openpilot's Paths.model_root()). jetlink + keeps its downloads in a directory of its own under it, `jetlink/`: the + model manager's cache clear deletes every file in the root it does not + recognise, a downloaded 1.75 GB ONNX included, and leaves directories alone.""" + + +class ModelSide(WorkerSide, Protocol): + """modeld: comma's model face, the engagement a swap waits out, and the + structured log line the link's telemetry goes to.""" + + def model_face(self) -> ModelFace: + """What a ModelState for comma's large model has to carry.""" + + def engagement(self) -> Callable[[int], bool]: + """A poller, made on the thread that calls it: (timeout_ms) -> engaged. Waits + up to timeout_ms for news and answers True when controls are engaged, or when + that is not known.""" + + def event(self, name: str, **fields: Any) -> None: + """A structured log line (cloudlog.event).""" + + +class BuildSide(Protocol): + """scons: comma's warp graph.""" + + def make_warp(self, cam_w: int, cam_h: int, model_w: int, model_h: int) -> tuple[Callable[..., Any], int]: + """The warp graph for this geometry, and the size of the NV12 frame it + reads. Imports comma's graph module before anything of tinygrad's.""" + + +class Openpilot(ModelSide, BuildSide, Protocol): + """The whole adapter: one object implements every side.""" + + +def load_adapter(module: str) -> Openpilot: + """The adapter an adapter module makes. An adapter module is the fork's + module that holds every openpilot import jetlink needs: its top level + imports only the standard library, and it has adapter() -> Openpilot and + owner_config() -> OwnerConfig. The entry points that run as their own + process (the provisioning run, the warp build) are told its name.""" + return importlib.import_module(module).adapter() + + +def members(protocol: type) -> dict[str, inspect.Signature | None]: + """Every member of `protocol`, inherited ones included: a method's signature + without self, or None for a data member.""" + found: dict[str, inspect.Signature | None] = {} + for cls in reversed(protocol.__mro__): + if cls is object or cls.__module__ == 'typing': + continue + for name in inspect.get_annotations(cls): + if not name.startswith('_'): + found[name] = None + for name, value in vars(cls).items(): + if not name.startswith('_') and inspect.isfunction(value): + sig = inspect.signature(value) + found[name] = sig.replace(parameters=list(sig.parameters.values())[1:]) + return found + + +def _shape(sig: inspect.Signature) -> list[tuple[str | None, Any, bool]]: + """What a caller depends on: each parameter's kind and whether it has a + default, and the name of one jetlink passes by keyword (keyword-only). The + other names are the adapter's to choose.""" + return [(p.name if p.kind is p.KEYWORD_ONLY else None, p.kind, p.default is not p.empty) + for p in sig.parameters.values()] + + +def _plain(sig: inspect.Signature) -> str: + """The parameters alone, as a caller writes them: '(key, value, block=False)'.""" + return str(sig.replace(parameters=[p.replace(annotation=p.empty) for p in sig.parameters.values()], + return_annotation=sig.empty)) + + +def conformance(obj: object, protocol: type) -> list[str]: + """Each member of `protocol` that `obj` lacks, or has with parameters a + caller would trip on (another count, kind or default, or another name for + one passed by keyword); [] when it conforms. One checker for both repos: + jetlink pins the interface with it, and the fork's tests run it against the + real adapter.""" + problems = [] + for name, expected in members(protocol).items(): + if not hasattr(obj, name): + problems.append(f"{name}: missing") + continue + if expected is None: + continue + value = getattr(obj, name) + if not callable(value): + problems.append(f"{name}: not callable") + continue + try: + actual = inspect.signature(value) + except (TypeError, ValueError): + problems.append(f"{name}: no signature to check") + continue + if _shape(actual) != _shape(expected): + problems.append(f"{name}{_plain(actual)}: expected {name}{_plain(expected)}") + return problems diff --git a/jetlink_repo/jetlink/openpilot/joining.py b/jetlink_repo/jetlink/openpilot/joining.py new file mode 100644 index 0000000000..4d1daa9fe5 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/joining.py @@ -0,0 +1,761 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +A ModelState that starts as the small model and upgrades to the Jetson. + +modeld's large-model load is a one-shot with BIG_MODEL_TIMEOUT and a silent, +one-way fallback. That suits a chestnut, powered from the comma. A Jetson is +on the ignition rail: cranking browns it out, so its boot starts about when +the comma goes onroad and takes 45 to 100 s, long after modeld gave up. + +So modeld is handed the small model it already loaded, and the Jetson is +swapped in underneath once the link, the engine and the warp are all there. +modeld re-reads `model` every frame, and `modelV2.big` keeps its meaning. + +From the join on, every frame goes to the Jetson whether or not it drives: +the small model drives on, and the large model's history and hidden state are +current and warm when the swap comes, so its first driving frame costs what +every frame costs. Those shadow frames are also the proof: the large model +is offered only once the host has answered the last PROVING_FRAMES of them +in time (model_state.keeping_up), so a host cold from a reconnect, or one +that is simply too slow, proves itself with the small model driving and not +at the wheel. The swap itself changes which model's output is published and +nothing else. While it drives, a reply that is not back in time is not +waited for: the previous frame's output is published again (a held frame, +model_state.HOLD_FRAME) and the camera frame is not dropped. Too many holds +hand the drive back to the small model, as a lost link does. + +Two rules the swap keeps: + +- tinygrad work happens on modeld's thread. The joining thread does link IO + only; building the JetlinkModelState unpickles a TinyJit, and doing that + next to the small model running frames on the same device is not safe. +- never swap while the plan is steering. The two models disagree by ~195 m of + planned path, and the swap costs a frame or two. Require fresh, fully + disengaged controls; standstill alone is not enough, longitudinal control + may still hold the brake. +""" +from __future__ import annotations + +import threading +import time + +from jetlink import protocol as P +from jetlink.comma import gadget +from jetlink.transport.priority import background_thread + +# after a join fails or the large model dies mid-drive. Long enough not to +# thrash a booting Jetson, short enough to catch one that finished a moment later +REJOIN_DELAY = 5.0 +# doubled per consecutive failure, reset by a join that lasted STABLE_SECONDS. +# A link that dies on its first frame every time must not cost a swap, a +# demote and an alert every few seconds +REJOIN_DELAY_MAX = 60.0 +STABLE_SECONDS = 60.0 +# after a join that held: a link that ran for minutes and then went is a USB +# drop, and the host re-enumerates a rebound gadget in under a second. The +# rest of the old delay was the driver's time, not the Jetson's +REJOIN_DELAY_QUICK = 1.0 +# failures in a streak retried that quickly before the doubling starts. A +# demote is mostly one late frame, a tail of the link or the host and not a +# dead one: on a phone's cable each one cost 5 to 60 s on the small model, +# five times in nine minutes on an iPad (2026-10-03). A link that fails on +# its heels this many times is backed off as before +QUICK_RETRIES = 3 +# after a link lost while it only shadowed: no swap, demote or alert to pay +# for, so no doubling. A phone that dropped its first frames again and again +# pulsed the icon for 1 to 60 s between tries (2026-10-04) +SHADOW_REJOIN_DELAY = 2.0 +# how often a backoff looks at the gadget. A host that configures it again +# after it went away is a replug, which the backoff is not for: one waited 16 s +# for a Jetson that was back in 0.4 (2026-09-29) +REPLUG_POLL = 0.25 +# link drops in one drive before the status names the cable. One is weather; +# the 2026-09-07 evening drive had six in twelve minutes, every one the +# USB-C port losing its host, and nothing the driver could see said so +DROPS_TO_BLAME_CABLE = 2 +ENGAGEMENT_POLL_MS = 100 +ENGAGEMENT_MAX_AGE = 0.25 +# a large-model frame is ~30 ms and the worst seen on the current stack ~55. +# One that took LATE_FRAME, or a second past SLOW_FRAME within LAG_WINDOW of +# the last, is a fault and is handled as a loss: modeld would otherwise count +# the dropped camera frames into modeldLagging with the slow model still +# steering. With a late reply held rather than waited for (model_state), a +# frame this long is the comma's own stall, a warp or a send, or a frame with +# nothing to hold yet. A frame past link.INFERENCE_TIMEOUT never returns; it fails +LATE_FRAME = 0.1 +SLOW_FRAME = 0.075 +LAG_WINDOW = 10.0 +# A host slower than the camera but never that slow (an iPhone that has warmed +# up: 55 ms a frame drops one camera frame in ten) still has modeld skipping +# frames, and selfdrived soft-disables on modeldLagging once modeld's filtered +# share of them (frameDropPerc) passes 1 %: three drops close together. modeld +# writes that share onto the model before every run(), and the large model +# hands back past DROP_LIMIT, short of the line: one dropped frame is forgiven, +# a second within about 6.5 s is not. modeld forgives the frame of a handover +# too, so the drops that decided it never reach selfdrived +DROP_LIMIT = 0.0075 +# the first frames after every swap are never counted as slow: the first after +# a join carries the history reset, and a Mac's is ~100 ms of CoreML warm-up +SETTLING_FRAMES = 3 +# the small model drives the first frames of every modeld start, even with the +# large model ready: its first run in a process costs ~1.3 s, which the first +# fallback frame paid (25 dropped frames, commIssue) when the large model had +# driven from frame one. Nothing is in control at a modeld start +SMALL_WARMUP_FRAMES = 3 +# shadowing this long without keeping up, the panel says the link is too slow +# to drive rather than 'checking' for the drive. The proof goes on: a host +# that warms up late still gets offered +SLOW_LINK_AFTER = 10.0 +# the panel's line while the large model shadows: once it is built, once it +# keeps up, and once it has not for SLOW_LINK_AFTER +CHECKING = 'checking the link keeps up' +PROVEN = 're-engage to switch' +SLOW = 'too slow to drive' +# why a demote happened, as the log says it +LOST = 'lost jetlink' +BEHIND = 'jetlink fell behind' +# why a built model was let go before it drove +NOT_UP = 'could not bring up the large model' +# and as the leave says it to the server (protocol.Msg.LEAVE) +LEAVING = {LOST: P.LEAVE_LOST, BEHIND: P.LEAVE_BEHIND} +# and as the panel's one line says it while the join tries again +RECONNECTING = {LOST: 'reconnecting', BEHIND: 'fell behind, reconnecting', NOT_UP: 'load failed, reconnecting'} + + +class JoiningModelState: + """Duck-types openpilot's modeld ModelState, with a second one inside. + + `progress` is where the join says what it is waiting on (status.Progress), + `engagement` makes the poller the swap window reads (the adapter's), and + `log` is cloudlog on a comma. + """ + + def __init__(self, small, connect, build, prepare=None, reset_small=None, *, progress, engagement, log): + self._small = small + self._active = small + self._connect = connect + self._build = build + self._reset_small = reset_small + self._progress = progress + self._engagement = engagement + self._log = log + + # whatever the swap would otherwise do on the frame loop, done now on + # modeld's main thread before the frame loop exists; at the swap it cost + # a 1.5 s frame + if prepare is not None: + try: + t0 = time.monotonic() + prepare() + log.warning("jetlink: prepared the large model ahead of the swap in %.2f s", time.monotonic() - t0) + except Exception: + log.exception("jetlink: could not prepare the large model ahead of the swap") + raise + + # handed over by the joining thread, consumed by the next frame, which + # builds the large model state from it. Only ever assigned under the lock + self._joined: tuple[object, object] | None = None + # the large model state once built: fed every frame, driving when it is + # _active. Only the frame thread assigns it + self._big = None + # what the frame thread let go of and why (LOST, BEHIND, or a build that + # failed), for the join thread to say goodbye to and close or keep. + # Assigned under the lock + self._retired: tuple[object, str] | None = None + self._lock = threading.Lock() + self._rejoin = threading.Event() + self._rejoin.set() + # earliest the join loop may try again. Without a backoff a fault that + # recurs on the first frame after every swap is a connect, build and demote + # every 700 ms for the drive, each costing modeld a 100 ms frame + self._rejoin_at = 0.0 + self._failures = 0 + self._joined_at = 0.0 + # links lost and lag demotes after a swap this drive. Only a lost link + # says anything about the cable + self._drops = 0 + self._lags = 0 + # changes of the model that drives, and decisions to change it: modeld + # compares it across run() (handovers) + self._handovers = 0 + # set on a frame the large model fell behind on; the next frame demotes. + # _slow_at is when the last slow frame was, for the second strike + self._lagging = False + self._slow_at: float | None = None + # modeld's share of dropped camera frames, for the frame about to run + self._frame_drop_ratio = 0.0 + # frames each model has run: the small one's since start, the large one's + # since it swapped in + self._small_frames = 0 + self._big_frames = 0 + # the panel's last line about the large model shadowing (CHECKING, PROVEN + # or SLOW). A report is a param write on the frame thread, so only a + # change is reported + self._proving = CHECKING + # whether the host had let go of the gadget when the last link was lost, + # and whether this failure streak has already skipped a backoff for a replug + self._host_left = False + self._replugged = False + + # assume engaged and moving until a message says otherwise, so a swap can + # never happen on no information + self._engaged = True + self._engagement_updated = 0.0 + self._stop = threading.Event() + + # whether the large model has produced a frame; a first inference that + # fails never announces readiness. Travels in modelV2.big and + # modelDataV2SP, not a param: a chestnut's load is over once, this never is + self._loading = True + + self._threads = [threading.Thread(target=self._join_loop, daemon=True), + threading.Thread(target=self._watch_engagement, daemon=True)] + for t in self._threads: + t.start() + + # -- what modeld reads ------------------------------------------------------ + # A read not defined here follows the model that is driving (__getattr__): + # vision_input_names, and modeld_v2's constants, smoothing and action + # function, which sunnypilot keeps on the ModelState so a custom small bundle + # can carry its own. Swaps and demotes happen inside run(), so what is read + # after it belongs to the model whose output it is. What the loop writes + # lands on both. The frame's inputs are built for the small bundle (its + # desire name, its optional slots); the large model reads the desire under + # any name and the loop always supplies what it needs, so a swap can land on + # any frame + + @property + def big_model_available(self) -> bool: + """Built, keeping up with the frames it shadows, and waiting to switch: + what big_model_state calls ready, and what the driver is offered.""" + return not self._stop.is_set() and self._shadowing and self._big.keeping_up + + @property + def _shadowing(self) -> bool: + # a large model built and fed every frame, with the small model driving + return self._big is not None and self._active is self._small + + @property + def chestnut(self) -> bool: + # modelV2.big. False while proxying, as the small model would report + return getattr(self._active, 'chestnut', False) + + @property + def handovers(self) -> int: + """Moves on every swap and demote, and on the frame that decides a lag + demote. modeld resets its dropped-frame filter when this changes across + run(), so the stall of a handover is not lag, as for a chestnut's fallback. + A count and not modelV2.big: a swap whose first frame fails and demotes in + the same run() leaves `chestnut` as it was.""" + return self._handovers + + @property + def big_model_state(self) -> str: + """modelDataV2SP.acceleratorState, one of its enum names (STATES).""" + if self._stop.is_set(): + return 'unavailable' + if self._active is not self._small: + return 'running' + if self.big_model_available: + # nothing left to wait for but a window. The icon says so rather than + # pulsing "loading" for the rest of a drive with no stop in it + return 'ready' + return 'retrying' if self._failures else 'joining' + + @property + def client(self): + # read by the status publisher on every send, so the telemetry follows the link + return getattr(self._active, 'client', None) + + @property + def desire_key(self) -> str: + return self._small.desire_key + + @property + def numpy_inputs(self): + return self._small.numpy_inputs + + @property + def lat_delay(self): + return self._active.lat_delay + + @lat_delay.setter + def lat_delay(self, value): + # modeld writes this every frame. Set on both, so a model that joins + # mid-drive does not start on a stale delay + self._small.lat_delay = value + if self._active is not self._small: + self._active.lat_delay = value + + @property + def PLANPLUS_CONTROL(self): + return self._active.PLANPLUS_CONTROL + + @PLANPLUS_CONTROL.setter + def PLANPLUS_CONTROL(self, value): + self._small.PLANPLUS_CONTROL = value + if self._active is not self._small: + self._active.PLANPLUS_CONTROL = value + + @property + def frame_drop_ratio(self) -> float: + return self._frame_drop_ratio + + @frame_drop_ratio.setter + def frame_drop_ratio(self, value): + # modeld's frameDropPerc / 100, written before every run(). Read here, by + # the lag rule; it lands on the small model too, as every write does + self._frame_drop_ratio = value + self._small.frame_drop_ratio = value + + def __getattr__(self, name): + # Only for names this class does not define. Without it, a comma or + # sunnypilot sync that adds one read was an AttributeError on the frame + # thread, which modeld re-raises: modeld dead for the drive, on jetlink + # devices only. Reads only; a new write lands here and nowhere else, so + # each one modeld makes has its own setter above + if name.startswith('_'): + # also what keeps __init__ from recursing before _active exists + raise AttributeError(name) + return getattr(self._active, name) + + # -- the frame path --------------------------------------------------------- + + def run(self, bufs, transforms, inputs, after_enqueue=None): + self._adopt() + if self._lagging: + # the last frame was the large model's last, published as it came + self._lagging = False + self._demote(BEHIND) + elif self._active is not self._small and self._frame_drop_ratio > DROP_LIMIT: + # modeld is behind the large model; this frame is the small model's + self._log.warning("jetlink: modeld dropped %.2f %% of camera frames behind the large model, " + "the small model drives from this one", self._frame_drop_ratio * 100) + self._demote(BEHIND) + self._note_proving() + self._maybe_swap() + if self._active is self._small: + self._small_frames += 1 + return self._run_small(bufs, transforms, inputs, after_enqueue) + return self._run_big(bufs, transforms, inputs, after_enqueue) + + def _run_small(self, bufs, transforms, inputs, after_enqueue): + """The small model's frame, and the large model's too when there is one: + warped now, sent once the small model's own work is on the GPU (its + after_enqueue), so the send overlaps it. A link that fails here is lost + before it drove: let go and reopened, with the small model's frame + unharmed.""" + big = self._big + if big is None: + return self._small.run(bufs, transforms, inputs, after_enqueue) + try: + big.prepare(bufs, transforms, inputs) + except Exception: + self._log.exception("jetlink: the large model's link failed with the small model driving, reopening") + self._retire(LOST) + return self._small.run(bufs, transforms, inputs, after_enqueue) + failed: list[BaseException] = [] + + def enqueued(): + try: + big.send() + except Exception as e: # never through the small model's run: modeld would make it fatal + failed.append(e) + if after_enqueue is not None: + after_enqueue() + + result = self._small.run(bufs, transforms, inputs, enqueued) + if not failed and not big.sent: + # a small model that did not call back; send now + try: + big.send() + except Exception as e: + failed.append(e) + if not failed: + # the proof: would this frame have been held? (model_state.check_shadow) + try: + big.check_shadow() + except Exception as e: + failed.append(e) + if failed: + self._log.error("jetlink: the large model's link failed with the small model driving, reopening", + exc_info=failed[0]) + self._retire(LOST) + return result + + def _run_big(self, bufs, transforms, inputs, after_enqueue): + big = self._big + started = time.monotonic() + try: + result = big.run(bufs, transforms, inputs, after_enqueue) + except Exception: + failed = time.monotonic() + self._log.exception("jetlink: large model failed mid-drive, back to the small model") + self._demote(LOST) + demoted = time.monotonic() + # re-run the frame rather than propagate: modeld's fallback is permanent, + # this one is retryable. after_enqueue is dropped, the large model may + # already have called it + result = self._small.run(bufs, transforms, inputs, None) + # all of it is one modeld frame, 42 to 107 ms on the 2026-09-29 drives + # with the small model warm. The dropped camera frames are forgiven + # (modeld sees the model change), but the log should say which part it was + done = time.monotonic() + self._log.warning("jetlink: fallback frame %.0f ms: link %.0f, demote %.0f, small model %.0f", + (done - started) * 1e3, (failed - started) * 1e3, + (demoted - failed) * 1e3, (done - demoted) * 1e3) + return result + took = time.monotonic() - started + self._big_frames += 1 + if self._loading: + # a connected engine can still fail its first inference; only announce + # readiness after a frame the caller can publish + self._loading = False + self._progress.clear() + self._log.warning("jetlink: large model joined mid-drive, modelV2.big is now true") + if self._big_frames > SETTLING_FRAMES: + why = getattr(big, 'behind', None) or (f'took {took * 1e3:.0f} ms' if self._fell_behind(took) else None) + if why: + # this frame's output is published as it came; its stall is forgiven now + self._lagging = True + self._handovers += 1 + self._log.warning("jetlink: large model %s, the small model drives from the next", why) + return result + + def _fell_behind(self, took: float) -> bool: + if took > LATE_FRAME: + return True + if took <= SLOW_FRAME: + return False + now = time.monotonic() + second = self._slow_at is not None and now - self._slow_at <= LAG_WINDOW + self._slow_at = now + return second + + @property + def _window_open(self) -> bool: + # standstill does not make an active longitudinal controller safe to swap + fresh = 0 <= time.monotonic() - self._engagement_updated < ENGAGEMENT_MAX_AGE + return fresh and not self._engaged + + def _adopt(self) -> None: + """Build the large model state from a join that has landed, on this + thread, which is where everything tinygrad touches must happen. From the + next frame on it is fed every frame (_run_small) and swapped in when the + window opens. A build that fails is backed off like a demote, or one that + fails the same way every time is a connect and a build per second for + the drive.""" + if self._joined is None: + return + with self._lock: + joined, self._joined = self._joined, None + if joined is None: + return + client, spec = joined + try: + t0 = time.monotonic() + big = self._build(client, spec) + big.lat_delay = self._small.lat_delay + self._log.warning("jetlink: built the large model state in %.0f ms; it runs every frame from here", + (time.monotonic() - t0) * 1000) + except Exception: + self._log.exception("jetlink: could not bring up the large model, staying small") + with self._lock: + self._retired = (client, NOT_UP) + self._back_off() + return + self._big = big + self._proving = CHECKING # the join thread said so at link ready + # the link is held from here, shadowing or driving: a shadow that ran for + # minutes and then went starts a new streak, as a drive that did + self._joined_at = time.monotonic() + + def _note_proving(self) -> None: + """The panel's line while the large model shadows, on a change only: + PROVEN once it keeps up, SLOW once it has not for SLOW_LINK_AFTER, and + CHECKING again after a hand-back. The first driven frame clears it.""" + if not self._shadowing: + return + if self._big.keeping_up: + line = PROVEN + elif time.monotonic() - self._joined_at > SLOW_LINK_AFTER: + line = SLOW + else: + line = CHECKING + if line != self._proving: + self._proving = line + self._log.warning("jetlink: large model %s", line) + self._report('connect', line) + + def _maybe_swap(self) -> None: + if not self._shadowing: + return + if not self._big.keeping_up or not self._window_open or self._small_frames < SMALL_WARMUP_FRAMES: + return + # nothing to build and nothing to reset (the module docstring says why) + self._big_frames = 0 + self._slow_at = None + self._handovers += 1 + self._active = self._big + + def _demote(self, why: str) -> None: + """Back to the small model, from a reset history. On the frame thread, so + nothing here waits: the join thread reads the port, reports and closes + the link, moments later.""" + self._retire(why) # while _shadowing still tells a drive from a shadow + self._active = self._small + self._handovers += 1 + self._loading = True + if self._reset_small is not None: + self._reset_small() + + def _retire(self, why: str) -> None: + """Let the large model go, driving or not, for the join thread to say + goodbye to and close or keep (_close_retired). Counted as a lost link or + a hand-back for lag, and backed off.""" + shadow = self._shadowing + big, self._big = self._big, None + if why == BEHIND: + self._lags += 1 + else: + self._drops += 1 + with self._lock: + self._retired = (big, why) + self._back_off(shadow) + + def _close_retired(self, retired, why: str) -> None: + """Say goodbye over the retired large model's link, then close it if it + is dead. A live one stays, on `Link`, for the next attempt, which is a + hello and a model request over it and not a reconnect: a model that fell + behind, or failed on the comma's side, leaves one. `retired` is the model + state, or the bare client of a build that failed.""" + self._leave(retired, LEAVING.get(why, why)) + if not getattr(getattr(retired, 'client', retired), 'dead', True): + return + try: + retired.close() + except Exception: + self._log.exception('jetlink: closing the retired link') + + def _leave(self, model, reason: str) -> None: + """Tell the server why the large model `model` stops using its link, with + what the comma measured over it (model_state.Trips). Off the frame thread.""" + self._say_leaving(getattr(model, 'client', None), reason, getattr(model, 'trips', None)) + + def _say_leaving(self, client, reason: str, trips=None) -> None: + """JetlinkClient.leave, which says nothing over a dead link.""" + leave = getattr(client, 'leave', None) + if leave is None: + return + measured = trips.summary() if trips is not None else {} + try: + leave(reason, drops=self._drops, lags=self._lags, **measured) + except Exception: + self._log.exception("jetlink: could not say why the link is left") + + def _back_off(self, shadow: bool = False) -> None: + """Push the next attempt out, further each time one fails on its heels. + + Each failed cycle is a swap frame, a demote frame and the alerts that go + with them. The first QUICK_RETRIES failures of a streak are retried in + REJOIN_DELAY_QUICK; the ones after double from REJOIN_DELAY. A join that + held for STABLE_SECONDS starts a new streak. A link lost while it only + shadowed waits SHADOW_REJOIN_DELAY. + """ + held = time.monotonic() - self._joined_at if self._joined_at else 0.0 + stable = bool(self._joined_at) and held > STABLE_SECONDS + self._failures = 1 if stable else self._failures + 1 + if stable: + self._replugged = False # a new streak may skip a backoff for a replug again + self._joined_at = 0.0 + if shadow: + delay = SHADOW_REJOIN_DELAY + elif self._failures <= QUICK_RETRIES: + delay = REJOIN_DELAY_QUICK + else: + delay = min(REJOIN_DELAY * 2 ** (self._failures - QUICK_RETRIES - 1), REJOIN_DELAY_MAX) + self._rejoin_at = time.monotonic() + delay + self._rejoin.set() + self._log.warning("jetlink: next attempt in %.0f s (failure %d, link held %.0f s, drop %d, lag %d this drive)", + delay, self._failures, held, self._drops, self._lags) + + # -- background ------------------------------------------------------------- + + def _report(self, stage: str, msg: str) -> None: + """Tell the UI what the join is waiting on. + + Offroad the record carries a provisioning run's progress; without this, a + Jetson that is not plugged in looked like one six seconds from loading. No + fraction to give, and the panel does not invent one. + """ + # the count is the diagnosis. A link that runs for minutes and then goes, + # again and again, is the cable, and the cable is the one thing the + # driver can do something about + self._progress.report(stage, 0.0, msg, drops=self._drops if self._drops >= DROPS_TO_BLAME_CABLE else 0) + + def _note_link_loss(self, why: str) -> None: + """Off the frame thread: what the comma's USB-C port sees now, next to the + failure (gadget.cc_orientation). A host still on the cable means the data + link alone went; the kernel logs the same edge as a Type-C disconnect.""" + cc = gadget.cc_orientation() + if cc is None: + port = "port state unknown" + else: + port = f"port sees a host (cc {cc})" if cc else "port sees no host (cc 0)" + # read now, before the teardown: a host back by the time the backoff starts + # has still been replugged + self._host_left = not gadget.host_attached() + self._log.warning("jetlink: %s, %s; drop %d, lag %d this drive", why, port, self._drops, self._lags) + self._report('connect', RECONNECTING[why]) + + def _join_loop(self) -> None: + """Open the link and get the engine ready. No tinygrad in here.""" + # off modeld's realtime core first. A thread started after + # config_realtime_process(7, 54) inherits SCHED_FIFO and the single-core + # affinity, and an equal-priority thread that wakes takes the core until + # it blocks. Measured with these threads left as created: exec p95 + # 90.8 ms, max 159.9, 5% frame drops, enough for modeldLagging + background_thread() + while not self._stop.is_set(): + # no timeout: once joined there is nothing to poll for, the frames find + # a dead link, and close() sets this. An idle wake per second is not + # free on modeld's core + self._rejoin.wait() + with self._lock: + retired, self._retired = self._retired, None + if retired is not None: + model, why = retired + if not self._stop.is_set(): + # before the teardown below, which can block: the port is read about + # when it let go. Not after close(), which has cleared the progress + self._note_link_loss(why) + # unbind and reader joins can block; only this thread does teardown, + # and it finishes before opening another link + self._close_retired(model, why) + if self._stop.is_set(): + return + self._rejoin.clear() + if self._wait_out_back_off(): + return + self._report('connect', 'waiting for jetlink') + try: + # the connect can take minutes when the picked model still has to be + # built, so it is handed the flag close() sets rather than polled + client, spec = self._connect(self._stop.is_set) + except Exception as e: + # expected while the Jetson boots. Not exception(): a stack trace every + # 5 s for the first minute of every drive is noise + self._log.warning("jetlink: not joined yet (%s), retrying in %.0fs", e, REJOIN_DELAY) + if self._stop.wait(REJOIN_DELAY): + return + self._rejoin.set() + continue + with self._lock: + if self._stop.is_set(): + client.close() + return + self._joined = (client, spec) + self._log.warning("jetlink: link ready, the large model shadows from the next frame") + self._report('connect', CHECKING) + + def _wait_out_back_off(self) -> bool: + """Until the next attempt is due, or a host configures the gadget again + after it went away. Once per failure streak: a host that flaps, a marginal + cable, would otherwise cycle the swap and its alerts with no backoff at + all. True once closed.""" + host_left, self._host_left = self._host_left or not gadget.host_attached(), False + while (left := self._rejoin_at - time.monotonic()) > 0: + if self._stop.wait(min(left, REPLUG_POLL)): + return True + attached = gadget.host_attached() + if attached and host_left and not self._replugged: + self._replugged = True + self._log.warning("jetlink: the host configured the gadget again, retrying now") + return False + host_left = host_left or not attached + return self._stop.is_set() + + def _watch_engagement(self) -> None: + background_thread() # off modeld's realtime core; see _join_loop + # made on this thread: the poller's sockets (a SubMaster) belong to the + # thread that made them + engaged = self._engagement() + while not self._stop.is_set(): + # rechecked on every poll, including ones with no news: that is when + # alive turns false. The frame thread expires the answer too + self._engaged = engaged(ENGAGEMENT_POLL_MS) + self._engagement_updated = time.monotonic() + + def close(self) -> None: + first = not self._stop.is_set() # the leave is said once; cleanups close twice + self._stop.set() + self._rejoin.set() + self._progress.clear() + with self._lock: + joined, self._joined = self._joined, None + if joined is not None: + if first: + self._say_leaving(joined[0], P.LEAVE_STOPPED) + joined[0].close() + big, self._big = self._big, None + close = getattr(big, 'close', None) + if close is not None: + if first: + self._leave(big, P.LEAVE_STOPPED) + close() + + +def join(parts, cam_w: int, cam_h: int, small) -> JoiningModelState: + """The model modeld runs: the small model now, the Jetson once it is there. + Returns straight away; the join runs in the background. + + The warp is loaded and warmed here, before the frame loop exists, rather + than at the swap on a driving frame. Sized from the recorded spec, which is + what the link will hand back; another geometry is rejected. + """ + from jetlink.openpilot import link as links + # here rather than in build(): the import then costs modeld's main thread + # before the frame loop, not the frame the swap lands on + from jetlink.openpilot.model_state import JetlinkModelState + op = parts.op + face = op.model_face() + ready: dict = {} + link = links.Link(parts.log) + + def prepare(): + from jetlink.openpilot.warp import prepare_reset, warm + # the gadget first, so the Jetson enumerates while the warp loads. Left to + # the join thread the bind landed ~3 s later, behind the small model's + # first frame, and one ignition had a 655 ms frame during the bind + links.present_early(link, background_thread) + cached = parts.spec.load() + if cached is not None: + img_h, img_w = cached.model_hw + geometry = (img_w * 2, img_h * 2) + else: + geometry = parts.warps.geometry()[2:] + try: + ready['reset_small'] = prepare_reset(small) + warp = parts.warps.load(cam_w, cam_h, *geometry) + warm(warp, face.frame_size(cam_w, cam_h)) + except Exception: + link.close() + raise + ready.update(warp=warp, geometry=geometry) + + def build(client, spec): + img_h, img_w = spec.model_hw + warp = ready.get('warp') if ready.get('geometry') == (img_w * 2, img_h * 2) else None + if warp is None: + raise RuntimeError('no prepared warp for the server model geometry') + return JetlinkModelState(cam_w, cam_h, client, spec, warp, face=face, log=parts.log, event=op.event) + + def connect(should_stop=None): + return links.open_link(parts, link, should_stop) + + return JoiningModelState(small, connect, build, prepare, reset_small=lambda: ready['reset_small'](), + progress=parts.progress, engagement=op.engagement, log=parts.log) diff --git a/jetlink_repo/jetlink/openpilot/link.py b/jetlink_repo/jetlink/openpilot/link.py new file mode 100644 index 0000000000..94643f54cf --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/link.py @@ -0,0 +1,324 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Opening the link over what the owner lends, and making the Jetson ready to +run the model that is picked. + +Both borrowers of the link do this. A provisioning run does it offroad, where +it can spend minutes downloading a gigabyte; modeld's join thread does it +onroad, where it cannot download but can upload a file the comma already has +and then wait out the build with the small model driving. What they share is +here: the lease and the client on it, the model's identity, the upload nobody +may make without proving the hash first, and the record of what the server +ended up with. +""" +from __future__ import annotations + +import functools +import threading +import time +from pathlib import Path + +from jetlink.comma import gadget + +# how long one attempt holds the gadget open waiting for a host. Not a deadline +# on the large model: JoiningModelState retries for the drive, since the Jetson +# boots after the comma is already onroad +CONNECT_TIMEOUT = 45.0 +CONNECT_DELAY = 0.5 +# four frame periods, and four times the slowest frame seen. A host that goes +# silent with no USB edge (a hung server, a phone's cable) costs one frame this +# long plus the small model's, inside modelV2's 0.5 s alive limit; 0.5 s here +# flashed commIssue on top of the fallback +INFERENCE_TIMEOUT = 0.2 +# how long the load may wait for the early gadget bind; a provisioning run may +# still be letting go of the endpoints +PRESENT_TIMEOUT = 5.0 +# An upload and a build on a busy box. The build itself is 102 to 294 s on this +# hardware; the ceiling is only there so a server that has stopped answering +# does not hold the caller for the rest of the day. +BUILD_TIMEOUT = 1800.0 + + +def _left(deadline: float | None, default: float) -> float: + """What is left of an absolute `deadline`, or `default` without one.""" + return default if deadline is None else max(0.0, deadline - time.monotonic()) + + +def connect(log, loan, deadline: float | None = None, name: str | None = None, wait: float | None = None): + """Open the link over what jetlinkd lent (jetlink.comma.lending): the + endpoint files, or on the cable the phone's next dial, which the loan + listens for (Loan.accept) for up to `wait` s, BORROW_TIMEOUT by default. + `deadline` is per frame, FRAME_TIMEOUT by default: modeld blocks on a frame + the way it blocks on a chestnut. `name` is what the server logs this + connection as: two comma processes share one gadget, and the Jetson's + journal has no clock to tell them apart by. + """ + from jetlink.client import FRAME_TIMEOUT, JetlinkClient + from jetlink.comma import lending + deadline = FRAME_TIMEOUT if deadline is None else deadline + if loan.cable: + sock = loan.accept(lending.BORROW_TIMEOUT if wait is None else wait) + peer = sock.getpeername()[0] + # the panels name the phone; the owner records it only while it holds the dial itself + gadget.note_link('cable', peer) + log.warning("jetlink: the phone dialed in from %s", peer) + return JetlinkClient.open_socket(sock, deadline=deadline, name=name) + return JetlinkClient.open_borrowed_ffs(loan.mount, loan.udc, bounce=loan.bounce, deadline=deadline, name=name) + + +class Link: + """modeld's end of the gadget: the lease, and the client that rides on it. + + Both are kept across join attempts. The lease lasts the drive, and opening + the gadget again per attempt is an unplug as the Jetson sees it, which, + while one boots and the join loop asks every few seconds, is an unplug a + cycle. So an attempt that cannot use the link leaves it here rather than + closing it, and only a deliberate close() lets go. On the cable the lease + is the listener, and a client that died is replaced by the phone's next + dial on it. + """ + + def __init__(self, log, name: str = 'modeld'): + self.log = log + self.name = name + self.client = None + self.loan = None + self._lock = threading.Lock() + self._abandoned = False + + def open(self, deadline: float | None = None): + """The client, opening one if we have not got one yet, or if the one held + is dead: a big model retired after a link loss closes its client, and + reusing it failed the next attempt with EBADF, 5 s after every loss.""" + if self.client is not None and self.client.dead: + self.close() + if self.client is None: + from jetlink.comma import lending + self.client = connect(self.log, name=self.name, loan=self._borrow(deadline), + wait=_left(deadline, lending.BORROW_TIMEOUT)) + return self.client + + def adopt(self, client) -> bool: + """Take a client another thread opened. False once we have given up waiting + for it, and then the caller closes what it opened.""" + with self._lock: + if self._abandoned: + return False + self.client = client + return True + + def abandon(self) -> None: + with self._lock: + self._abandoned = True + + def note_server(self, hello: dict) -> None: + """Pass the server's hello on to the owner over the lease, so every + drive's join refreshes what it knows of the far end (Loan.note_server). + Raises when that let the lease go: a client whose bounce goes over a + closed loan cannot free a stuck write, so the attempt starts over.""" + if self.loan is not None and not self.loan.note_server(hello) and self.loan.closed: + from jetlink.transport.base import LinkError + raise LinkError("the owner stopped answering on the loan") + + def close(self) -> None: + """Let the link go. The lease stays: jetlinkd should hold the gadget for + the whole drive, however many times the join has to start over.""" + client, self.client = self.client, None + if client is not None: + try: + client.close() + except Exception: + self.log.exception("jetlink: error closing the link") + + def _borrow(self, deadline: float | None): + """The lease on the gadget jetlinkd owns. Raises without one: only the + owner ever holds ep0, so the join loop asks again with the small model + driving. An owner whose lender cannot listen says so in the offroad alert.""" + from jetlink.comma import lending + if self.loan is not None and not self.loan.closed: + return self.loan + # bounded by whatever the caller has left: an early present that spends its + # whole budget here has nothing left to open the link with + self.loan = lending.borrow(self.name, timeout=_left(deadline, lending.BORROW_TIMEOUT)) + if self.loan is None: + raise TimeoutError("jetlinkd lent no link") + return self.loan + + +def present_early(link: Link, background) -> None: + """Take the link now, from a thread that is not modeld's. + + modeld's main thread is already SCHED_FIFO 54 on core 7, and the FunctionFS + reader the open creates would inherit that and preempt the frame loop, so the + helper first calls `background` to get off it. Bounded, so a hung open + cannot hold modeld's load; a helper that finishes late closes what it opened. + """ + deadline = time.monotonic() + PRESENT_TIMEOUT + + def present(): + background() + # retried: a provisioning run may still have the endpoints open, and a + # single try fails in milliseconds + while True: + try: + client = link.open(deadline) + break + except Exception as e: + if time.monotonic() >= deadline: + link.log.warning("jetlink: could not present the gadget early (%s), the join will", e) + return + time.sleep(0.2) + if not link.adopt(client): + client.close() + + t = threading.Thread(target=present, name='jetlink-present', daemon=True) + t.start() + t.join(max(0.0, deadline - time.monotonic()) + 0.5) + if t.is_alive(): + link.abandon() + link.log.warning("jetlink: presenting the gadget took over %.0f s, the join will", PRESENT_TIMEOUT) + + +def connect_patiently(link: Link): + """Open the link, tolerating a busy gadget or a Jetson that is still booting.""" + deadline = time.monotonic() + CONNECT_TIMEOUT + last = None + while True: + try: + client = link.open() + except Exception as e: + client, last = None, e + if client is not None: + # over a phone's cable wait_for_host returns at once and a TCP client's + # rebind is a no-op, so its network interface is never bounced + if gadget.wait_for_host(max(0.0, deadline - time.monotonic()), bounce=client.rebind, + report=lambda: link.log.warning("jetlink: gadget up, waiting for the jetson to enumerate")): + return client + # the link stays on `link`, still bound, for the next attempt + raise TimeoutError(f"no jetson attached within {CONNECT_TIMEOUT:.0f}s") + if time.monotonic() > deadline: + # nothing is held here: this is a gadget we could not open at all, + # usually a provisioning run still finishing an exchange on the endpoints + raise last if last is not None else TimeoutError("could not open the link") + link.log.warning("jetlink: link not ready (%s), retrying", last) + time.sleep(CONNECT_DELAY) + + +# -- making the Jetson ready ------------------------------------------------------ + +def identity(parts, entry: dict) -> tuple[str, int]: + """The picked model's sha256 and byte count, from the catalog model's LFS + pointer, so the comma can name the model without holding or hashing the + ONNX. Looked up once per model ever and kept in a param: the one part of + provisioning that needs the internet.""" + sha256, nbytes = entry.get('oid'), entry.get('size') + if sha256 and nbytes: + return sha256, int(nbytes) + parts.progress.report('connect', 0.0, 'finding model') + return parts.models.resolve_pointer(entry['ref']) + + +# What has already been hashed in this process, keyed on the file as it was +# then. A join that fails and tries again would otherwise read a gigabyte off +# the disk every time, on a thread that is now doing it next to a running frame +# loop. +_hashed: dict[tuple[str, int, int], str] = {} + + +def verified_upload(log, model_path: Path | None, sha256: str, nbytes: int) -> Path | None: + """The file to upload, once its hash is proven to match the registry: under + a sha the bytes do not have, the Jetson's plan would lie about its contents.""" + if model_path is None: + return None + try: + st = model_path.stat() + if st.st_size != nbytes: + log.error("jetlink: %s is %d bytes, the registry says %d; not uploading it", + model_path.name, st.st_size, nbytes) + return None + key = (str(model_path), st.st_size, st.st_mtime_ns) + have = _hashed.get(key) + if have is None: + from jetlink.spec import sha256_file + have, _ = sha256_file(str(model_path)) + _hashed[key] = have + if have != sha256: + log.error("jetlink: %s hashes to %s, the registry says %s; not uploading it", + model_path.name, have[:16], sha256[:16]) + return None + except OSError: + return None + return model_path + + +def ensure(parts, client, sha256: str, nbytes: int, model_path: Path | None, *, + progress=None, should_stop=None, build_timeout: float = BUILD_TIMEOUT): + """Make the server ready for this model and remember what it answered. + Asks without the file first: the server answers from the sha alone when it + has the model, which is every parked poll and every join. EngineMissing means + the Jetson has neither the plan nor the bytes, and neither has the caller.""" + from jetlink.client import EngineMissing + ask = functools.partial(client.ensure_engine, sha256, nbytes, progress=progress, + build_timeout=build_timeout, should_stop=should_stop) + try: + spec = ask(onnx_path=None) + except EngineMissing: + upload = verified_upload(parts.log, model_path, sha256, nbytes) + if upload is None: + raise + spec = ask(onnx_path=upload) + parts.spec.store(spec) + return spec + + +def open_link(parts, link: Link, should_stop=None): + """Get a client and a spec. Link IO only, so it is safe off modeld's thread; + everything that touches tinygrad stays in the joining state's build. + + The picked model is built here if the Jetson has not got it, minutes with + the small model driving: a provisioning run works offroad only, so a model + picked in the driveway cost the whole drive otherwise. Only reached with + the small model driving (the join loop stops asking once joined), so a build + here never unloads an engine that is steering. Whatever this attempt cannot + use stays on `link`, still open, for the next one. + """ + from jetlink.client import EngineMissing + + selected = parts.models.selected_model() + if selected is None: + raise RuntimeError('no large model has been picked yet') + + # the endpoints may still be held by a provisioning run, and the Jetson may still be + # booting; both resolve on their own + client = connect_patiently(link) + try: + hello = client.hello(timeout=10.0) + parts.log.warning("jetlink: %s trt %s, engine %s, loaded %s", + hello.get('device'), hello.get('trt_version'), + hello.get('engine_state'), str(hello.get('loaded'))[:16]) + link.note_server(hello) + sha256, nbytes = identity(parts, selected) + if not parts.spec.engine_ready_for(sha256): + parts.log.warning("jetlink: %s is not built yet, building it with the small model driving", + selected.get('name', sha256[:16])) + try: + # normally one round trip, since the provisioning run left the engine loaded. A + # server that restarted reloads from the plan cache, 13 to 25 s; one + # that has never seen this model builds it, 102 to 294 s + spec = ensure(parts, client, sha256, nbytes, parts.models.shipped_model_path(), + progress=parts.progress.report_with_eta, should_stop=should_stop) + except EngineMissing: + # neither end has the bytes. Fetching them needs the internet and a + # gigabyte of it, which is a parked job; clear the record so the next + # parked period provisions again + parts.spec.clear_ready() + raise + client.deadline = INFERENCE_TIMEOUT + return client, spec + except BaseException: + link.close() + raise diff --git a/jetlink_repo/jetlink/openpilot/model_state.py b/jetlink_repo/jetlink/openpilot/model_state.py new file mode 100644 index 0000000000..4067245874 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/model_state.py @@ -0,0 +1,421 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +A ModelState whose policy runs on the Jetson. + +Everything that touches the car stays on the comma: cameras, calibration, the +warp, the parser, controlsd, panda, CAN. The Jetson is a pure function, warped +frames and context in, 18452 floats out, and holds no control state. + +The split is at `run_policy`. The warp stays on the comma's GPU: its input is a +2 MB camera buffer already there and its output the 393 KB the link carries +anyway. The history queues live on the Jetson, shipping them would cost ~10 MB +a frame instead of ~0.5 MB. Upstream fused warp and policy into one JIT, so +the fork's build compiles a standalone warp; see warp. + +A model that keeps its own history (openpilot #38916, Cinque Terre V3 on) +takes the same warped frame and the same scalars; its hidden state never +leaves the Jetson, so there is no prev_feat to send back. The spec says which. + +What openpilot's modeld reads off a ModelState comes from the fork's adapter +(ModelFace): comma's constants, parser, smoothing and action function. +tinygrad is imported when a model is built, never at module level: only +modeld and the build have it. +""" +from __future__ import annotations + +import os +import time +from collections import deque +from collections.abc import Callable +from dataclasses import dataclass + +import numpy as np + +from jetlink.openpilot.warp import call_warp + +SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') +SLOW_FRAME = 0.05 # the full 20 Hz budget, not just the largest outliers +# How far into a frame end() waits for the reply before publishing the +# previous frame's output again (a held frame). Past this the camera frame +# would be dropped: modeld's own work around run() takes the rest of the 50 ms. +# A held frame is a plan one frame old, which the 10 s plan and the actuator +# delay both dwarf, where a dropped frame is the same stale plan plus a count +# toward selfdrived's modeldLagging. None waits the client's deadline, as before +HOLD_FRAME = 0.046 +# Held frames the large model may make before it is behind rather than +# tailing: this many in a row, or more than HOLDS_ALLOWED within HOLD_WINDOW. +# A host at 40 ms a frame with a tail past 46 (an iPhone) holds a few frames +# in a hundred, each a plan one frame old; one at 60 ms holds every frame. +# More in a row than the client's deadline (link.INFERENCE_TIMEOUT, 0.2 s) +# lets a quiet host run to, so a run of holds is always a host that answers, +# late; one that has stopped answering fails the link first +# (JetlinkClient.infer_begin), as a lost link +HOLDS_IN_A_ROW = 5 +HOLDS_ALLOWED = 20 +HOLD_WINDOW = 10.0 +# How long a host may answer none of its shadow frames before the link is +# abandoned, where a driving one gets the client's deadline +# (link.INFERENCE_TIMEOUT, 0.2 s). The small model drives meanwhile, and a +# phone's first frames after a join are Neural Engine warm-up: at 0.2 s an +# iPhone 18 Pro lost 11 of 18 joins inside 0.33 s (2026-10-04) +SHADOW_SILENCE = 1.0 +# frames between asks for the server's telemetry, which rides on the response: +# every second one, as modeld sent a chestnut's state (20 Hz over 10 Hz) +TELEMETRY_EVERY = 2 +# telemetry goes to the log at most this often +TELEMETRY_PERIOD = 1.0 +# frames Trips keeps the timings of: a minute at 20 Hz +TRIPS_KEPT = 1200 +# shadow frames in a row that would not have been held (check_shadow) before +# the large model may drive (joining reads `keeping_up`): a second at 20 Hz. +# A host cold from a reconnect proves itself with the small model driving, +# not at the wheel, where an iPhone's every return was a soft disable +# within a second (2026-10-04) +PROVING_FRAMES = 20 + + +class Trips: + """What the comma measures of its frames over the link: the whole run, warp + to parsed output, which is the frame as modeld waits on it, against the + server's own total, which is all the server can see. Summarised for the + leave (JetlinkClient.leave), so a phone's log carries the half of the frame + budget it cannot measure. Bounded to the last TRIPS_KEPT frames.""" + + def __init__(self): + self.sent = 0 # frames sent; the ones never ended ran with the small model driving + self.frames = 0 # frames waited for and published + self.over = 0 # frames past SLOW_FRAME, the whole 20 Hz budget + self.held = 0 # frames whose reply was late and published the previous one + self.started = time.monotonic() + self._whole_ms: deque[float] = deque(maxlen=TRIPS_KEPT) + self._server_ms: deque[float] = deque(maxlen=TRIPS_KEPT) + + def record(self, whole_s: float, server_us: int) -> None: + self.frames += 1 + if whole_s > SLOW_FRAME: + self.over += 1 + self._whole_ms.append(whole_s * 1e3) + self._server_ms.append(server_us / 1e3) + + def summary(self) -> dict: + """frames, over, held, shadowed, span_s, and over the frames kept: p50_ms, + p99_ms, max_ms of the whole frame and server_ms, the server's mean total.""" + out = {'frames': self.frames, 'over': self.over, 'held': self.held, 'shadowed': self.sent - self.frames, + 'span_s': round(time.monotonic() - self.started, 1)} + if self._whole_ms: + whole = sorted(self._whole_ms) + out.update(p50_ms=round(whole[len(whole) // 2], 1), p99_ms=round(whole[int(0.99 * (len(whole) - 1))], 1), + max_ms=round(whole[-1], 1), server_ms=round(sum(self._server_ms) / len(self._server_ms), 1)) + return out + + +@dataclass +class Frame: + """One frame on its way to the host: when the warp and the readback + finished, the bytes the wire takes, then its seq and send time once sent.""" + t0: float + t1: float + t2: float + data: object + seq: int | None = None + sent: float = 0.0 + telemetry: bool = False + + +class JetlinkModelState: + """Duck-types openpilot's modeld ModelState, and modeld_v2's where it differs. + + The small model is whatever bundle the user picked, on whichever modeld that + bundle needs. sunnypilot's modeld_v2 reads constants, smoothing and the + action function off the ModelState; stock modeld has them as module + constants. This is comma's large model, so they are comma's. + + A frame is prepare(), send() and end(). run() is all three, as modeld calls + it. The joining model (joining.JoiningModelState) prepares and sends every + frame the small model drives and never ends them, so the host's model is + warm and current at the swap; it reads `behind` after each driven frame. + """ + + prev_desire: np.ndarray # for tracking the rising edge of the pulse + + def __init__(self, cam_w: int, cam_h: int, client, spec, warp, *, face, log, event): + from tinygrad.device import Device + from tinygrad.tensor import Tensor + self._tensor = Tensor + self._log = log + # (name, **fields): a structured log line, cloudlog.event on a comma + self._event = event + self.face = face + # the joining model sets it from the small model before the first frame, + # and modeld writes it every frame after + self.lat_delay = 0.0 + self.constants = face.constants + self.LAT_SMOOTH_SECONDS = face.lat_smooth_seconds + self.LONG_SMOOTH_SECONDS = face.long_smooth_seconds + self.PLANPLUS_CONTROL = 1.0 + self.get_action_from_model = face.get_action_from_model + + self.client = client + self.spec = spec + # Over a phone's cable, hand the socket the warp's GPU mapping itself: the + # kernel copies it while the first segments are already on the wire, where + # copying it here first held the send back 2.5 ms. On the Mac stand-in the + # comma's side of a frame was 0.6 ms faster at p50 and 2 ms at p99. USB + # keeps the host copy it was measured with. The client's transport says + # which, as the owner lent it + self.send_from_gpu = client.t.link_info().get('kind') == 'cable' + # not chestnut hardware, but the same role: modelV2.big, the UI and the + # model manager key off this flag + self.chestnut = True + + # a warm warp, loaded ahead: the first call costs ~2 s and this can run on + # modeld's frame thread (warp.warm) + self.warp = warp + + self.input_shapes = spec.input_shapes + self.output_slices = spec.output_slices + # from the spec, not ModelConstants: the server derives its history stride + # from the same field + self.frame_skip = spec.frame_skip + + # the warp's own two inputs stay separate NPY tensors; everything past the + # warp is the server's. Writing the numpy array is what feeds the JIT + self.npy = {'tfm': np.zeros((3, 3), dtype=np.float32), + 'big_tfm': np.zeros((3, 3), dtype=np.float32)} + self.warp_inputs = {k: Tensor(v, device='NPY').realize() for k, v in self.npy.items()} + + # compile_modeld.make_input_queues' packed_npy_inputs, minus the GPU queues + # the server owns; for a stateful graph, minus prev_feat too + self.packed = np.zeros(spec.packed_nelem, dtype=np.float32) + self.npy.update({k: self.packed[at].reshape(shape) for k, (at, shape) in spec.packed_layout.items()}) + + # read once: it must be the device the cached JIT was compiled against + self.warp_dev = Device.DEFAULT + self.prev_desire = np.zeros(face.desire_len, dtype=np.float32) + self.parser = face.parser() + self.frame_size = face.frame_size(cam_w, cam_h) + # the camera buffers modeld hands over, whatever the graph calls its inputs + self.vision_input_names = ['img', 'big_img'] + self.full_frames: dict = {} + self._blob_cache: dict = {} + self._need_reset = True + self._frame_id = 0 + self._last_logged = 0.0 + self._last_hold_logged = 0.0 + self.trips = Trips() + # the frame prepared and not yet ended, if any + self._frame: Frame | None = None + # the newest output parsed, with what it parsed to: a held frame + # publishes the dict again rather than parsing the array a second time, + # which it must not, the parser's softmax working in place + self._parsed: tuple[np.ndarray, dict] | None = None + # when the large model held a frame, within HOLD_WINDOW, and how many in + # a row; `behind` says why it should hand back after the last frame + self._holds: deque[float] = deque() + self._holds_in_a_row = 0 + self.behind: str | None = None + # shadow frames in a row that would not have been held + self._kept_up = 0 + + def slice_outputs(self, model_outputs: np.ndarray, output_slices: dict[str, slice]) -> dict[str, np.ndarray]: + return {k: model_outputs[np.newaxis, v] for k, v in output_slices.items()} + + def log_telemetry(self) -> None: + """The server's health, piggybacked on the previous response, to the log + at 1 Hz. chestnutState is comma's board on the wire and a Jetson's + telemetry has no message of its own yet.""" + telemetry = self.client.last_state + if not telemetry: + return + now = time.monotonic() + if now - self._last_logged < TELEMETRY_PERIOD: + return + self._last_logged = now + self._event("jetlinkTelemetry", dead=bool(self.client.dead), **telemetry) + + def prepare(self, bufs: dict, transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray]) -> None: + """Take this frame up: read what the host has answered so far, then warp + the frame for it. send() puts it on the link and end() waits for its + answer. A frame sent and never ended ran on the host with the small model + driving (a shadow frame); its timings reach the log here, at the next.""" + frame = self._frame + if frame is not None and frame.seq is not None and (self._frame_id <= 3 or frame.sent - frame.t0 > SLOW_FRAME / 2): + # the frame before was sent and never ended: a shadow frame + self._log.warning("jetlink: shadow frame %d warp %.1f data %.1f send %.1f ms; %d unanswered", self._frame_id, + (frame.t1 - frame.t0) * 1e3, (frame.t2 - frame.t1) * 1e3, (frame.sent - frame.t2) * 1e3, + self.client.unanswered) + self.client.drain() + self.log_telemetry() + + for key in bufs.keys(): + ptr = np.frombuffer(bufs[key].data, dtype=np.uint8).ctypes.data + cache_key = (key, ptr) + if cache_key not in self._blob_cache: + self._blob_cache[cache_key] = self._tensor.from_blob(ptr, (self.frame_size,), dtype='uint8', device=self.warp_dev) + self.full_frames[key] = self._blob_cache[cache_key] + + # Model decides when action is completed, so desire input is just a pulse triggered on rising edge. + # Under whichever name the loop keyed it: stock modeld's desire_pulse, or a modeld_v2 bundle's own + desire = inputs[next(k for k in inputs if k.startswith('desire'))] + desire[0] = 0 + self.npy['desire'][:] = np.where(desire - self.prev_desire > .99, desire, 0) + self.prev_desire[:] = desire + self.npy['traffic_convention'][:] = inputs['traffic_convention'] + self.npy['action_t'][:] = inputs['action_t'] + self.npy['tfm'][:, :] = transforms['img'][:, :] + self.npy['big_tfm'][:, :] = transforms['big_img'][:, :] + + t0 = time.perf_counter() + warped = call_warp(self.warp, **self.warp_inputs, frame=self.full_frames['img'], big_frame=self.full_frames['big_img']) + t1 = time.perf_counter() + # .data() rather than .numpy(): same ~2.5 ms mean (a write-combined GPU + # mapping read), but no per-frame allocation and no 52 ms outlier. The + # mapping is safe to send: infer_begin returns only once the socket has + # copied all of it, before the next warp can write it + if self.send_from_gpu: + data = warped._buffer().as_memoryview(allow_zero_copy=True) + else: + data = warped.data() + self._frame = Frame(t0, t1, time.perf_counter(), data) + + def send(self, want_telemetry: bool = False, silence: float | None = SHADOW_SILENCE) -> None: + """The prepared frame to the host. It asks for the server's telemetry + when the log is due for it, or when told to: modeld asks on the frames it + would have sent a chestnut's state on. Sent from outside, it is a shadow + frame and a quiet host has `silence`; run() sends with the client's + deadline.""" + frame = self._frame + self._frame_id += 1 + frame.telemetry = want_telemetry or time.monotonic() - self._last_logged >= TELEMETRY_PERIOD + frame.seq = self.client.infer_begin(frame.data, self.packed, self._frame_id, reset=self._need_reset, + want_state=frame.telemetry, silence=silence) + frame.sent = time.perf_counter() + frame.data = None + self._need_reset = False + self.trips.sent += 1 + + @property + def sent(self) -> bool: + """Has the prepared frame gone out?""" + return self._frame is not None and self._frame.seq is not None + + def end(self, after_enqueue: Callable[[], None] | None = None) -> dict[str, np.ndarray]: + """The sent frame's output, as modeld blocks on a chestnut's, but not past + HOLD_FRAME once there is an output to publish again: a frame that long + is a dropped camera frame. Only a stall past the client's deadline raises, + into the joining state's demotion to the small model.""" + frame, self._frame = self._frame, None + # publish health while the Jetson works + if after_enqueue is not None: + after_enqueue() + waiting_from = time.perf_counter() + hold = None + if HOLD_FRAME and self.client.last_output is not None: + hold = self._hold_left(frame) + model_output = self.client.infer_end(frame.seq, hold=hold) + t4 = time.perf_counter() + held = model_output is None + self.behind = self._note_hold(held) + outputs = self._outputs(self.client.last_output if held else model_output) + self.trips.record(t4 - frame.t0, self.client.last_timings[2]) + # a frame past the budget is a dropped camera frame; send against reply + # says which end it was. Held frames are a steady few on a marginal host, + # so after the first they go to the log at TELEMETRY_PERIOD + log_hold = held and (self.trips.held <= 3 or t4 - self._last_hold_logged >= TELEMETRY_PERIOD) + if self._frame_id <= 3 or t4 - frame.t0 > SLOW_FRAME or log_hold: + if held: + self._last_hold_logged = t4 + # persisted on the comma so a drive can separate server execution from + # receive stalls once the Jetson is offline; server total excludes USB + gpu_us, queue_us, total_us = self.client.last_timings + receive = getattr(self.client.t, 'last_receive', {}) + self._log.warning("jetlink: frame %d warp %.1f data %.1f send %.1f wait %.1f ms%s; " + "server gpu %.1f queue %.1f total %.1f ms; ffs maxima prepare %.1f read_wait %.1f handoff %.1f ms", + self._frame_id, (frame.t1 - frame.t0) * 1e3, (frame.t2 - frame.t1) * 1e3, + (frame.sent - frame.t2) * 1e3, (t4 - waiting_from) * 1e3, + f', held ({self.trips.held} so far)' if held else '', + gpu_us / 1e3, queue_us / 1e3, total_us / 1e3, receive.get('prepare', 0.0) * 1e3, + receive.get('read_wait', 0.0) * 1e3, receive.get('handoff', 0.0) * 1e3) + return outputs + + def run(self, bufs: dict, transforms: dict[str, np.ndarray], + inputs: dict[str, np.ndarray], after_enqueue: Callable[[], None] | None = None) -> dict[str, np.ndarray]: + self.prepare(bufs, transforms, inputs) + self.send(after_enqueue is not None, silence=None) + return self.end(after_enqueue) + + @property + def keeping_up(self) -> bool: + """Would the host have driven the last PROVING_FRAMES shadow frames + without a hold? What the joining model swaps on; nothing while it drives.""" + return self._kept_up >= PROVING_FRAMES + + def check_shadow(self) -> None: + """Score the shadow frame just sent as end() would have, now that the + small model's own frame is done: kept up when its reply is back HOLD_FRAME + or less after its warp began, waited for until then as end() waits. The + joining model calls this once per shadow frame; a reply not back by then + is read by the next prepare(), and starts the proof over. The shadow + frame went out after the small model's work was queued, later than a + driven one, so a host that passes here does not hold at the wheel.""" + frame = self._frame + if frame is None or frame.seq is None: + return + self.client.drain() + left = self._hold_left(frame) + if not self.client.unanswered: + kept = left >= 0 + elif self.client.unanswered == 1 and left > 0: + kept = self.client.infer_end(frame.seq, hold=left) is not None + else: + # past the hold, or behind older frames: replies come in order, so + # this one cannot make it, and waiting would only hold the small model + kept = False + self._kept_up = self._kept_up + 1 if kept else 0 + + @staticmethod + def _hold_left(frame) -> float: + """What is left of the hold for `frame`, from its warp's start.""" + return (HOLD_FRAME or SLOW_FRAME) - (time.perf_counter() - frame.t0) + + def _note_hold(self, held: bool) -> str | None: + """Why the large model should hand back after this frame, if it should: + HOLDS_IN_A_ROW held frames running, or more than HOLDS_ALLOWED in + HOLD_WINDOW. None while it is keeping up.""" + if not held: + self._holds_in_a_row = 0 + return None + self.trips.held += 1 + now = time.monotonic() + self._holds.append(now) + while now - self._holds[0] > HOLD_WINDOW: + self._holds.popleft() + self._holds_in_a_row += 1 + if self._holds_in_a_row >= HOLDS_IN_A_ROW: + return f'held {self._holds_in_a_row} frames in a row' + if len(self._holds) > HOLDS_ALLOWED: + return f'held {len(self._holds)} frames in {HOLD_WINDOW:.0f} s' + return None + + def _outputs(self, model_output: np.ndarray) -> dict[str, np.ndarray]: + """The output as modeld reads it. An array parsed before (a held frame's) + gives the same dict again: the parser's softmax works in place on it.""" + if self._parsed is not None and self._parsed[0] is model_output: + return self._parsed[1] + # the non-finite check runs on the server (Status.NOT_FINITE -> LinkError), + # so modeld's big->small failover fires as it does for a chestnut + outputs_dict = self.parser.parse_outputs(self.slice_outputs(model_output, self.output_slices)) + self.spec.feed_back(self.packed, model_output) + if SEND_RAW_PRED: + outputs_dict['raw_pred'] = model_output.copy() + self._parsed = (model_output, outputs_dict) + return outputs_dict + + def close(self) -> None: + """Let go of the link; the joining state calls this on a model it retires.""" + self.client.close() + diff --git a/jetlink_repo/jetlink/openpilot/models.py b/jetlink_repo/jetlink/openpilot/models.py new file mode 100644 index 0000000000..2ef4f957d5 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/models.py @@ -0,0 +1,244 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Which large model runs, and where its bytes are. + +The large model is the model manager's big-model pick, the same slot a +chestnut runs from (Keys.big_model). Its bundles are tinygrad pkls for a GPU +the Jetson does not have, but each one names the comma commit it was compiled +from, and that commit's ONNX in comma's LFS is what the Jetson runs. The +catalog says what exists, the slot says which one, the pointer at the commit +says which bytes. A commit that ships a precompiled pkl instead names its +export, and jetlink's registry follows that to comma's model repo; see +jetlink.registry.lfs. + +Without a model manager (Keys.big_model and Keys.catalog None) there is no +pick and no catalog, and the large model is jetlink's own default, named by its +ref: its pointer resolves from the ref as any pick's does. +""" +from __future__ import annotations + +import re +import time +from pathlib import Path + +from jetlink.registry.catalog import DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF + +POINTER_TIMEOUT = 10.0 +_REF = re.compile(r'[0-9a-f]{40}') +# the build date a catalog name ends in, " (September 17, 2026)"; any other parenthesis stays +_TRAILING_DATE = re.compile(r' \([A-Za-z]+ \d{1,2}, \d{4}\)$') +# the index and the slot are JSON params, and the UI names the active model +# every frame; the status line can lag a new pick by this long +INDEX_TTL = 2.0 + + +class Models: + def __init__(self, op): + self.op = op + self._index_cache: tuple[float, list[dict], dict[str, dict]] | None = None + self._slot_cache: tuple[float, dict | None] | None = None + + def catalog(self) -> list[dict]: + """sunnypilot's big-model bundles as {name, ref}, newest first, as the model + manager's own picker lists them. From the cached JSON rather than the parsed + bundles: parsing builds capnp objects and writes chunk manifests, for two + fields. Never raises.""" + try: + key = self.op.keys.catalog + bundles = ((self.op.get(key) if key else None) or {}).get('bundles', []) + found = [b for b in bundles if _REF.fullmatch(str(b.get('ref'))) + and int(b.get('minimum_selector_version', 0)) == self.op.catalog_selector] + except Exception: + self.op.log.exception("jetlink: could not read the big-model catalog") + return [] + found.sort(key=lambda b: int(b.get('index', 0)), reverse=True) + return [{'name': str(b.get('display_name') or b['ref'][:10]), 'ref': b['ref']} for b in found] + + def pointers(self) -> dict[str, dict]: + value = self.op.get(self.op.keys.pointers) + return value if isinstance(value, dict) else {} + + def fetch_pointer(self, ref: str) -> tuple[str, int]: + """The oid and size of the ONNX a comma commit names, in its tree or, for a + precompiled-pkl commit, in comma's model repo. The Jetson's registry does + the same lookup, so the two ends agree on every model's identity.""" + from jetlink.registry.lfs import fetch_pointer as registry_fetch_pointer + pointer = registry_fetch_pointer(ref, timeout=POINTER_TIMEOUT) + return pointer.oid, pointer.size + + def resolve_pointer(self, ref: str) -> tuple[str, int]: + """The oid and size behind a catalog model, fetched the first time and kept for good.""" + known = self.pointers() + if ref in known: + return known[ref]['oid'], int(known[ref]['size']) + oid, size = self.fetch_pointer(ref) + known[ref] = {'oid': oid, 'size': size} + # blocking: the next lookup reads this back, and a put still in flight would be lost under it + self.op.put(self.op.keys.pointers, known, block=True) + self._index_cache = None + self.op.log.warning("jetlink: %s is %s, %d MB", ref[:10], oid[:16], size >> 20) + return oid, size + + @staticmethod + def _row(name: str, ref: str, known: dict[str, dict]) -> dict: + p = known.get(ref) or {} + return {'name': name, 'ref': ref, 'oid': p.get('oid'), 'size': int(p['size']) if p.get('size') else None} + + def _index(self) -> tuple[list[dict], dict[str, dict]]: + """The catalog's rows and the pointers behind them, read together once per INDEX_TTL.""" + now = time.monotonic() + if self._index_cache is None or now - self._index_cache[0] >= INDEX_TTL: + known = self.pointers() + self._index_cache = (now, [self._row(b['name'], b['ref'], known) for b in self.catalog()], known) + return self._index_cache[1], self._index_cache[2] + + def model_index(self) -> list[dict]: + """Every catalog model, {name, ref, oid, size}. oid and size are None until + the model has been selected and resolved. No network.""" + return self._index()[0] + + def selected_slot(self) -> dict | None: + """The big-model slot's pick as {name, ref}, from the raw param rather than a + parsed bundle, and memoised: the UI asks every frame.""" + now = time.monotonic() + if self._slot_cache is not None and now - self._slot_cache[0] < INDEX_TTL: + return self._slot_cache[1] + key = self.op.keys.big_model + slot = self.op.get(key) if key else None + ref = slot.get('ref') if isinstance(slot, dict) else None + pick = None + if isinstance(ref, str) and ref: + pick = {'name': str(slot.get('displayName') or ref[:10]), 'ref': ref} + self._slot_cache = (now, pick) + return pick + + def default_model(self) -> dict | None: + """What runs with no pick: jetlink's default big model, else the newest the + catalog lists. Not the chestnut's default, which is the model in the tree. + A fork with no catalog at all (Keys.catalog None) gets jetlink's default by + its ref; one whose catalog has not been fetched yet waits for it, as ever.""" + if self.op.keys.catalog is None: + return self._row(DEFAULT_BIG_MODEL_NAME, DEFAULT_BIG_MODEL_REF, self._index()[1]) + models = self.model_index() + return next((m for m in models if m['ref'] == DEFAULT_BIG_MODEL_REF), models[0] if models else None) + + def default_model_name(self) -> str | None: + """default_model()'s name without the catalog's trailing build date, the + form the chestnut's DEFAULT_BIG_MODEL has. Per frame from the UI, off the + index's cache.""" + model = self.default_model() + return _TRAILING_DATE.sub('', model['name']) if model else None + + def selected_model(self) -> dict | None: + """The slot's pick, listed in the catalog or not: a ref is enough to find its + pointer. With no pick, default_model().""" + if (pick := self.selected_slot()) is not None: + return self._row(pick['name'], pick['ref'], self._index()[1]) + return self.default_model() + + def selected_model_name(self) -> str | None: + selected = self.selected_model() + return selected['name'] if selected else None + + def model_dir(self) -> Path: + """Ours, under the model manager's root: its cache clear removes every file + it does not recognise and leaves directories alone.""" + return Path(self.op.model_root()) / 'jetlink' + + @staticmethod + def model_file_name(model: dict) -> str: + """One file per model, so switching back does not re-download.""" + return f"{model['oid'][:16]}.onnx" + + def shipped_model_path(self) -> Path | None: + """The chosen large model, if it has been fetched. Keyed on the oid, not + the in-tree pointer, which moves with upstream syncs; the size is the cheap + check that the file is the one we mean.""" + model = self.selected_model() + if model is None or not model['oid']: + return None + path = self.model_dir() / self.model_file_name(model) + if path.is_file() and path.stat().st_size == model['size']: + return path + return None + + def lfs_endpoints(self) -> list[str]: + """Where a model's bytes are asked for, nearest first: the LFS server this + checkout's .lfsconfig names (a release ships none), then the ones the Jetson + asks.""" + from jetlink.registry.lfs import LFS_ENDPOINTS + out = [] + try: + for line in (Path(self.op.basedir) / '.lfsconfig').read_text().splitlines(): + key, sep, value = line.strip().partition('=') + if sep and key.strip() == 'url' and value.strip(): + out.append(value.strip().removesuffix('/')) + break + except OSError: + pass + return out + [e for e in LFS_ENDPOINTS if e not in out] + + def fetch_shipped_model(self, progress=None, should_stop=None) -> Path | None: + """Download the chosen large model if it is not here yet; None when nothing + is chosen. The registry streams it to a .part file and hashes it on the way, + so only the whole model ever takes the name.""" + from jetlink.registry.catalog import NetworkError + from jetlink.registry.lfs import Pointer, lfs_download, lfs_resolve + model = self.selected_model() + if model is None or not model['oid']: + return None + dest = self.model_dir() / self.model_file_name(model) + if dest.is_file() and dest.stat().st_size == model['size']: + return dest + pointer = Pointer(model['oid'], int(model['size'])) + for endpoint in self.lfs_endpoints(): + href = lfs_resolve(endpoint, pointer) + if href is None: + continue + self.op.log.warning("jetlink: fetching the large model (%d MB) from %s", pointer.size >> 20, endpoint) + return lfs_download(href, pointer, dest, progress=progress, should_stop=should_stop) + raise NetworkError(f"no LFS server has {pointer.oid[:16]}") + + # -- the model manager's catalog -------------------------------------------- + + def big_catalog(self, catalog: dict) -> dict: + """The big-model catalog with every newer one sunnypilot has published folded in. + A Jetson runs the commit's ONNX, so a model sunnypilot only builds for its next + runtime is still one it can run; see jetlink.registry.catalog.fetch_catalogs. + Never raises: a probe that fails keeps what the last one found.""" + try: + from jetlink.registry.catalog import fetch_catalogs, merge_catalogs + merged = merge_catalogs([catalog, fetch_catalogs()], selector=self.op.catalog_selector) + added = len(merged.get('bundles', [])) - len(catalog.get('bundles', [])) + if added <= 0: + return catalog + self.op.log.warning("jetlink: %d model(s) only newer catalogs list", added) + return merged + except Exception: + self.op.log.exception("jetlink: could not check for newer catalogs") + return self._found_before(catalog) + + def _found_before(self, catalog: dict) -> dict: + """The catalog with the models the last probe found folded in again, from the + model manager's cached copy: every entry there the fetched catalog does not + list. Not only merge_catalogs' entries with no artifacts: a newer catalog at + the same selector version comes through with its builds (v26 lists Cinque + Terre V3 that way). Dropped with a probe that failed, a pick only a newer + catalog lists would be reset at the manager's next start and the owner would + provision the default in its place; a refresh on a flaky network is enough. + A model sunnypilot withdrew stays listed until a probe works.""" + try: + listed = {b.get('ref') for b in catalog.get('bundles', [])} + key = self.op.keys.catalog + cached = ((self.op.get(key) if key else None) or {}).get('bundles', []) + kept = [b for b in cached if isinstance(b, dict) and b.get('ref') not in listed] + except Exception: + return catalog + if not kept: + return catalog + self.op.log.warning("jetlink: keeping %d model(s) the last probe found", len(kept)) + return {**catalog, 'bundles': [*catalog.get('bundles', []), *kept]} diff --git a/jetlink_repo/jetlink/openpilot/owner.py b/jetlink_repo/jetlink/openpilot/owner.py new file mode 100644 index 0000000000..49e0362406 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/owner.py @@ -0,0 +1,52 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +jetlinkd, the resident gadget owner, as an openpilot device runs it. + +manager runs the fork's adapter module under the name jetlinkd, and its main() +builds an OwnerConfig and calls main() here. The owner itself is +jetlink.comma.owner; this hands it the settings, read off the params files the +config names, the chestnut's USB ids, and the provisioning run to start when +there is work. + +The owner stays resident for the whole drive at about 10 MB, so this module +and everything it imports is the standard library and jetlink's light half; +tests/openpilot/test_imports.py holds the line. + +manager starts it again if it dies (the fork's process_config), and every +start adopts what the last owner left. A crash loop would re-enumerate the +Jetson at every start, so the owner's own backoff holds a start back once +owners keep dying (jetlink.comma.owner.note_start). +""" +from __future__ import annotations + +import signal +import sys + +from jetlink.comma import gadget +from jetlink.comma import owner as comma_owner +from jetlink.openpilot.interface import OwnerConfig +from jetlink.openpilot.settings import FileParams, Settings + +# the provisioning run's module. Named here and nowhere in the fork: the fork +# names its adapter, and moving the run is jetlink's business alone +WORKER = 'jetlink.openpilot.provision' + + +def worker(config: OwnerConfig) -> list[str]: + """One provisioning run's argv, on this interpreter, over the fork's adapter.""" + return [sys.executable, '-m', WORKER, '--adapter', config.adapter] + + +def main(config: OwnerConfig) -> None: + """Hold the gadget until manager stops this process (SIGINT, or SIGTERM).""" + gadget.set_logger(comma_owner.logger(config.log_file)) + owner = comma_owner.Owner(worker(config), cwd=str(config.cwd), env=dict(config.env), + settings=Settings(FileParams(config.params_dir), config.keys), + chestnut_ids=config.chestnut_ids) + signal.signal(signal.SIGTERM, owner.request_stop) + signal.signal(signal.SIGINT, owner.request_stop) + owner.run() diff --git a/jetlink_repo/jetlink/openpilot/parts.py b/jetlink_repo/jetlink/openpilot/parts.py new file mode 100644 index 0000000000..6c47f0f152 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/parts.py @@ -0,0 +1,93 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +This process's jetlink, inside: the fork's adapter and what jetlink.openpilot +keeps over it. The fork holds a Jetlink (the API) and never sees these; the +modules here pass them to each other. + +One per process, from bind(). Each part is made on first use, so a reader +such as manager's should_run loads no more than the settings, and none of it +is on the resident owner's path. +""" +from __future__ import annotations + +import time +from functools import cached_property +from pathlib import Path + +from jetlink.comma import gadget +from jetlink.openpilot.settings import FileParams, Settings + +# the chestnut runs the big model natively and the link stays off beside it, +# whatever the setting says. Cached: the UI asks five times a second and the +# answer is a walk of the USB bus +CHESTNUT_TTL = 2.0 + + +def for_this_process(op) -> Parts: + """The parts over the fork's adapter, with jetlink.comma's log pointed at + op.log: a heavy process's lines belong in the drive's log, while the owner's + go to its own file.""" + gadget.set_logger(op.log) + return Parts(op) + + +class Parts: + def __init__(self, op): + self.op = op + self._settings: Settings | None = None + self._chestnut: tuple[float, bool] | None = None + + @property + def log(self): + return self.op.log + + @property + def settings(self) -> Settings: + """The link setting and whether the car is parked, off the params files, + as the owner reads them. Follows the directory the adapter names, which + moves with OPENPILOT_PREFIX as Params does.""" + directory = Path(self.op.params_dir()) + if self._settings is None or self._settings.params.directory != directory: + self._settings = Settings(FileParams(directory), self.op.keys) + return self._settings + + @cached_property + def models(self): + from jetlink.openpilot.models import Models + return Models(self.op) + + @cached_property + def spec(self): + from jetlink.openpilot.state import SpecRecord + return SpecRecord(self.op) + + @cached_property + def progress(self): + from jetlink.openpilot.status import Progress + return Progress(self.op, size=lambda: (self.models.selected_model() or {}).get('size')) + + @cached_property + def presence(self): + from jetlink.openpilot.status import Presence + return Presence() + + @cached_property + def warps(self): + from jetlink.openpilot.warp import Warps + return Warps(self.op) + + def chestnut_fitted(self) -> bool: + now = time.monotonic() + if self._chestnut is None or now - self._chestnut[0] > CHESTNUT_TTL: + self._chestnut = (now, bool(self.op.chestnut_present())) + return self._chestnut[1] + + def enabled(self, mode: str | None = None) -> bool: + """The link setting is on and no chestnut is fitted; `mode` is the setting + when the caller has read it already.""" + mode = self.settings.mode() if mode is None else mode + return mode != 'off' and not self.chestnut_fitted() diff --git a/jetlink_repo/jetlink/openpilot/provision.py b/jetlink_repo/jetlink/openpilot/provision.py new file mode 100644 index 0000000000..58a3d613ca --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/provision.py @@ -0,0 +1,263 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +One provisioning run: `python -m jetlink.openpilot.provision --adapter MODULE`. + +The heavy half of jetlink, numpy and the client, which is why it is a run and +not a daemon. The owner (jetlink.openpilot.owner, as jetlinkd) holds the gadget +for the whole time the link is enabled and starts a run when something +changes; the run borrows the endpoint files from it exactly as modeld does +(jetlink.comma.lending), so the gadget never leaves the bus and a parked car +keeps one resident jetlink process of about 13 MB instead of a run's 47. Only +the owner ever holds ep0: a run that gets no loan logs it and exits, and the +owner starts another. + +The result is cached on the Jetson, recorded in the spec record and left +loaded on the server. The owner stops a run at the onroad transition with +SIGTERM, so every long wait polls `stop`; the server's build thread carries on +regardless and modeld picks the engine up over its own link. + +What the owner cannot learn for itself it hears from the run: the server's +hello, passed on over the loan (lending.Loan.note_server), and whether the +round left work undone, which is the exit status: 0 when nothing is left, +anything else to be tried again once the owner's backoff has passed. +""" +from __future__ import annotations + +import argparse +import signal + +from jetlink import protocol as P +from jetlink.comma import gadget, lending +from jetlink.openpilot import link + +# how long to wait for the Jetson to enumerate before giving up on this run. +# The owner presented the gadget; a box that is asleep answers the bind in +# about 8 s, one that is off never does and the next run will find it +WAKE_TIMEOUT = 20.0 + + +class ProvisioningRun: + def __init__(self, parts): + self.parts = parts + self.log = parts.log + self.client = None + self.loan = None + self.stop = False + self.fetch_failed = False + + # -- lifecycle ------------------------------------------------------------ + + def request_stop(self, *_) -> None: + self.stop = True + + def close_link(self) -> None: + """Always go through this: a FunctionFS owner that exits without closing + can wedge the driver until a reboot.""" + client, self.client = self.client, None + if client is not None: + try: + client.leave(P.LEAVE_PROVISIONED) + client.close() + except Exception: + self.log.exception("jetlink: error closing the link") + + def open_link(self) -> bool: + """Borrow the link from the owner that started this run. Without a loan + there is nothing to open: only the owner ever holds ep0.""" + if self.client is not None: + return True + try: + self.loan = lending.borrow('provision') + if self.loan is None: + self.log.error("jetlink: the owner lent no link, nothing to provision over") + return False + self.client = link.connect(self.log, self.loan, deadline=5.0, name='provision') + return True + except Exception: + self.log.exception("jetlink: could not open the link") + return False + + # -- provisioning --------------------------------------------------------- + + def fetch_model(self): + """Download the pinned large model, once. + + Minutes on a slow link, so it reports progress and stops when the owner + stops the run; it is the one call in a run that blocks for long. + """ + if self.fetch_failed: + return None + try: + path = self.parts.models.fetch_shipped_model( + progress=lambda frac: self.parts.progress.report('download', frac, 'downloading'), + should_stop=lambda: self.stop, + ) + except Exception: + self.log.exception("jetlink: could not fetch the large model") + self.parts.progress.report('failed', 1.0, 'could not download the large model') + # one attempt per run; retrying a gigabyte on a loop is worse than staying small + self.fetch_failed = True + return None + return path + + def provision(self) -> bool: + """Make the Jetson ready for the selected model; a host is attached. The + file is fetched only when the server asks for the bytes: the Jetson keeps + its own copy of every ONNX and never prunes it.""" + from jetlink.client import EngineMissing + parts = self.parts + entry = parts.models.selected_model() + if entry is None: + # no catalog yet; not an error + parts.spec.clear_ready() + parts.progress.clear() + return False + sha256, nbytes = link.identity(parts, entry) + + # only needed if the server turns out not to have this model; None is a + # legitimate state here, see EngineMissing below + model_path = parts.models.shipped_model_path() + + self.log.warning("jetlink: provisioning %s (%d MB, sha %s)", + entry.get('name', sha256[:16]), nbytes >> 20, sha256[:16]) + parts.progress.report('connect', 0.0, 'connecting') + + def ensure(path): + return link.ensure(parts, self.client, sha256, nbytes, path, progress=parts.progress.report_with_eta, + should_stop=lambda: self.stop) + + hello = self.hello() + self.log.warning("jetlink: server %s trt %s", hello.get('device'), hello.get('trt_version')) + try: + spec = ensure(model_path) + except EngineMissing: + # the server has nothing to build from, and neither have we: fetch the + # model and hand it over in this run. The owner lends the link for the + # whole run, so leaving after the download only put the build a + # WORKER_BACKOFF later, five minutes of a finished download doing nothing + if model_path is not None or (model_path := self.fetch_model()) is None: + raise + # a download of minutes can outlast the session; a hello starts a new one + self.hello() + spec = ensure(model_path) + + parts.progress.report('ready', 1.0, 'engine ready') + self.log.warning("jetlink: engine ready for %s", spec.sha256[:16]) + return True + + def hello(self) -> dict: + """Say hello, and pass the answer on to the owner, which needs its + sleep_after to decide whether letting go of the gadget is worth what it + costs and never speaks the protocol to learn it.""" + hello = self.client.hello(timeout=10.0) + if self.loan is not None: + self.loan.note_server(hello) + return hello + + # -- the parked car ------------------------------------------------------- + + def has_work(self) -> bool: + """Is there a reason to wake the Jetson? Only things the link can fix count.""" + spec = self.parts.spec.load() + if spec is None or not self.parts.spec.engine_ready_for(spec.sha256): + return True + selected = self.parts.models.selected_model() + return selected is not None and selected.get('oid') != spec.sha256 + + def shutdown_jetson(self, reason: str) -> None: + """hardwared is shutting the comma down and wants the Jetson off too. + The request file is removed whatever happens: hardwared is waiting on it.""" + self.log.warning("jetlink: shutting the jetson down: %s", reason) + try: + if not self.open_link(): + raise RuntimeError("could not open the link") + if not self.wait_for_jetson(): + raise TimeoutError(f"no jetson attached within {WAKE_TIMEOUT:.0f} s") + # a hello first: the gadget can have stayed bound since the last + # borrower, and the server's session with it. This client's seqs start + # at 1 again, and the session drops anything at or below the last seq + # it saw as a replay; only a hello starts it over + self.hello() + resp = self.client.shutdown(reason, timeout=5.0) + self.log.warning("jetlink: jetson answered the shutdown request: %s", resp) + except Exception: + self.log.exception("jetlink: could not shut the jetson down") + finally: + gadget.finish_shutdown() + + # -- one run -------------------------------------------------------------- + + def wait_for_jetson(self) -> bool: + """Has a host enumerated within WAKE_TIMEOUT? A sleeping Jetson wakes to the bind.""" + return gadget.wait_for_host(WAKE_TIMEOUT, bounce=self.bounce, should_stop=lambda: self.stop, + mode=self.parts.settings.mode()) + + def bounce(self) -> bool: + """Ask the owner to bounce the gadget, over the lease. On a phone's cable + the client's rebind is a no-op: nothing is stuck in an endpoint file.""" + try: + return bool(self.client.rebind()) if self.client is not None else False + except Exception: + self.log.exception("jetlink: could not bounce the gadget") + return False + + def run(self) -> bool: + """One provisioning round. True when there is nothing left to do, which + main() makes the exit status the owner reads.""" + # the setting alone, as the owner that started this run reads it + if self.parts.settings.mode() == 'off': + return True + + reason = gadget.pending_shutdown() + if reason is not None: + self.shutdown_jetson(reason) + return True + + # without a warp for this camera the engine would never run; the offroad + # alert says so, and waking the Jetson to build one would not change it + if not self.parts.warps.built(): + self.log.warning("jetlink: no warp built for this camera, nothing to provision for") + return True + + if not self.has_work(): + self.log.warning("jetlink: nothing to provision") + return True + + finished = False + try: + if not self.open_link(): + return False + if not self.wait_for_jetson(): + self.log.warning("jetlink: no jetson within %.0f s, leaving it for the next run", WAKE_TIMEOUT) + return False + finished = self.provision() + except Exception: + self.log.exception("jetlink: provisioning failed") + self.parts.progress.report('failed', 1.0, 'see the log') + finally: + self.close_link() + return finished + + +def main(argv: list[str] | None = None) -> None: + """One provisioning round over the fork's adapter, as the owner starts it. + Exits 0 when nothing is left to do, and 1 when the owner should try again.""" + from jetlink.openpilot.interface import load_adapter + from jetlink.openpilot.parts import for_this_process + p = argparse.ArgumentParser(prog='python -m jetlink.openpilot.provision', description=main.__doc__) + p.add_argument('--adapter', required=True, help="the fork's adapter module") + args = p.parse_args(argv) + d = ProvisioningRun(for_this_process(load_adapter(args.adapter))) + # the owner stops this at the onroad transition; closing the link properly is + # what keeps the driver healthy for modeld + signal.signal(signal.SIGTERM, d.request_stop) + signal.signal(signal.SIGINT, d.request_stop) + raise SystemExit(0 if d.run() else 1) + + +if __name__ == "__main__": + main() diff --git a/jetlink_repo/jetlink/openpilot/settings.py b/jetlink_repo/jetlink/openpilot/settings.py new file mode 100644 index 0000000000..4ebeb30f4c --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/settings.py @@ -0,0 +1,105 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The settings jetlink reads straight off openpilot's params files: the Jetlink +setting and whether the car is parked, which every process reads, and whether +the owner charges an iPhone. + +The owner cannot construct openpilot's Params: it imports swaglog, which costs +28 MB and brings numpy, capnp and zmq with it. So the fork's adapter says where +the store is and what the keys are called (OwnerConfig), and this reads the +files. params.cc writes a value to a temporary file, fsyncs it and renames it +over the key, so a plain read gets the old value or the new one and never a +torn one. The byte format is params.cc's put: b"1" or b"0" for a BOOL, +decimal ASCII for an INT. The fork's tests write with the real Params and read +back through here, so a change on either side fails there. + +Every other process reads these two the same way, so they agree with the owner; +JSON values go through the adapter's get and put, which are the real Params. + +The standard library only: the owner imports this module. +""" +from __future__ import annotations + +import os +from pathlib import Path + +from jetlink.openpilot.interface import MODES, Keys + + +class FileParams: + """openpilot's params store, read as files. Nothing here writes.""" + + def __init__(self, directory: Path): + self.directory = Path(directory) + + def path(self, key: str) -> Path: + return self.directory / key + + def raw(self, key: str) -> bytes | None: + """A param's bytes, or None if it is unset or unreadable.""" + try: + return self.path(key).read_bytes() + except OSError: + return None + + def get_bool(self, key: str) -> bool | None: + """A param openpilot stores with put_bool. None when it is unset.""" + value = self.raw(key) + if value is None: + return None + return value.strip() in (b'1', b'true', b'True') + + def get_int(self, key: str) -> int | None: + """An INT param. None when it is unset or not a number.""" + value = self.raw(key) + try: + return int(value) + except (TypeError, ValueError): + return None + + +class Settings: + """What jetlink reads of openpilot's params without the Params library.""" + + def __init__(self, params: FileParams, keys: Keys): + self.params = params + self.keys = keys + # the pick and the built model's record: a change to either is a reason + # for the owner to look again. A stat, not a read: the owner does not parse + # the catalog or the spec, it only notices they moved. Paths once, since + # the owner stats them twice a second + watched = (keys.big_model, keys.spec) + self._watched = {k: str(params.path(k)) for k in watched if k} + + def mode(self) -> str: + """The Jetlink setting, one of MODES. Unset or unreadable is 'off'; manager + writes the default before anything runs.""" + index = self.params.get_int(self.keys.link) + return MODES[index] if index is not None and 0 <= index < len(MODES) else 'off' + + def offroad(self) -> bool: + """Is the car parked? The owner runs onroad too, to keep hold of the gadget, + and everything else jetlink does belongs to a parked car. A missing param + is manager not having written one yet, which reads as parked.""" + value = self.params.get_bool(self.keys.offroad) + return True if value is None else value + + def charge_phone(self) -> bool: + """Is an iPhone on a direct cable asked to charge from the comma? Off when + the param is unset or the fork declares none.""" + key = self.keys.charge_phone + return bool(key) and self.params.get_bool(key) is True + + def marks(self) -> dict[str, int]: + """When each watched param last changed, by key; 0 for one that is unset.""" + out = {} + for key, path in self._watched.items(): + try: + out[key] = os.stat(path).st_mtime_ns + except OSError: + out[key] = 0 + return out diff --git a/jetlink_repo/jetlink/openpilot/state.py b/jetlink_repo/jetlink/openpilot/state.py new file mode 100644 index 0000000000..04bfee8bce --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/state.py @@ -0,0 +1,54 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The picked model's spec, and whether its engine is built, in one record. + +Reading the shapes and output slices means parsing a 766 MB ONNX. The server +does that when it builds the engine and answers with the spec; provisioning +keeps the answer here, and modeld reads it and never touches the file. The +record also says whether the server has built the engine for the sha it names, +so the spec and the readiness can never name different models. It must +survive a reboot, or every ignition rebuilds a 160 s engine. + +A param today (Keys.spec), through the adapter's get and put. +""" +from __future__ import annotations + + +class SpecRecord: + def __init__(self, op): + self.op = op + + def _raw(self) -> dict | None: + # get tolerates a params library older than the key + value = self.op.get(self.op.keys.spec) + return value if isinstance(value, dict) else None + + def load(self): + """The recorded ModelSpec, or None if there is not a usable one.""" + from jetlink.spec import ModelSpec + try: + d = self._raw() + return ModelSpec.from_dict(d) if d else None + except Exception: + self.op.log.exception("jetlink: cached spec is unreadable") + return None + + def store(self, spec) -> None: + """The server has built the engine for this spec and answered with it.""" + self.op.put(self.op.keys.spec, {**spec.to_dict(), 'ready': True}) + + def engine_ready_for(self, sha256: str | None) -> bool: + """Has the server built the engine for this model? Params only.""" + d = self._raw() + return bool(sha256) and d is not None and d.get('sha256') == sha256 and d.get('ready') is True + + def clear_ready(self) -> None: + """The engine is no longer known to be built. The spec stays: it still + sizes the warp, and the next provisioning run asks again.""" + d = self._raw() + if d is not None and d.get('ready'): + self.op.put(self.op.keys.spec, {**d, 'ready': False}) diff --git a/jetlink_repo/jetlink/openpilot/status.py b/jetlink_repo/jetlink/openpilot/status.py new file mode 100644 index 0000000000..a61350901a --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/status.py @@ -0,0 +1,285 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +What the UI, hardwared and manager are told, in one snapshot, and the +progress the panels show while something provisions or joins. + +Every answer here is files and params, no link IO: the UI asks five times a +second. The fork maps the snapshot onto its own widgets; the one mapping that +is jetlink's knowledge, which icon a state is, is Status.icon. + +Light at module level: the package imports it, and the owner imports the package. +""" +from __future__ import annotations + +import time +from typing import NamedTuple + +from jetlink.comma import gadget + +# -- progress ----------------------------------------------------------------- +# Written by the provisioning run and the joining state, read by the UI: a +# param today (Keys.progress), because the writer is another process. + +# Each report is a file write, and the UI reads it at 5 Hz. An upload reports +# once per 4 MB chunk, 440 of them for a 1.7 GB model, and onroad that is IO a +# recording would have to share the disk with. +PROGRESS_MIN_INTERVAL = 0.25 + + +def estimated_build_seconds(size: int) -> int: + """Orin Nano Super, TensorRT 10.3: the 766 MB models built in 102 to 166 s, + the 1.75 GB ones in 230 to 294 s.""" + return int(60 + 130 * size / 1e9) + + +def _eta(seconds: float) -> str: + if seconds >= 90: + return f"about {seconds / 60:.0f} min left" + return f"about {max(seconds, 1):.0f}s left" + + +class Progress: + def __init__(self, op, size=None): + self.op = op + # () -> the picked model's size in bytes, or None: what the build's estimate is made from + self._size = size + self._last = ('', 0.0) + + def read(self) -> dict | None: + """{stage, frac, msg} while something provisions, else None. Never raises.""" + try: + value = self.op.get(self.op.keys.progress) + except Exception: + return None + return value if isinstance(value, dict) else None + + def report(self, stage: str, frac: float, msg: str = '', drops: int = 0) -> None: + """Never raises: called from except handlers. A fraction's ticks are held to + 4 Hz within a stage; a report with no fraction (a join's state) and the end + of a stage always go through, so the panel's last word is never dropped. A + join says 'waiting for jetlink' and 'checking the link keeps up' 50 ms apart; + holding the second left the first on the panel for the rest of the drive. + `msg` is the panel's one short line; `drops`, once the link has dropped often + enough to blame the cable, is the panel's to word.""" + last_stage, last_at = self._last + now = time.monotonic() + if 0.0 < frac < 1.0 and stage == last_stage and now - last_at < PROGRESS_MIN_INTERVAL: + return + self._last = (stage, now) + try: + self.op.put(self.op.keys.progress, {'stage': stage, 'frac': round(frac, 4), 'msg': msg, 'drops': drops}) + except Exception: + self.op.log.exception("jetlink: could not report progress") + + def clear(self) -> None: + try: + self.op.remove(self.op.keys.progress) + except Exception: + self.op.log.exception("jetlink: could not clear progress") + + def report_with_eta(self, stage: str, frac: float, msg: str = '') -> None: + """Progress, with how long the build still has to run, from the model's + size. Only the build: the upload reports MB of MB and a connect has nothing + to predict.""" + if stage == 'build': + size = self._size() if self._size is not None else None + if size: + msg = _eta(estimated_build_seconds(size) * max(0.0, 1.0 - frac)) + self.report(stage, frac, msg) + + +# -- what is on the other end ------------------------------------------------- +# The owner (jetlink.comma.owner) writes all of this into its status record +# every step, presence and its hold included, so every reader agrees with it +# and with each other. Without a live record (no owner since boot, one that +# stopped cleanly, or one too old to write it) each reader reads the gadget's +# files itself, as below. + +# the offroad alert for a link that is on while nobody holds the gadget: the +# owner's heartbeat stopped. manager starts it again, and its crash-loop +# backoff says so in its record's error while it waits +STOPPED = "service stopped" + + +def owner_record() -> tuple[dict | None, dict | None]: + """The owner's status record, and the same record if it is live, else None. + A stale record from an owner that was asked to stop is no record: manager + SIGKILLed it in a slow teardown, and nothing is wrong.""" + record = gadget.owner_status() + if record is None or gadget.owner_alive(record): + return record, record + return (None, None) if record.get('stopping') else (record, None) + + +class Presence: + """Presence as a reader works it out from the files, for when the owner's + record is not there to say.""" + + def __init__(self): + self._last_configured = 0.0 + + def present(self) -> bool: + """Is a host on the other end now: configured us, or did within + PRESENCE_HOLD? A phone on the cable is a host on the gadget like any other.""" + if gadget.dormant(): + # no enumeration during suspend; the CC line still tells a sleeping host from an unplugged one + return gadget.port_has_host() + now = time.monotonic() + if gadget.host_attached(): + self._last_configured = now + return True + return now - self._last_configured < gadget.PRESENCE_HOLD + + +def link_transport(mode: str | None = None, live: dict | None = None) -> str: + """What carries the link, for the panels: a host on the vendor interface, + or an iPhone dialed in over the network one. `live` is the owner's record; + without it, `mode` stands in until the owner has said. Never raises.""" + try: + if live is not None and live.get('link') in ('usb', 'cable'): + kind, peer = live['link'], live.get('peer') + else: + kind, peer = gadget.link_state() + kind = kind or gadget.link_kind(mode) + if kind == 'cable' and not peer: + # the owner records the phone while it holds the dial; a borrower that + # took the dial itself noted it in the link record + peer = gadget.link_state()[1] + if kind == 'cable': + return f"iOS over USB ({peer})" if peer else "iOS over USB" + except Exception: + pass + return "USB" + + +def usb_port() -> str | None: + """What the comma's USB-C port controller sees on the CC pin: 'empty', or + 'host' for a cable with something live behind it (it cannot say what). None + where the kernel does not expose it, rather than claiming an empty port.""" + cc = gadget.cc_orientation() + if cc is None: + return None + return 'host' if cc else 'empty' + + +# -- whether the link can run -------------------------------------------------- + +# the offroad alert's text for a device whose build made no warp for its camera +NO_WARP = "no warp built for this camera" + + +def unavailable(parts, record: dict | None) -> str | None: + """Why an enabled link cannot run the large model, or None: files only. + `record` is the owner's status record: its error when it is live, STOPPED + when its heartbeat is not, and the gadget's own files without one.""" + if record is None: + error = gadget.gadget_error() + elif not gadget.owner_alive(record): + return STOPPED + else: + error = record.get('error') + if error: + return str(error) + return None if parts.warps.built() else NO_WARP + + +def _built_for_the_pick(parts) -> bool: + spec = parts.spec.load() + selected = parts.models.selected_model() + return (spec is not None and selected is not None and spec.sha256 == selected['oid'] + and parts.spec.engine_ready_for(spec.sha256)) + + +def reason(parts, mode: str) -> str | None: + """Why the link cannot run, for someone who asked for it only: with it off, + a device that cannot present the gadget simply does not offer the feature.""" + return unavailable(parts, owner_record()[0]) if parts.enabled(mode) else None + + +# -- the snapshot ---------------------------------------------------------------- + +class Status(NamedTuple): + """Everything a reader shows, taken at once. Fields are only ever added.""" + enabled: bool # the setting is on and no chestnut is fitted + mode: str # the Jetlink setting, one of MODES + transport: str # 'USB', or 'iOS over USB ()' + present: bool # a host is on the gadget now, or asleep and known to be there + port: str | None # 'host' or 'empty' off the CC pin; None where the kernel does not say + ready: bool # the picked model's engine is built (records only, no link IO) + reason: str | None # why an enabled link cannot run: the offroad alert's text + progress: dict | None # {stage, frac, msg} while provisioning or joining + model: str | None # the big model it will run: the pick, else jetlink's default + default_model: str | None # jetlink's default, named like the chestnut's (no build date) + + @property + def active_model(self) -> str | None: + """model, once it can run.""" + return self.model if self.ready else None + + def icon(self, started: bool, model_seen: bool, running_big: bool, state: str) -> str: + """The chestnut icon's state for the link: a ChestnutState value. + + Offroad it is progress and the records; onroad, modelV2 and the joining + model's state (modelDataV2SP.acceleratorState). `model_seen` is a modelV2 + since this drive started, `running_big` a live one that says big, and + `state` the acceleratorState name. + """ + if not started: + stage = str((self.progress or {}).get('stage', '')) + if not self.present: + return 'disconnected' + if stage and stage != 'ready': + return 'failed' if stage == 'failed' else 'loading' + return 'ready' if self.ready else 'uncompiled' + + if model_seen and running_big: + return 'active' + if not self.present: + return 'disconnected' + # attached, a pending join is loading, not a failed model + if state in ('joining', 'retrying') or not model_seen: + return 'loading' + # the engine is up and only the swap window is missing: it opens when + # nothing is in control, so a driver who stays engaged keeps the small model + if state == 'ready': + return 'waiting' + if not self.ready: + return 'uncompiled' + if state == 'running': + return 'active' + return 'failed' + + +def read(parts, mode: str) -> Status: + """Everything at once, over the link setting the caller read: each file once.""" + record, live = owner_record() + enabled = parts.enabled(mode) + reason = unavailable(parts, record) if enabled else None + return Status( + enabled=enabled, + mode=mode, + transport=link_transport(mode, live), + present=bool(live.get('present')) if live is not None else parts.presence.present(), + port=usb_port(), + ready=enabled and reason is None and _built_for_the_pick(parts), + reason=reason, + progress=parts.progress.read(), + model=parts.models.selected_model_name(), + default_model=parts.models.default_model_name(), + ) + + +def failed(error: str, mode: str) -> Status: + """What a reader gets when the snapshot itself failed: nothing to show, the + setting as it is, and, for someone who asked for the link, the failure where + the offroad alert puts it. A device with the link off is never nagged.""" + return Status(enabled=False, mode=mode, transport='USB', present=False, port=None, ready=False, + reason=failure(error, mode), progress=None, model=None, default_model=None) + + +def failure(error: str, mode: str) -> str | None: + return f"jetlink status failed: {error}" if mode != 'off' else None diff --git a/jetlink_repo/jetlink/openpilot/warp.py b/jetlink_repo/jetlink/openpilot/warp.py new file mode 100644 index 0000000000..4659a923f9 --- /dev/null +++ b/jetlink_repo/jetlink/openpilot/warp.py @@ -0,0 +1,253 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +The comma-side warp JIT: how it is built, where it lives, what loads it, and +the small model's reset for a fallback. + +The warp stays on the comma (see model_state), and upstream's fused run_model +JIT (openpilot #38684) has no warp to borrow, so comma's make_warp graph is +JIT-compiled as a build target: the fork's build runs +`python -m jetlink.openpilot.warp` once per camera it builds for. A source +build makes the one for its own camera, a prebuilt release one for every +camera it installs on. Nothing compiles one at runtime: in modeld the ~9 s +compile would hold back the first frame on every ignition, and in a +provisioning run, which only runs offroad, it was lost to ignition. A device +without one runs the small model. + +load() is what stands between a bad pickle and the car. + +tinygrad is the fork's, and only modeld and the build have it: every import of +it here is inside the function that needs it. +""" +from __future__ import annotations + +import argparse +import pickle +import sys +from pathlib import Path + +# what TinyJit records for the keyword call in call_warp: sorted(kwargs) +WARP_INPUT_NAMES = ['big_frame', 'big_tfm', 'frame', 'tfm'] + + +def call_warp(warp, tfm, big_tfm, frame, big_frame): + """Call a warp JIT. Every caller goes through here, capture included. + + TinyJit names inputs from enumerate(args) plus sorted(kwargs) and refuses a + call whose names differ from the capture. A positional compile and a keyword + call raised JitError on the first frame of a drive. + """ + return warp(tfm=tfm, big_tfm=big_tfm, frame=frame, big_frame=big_frame) + + +def init_device(log) -> None: + """Bring the GPU up now, on the caller's thread. + + tinygrad initialises the device on its first kernel run and spawns a libusb + event thread doing it. A thread created after config_realtime_process(7, 54) + inherits SCHED_FIFO 54 and the core-7 pin and preempts the frame loop; that + cost 5% of frames once. Failure is not a reason to refuse the accelerator. + """ + try: + from tinygrad.tensor import Tensor + Tensor([0.0]).realize() + except Exception: + log.exception("jetlink: could not bring the gpu up before modeld goes realtime") + # the same trap for tinygrad's compile pool (engine/worker.py): created on + # the first compile, after modeld goes realtime, its handler threads sat at + # SCHED_FIFO 54 on core 7. An older tinygrad or PARALLEL=0 is not a failure + try: + from tinygrad.engine.worker import get_worker_pool + get_worker_pool() + except Exception: + log.exception("jetlink: could not start tinygrad's compile pool before modeld goes realtime") + + +class Warps: + """The warps the build made for this device, as the fork's adapter says where.""" + + def __init__(self, op): + self.op = op + # nothing compiles a warp at runtime and the build runs before manager, so + # the answer holds for the life of the process; the UI asks at 5 Hz + self._built: bool | None = None + + def geometry(self) -> tuple[int, int, int, int]: + """(cam_w, cam_h, model_w, model_h) for this device: the choice modeld's + build makes, so the warp built is the one modeld asks for. If they + disagree, load() raises and the drive is small-model.""" + return tuple(self.op.camera()) + + def path(self, cam_w: int, cam_h: int, model_w: int, model_h: int) -> Path: + return Path(self.op.warp_path(cam_w, cam_h, model_w, model_h)) + + def is_cached(self, cam_w: int, cam_h: int, model_w: int, model_h: int) -> bool: + """Is there a warp for this geometry? + + Presence only. Staleness is the build's job: the target depends on tinygrad + and the capture sources. A pickle from an incompatible tinygrad raises in load(). + """ + return self.path(cam_w, cam_h, model_w, model_h).is_file() + + def built(self) -> bool: + """Is there a warp for this device's camera? Without one the link cannot + run the large model, which the offroad alert says (status.NO_WARP).""" + if self._built is None: + self._built = self.is_cached(*self.geometry()) + return self._built + + def load(self, cam_w: int, cam_h: int, model_w: int, model_h: int): + """The built warp JIT. Raises if it is not there or is stale. + + modeld's big-model load is wrapped in the fallback to the small model, and a + warp that cannot be trusted must not reach the car. + """ + if not self.is_cached(cam_w, cam_h, model_w, model_h): + raise RuntimeError(f"no warp built for {cam_w}x{cam_h} -> {model_w}x{model_h}; " + "the fork's build makes it (python -m jetlink.openpilot.warp)") + with open(self.path(cam_w, cam_h, model_w, model_h), 'rb') as f: + warp = pickle.load(f) + + # a JIT pickled before TinyJit captured loads fine and computes nothing; one + # captured with a different call convention raises JitError on the first + # frame of a drive. Both have happened + captured = getattr(warp, 'captured', None) + if captured is None: + raise RuntimeError("cached warp was pickled before it captured; it computes nothing") + names = list(getattr(captured, 'expected_names', [])) + if names != WARP_INPUT_NAMES: + raise RuntimeError(f"cached warp expects {names}, call_warp passes {WARP_INPUT_NAMES}") + return warp + + +def warm(warp, frame_size: int) -> None: + """Run a loaded warp JIT until it is cheap to call. `frame_size` is the + camera's NV12 buffer size (ModelFace.frame_size). + + Measured: loading 0.3 s, the first call 1.9 s, the second 5 ms. Paid on + modeld's frame loop that was ~26 dropped frames and 16 s of modeldLagging + after every join. + """ + import numpy as np + from tinygrad.device import Device + from tinygrad.tensor import Tensor + + frames = [np.zeros(frame_size, dtype=np.uint8) for _ in range(2)] + blobs = [Tensor.from_blob(f.ctypes.data, (frame_size,), dtype='uint8', device=Device.DEFAULT) for f in frames] + eye = [np.eye(3, dtype=np.float32) for _ in range(2)] + tfm, big_tfm = (Tensor(e, device='NPY').realize() for e in eye) + for _ in range(2): + call_warp(warp, tfm, big_tfm, blobs[0], blobs[1]).realize() + Device.default.synchronize() + + +def prepare_reset(model): + """Capture the small model's reset before driving, keeping its JIT's buffer identities. + + Its history is stale after the Jetson ran, so a fallback starts from the same + zero history as modeld startup. Nothing is allocated or compiled on the + failure frame. + + The small model is whatever bundle the user picked, stock modeld's or a + modeld_v2 one, so the history is whatever GPU queues it has; the NPY tensors + are zeroed through their numpy views. Duck-typed on openpilot's ModelState: + input_queues, numpy_inputs (modeld_v2) or npy (modeld), prev_desire. The + fork's tests pin those names. + """ + from tinygrad import Tensor, TinyJit + + queues = tuple(q for q in model.input_queues.values() if q.device != 'NPY') + npy = model.numpy_inputs if hasattr(model, 'numpy_inputs') else model.npy + + @TinyJit + def clear(): + Tensor.realize(*(q.assign(0) for q in queues)) + + for _ in range(3): + clear() + + def reset(): + clear() + model.prev_desire.fill(0) + for array in npy.values(): + array.fill(0) + + return reset + + +# -- the build ------------------------------------------------------------------ + +def compile_warp(graph, frame_size: int, out: Path) -> Path: + """JIT the warp graph and pickle it to `out`. Holds the GPU while it runs. + + Three runs before pickling: TinyJit captures on the second call, and a + pickle taken earlier is an empty jit that silently does nothing. + """ + import numpy as np + from tinygrad.device import Device + from tinygrad.engine.jit import TinyJit + from tinygrad.tensor import Tensor + + warp_jit = TinyJit(graph, prune=True) + + # one set of input tensors: TinyJit captures against the buffers it is + # first handed. Random so nothing constant-folds + rng = np.random.default_rng(42) + tfm_npy, big_tfm_npy = np.eye(3, dtype=np.float32), np.eye(3, dtype=np.float32) + tfm = Tensor(tfm_npy, device='NPY') + big_tfm = Tensor(big_tfm_npy, device='NPY') + frame = Tensor.randint(frame_size, low=0, high=256, dtype='uint8', device=Device.DEFAULT).realize() + big_frame = Tensor.randint(frame_size, low=0, high=256, dtype='uint8', device=Device.DEFAULT).realize() + for _ in range(3): + tfm_npy[:] = rng.standard_normal((3, 3)).astype(np.float32) + big_tfm_npy[:] = rng.standard_normal((3, 3)).astype(np.float32) + call_warp(warp_jit, tfm, big_tfm, frame, big_frame).realize() + Device.default.synchronize() + + out = Path(out) + out.parent.mkdir(parents=True, exist_ok=True) + # through a temporary: a compile killed mid-write leaves nothing under the + # name load() opens + tmp = out.with_suffix('.pkl.tmp') + with open(tmp, 'wb') as f: + pickle.dump(warp_jit, f) + tmp.replace(out) + return out + + +def size(text: str) -> tuple[int, int]: + """'WxH' as (w, h).""" + w, sep, h = text.lower().partition('x') + if not sep: + raise argparse.ArgumentTypeError(f"expected WxH, not {text!r}") + try: + return int(w), int(h) + except ValueError: + raise argparse.ArgumentTypeError(f"expected WxH, not {text!r}") from None + + +def main(argv: list[str] | None = None) -> None: + """Build the warp for one camera, as the fork's build runs it.""" + from jetlink.openpilot.interface import load_adapter + p = argparse.ArgumentParser(prog='python -m jetlink.openpilot.warp', description=main.__doc__) + p.add_argument('--adapter', required=True, help="the fork's adapter module") + p.add_argument('--camera', type=size, required=True, help='camera resolution, WxH') + p.add_argument('--model', type=size, required=True, help='model input, WxH') + p.add_argument('--output', type=Path, required=True) + args = p.parse_args(argv) + + (cam_w, cam_h), (model_w, model_h) = args.camera, args.model + op = load_adapter(args.adapter) + print(f"Compiling jetlink warp for {cam_w}x{cam_h} -> {model_w}x{model_h}...") + # before anything of tinygrad's is imported here: comma's graph module + # patches tinygrad's firmware fetch as it loads + graph, frame_size = op.make_warp(cam_w, cam_h, model_w, model_h) + out = compile_warp(graph, frame_size, args.output) + print(f" Saved to {out}") + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/jetlink_repo/jetlink/protocol.py b/jetlink_repo/jetlink/protocol.py new file mode 100644 index 0000000000..6fd0b1b896 --- /dev/null +++ b/jetlink_repo/jetlink/protocol.py @@ -0,0 +1,155 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Wire protocol. + +Every message is a 32-byte header and an opaque payload. 32 keeps float32 +payloads 8-byte aligned, so both ends can view the receive buffer without +copying. + +INFER is a fixed struct plus two raw arrays sized at handshake, not json: one +vectored write out, one read into a preallocated buffer back. +""" +from __future__ import annotations + +import struct +from enum import IntEnum + +MAGIC = 0x4B4E4C4A # b'JLNK' +# The comma package and the server are updated together, so each speaks exactly +# one version, and a header of any other is a broken stream. 3 keeps the hidden +# state on the server: INFER_REQ has no prev_feat and INFER_RESP no +# hidden_state slice, which takes the big models' reply from 72 KB (five 16 KB +# reads on the comma) to 8 KB (one). Tensors are unchanged, so existing plans +# still load. +VERSION = 3 + +# A bulk transfer ends on a short packet, so a message that is an exact multiple +# of the packet size never terminates the peer's read and arrives a frame late; +# the sender appends a pad byte and sets Flag.PADDED. 512 is a high-speed packet +# and divides a SuperSpeed one, so every message ends short on a USB 2 link too: +# an Android phone, or a cable that fell back to high speed. A reader takes the +# pad byte whenever the flag says so, at any length. +PACKET_MULTIPLE = 512 + +# The gadget pads every message to a burst so it never ends on a short packet: +# dwc3 flushed its TX FIFO past one about once in 400 frames, and the host read +# those bytes as the next header. Host to device keeps the PADDED byte instead. +GADGET_TX_ALIGN = 16 * 1024 + +# The comma's gadget as a USB host finds it: these IDs (pid.codes' test +# allocation; get a real PID before distributing this), which +# scripts/comma/jetlink-root.sh presents, then the link's interface by its class +# triple (transport/ffs.py), because the comma set to iOS presents a composite +# gadget and a reordered config could move the number. Endpoint addresses are +# read from the descriptors, never assumed: FunctionFS renumbers them at bind. +USB_VID = 0x1209 +USB_PID = 0x0001 +USB_VENDOR_CLASS = (0xFF, 0xFF, 0xFF) +USB_MAX_PACKET = 1024 # SuperSpeed bulk + +# magic, version, msg_type, seq, flags, length, reserved, 4 pad +HEADER_FMT = ' server describes itself + HELLO_RESP = 2 # json: server caps, loaded engine, shapes + ENGINE_REQ = 3 # json: {sha256, nbytes, frame_skip} -> make this model ready + ENGINE_RESP = 4 # json: {state: ready|need_upload|building|failed, spec when ready, ...} + UPLOAD_CHUNK = 5 # u64 offset + bytes + UPLOAD_DONE = 6 # json: {sha256} + PROGRESS = 7 # json: {stage, frac, msg} - unsolicited, server -> client + INFER_REQ = 8 # InferHeader + warped(u8) + packed(f32) + INFER_RESP = 9 # InferRespHeader + outputs(f32) but hidden_state, which stays on the server + # 10, 11 were RESET_REQ/RESP; queues are cleared with Flag.RESET_QUEUES + STATE_REQ = 12 # telemetry + STATE_RESP = 13 # json + ERROR = 14 # json: {error, detail} + PING = 15 # on a connection with no hello and no ENGINE_REQ: ERROR no_hello + PONG = 16 + SHUTDOWN_REQ = 17 # json: {reason} -> power the Jetson off for good; see JetlinkClient.shutdown + SHUTDOWN_RESP = 18 # json: {ok, detail} + LEAVE = 19 # json: {reason, ...what the client measured}; client -> server, no reply. + # The client stops using the link: it handed the model back or is exiting. + # The connection may stay for a later HELLO_REQ. An older server answers + # ERROR unknown_message, which the client discards (JetlinkClient.leave) + + +# Why a client stops using the link, LEAVE's json 'reason' (JetlinkClient.leave); +# the server logs each in words (Session.onLeave) and anything else verbatim +LEAVE_BEHIND = 'behind' # modeld fell behind the large model +LEAVE_LOST = 'lost' # the comma lost the link, said when it can still carry it +LEAVE_STOPPED = 'stopped' # modeld stopped +LEAVE_PROVISIONED = 'provisioned' # the provisioning run finished +LEAVE_REASONS = (LEAVE_BEHIND, LEAVE_LOST, LEAVE_STOPPED, LEAVE_PROVISIONED) + + +class Flag(IntEnum): + RESET_QUEUES = 1 << 0 # on INFER_REQ: warm-start, clear history before this frame + WANT_STATE = 1 << 1 # on INFER_REQ: append telemetry json to the response. + # Piggybacked because at 20 Hz a separate exchange + # would race a frame. + WANT_HIDDEN = 1 << 2 # on INFER_REQ: keep hidden_state in the response, for + # a caller logging the whole output vector. 64 KB more + # on the big models; nothing else needs it. + PADDED = 1 << 7 # one pad byte follows the payload; see PACKET_MULTIPLE + + +# INFER_REQ: frame_id, flags. Sizes of the two arrays come from the handshake. +INFER_REQ_FMT = ' bytes: + return _header.pack(MAGIC, VERSION, int(msg_type), seq, flags, length, reserved) + + +def unpack_header(buf) -> tuple[int, int, int, int, int, int, int]: + magic, version, msg_type, seq, flags, length, reserved = _header.unpack_from(buf) + if magic != MAGIC: + raise ProtocolError(f"bad magic 0x{magic:08x} (link desynced or not a jetlink peer)") + if version != VERSION: + raise ProtocolError(f"peer speaks protocol v{version}, we speak v{VERSION}") + return magic, version, msg_type, seq, flags, length, reserved + + +def pack_infer_req(frame_id: int, flags: int = 0) -> bytes: + return _infer_req.pack(frame_id, flags) + + +def unpack_infer_req(buf, offset: int = 0) -> tuple[int, int]: + return _infer_req.unpack_from(buf, offset) + + +def pack_infer_resp(frame_id: int, status: int, gpu_us: int, queue_us: int, total_us: int) -> bytes: + return _infer_resp.pack(frame_id, status, gpu_us, queue_us, total_us) + + +def unpack_infer_resp(buf, offset: int = 0) -> tuple[int, int, int, int, int]: + return _infer_resp.unpack_from(buf, offset) + + +class Status(IntEnum): + OK = 0 + NOT_READY = 1 # no engine loaded + BAD_SHAPE = 2 + INFER_FAILED = 3 + NOT_FINITE = 4 # model produced NaN/Inf; caller must fall back diff --git a/jetlink_repo/jetlink/queues.py b/jetlink_repo/jetlink/queues.py new file mode 100644 index 0000000000..3ef55ac369 --- /dev/null +++ b/jetlink_repo/jetlink/queues.py @@ -0,0 +1,98 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +A queued graph's history buffers, in numpy: the reference the server's staging +is held to. + +openpilot folds these into the tinygrad JIT on the GPU. Shipping the history +across the link would cost ~10 MB a frame, so the queues live on the server and +the comma sends only the newest warped frame and the packed scalars. The hidden +state the graph returns stays there too: the next frame pushes it into the +features queue, where modeld's prev_feat went. The fork's tests/test_queues.py +checks this against the tinygrad original, and the staging conformance fixture +(JetlinkKit/Scripts/make_conformance_fixtures.py) holds the Swift server's +queues to this, bit for bit. + +openpilot's rolling `cat(buf[1:], new)`, stored in float16 rather than cast at +the model boundary, which gives the same values since each row is cast once. +Pass dtype=np.float32 for openpilot's exact intermediates. + +A stateful graph (openpilot #38916) does the queueing itself, so there is +nothing here for one of those. +""" +from __future__ import annotations + +import numpy as np + +from jetlink.spec import DRIVING_OUTPUT, ModelSpec + + +class PolicyQueues: + """The server's state for one queued model. Everything `run_policy` owned in + the JIT, and the prev_feat modeld carried from one frame's output to the next.""" + + def __init__(self, spec: ModelSpec, dtype=np.float16): + self.spec = spec + hidden, size = spec.hidden_range, int(np.prod(spec.prev_feat_shape)) + if hidden is None or hidden[1] - hidden[0] != size: + raise ValueError(f"a queued graph needs a hidden_state output of {size} floats to feed back; " + f"its output_slices give {hidden}") + self._hidden = slice(*hidden) + self.img_q = np.zeros(spec.img_buf_shape, dtype) + self.big_img_q = np.zeros(spec.img_buf_shape, dtype) + self.feat_q = np.zeros(spec.feat_q_shape, dtype) + self.desire_q = np.zeros(spec.desire_q_shape, dtype) + # float32, as the comma held it: the values the reply carried and the + # comma sent back, cast into the queue at the same point + self.prev_feat = np.zeros(spec.prev_feat_shape, np.float32) + + def reset(self) -> None: + """RESET_QUEUES: empty history, and nothing to feed back.""" + for q in (self.img_q, self.big_img_q, self.feat_q, self.desire_q, self.prev_feat): + q[...] = 0 + + def new_client(self) -> None: + """A hello: the next frame feeds back zeros, as a new modeld's prev_feat + did. The queues stay until the client resets them.""" + self.prev_feat[...] = 0 + + def after_run(self, outputs: dict[str, np.ndarray], dest=None) -> None: + """Keep this frame's hidden state for the next one. + + Only after a frame whose outputs are all finite: modeld fed back only + what infer_end returned, and it raises on NOT_FINITE and on a failed run. + `dest` is unused; the fork's test_queues still passes it. + """ + self.prev_feat.reshape(-1)[...] = np.asarray(outputs[DRIVING_OUTPUT], np.float32).reshape(-1)[self._hidden] + + def step(self, warped: np.ndarray, packed: np.ndarray) -> dict[str, np.ndarray]: + """Advance the queues one frame and return the model's inputs, as new arrays. + + warped: (2, 6, H, W) uint8 from openpilot's warp. packed: flat float32, + laid out per ModelSpec.packed_shapes. The features queue takes prev_feat, + the hidden state after_run kept. + """ + spec, fs = self.spec, self.spec.frame_skip + if warped.shape != spec.warped_shape: + raise ValueError(f"warped {warped.shape} != {spec.warped_shape}") + if packed.size != spec.packed_nelem: + raise ValueError(f"packed {packed.size} != {spec.packed_nelem}") + desire, traffic_convention, action_t = (packed[at] for at, _ in spec.packed_layout.values()) + for q, row in ((self.img_q, warped[0]), (self.big_img_q, warped[1]), + (self.desire_q, desire), (self.feat_q, self.prev_feat)): + q[:-1] = q[1:] + q[-1] = row.reshape(q.shape[1:]) + shapes, dtype, des = spec.input_shapes, self.img_q.dtype, self.desire_q + # features_buffer is declared 4-D (1,32,32,512); the queue's (32,1,16384) + # rows are the same values, so a reshape covers it + return { + 'img': self.img_q[::fs].copy().reshape(shapes['img']), + 'big_img': self.big_img_q[::fs].copy().reshape(shapes['big_img']), + 'desire_pulse': des.reshape(-1, fs, *des.shape[1:]).max(axis=1).reshape(shapes['desire_pulse']), + 'traffic_convention': traffic_convention.astype(dtype).reshape(shapes['traffic_convention']), + 'action_t': action_t.astype(dtype).reshape(shapes['action_t']), + 'features_buffer': self.feat_q[::fs].copy().reshape(shapes['features_buffer']), + } diff --git a/jetlink_repo/jetlink/registry/__init__.py b/jetlink_repo/jetlink/registry/__init__.py new file mode 100644 index 0000000000..cc8e725d68 --- /dev/null +++ b/jetlink_repo/jetlink/registry/__init__.py @@ -0,0 +1,13 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Which large models exist and how to get their bytes: sunnypilot's catalog +(catalog) and a model's ONNX by its git-lfs pointer (lfs). + +The comma imports both, so this package stays stdlib only and imports nothing +itself. The server's side of it is JetlinkRegistry in Swift, held to these +modules by the registry conformance fixture (docs/conformance.md). +""" diff --git a/jetlink_repo/jetlink/registry/catalog.py b/jetlink_repo/jetlink/registry/catalog.py new file mode 100644 index 0000000000..3a870c0e5b --- /dev/null +++ b/jetlink_repo/jetlink/registry/catalog.py @@ -0,0 +1,189 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +sunnypilot's big-model catalog: which large models exist and which commit each one is. + +The catalog is the same JSON the comma's model manager caches for its chestnut +slot, so a model picked here is a model the comma can ask for. A bundle's +artifacts are tinygrad pkls for a GPU we do not have; the only field that +matters is `ref`, the comma openpilot commit the bundle was compiled from, +because that commit's ONNX is what a jetlink server runs. + +Stdlib only, and every network call takes an `opener` so tests stay offline. +""" +from __future__ import annotations + +import json +import re +import urllib.error +import urllib.request +from dataclasses import dataclass + + +CATALOG_URL_TEMPLATE = 'https://raw.githubusercontent.com/sunnypilot/sunnypilot-models/refs/heads/gh-pages/docs/driving_models_chestnut_v{version}.json' +# v26 is v25 plus Cinque Terre V3, at the same selector version. v27 onwards +# are selector 20 and a newer tinygrad, for sunnypilot's next sync. +CATALOG_VERSION = 26 +CATALOG_URL = CATALOG_URL_TEMPLATE.format(version=CATALOG_VERSION) +# sunnypilot publishes a new catalog version for new models or a new runtime, +# and keeps the old ones. The versions after the pinned one are probed up to +# the first that is not there, so a model published later is listed without a +# release of this package. See merge_catalogs. +PROBE_LIMIT = 10 +# The selector version the fork requires (REQUIRED_JSON_VERSION on the comma). +# It is a string in the JSON; bundles at any other version describe fields we +# would misread. +REQUIRED_SELECTOR_VERSION = 19 +# Cinque Terre V3 Model (September 17, 2026) +DEFAULT_BIG_MODEL_REF = 'bf3e3631b3f91d92a1020a5e0dd4298b93ff4244' +# its name as sunnypilot's catalog lists it, without the build date: what a +# comma with no catalog to name it from shows (jetlink.openpilot, on a fork +# without sunnypilot's model manager) +DEFAULT_BIG_MODEL_NAME = 'Cinque Terre V3 Model' +CATALOG_TIMEOUT = 10.0 + +_REF = re.compile(r'[0-9a-f]{40}') +_SHA256 = re.compile(r'[0-9a-f]{64}') + + +class RegistryError(Exception): + """Anything the registry refuses to do.""" + + +class NetworkError(RegistryError): + """A server could not be reached or did not answer sensibly.""" + + +class VerifyError(RegistryError): + """Bytes arrived, but not the bytes that were asked for.""" + + +class NotFound(NetworkError): + """The server answered that there is nothing at the URL.""" + + +@dataclass(frozen=True) +class CatalogModel: + name: str + short_name: str + ref: str + build_time: str + index: int + + +def is_ref(value: str) -> bool: + """A comma openpilot commit: 40 lowercase hex characters.""" + return isinstance(value, str) and _REF.fullmatch(value) is not None + + +def is_sha256(value: str) -> bool: + """A model identity: 64 lowercase hex characters, which is also the LFS oid.""" + return isinstance(value, str) and _SHA256.fullmatch(value) is not None + + +def parse_catalog(data: dict) -> list[CatalogModel]: + """The big-model bundles, newest first. No IO, and no bundle can raise. + + A malformed entry in a catalog served to every comma must cost that entry and + nothing else, so each bundle is parsed in its own try. + """ + found: dict[str, CatalogModel] = {} + for bundle in _bundles(data): + try: + ref = bundle.get('ref') + if not is_ref(ref) or ref in found: + continue + if _selector(bundle) != REQUIRED_SELECTOR_VERSION or not bundle.get('is_big'): + continue + found[ref] = CatalogModel(name=str(bundle.get('display_name') or ref[:10]), short_name=str(bundle.get('short_name') or ''), + ref=ref, build_time=str(bundle.get('build_time') or ''), index=int(bundle.get('index', 0))) + except (AttributeError, TypeError, ValueError): + continue + return sorted(found.values(), key=lambda m: m.index, reverse=True) + + +def http_get(url: str, timeout: float, opener=None, limit: int = -1) -> bytes: + """The body at `url`, at most `limit` bytes. A 404 is NotFound, and every + other failure a NetworkError, so a caller only catches what it treats apart.""" + opener = opener or urllib.request.urlopen + try: + with opener(url, timeout=timeout) as response: + return response.read(limit) + except urllib.error.HTTPError as e: + raise (NotFound if e.code == 404 else NetworkError)(f"could not fetch {url}: {e}") from e + except (OSError, ValueError) as e: + raise NetworkError(f"could not fetch {url}: {e}") from e + + +def http_json(url: str, timeout: float, opener=None, kind: type = dict): + """http_get, parsed, and of the type asked for.""" + body = http_get(url, timeout, opener) + try: + data = json.loads(body.decode()) + except ValueError as e: + raise NetworkError(f"{url} did not serve JSON: {e}") from e + if not isinstance(data, kind): + raise NetworkError(f"{url} did not serve a JSON {kind.__name__}") + return data + + +def fetch_catalog(url: str = CATALOG_URL, timeout: float = CATALOG_TIMEOUT, opener=None) -> dict: + """The catalog JSON. Every failure, transport or content, is a NetworkError.""" + return http_json(url, timeout, opener) + + +def fetch_catalogs(opener=None) -> dict: + """The pinned catalog merged with every one sunnypilot has published since, + whose commits the server runs too. The probe stops at the first version that + is not there. Any other failure raises: a short list would drop a model + found only past it from whatever the caller caches, so the caller keeps its + last complete one instead.""" + found = [fetch_catalog(CATALOG_URL, opener=opener)] + for v in range(CATALOG_VERSION + 1, CATALOG_VERSION + 1 + PROBE_LIMIT): + try: + found.append(fetch_catalog(CATALOG_URL_TEMPLATE.format(version=v), opener=opener)) + except NotFound: + break + return merge_catalogs(found) + + +def merge_catalogs(catalogs: list[dict], selector: int = REQUIRED_SELECTOR_VERSION) -> dict: + """One catalog in the first one's shape, listing every big model of them all. + + A model some catalog lists at `selector` comes through as published, so a + chestnut can still fetch its build. One listed only at another selector + version, which is where sunnypilot puts every model once it moves runtimes, + comes from the newest catalog that has it, retyped to `selector` and with no + artifacts: an accelerator runs the commit's ONNX and needs nothing else from + the entry, and a chestnut has nothing to download. + """ + if not catalogs: + return {} + kept: dict[str, dict] = {} + others: dict[str, dict] = {} + for data in catalogs: + for bundle in _bundles(data): + ref = bundle.get('ref') + if not is_ref(ref): + continue + if _selector(bundle) == selector: + kept.setdefault(ref, bundle) + elif bundle.get('is_big'): + others[ref] = bundle # the newest catalog's entry wins + extra = [{**b, 'minimum_selector_version': str(selector), 'models': []} for ref, b in others.items() if ref not in kept] + return {**catalogs[0], 'bundles': [*kept.values(), *extra]} + + +def _bundles(data: dict) -> list[dict]: + bundles = data.get('bundles') if isinstance(data, dict) else None + return [b for b in bundles if isinstance(b, dict)] if isinstance(bundles, list) else [] + + +def _selector(bundle: dict) -> int | None: + try: + return int(bundle.get('minimum_selector_version', 0)) + except (TypeError, ValueError): + return None diff --git a/jetlink_repo/jetlink/registry/lfs.py b/jetlink_repo/jetlink/registry/lfs.py new file mode 100644 index 0000000000..74947f0dc3 --- /dev/null +++ b/jetlink_repo/jetlink/registry/lfs.py @@ -0,0 +1,245 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Getting a large model's ONNX by its git-lfs oid. + +comma overwrites one file per model, so the commit is the only name a big +model's ONNX has. GitHub's raw host serves the LFS pointer for any commit it +holds, merged or not, and that pointer carries the oid and size the comma will +ask a jetlink server for. The bytes themselves are on comma's LFS servers, +which is GitLab and not GitHub: each is asked in turn because which one has an +object varies with the model's age. + +Since openpilot #38930 (2026-09-16) a commit can ship a precompiled tinygrad +pkl instead, with no ONNX in the tree at all; Cinque Terre V3 is one. Its +subject names the export ("Use f78ed37d for the precompiled eGPU driving +model") and the ONNX is in comma's HuggingFace model repo, in the folder that +id starts. sunnypilot's model builds find it the same way. That repo speaks +the LFS batch protocol too, so it is one more endpoint to ask. + +The zoompilot fork downloads through this module too, asking its own +.lfsconfig endpoint first. No openpilot imports. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import re +import shutil +import urllib.parse +import urllib.request +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +from jetlink.registry.catalog import NetworkError, NotFound, RegistryError, VerifyError, http_get, http_json, is_sha256 + +log = logging.getLogger('jetlink.registry') + +BIG_ONNX = 'big_driving_supercombo.onnx' +POINTER_URL = 'https://raw.githubusercontent.com/commaai/openpilot/{ref}/openpilot/selfdrive/modeld/models/' + BIG_ONNX +# a commit's subject without the API, whose anonymous limit a car behind CGNAT shares +COMMIT_PATCH_URL = 'https://github.com/commaai/openpilot/commit/{ref}.patch' +DRIVING_MODELS_REPO = 'commaai/openpilot_driving_models' +DRIVING_MODELS_TREE_URL = f'https://huggingface.co/api/models/{DRIVING_MODELS_REPO}/tree/main' +LFS_ENDPOINTS = ( + 'https://gitlab.com/commaai/openpilot-lfs.git/info/lfs', # every object, older and PR-branch models included + 'https://huggingface.co/commaai/openpilot-lfs.git/info/lfs', # where comma is moving them; the current ones + f'https://huggingface.co/{DRIVING_MODELS_REPO}.git/info/lfs', # the exports behind a precompiled pkl +) +# an export folder is a uuid; the subject names its first eight +_EXPORT_ID = re.compile(r'\b[0-9a-f]{8}\b') +LFS_MEDIA_TYPE = 'application/vnd.git-lfs+json' +POINTER_TIMEOUT = 10.0 +CONNECT_TIMEOUT = 30.0 +# A pointer is 134 bytes. Anything larger is the ONNX itself, served by a host +# that resolved the LFS filter for us, and reading a gigabyte to find that out +# is not on. +POINTER_MAX = 4096 +CHUNK = 4 << 20 +FREE_SLACK = 64 << 20 + +ProgressFn = Callable[[float], None] +StopFn = Callable[[], bool] + + +@dataclass(frozen=True) +class Pointer: + oid: str # the ONNX SHA-256 + size: int + + +def parse_pointer_text(text: str) -> Pointer | None: + """The oid and size in a git-lfs pointer's text, or None if it is not one.""" + if not isinstance(text, str) or len(text.encode('utf-8', 'replace')) > POINTER_MAX: + return None + oid = None + size = None + for line in text.splitlines(): + key, _, value = line.partition(' ') + if key == 'oid': + oid = value.removeprefix('sha256:').strip() + elif key == 'size': + try: + size = int(value) + except ValueError: + return None + if not is_sha256(oid) or size is None or size <= 0: + return None + return Pointer(oid, size) + + +def fetch_pointer(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> Pointer: + """The oid and size of the ONNX at a comma commit: in its tree, or for a + commit that ships a precompiled pkl instead, the export its subject names.""" + try: + text = http_get(POINTER_URL.format(ref=ref), timeout, opener, POINTER_MAX).decode('utf-8', 'replace') + except NotFound: + return fetch_export_pointer(ref, timeout=timeout, opener=opener) + pointer = parse_pointer_text(text) + if pointer is None: + raise RegistryError(f"{ref[:10]} did not serve an lfs pointer") + return pointer + + +def commit_subject(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> str: + """A comma commit's subject line, from the head of its patch.""" + url = COMMIT_PATCH_URL.format(ref=ref) + lines = http_get(url, timeout, opener, POINTER_MAX).decode('utf-8', 'replace').splitlines() + for i, line in enumerate(lines): + if line.startswith('Subject:'): + subject = [line.removeprefix('Subject:').strip()] + # a long subject is folded onto indented lines; the headers end at a blank one + for cont in lines[i + 1:]: + if not cont[:1].isspace() or not cont.strip(): + break + subject.append(cont.strip()) + return re.sub(r'^\[PATCH[^\]]*\]\s*', '', ' '.join(subject)) + raise RegistryError(f"{url} has no subject line") + + +def _tree(path: str, timeout: float, opener) -> list[dict]: + url = DRIVING_MODELS_TREE_URL + (f"/{urllib.parse.quote(path)}?recursive=true" if path else '') + return [e for e in http_json(url, timeout, opener, list) if isinstance(e, dict) and isinstance(e.get('path'), str)] + + +def fetch_export_pointer(ref: str, timeout: float = POINTER_TIMEOUT, opener=None) -> Pointer: + """The big ONNX a precompiled-pkl commit was built from, in comma's model repo.""" + subject = commit_subject(ref, timeout=timeout, opener=opener) + ids = list(dict.fromkeys(_EXPORT_ID.findall(subject))) + if not ids: + raise RegistryError(f"{ref[:10]} has no {BIG_ONNX} and its subject names no export: {subject!r}") + folders = [e['path'] for e in _tree('', timeout, opener) if e.get('type') == 'directory'] + for export in ids: + matches = [f for f in folders if f.startswith(export)] + if not matches: + continue + if len(matches) > 1: + raise RegistryError(f"{ref[:10]}: {export} starts {len(matches)} folders in {DRIVING_MODELS_REPO}") + files = [e for e in _tree(matches[0], timeout, opener) + if e.get('type') == 'file' and e['path'].rsplit('/', 1)[-1] == BIG_ONNX] + if len(files) > 1: + # a subject that names the checkpoint, as '1a421175-.../12864' does, picks one + files = [f for f in files if f['path'].rsplit('/', 1)[0] in subject] or files + if len(files) != 1: + raise RegistryError(f"{ref[:10]}: {len(files)} copies of {BIG_ONNX} under {matches[0]}") + lfs = files[0].get('lfs') or {} + oid, size = lfs.get('oid'), lfs.get('size') + if not is_sha256(oid) or not isinstance(size, int) or size <= 0: + raise RegistryError(f"{files[0]['path']} is not an lfs object") + log.info("%s names export %s: %s", ref[:10], export, files[0]['path']) + return Pointer(oid, size) + raise RegistryError(f"{ref[:10]}: no folder in {DRIVING_MODELS_REPO} for {', '.join(ids)}") + + +def lfs_resolve(endpoint: str, pointer: Pointer, timeout: float = CONNECT_TIMEOUT, opener=None) -> str | None: + """Ask one LFS server for a download href, or None if it does not have it. + + A server that is down is not different from a server that lacks the object: + either way the caller moves to the next one, so nothing raises here. + """ + opener = opener or urllib.request.urlopen + body = json.dumps({ + 'operation': 'download', + 'transfers': ['basic'], + 'objects': [{'oid': pointer.oid, 'size': pointer.size}], + }).encode() + request = urllib.request.Request(f"{endpoint}/objects/batch", data=body, method='POST', + headers={'Accept': LFS_MEDIA_TYPE, 'Content-Type': LFS_MEDIA_TYPE}) + try: + with opener(request, timeout=timeout) as response: + payload = json.loads(response.read().decode()) + except (OSError, ValueError) as e: + log.warning("lfs batch failed at %s: %s", endpoint, e) + return None + + for obj in (payload.get('objects') or []) if isinstance(payload, dict) else []: + if not isinstance(obj, dict) or obj.get('oid') != pointer.oid: + continue + if 'error' in obj: + log.warning("%s has no %s (%s)", endpoint, pointer.oid[:16], (obj['error'] or {}).get('message')) + return None + href = ((obj.get('actions') or {}).get('download') or {}).get('href') + if href: + return str(href) + return None + + +def lfs_download(href: str, pointer: Pointer, dest: Path, progress: ProgressFn | None = None, + should_stop: StopFn | None = None, opener=None) -> Path: + """Stream to a .part file, hashing as we go, and only then take the name. + + A half-written model must never sit where the next start would hand it to a + backend to build from. + """ + opener = opener or urllib.request.urlopen + dest = Path(dest) + dest.parent.mkdir(parents=True, exist_ok=True) + free = shutil.disk_usage(dest.parent).free + if free < pointer.size + FREE_SLACK: + raise RegistryError(f"need {pointer.size >> 20} MB for the model, {free >> 20} MB free") + + part = dest.with_name(dest.name + '.part') + digest = hashlib.sha256() + written = 0 + # Whole percent only: a gigabyte at 4 MB a chunk would call this a few + # hundred times and the callback may write a param or a socket line. + reported = -1 + try: + with opener(href, timeout=CONNECT_TIMEOUT) as response, open(part, 'wb') as out: + while True: + if should_stop is not None and should_stop(): + raise RegistryError('download cancelled') + chunk = response.read(CHUNK) + if not chunk: + break + out.write(chunk) + digest.update(chunk) + written += len(chunk) + if progress is not None and pointer.size: + percent = int(100 * written / pointer.size) + if percent != reported: + reported = percent + progress(min(1.0, written / pointer.size)) + except RegistryError: + part.unlink(missing_ok=True) + raise + except Exception as e: + part.unlink(missing_ok=True) + raise NetworkError(f"could not download {pointer.oid[:16]}: {e}") from e + + if written != pointer.size: + part.unlink(missing_ok=True) + raise VerifyError(f"{pointer.oid[:16]} is {written} bytes, expected {pointer.size}") + if digest.hexdigest() != pointer.oid: + part.unlink(missing_ok=True) + raise VerifyError(f"downloaded bytes hash to {digest.hexdigest()[:16]}, expected {pointer.oid[:16]}") + + part.replace(dest) + if progress is not None: + progress(1.0) + return dest diff --git a/jetlink_repo/jetlink/spec.py b/jetlink_repo/jetlink/spec.py new file mode 100644 index 0000000000..1fcf954e20 --- /dev/null +++ b/jetlink_repo/jetlink/spec.py @@ -0,0 +1,250 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +Every size on the wire, derived from the model's ONNX metadata. + +Two graph layouts are in the wild. The queued one takes its history as +stacked inputs (img, features_buffer, desire_pulse) and the server keeps the +queues; see queues.PolicyQueues. It mirrors get_policy_npy_shapes / +make_input_queues in openpilot's selfdrive/modeld/compile_modeld.py up to +openpilot #38916. feat_dim is prod(fb[2:]); older forks use fb[2], which gives +32 rather than 16384 for the big model's (1,32,32,512) features_buffer. +tests/test_queues.py catches the drift. + +The stateful one (openpilot #38916, 2026-09-15; Cinque Terre V3 onwards) +carries its history in the graph: the newest warped frame goes in as new_img, +each queue goes in as state_ and comes back advanced as next_state_, and +the hidden state never leaves the graph. + +Either way the wire carries the newest frame and three scalars, and the reply +the outputs less hidden_state (protocol 3): the queued graph's server feeds its +own hidden state back, which modeld did through prev_feat until then. +""" +from __future__ import annotations + +import hashlib +import json +import math +from dataclasses import dataclass +from functools import cached_property +from pathlib import Path + +from jetlink.onnx_meta import parse_file +from jetlink.protocol import INFER_REQ_SIZE, INFER_RESP_SIZE + +# openpilot ModelConstants; duplicated so the server needs no openpilot import +MODEL_RUN_FREQ = 20 +MODEL_CONTEXT_FREQ = 5 +DEFAULT_FRAME_SKIP = MODEL_RUN_FREQ // MODEL_CONTEXT_FREQ # 4 + +CHUNK = 4 << 20 # model upload chunk + +# the driving output every layout has, 18452 floats in openpilot's layout +DRIVING_OUTPUT = 'outputs' +# the slice of it that stays on the server; 16384 of the 18452 +HIDDEN_STATE = 'hidden_state' +# the input only a stateful graph has; see the module docstring +STATEFUL_FRAME = 'new_img' +STATE_OUTPUT_PREFIX = 'next_' + + +@dataclass(frozen=True) +class ModelSpec: + """Everything both ends need to agree on, derived from the ONNX.""" + sha256: str + nbytes: int + frame_skip: int + input_shapes: dict[str, tuple[int, ...]] + output_shapes: dict[str, tuple[int, ...]] + output_slices: dict[str, slice] + checkpoint: str | None + + # --- layout --- + @property + def stateful(self) -> bool: + """The graph keeps its own history (openpilot #38916).""" + return STATEFUL_FRAME in self.input_shapes + + @property + def state_pairs(self) -> dict[str, str]: + """state_ input -> the next_state_ output that feeds it next frame. + Empty for a queued graph. Same rule as openpilot's ModelState.""" + return {n: STATE_OUTPUT_PREFIX + n for n in self.input_shapes + if STATE_OUTPUT_PREFIX + n in self.output_shapes} + + # --- vision --- + @property + def img_shape(self) -> tuple[int, ...]: + return self.input_shapes['img'] # (1, 12, H, W); queued graphs only + + @property + def n_frames(self) -> int: + return self.img_shape[1] // 6 + + @property + def model_hw(self) -> tuple[int, int]: + # new_img is (2, 6, H, W) and img (1, 12, H, W): the last two either way + shape = self.input_shapes[STATEFUL_FRAME] if self.stateful else self.img_shape + return shape[-2], shape[-1] + + @property + def img_buf_shape(self) -> tuple[int, int, int, int]: + h, w = self.model_hw + return (self.frame_skip * (self.n_frames - 1) + 1, 6, h, w) + + @property + def warped_shape(self) -> tuple[int, int, int, int]: + """What `warp` on the comma produces: narrow and wide stacked.""" + h, w = self.model_hw + return (2, 6, h, w) + + @property + def warped_nbytes(self) -> int: + return math.prod(self.warped_shape) # uint8 + + # --- recurrent / scalar inputs --- + @property + def feat_dim(self) -> int: + """Flattened per-frame feature size. (1,32,32,512) -> 16384.""" + fb = self.input_shapes['features_buffer'] + return math.prod(fb[2:]) + + @property + def packed_shapes(self) -> dict[str, tuple[int, ...]]: + """The floats a frame sends after the image: compile_modeld's + packed_npy_inputs less prev_feat, which the server keeps.""" + scalars = {'traffic_convention': tuple(self.input_shapes['traffic_convention']), + 'action_t': tuple(self.input_shapes['action_t'])} + if self.stateful: + # the pulse is the graph's own input + return {'desire': (math.prod(self.input_shapes['desire']),), **scalars} + return {'desire': (self.input_shapes['desire_pulse'][2],), **scalars} + + @property + def prev_feat_shape(self) -> tuple[int, int]: + """The hidden state a queued graph's server feeds back each frame.""" + return (self.input_shapes['features_buffer'][0], self.feat_dim) + + @property + def packed_layout(self) -> dict[str, tuple[slice, tuple[int, ...]]]: + """Where each of packed_shapes sits in the flat floats: name -> (slice, shape).""" + out, offset = {}, 0 + for name, shape in self.packed_shapes.items(): + out[name] = (slice(offset, offset + math.prod(shape)), shape) + offset += math.prod(shape) + return out + + @property + def packed_nelem(self) -> int: + return sum(math.prod(s) for s in self.packed_shapes.values()) + + def feed_back(self, packed, output) -> None: + """Nothing: the server feeds the hidden state back itself (protocol 3). + Kept for the modeld glue that still calls it after every frame.""" + + @property + def packed_nbytes(self) -> int: + return self.packed_nelem * 4 # float32 + + @property + def feat_q_shape(self) -> tuple[int, int, int]: + fb = self.input_shapes['features_buffer'] + return (self.frame_skip * fb[1], fb[0], self.feat_dim) + + @property + def desire_q_shape(self) -> tuple[int, int, int]: + dp = self.input_shapes['desire_pulse'] + return (self.frame_skip * dp[1], dp[0], dp[2]) + + # --- output --- + @property + def output_nelem(self) -> int: + return math.prod(self.output_shapes[DRIVING_OUTPUT]) + + @property + def output_nbytes(self) -> int: + return self.output_nelem * 4 # we return float32, as openpilot's JIT does + + # cached: the client reads these on every frame, and a frozen dataclass + # without slots has a __dict__ to keep them in + @cached_property + def hidden_range(self) -> tuple[int, int] | None: + """[start, stop) of hidden_state in the output: what the reply leaves out. + Its ends resolve as Python slicing resolves them, an open end or one + counted from the back included, which is also how the Swift server reads + them (NamedSlice.range(in:)). None when the model names no such slice or + it takes nothing, and then the reply is whole.""" + s = self.output_slices.get(HIDDEN_STATE) + if s is None or s.step not in (None, 1): + return None + start, stop, _ = s.indices(self.output_nelem) + return (start, stop) if start < stop else None + + @cached_property + def reply_nelem(self) -> int: + """The floats an INFER_RESP carries: the output less hidden_state.""" + h = self.hidden_range + return self.output_nelem - (h[1] - h[0] if h else 0) + + # --- wire sizes --- + @property + def infer_req_nbytes(self) -> int: + return INFER_REQ_SIZE + self.warped_nbytes + self.packed_nbytes + + @property + def infer_resp_nbytes(self) -> int: + """Without telemetry; with Flag.WANT_HIDDEN it is INFER_RESP_SIZE + + output_nbytes.""" + return INFER_RESP_SIZE + self.reply_nelem * 4 + + # -- the wire form of a spec --------------------------------------------- + # One encoder and one decoder: both ends and the bench need this, and copies + # would drift the moment a field is added. + + def to_dict(self) -> dict: + return { + 'sha256': self.sha256, + 'nbytes': self.nbytes, + 'frame_skip': self.frame_skip, + 'checkpoint': self.checkpoint, + 'input_shapes': {k: list(v) for k, v in self.input_shapes.items()}, + 'output_shapes': {k: list(v) for k, v in self.output_shapes.items()}, + 'output_slices': {k: [v.start, v.stop] for k, v in self.output_slices.items()}, + } + + @classmethod + def from_dict(cls, d: dict) -> ModelSpec: + return cls( + sha256=d['sha256'], nbytes=d['nbytes'], + frame_skip=d.get('frame_skip', DEFAULT_FRAME_SKIP), + input_shapes={k: tuple(v) for k, v in d['input_shapes'].items()}, + output_shapes={k: tuple(v) for k, v in d['output_shapes'].items()}, + output_slices={k: slice(*v) for k, v in d['output_slices'].items()}, + checkpoint=d.get('checkpoint')) + + @classmethod + def load(cls, path: str | Path) -> ModelSpec: + """A spec saved as to_dict's JSON.""" + return cls.from_dict(json.loads(Path(path).read_text())) + + +def sha256_file(path: str, bufsize: int = 1 << 20) -> tuple[str, int]: + h = hashlib.sha256() + n = 0 + with open(path, 'rb') as f: + while chunk := f.read(bufsize): + h.update(chunk) + n += len(chunk) + return h.hexdigest(), n + + +def spec_from_onnx(path: str, frame_skip: int = DEFAULT_FRAME_SKIP, + sha256: str | None = None, nbytes: int | None = None) -> ModelSpec: + meta = parse_file(path) + if sha256 is None or nbytes is None: + sha256, nbytes = sha256_file(path) + return ModelSpec(sha256=sha256, nbytes=nbytes, frame_skip=frame_skip, input_shapes=dict(meta.inputs), + output_shapes=dict(meta.outputs), output_slices=meta.output_slices, checkpoint=meta.model_checkpoint) diff --git a/jetlink_repo/jetlink/transport/__init__.py b/jetlink_repo/jetlink/transport/__init__.py new file mode 100644 index 0000000000..dba89a4763 --- /dev/null +++ b/jetlink_repo/jetlink/transport/__init__.py @@ -0,0 +1,9 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. +""" +from jetlink.transport.base import LinkError, LinkTimeout, Message, Transport + +__all__ = ['LinkError', 'LinkTimeout', 'Message', 'Transport'] diff --git a/jetlink_repo/jetlink/transport/base.py b/jetlink_repo/jetlink/transport/base.py new file mode 100644 index 0000000000..a8293a2349 --- /dev/null +++ b/jetlink_repo/jetlink/transport/base.py @@ -0,0 +1,334 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. +""" +from __future__ import annotations + +import time +from abc import ABC, abstractmethod +from dataclasses import dataclass + +from jetlink import protocol as P + +# Stops a corrupt length field making RxBuffer allocate gigabytes. A big +# model's request is 393 KB. +MAX_MESSAGE = 16 << 20 +_PAD = bytes(P.GADGET_TX_ALIGN) + + +# The USB generation of each speed Linux names, as the server's link event +# shows it. The comma's hello carries the speed (Transport.link_info), because +# only its end always knows: a phone's cable is TCP over USB, and a host may +# not see the bus speed. The server maps it with this (Pinned.usbSpeedMedia). +USB_MEDIA = {'super-speed-plus': 'usb3', 'super-speed': 'usb3', 'high-speed': 'usb2', + 'full-speed': 'usb1', 'low-speed': 'usb1'} +UDC_SYSFS = '/sys/class/udc' + + +def udc_speed(udc: str | None = None, root: str | None = None) -> str | None: + """The speed a device controller negotiated with its host, as Linux names it + (super-speed, high-speed, ...), from `udc` or the first controller; None + without one, or before a host has configured it.""" + import os + root = root or UDC_SYSFS + try: + name = udc or sorted(os.listdir(root))[0] + with open(os.path.join(root, name, 'current_speed')) as f: + speed = f.read().strip() + except (OSError, IndexError): + return None + return speed if speed in USB_MEDIA else None + + +def usb_link_info(kind: str, speed: str | None) -> dict: + return {'kind': kind, **({'usb_speed': speed} if speed else {})} + + +class LinkError(IOError): + """The link is unusable. Callers treat this as 'fall back to the small model'.""" + + +class LinkTimeout(LinkError): + """No complete message arrived in time. The stream is still in sync.""" + + +@dataclass +class Message: + msg_type: int + seq: int + flags: int + payload: memoryview # valid only until the next recv() on this transport + + +class Transport(ABC): + """One framed, ordered, reliable message channel. + + Implementations must preserve message boundaries and ordering. recv() hands + back a view into a reusable buffer: copy anything you need to keep. + """ + + def link_info(self) -> dict: + """What the comma's hello says about this link: its kind (usb, cable or + tcp) and, over USB, the speed the controller negotiated. Empty when there + is nothing to say.""" + return {} + + @abstractmethod + def send(self, msg_type: int, seq: int, parts=(), flags: int = 0, timeout: float | None = None) -> None: + """Send one message. `parts` is an iterable of buffers, sent as one message.""" + + @abstractmethod + def recv(self, timeout: float | None = None) -> Message: + """Block for one message. Raises LinkTimeout if `timeout` elapses.""" + + @abstractmethod + def close(self) -> None: + ... + + def send_json(self, msg_type: int, seq: int, obj, flags: int = 0, timeout: float | None = None) -> None: + import json + self.send(msg_type, seq, (json.dumps(obj).encode(),), flags, timeout) + + @property + def lendable(self) -> bool: + """Could another process take over the IO on this link right now? Only a + gadget can be handed over; see FfsTransport.lendable.""" + return False + + def rebind(self) -> bool: + """Bounce the link so the peer sees it arrive again, if that means + anything here. Only the gadget transport can; see FfsTransport.rebind.""" + return False + + def release_endpoints(self) -> bool: + """Give up the IO without giving up the link, where the two are separable. + Only the gadget transport can; see FfsTransport.release_endpoints.""" + return False + + +class RxBuffer: + """Receive buffer for a byte stream carrying framed messages. + + Reads land in here and messages are handed out as views, so the steady state + neither allocates nor copies. Resumable: a read that times out mid-message + leaves the bytes in place, so a missed deadline costs a frame, not the stream. + """ + + def __init__(self, size: int = 1 << 20): + self.buf = bytearray(size) + self.view = memoryview(self.buf) + self.start = 0 # first byte not yet consumed + self.end = 0 # one past the last byte read + + @property + def available(self) -> int: + return self.end - self.start + + def reserve(self, need: int) -> None: + """Guarantee room for `need` unconsumed bytes, compacting or growing.""" + if self.start and self.start + need > len(self.buf): + # Through the bytearray, not the view: the ranges overlap and a + # memoryview slice assignment is a memcpy. Compaction is rare. + self.buf[:self.available] = self.buf[self.start:self.end] + self.end -= self.start + self.start = 0 + if need > len(self.buf): + grown = bytearray(max(need, len(self.buf) * 2)) + grown[:self.available] = self.view[self.start:self.end] + self.buf, self.view = grown, memoryview(grown) + self.end -= self.start + self.start = 0 + + def writable(self) -> memoryview: + return self.view[self.end:] + + def committed(self, n: int) -> None: + self.end += n + + def take(self, n: int) -> memoryview: + out = self.view[self.start:self.start + n] + self.start += n + return out + + def consumed(self) -> None: + """Call once a message has been fully handed out.""" + if self.start == self.end: + self.start = self.end = 0 + + +class StreamTransport(Transport): + """Framing over any ordered byte stream. + + TCP, USB bulk and FunctionFS are all streams, so they all need exactly this. + Subclasses supply only the two primitives that differ. + """ + + # Extra capacity past the current message, for transports whose reads need a + # minimum buffer size (a bulk OUT endpoint wants a whole packet). + read_slack = 0 + # Bulk endpoints reject a read whose buffer is not a whole number of packets. + # 0 means "no constraint" (TCP). + packet_size = 0 + read_chunk = 1 << 20 + # Pad sent messages to a multiple of this and expect the peer's padded + # likewise; 0 uses the one-byte PADDED rule. See protocol.GADGET_TX_ALIGN. + tx_align = 0 + rx_align = 0 + # Largest single write. FunctionFS allocates a contiguous buffer per writev, + # so a 4 MB upload chunk hits ENOMEM on a fragmented device where a few + # hundred KB of inference does not. 0 means no cap (TCP). + write_chunk = 0 + + def __init__(self, rx_size: int = 1 << 20): + self.rx = RxBuffer(rx_size) + self._desynced = False + self._send_deadline: float | None = None + + def _write_timeout(self, default: float | None = None) -> float | None: + if self._send_deadline is None: + return default + remaining = self._send_deadline - time.monotonic() + if remaining <= 0: + raise LinkError('send timed out; link abandoned') + return remaining + + # -- primitives a subclass must provide ---------------------------------- + + @abstractmethod + def _write(self, bufs: list[memoryview]) -> int: + """Write from one or more buffers. Returns bytes written (may be partial).""" + + @abstractmethod + def _read_into(self, dest: memoryview, timeout: float | None) -> int: + """Read up to len(dest) bytes, returning how many arrived. + + May return short, including 0. Must not raise on a timeout, and must not + drop what did arrive: that is how a stream silently desyncs. + """ + + # -- framing ------------------------------------------------------------- + + def send(self, msg_type: int, seq: int, parts=(), flags: int = 0, timeout: float | None = None) -> None: + # cast('B'): slicing a float32 view in advance() would step by elements + bufs = [memoryview(p).cast('B') for p in parts] + length = sum(b.nbytes for b in bufs) + if self.tx_align: + # Never a short packet in this direction; see protocol.GADGET_TX_ALIGN. + pad = -(P.HEADER_SIZE + length) % self.tx_align + if pad: + bufs.append(memoryview(_PAD)[:pad]) + elif (P.HEADER_SIZE + length) % P.PACKET_MULTIPLE == 0: + # A bulk transfer only ends on a short packet; see protocol.PACKET_MULTIPLE. + flags |= P.Flag.PADDED + bufs.append(memoryview(_PAD)[:1]) + header = P.pack_header(msg_type, seq, length, flags) + bufs.insert(0, memoryview(header)) + self._send_deadline = None if timeout is None else time.monotonic() + timeout + try: + while bufs: + self._write_timeout() + n = self._write(take(bufs, self.write_chunk) if self.write_chunk else bufs) + if n <= 0: + raise LinkError("peer went away during send") + bufs = advance(bufs, n) + finally: + self._send_deadline = None + + def _clamp_read(self, dest: memoryview) -> int: + """How many bytes this transport may ask for in one read.""" + n = min(dest.nbytes, self.read_chunk) + return (n // self.packet_size) * self.packet_size if self.packet_size else n + + def _fill(self, need: int, timeout: float | None) -> None: + """Read until `need` bytes are buffered, or the deadline passes. + + The deadline is per message, not per read, or one response could take + several times the caller's budget. Partial reads are kept, so a missed + deadline costs a frame and leaves the stream in sync. A deadline that has + passed still gets one read that does not wait, so a timeout of 0 takes + whatever has already arrived and nothing else: that is how a caller + polls for a reply without blocking (JetlinkClient.drain). + """ + self.rx.reserve(need + self.read_slack) + end = None if timeout is None else time.monotonic() + timeout + while self.rx.available < need: + dest = self.rx.writable()[:self._read_limit(need - self.rx.available)] + if self._clamp_read(dest) == 0: + # No room for a whole packet: every read returns 0 and this loop spins + # while the peer blocks. read_slack is too small; say so, do not hang. + raise LinkError(f"no room to read the rest of a {need} byte message " + f"({self.rx.available} in hand); read_slack too small") + remaining = None if end is None else max(0.0, end - time.monotonic()) + n = self._read_into(dest, remaining) + if n: + self.rx.committed(n) + elif remaining == 0.0: + raise LinkTimeout(f"only {self.rx.available} of {need} bytes arrived in time") + + def _read_limit(self, missing: int) -> int: + """`missing` bytes of the current message, rounded up to a whole packet. + + Keeps the USB host in sync: the gadget's messages are burst-aligned, so a + read for exactly what is left never stays outstanding past the end of a + message. Reading further desynced about once in 400 frames. + """ + if self.packet_size: + return -(-missing // self.packet_size) * self.packet_size + return missing + + def recv(self, timeout: float | None = None) -> Message: + if self._desynced: + raise LinkError("stream desynced; the link must be reopened") + end = None if timeout is None else time.monotonic() + timeout + self._fill(P.HEADER_SIZE, timeout) + try: + _, _, msg_type, seq, flags, length, _reserved = P.unpack_header( + self.rx.view[self.rx.start:self.rx.start + P.HEADER_SIZE]) + if length > MAX_MESSAGE: + raise P.ProtocolError(f"message claims {length} bytes, over the {MAX_MESSAGE} cap") + except P.ProtocolError as e: + # Nothing resynchronises a byte stream mid-message. Latch it and report a + # LinkError, which callers reconnect on and modeld answers with the + # small model; a ProtocolError is something no caller catches. + self._desynced = True + raise LinkError(f"protocol error, link unusable: {e}") from e + # the remainder of the budget, so one recv cannot block for twice what it + # was given + if self.rx_align: + pad = -(P.HEADER_SIZE + length) % self.rx_align + else: + pad = 1 if flags & P.Flag.PADDED else 0 + self._fill(P.HEADER_SIZE + length + pad, + None if end is None else max(0.0, end - time.monotonic())) + self.rx.take(P.HEADER_SIZE) + payload = self.rx.take(length) + self.rx.take(pad) + self.rx.consumed() + return Message(msg_type, seq, flags, payload) + + +def take(bufs: list[memoryview], n: int) -> list[memoryview]: + """The first `n` bytes across a list of buffers, without copying.""" + out: list[memoryview] = [] + for mv in bufs: + if n <= 0: + break + out.append(mv if mv.nbytes <= n else mv[:n]) + n -= out[-1].nbytes + return out + + +def advance(bufs: list[memoryview], n: int) -> list[memoryview]: + """Drop the first `n` bytes across a list of buffers, returning what is left.""" + out: list[memoryview] = [] + for mv in bufs: + if n: + if n >= mv.nbytes: + n -= mv.nbytes + continue + mv = mv[n:] + n = 0 + out.append(mv) + return out diff --git a/jetlink_repo/jetlink/transport/ffs.py b/jetlink_repo/jetlink/transport/ffs.py new file mode 100644 index 0000000000..5bd71366dd --- /dev/null +++ b/jetlink_repo/jetlink/transport/ffs.py @@ -0,0 +1,717 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +USB gadget side of the link: a FunctionFS vendor-specific bulk function. + +On a comma this is the comma end. The roles look backwards and are not: AGNOS +has CONFIG_USB_F_FS built in, while a host needs no kernel driver at all, which +an L4T rootfs stripped of the gadget modules does not have. See +docs/transport.md. + +Bring-up (scripts/comma/jetlink-root.sh gadget does all of this): + configfs gadget -> functions/ffs.jetlink -> mount -t functionfs + write descriptors + strings to ep0 -> ep1 (OUT) and ep2 (IN) appear + echo > UDC +""" +from __future__ import annotations + +import errno +import logging +import os +import signal +import struct +import threading +import time +from collections import deque + +from jetlink import protocol as P +from jetlink.transport.base import UDC_SYSFS, LinkError, LinkTimeout, StreamTransport, take, udc_speed, usb_link_info +from jetlink.transport.priority import background_thread, widen_affinity +from jetlink.transport.watchdog import WriteWatchdog + +# --- FunctionFS ABI ------------------------------------------------------- + +FUNCTIONFS_DESCRIPTORS_MAGIC_V2 = 3 +FUNCTIONFS_STRINGS_MAGIC = 2 + +FLAG_HAS_FS = 1 << 0 +FLAG_HAS_HS = 1 << 1 +FLAG_HAS_SS = 1 << 2 + +USB_DT_INTERFACE = 0x04 +USB_DT_ENDPOINT = 0x05 +USB_DT_SS_ENDPOINT_COMP = 0x30 + +USB_ENDPOINT_XFER_BULK = 0x02 +EP_OUT = 0x01 # host -> device +EP_IN = 0x82 # device -> host + +# FunctionFS kmallocs a contiguous buffer per read: order-5 and order-6 failures +# in ffs_epfile_read_iter stalled replies 100-300 ms, and order-2 does not fail. +# Do not go smaller to dodge the rare slow success: one page per read is ~19 +# syscalls for a 74 KB reply and cost ~6 ms a frame. +READ_CHUNK = 16 * P.USB_MAX_PACKET +# How much the reader may queue before it stops. Only bounds memory if the +# consumer stalls: inference never needs more than one response. +MAX_QUEUED = 8 << 20 +# Read buffers kept for reuse. Steady state has one or two in flight, so this is +# only a ceiling for a transient backlog. 16 * 16 KB = 256 KB. +FREE_BUFS = 16 +# Just below modeld's frame loop (54), so the loop wins a contended core and the +# reader still preempts everything else. See _raise_reader_priority. +READER_RT_PRIORITY = 51 + +log = logging.getLogger('jetlink') + +# Enumerated is not ready: the host still has to open the device and claim the +# interface, and until then the endpoints return EIO. Its sysfs poll makes that +# a second or two, so wait rather than fail the first frame after connect. +EP_READY_TIMEOUT = 10.0 +# How long opening the endpoint files waits for a host to configure the gadget; +# EP_READY_TIMEOUT covers the claim that follows. +EP_OPEN_TIMEOUT = 10.0 + +# FunctionFS writes cannot time out: the request sits queued until the host +# drains it, measured at 90 s while a Jetson booted and three minutes on a +# drive. Unbinding the UDC dequeues it, the writev returns ESHUTDOWN, and the +# caller retries. A real write is ~3.6 ms. +WRITE_TIMEOUT = 15.0 +# where the comma's gadget lives: jetlink-root.sh gadget builds it there and +# mounts its FunctionFS instance here +GADGET = '/sys/kernel/config/usb_gadget/jetlink' +MOUNT = '/dev/ffs-jetlink' +# How long close() waits for the reader thread after unbinding, which is what +# wakes it. A read the kernel will not complete is left to die with the process. +READER_JOIN_TIMEOUT = 1.0 +# How long a rebind leaves the gadget off the bus. Long enough that the host +# sees a disconnect and not a glitch; a dwc3 pull-down is immediate. +REBIND_SETTLE = 0.5 +_NOT_READY = (errno.EIO, errno.ESHUTDOWN, errno.ENODEV) + +# A signal inside a FunctionFS transfer is not a retry: ffs_epfile_io returns +# EINTR mid-transfer, os.writev re-issues the whole call, and the host sees the +# start of the message twice. modeld's msgq raises SIGUSR2 ~160 times a second, +# which cost one link failure per 30 s of driving. Nothing here waits on one. +_IO_SIGNALS = signal.valid_signals() + + +def _interface_desc(n_endpoints: int = 2, i_interface: int = 1) -> bytes: + return struct.pack(' bytes: + return struct.pack(' bytes: + """SuperSpeed endpoint companion. + + bMaxBurst counts *additional* packets, so 15 is a 16 KB burst. Without + bursting a 459 KB request takes 167 ms, over the whole frame budget. + """ + return struct.pack(' bytes: + fs = _interface_desc() + _endpoint_desc(EP_OUT, 64) + _endpoint_desc(EP_IN, 64) + hs = _interface_desc() + _endpoint_desc(EP_OUT, 512) + _endpoint_desc(EP_IN, 512) + ss = (_interface_desc() + + _endpoint_desc(EP_OUT, P.USB_MAX_PACKET) + _ss_companion() + + _endpoint_desc(EP_IN, P.USB_MAX_PACKET) + _ss_companion()) + + # struct usb_functionfs_descs_head_v2: magic, length, flags, then ALL the + # per-speed counts together (one __le32 per flag set, in flag order), and only + # then the descriptor blocks. Interleaving count/block per speed gets EINVAL. + body = struct.pack(' bytes: + s = name.encode() + b'\0' + return (struct.pack(' dict: + return usb_link_info('usb', udc_speed(self.bound_udc)) + + @classmethod + def borrowed(cls, mount: str, udc: str, bounce=None, owner_gadget: str | None = None) -> FfsTransport: + """A transport over a gadget another process owns. + + That process holds ep0 and the UDC bind for its whole life, so the gadget + does not leave the bus when the link changes hands: this end only opens the + endpoint files and moves bytes, and nothing here writes descriptors, binds + or unbinds. The endpoint files open again once nothing is reading them (a + read a dead link left queued keeps them busy: see _open_after_bounce); ep0 + never does, which is why the owner keeps it. + + The ep0 rule is unchanged and `udc` is what keeps it: the endpoints are + still opened only once that controller reads configured. `bounce` is how a + write with no reader is freed, since the unbind that frees it belongs to + the owner. `owner_gadget` is the last resort for when the owner does not + answer at all; see _abort_write. + """ + if not udc: + # _configured() would answer "no controller of ours to ask, assume + # ready" and _ensure_epfiles would open an endpoint no host has enabled, + # which costs the gadget until the comma is rebooted. + raise LinkError('a borrowed gadget must name its controller') + t = cls.__new__(cls) + t._prepare(mount, gadget=None) + t.bound_udc = udc + t._bounce = bounce + t._owner_gadget = owner_gadget + return t + + def _prepare(self, mount: str, gadget: str | None) -> None: + # This end only receives replies: an INFER_RESP is 8 KB, or 74 KB when + # WANT_HIDDEN asks for the whole output, and the upload goes the other way. + # 256 KB leaves margin the memory-tight comma can spare, and RxBuffer grows + # past it on demand. + super().__init__(rx_size=256 << 10) + self.mount = mount + self.gadget = gadget + self.bound_udc: str | None = None + self._bounce = None + self._owner_gadget: str | None = None + self.ep0 = self.ep_out = self.ep_in = -1 + self._state_fd = -1 # held-open UDC 'state' fd; see _udc_state + self._ready_deadline: float | None = None + self._read_size = READ_CHUNK + self._cv = threading.Condition() + self._chunks: deque[tuple[memoryview, float, float, float]] = deque() + self._free: deque[bytearray] = deque() # read buffers the consumer handed back; see _read_loop + self.last_receive = {'prepare': 0.0, 'read_wait': 0.0, 'handoff': 0.0} + self._queued = 0 + self._reader_error: str | None = None + self._closing = False + self._had_host = False + self._open_lock = threading.Lock() + self._write_aborted = False + self._reader: threading.Thread | None = None + self._write_guard = WriteWatchdog(self._abort_write) + + def _udc_state(self) -> str | None: + """The controller's gadget state, off a held-open fd. + + The reader checks this before every readv and cannot skip it (see + _read_loop). Opening the sysfs file each time allocates, and under memory + pressure that reclaim stalled a frame 25 ms; a pread at offset 0 still + re-runs the attribute's show(), so the value is current. pread, not lseek + and read: the reader and the frame thread both ask during the claim + window, and two lseek+read pairs interleaved on one fd hand one of them + an empty read, which _host_gone would take for a host that left. The fd + is dropped on unbind and reopened lazily in case the controller differs. + """ + if self.bound_udc is None: + return None + if self._state_fd < 0: + try: + self._state_fd = os.open(os.path.join(UDC_SYSFS, self.bound_udc, 'state'), os.O_RDONLY) + except OSError: + return None + try: + state = os.pread(self._state_fd, 64, 0).decode().strip() + except OSError: + _close_quietly(self._state_fd) + self._state_fd = -1 + return None + return state or None + + def _configured(self) -> bool: + """Has a host set our configuration? Only then are the endpoints enabled.""" + if self.bound_udc is None: + return self.gadget is None # no controller of ours to ask; assume ready + return self._udc_state() == 'configured' + + def _udc_note(self) -> str: + """The controller's state, for a failure message. + + The same ENODEV comes from a cable falling out ("not attached"), from a + host resetting or re-enumerating us ("default", "addressed") and from a + bus the host suspended ("suspended"). A drive's worth of failures read + the same without it; with it the log says which layer let go. + """ + state = self._udc_state() + return f" (udc: {state})" if state else '' + + def _ensure_epfiles(self) -> None: + """Open ep1/ep2 and start the reader, once a host has enabled them. + + Never touch an endpoint file before that. ffs_epfile_io sleeps in + wait_event_interruptible until epfile->ep is set, and unbind does not wake + it; the sleeping syscall holds the struct file, so ffs_epfile_release never + runs and every ep0 open after it answers EBUSY, to this process and the next + one, until the comma is rebooted. The UDC reading configured is the same + edge that sets epfile->ep. + """ + if self.ep_out >= 0: + return + with self._open_lock: + if self.ep_out >= 0: + return + deadline = time.monotonic() + EP_OPEN_TIMEOUT + self._wait_configured(deadline) + try: + self.ep_out = os.open(os.path.join(self.mount, 'ep1'), os.O_RDWR) + except OSError as e: + if e.errno != errno.EBUSY or self._bounce is None: + raise + self.ep_out = self._open_after_bounce(deadline) + self.ep_in = os.open(os.path.join(self.mount, 'ep2'), os.O_RDWR) + self._had_host = True + self._reader = threading.Thread(target=self._read_loop, name='jetlink-ffs-read', daemon=True) + self._reader.start() + + def _wait_configured(self, deadline: float) -> None: + while not self._configured(): + if self._closing: + raise LinkError("gadget closing") + if time.monotonic() >= deadline: + raise LinkTimeout(f"no host configured the gadget within {EP_OPEN_TIMEOUT:.0f}s") + time.sleep(0.02) + + def _open_after_bounce(self, deadline: float) -> int: + """ep1 answered EBUSY: a transport this process closed after its link died + left a read queued on it, and FunctionFS keeps that read, and the endpoint + with it, until an unbind completes it. Seen when a Jetson's server + restarted mid-drive: some rejoins met EBUSY until the drive ended. The + unbind is the owner's, so a borrower asks for one, then opens once the + host has the gadget back and the stale read has let go.""" + log.warning("jetlink: ep1 is held by a read the last link left queued, asking the owner to bounce the gadget") + if not self._bounce(): + raise LinkError("ep1 is busy and the owner would not bounce the gadget") + while True: + self._wait_configured(deadline) + try: + return os.open(os.path.join(self.mount, 'ep1'), os.O_RDWR) + except OSError as e: + if e.errno != errno.EBUSY or time.monotonic() >= deadline: + raise + time.sleep(0.05) + + def release_endpoints(self) -> bool: + """Put the endpoint files down, keeping ep0 and the descriptors. + + An exchange leaves a read queued on the endpoint, and FunctionFS keeps it + queued until something completes it: nobody else may read that endpoint + until it does, and the only thing that dequeues it is the unbind. So the + owner gives the endpoints up like this and binds straight back - one + re-enumeration - without ever letting go of ep0, which is the whole point, + since the gadget exists only while somebody holds it. + + False when there is nothing to give up. Afterwards this transport is bare + and bound, and using it again reopens the endpoints as any first use does. + """ + if self.gadget is None or self._closing or self.ep_out < 0: + return False + udc, self._had_host = self.bound_udc, False + self.unbind() # completes the queued read with ESHUTDOWN + reader, self._reader = self._reader, None + if reader is not None and reader is not threading.current_thread(): + reader.join(READER_JOIN_TIMEOUT) + self._close_fds(('ep_in', 'ep_out'), reader) + # Whatever the last exchange left behind goes with the endpoints: half a + # message would frame the next one's first reply as garbage. + with self._cv: + self._chunks.clear() + self._free.clear() + self._queued = 0 + self._reader_error = None + self._cv.notify_all() + self.rx.start = self.rx.end = 0 + self._ready_deadline = None + self._write_aborted = False + time.sleep(REBIND_SETTLE) + self.bind(udc) + return True + + def bind(self, udc: str | None = None) -> None: + if self.gadget is None: + raise LinkError("no gadget path given") + if udc is None: + udcs = sorted(os.listdir(UDC_SYSFS)) + if not udcs: + raise LinkError("no USB device controller found") + udc = udcs[0] + with open(os.path.join(self.gadget, 'UDC'), 'w') as f: + f.write(udc + '\n') + self.bound_udc = udc + + @property + def lendable(self) -> bool: + """Bound to a controller, with no endpoint file open on this end. + + The state another process can take the endpoints over from. FunctionFS + keeps a queued read queued until something completes it, so a reader here + would sit in front of the borrower and take its reply; and without a bound + controller there is nothing to take over. + """ + return bool(self.gadget is not None and self.bound_udc and self.ep_out < 0 + and not self._closing) + + def rebind(self) -> bool: + """One unplug and replug, as the host sees it. + + For a host that answered our bind with a bus reset and then stopped: the + UDC sits in `default` or `addressed` and only another edge moves it. A + Jetson whose hubs are not armed for remote wakeup does exactly that. + + Refused while this end has the endpoint files open. Unbinding then + completes the reader's request with ESHUTDOWN and the transport is done; + the caller wants a working link, not a freshly bound dead one. + """ + if self._closing or self.ep_out >= 0: + return False + if self._bounce is not None: + return bool(self._bounce()) # a borrowed gadget: the bind is the owner's + if not self.lendable: + return False + udc = self.bound_udc + self.unbind() + time.sleep(REBIND_SETTLE) + self.bind(udc) + return True + + def unbind(self, gadget: str | None = None) -> None: + """Take the gadget off the bus. `gadget` overrides our own path, which is + how a borrower reaches an owner that has stopped answering.""" + gadget = gadget or self.gadget + if gadget is None or self.bound_udc is None: + return + try: + with open(os.path.join(gadget, 'UDC'), 'w') as f: + f.write('\n') + except OSError: + pass + self.bound_udc = None + if self._state_fd >= 0: # reopens lazily against whatever udc we bind next + _close_quietly(self._state_fd) + self._state_fd = -1 + + def _shrink(self, attr: str) -> bool: + """Halve a transfer size after the kernel refused to allocate for one. + + ENOMEM here is contiguous memory, so it tracks fragmentation: a size that + worked at boot can fail an hour in, which is what a 1.7 GB model does to a + comma. Slow beats broken, and both directions need it. + """ + floor = (self.packet_size or P.USB_MAX_PACKET) * 16 + current = getattr(self, attr) + if current <= floor: + return False + setattr(self, attr, max(floor, current // 2)) + if attr == '_read_size': + self._free.clear() # pooled buffers are the old, larger size now + log.warning("jetlink: gadget could not allocate for %s, dropping to %d KB", + attr, getattr(self, attr) >> 10) + return True + + def _shrink_write(self) -> bool: + return self._shrink('write_chunk') + + def _abort_write(self) -> None: + """Take the link down so a write nobody is reading returns. See WRITE_TIMEOUT. + + On the watchdog thread: the one stuck in writev cannot act. + """ + self._write_aborted = True + log.warning("jetlink: no reader for %.3f s, dropping the gadget to free the write", + getattr(self, '_write_budget', WRITE_TIMEOUT)) + if self._bounce is None: + return self.unbind() + # A borrowed gadget: the unbind that dequeues this belongs to whoever owns + # ep0, so ask. If nobody answers - the owner died, or is wedged itself - + # take the gadget down from here anyway. This runs while a frame thread is + # inside a writev the kernel will never return from on its own, and a + # re-enumeration costs one rejoin where a wedged modeld costs the whole + # drive, the small model included. + if self._bounce(): + return + log.error("jetlink: the gadget's owner did not answer; dropping the link from here") + self.unbind(self._owner_gadget) + + def send(self, *args, **kwargs) -> None: + # Reset the quantum at each message: a latched shrink would split every + # later request across two writes and re-arm the dwc3 replay (see + # write_chunk). The pressure that forced it is usually gone by the next + # frame. + self.write_chunk = type(self).write_chunk + super().send(*args, **kwargs) + + def _write(self, bufs: list[memoryview]) -> int: + self._ensure_epfiles() + self._write_budget = self._write_timeout(WRITE_TIMEOUT) + # See _IO_SIGNALS: a signal here duplicates data on the wire. + was = signal.pthread_sigmask(signal.SIG_BLOCK, _IO_SIGNALS) + try: + if not self._write_guard.arm(self._write_budget): + raise LinkError('gadget write watchdog already expired or closed') + while True: + try: + n = os.writev(self.ep_in, bufs) + self._had_host = True + return n + except OSError as e: + if self._write_aborted: + # Say whose doing it was: ENODEV alone reads like a cable falling + # out of a socket. + raise LinkError(f"gadget write had no reader for {self._write_budget:.3f}s") from e + # FunctionFS submits a write as one request, so a failed writev put + # nothing on the wire and is safe to retry. + if e.errno in _NOT_READY and self._wait_for_host_ready(): + continue + if e.errno == errno.ENOMEM and self._shrink_write(): + # A short write is fine: send() loops until the message is out. + bufs = take(bufs, self.write_chunk) + continue + raise LinkError(f"gadget write failed: {e}{self._udc_note()}") from e + finally: + completed = self._write_guard.disarm() + signal.pthread_sigmask(signal.SIG_SETMASK, was) + if not completed: + raise LinkError('gadget write had no reader before deadline; link abandoned') + + # -- the reader thread --------------------------------------------------- + + def _widen_affinity(self) -> None: + """Move the reader off the core modeld pinned its frame loop to. + + A thread created under config_realtime_process(7, 54) inherits SCHED_FIFO 54 + and the single-core pin, and sharing that core serialises the reader with + the frame loop. Usually a no-op: the creator is normally a background thread + already. See priority.widen_affinity. + """ + widen_affinity() + + def _raise_reader_priority(self) -> None: + """Realtime priority, so a completed read leaves the endpoint at once. + + The creator has dropped to SCHED_OTHER, and under recording plus onroad load + the wait to be scheduled after a transfer was 17 ms mean, 23 ms max - over + budget on read_wait alone. The reader copies a chunk and blocks again, so it + never holds a core. Best effort, and open-coded because jetlink must not + import openpilot. + """ + try: + os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(READER_RT_PRIORITY)) + except (OSError, AttributeError, ValueError): + pass + + def _read_loop(self) -> None: + # As in _write: an interrupted read drops the packets it already took. This + # thread handles no signals, so mask them for its whole life. + signal.pthread_sigmask(signal.SIG_BLOCK, _IO_SIGNALS) + self._widen_affinity() + self._raise_reader_priority() + # Fill the pool up front: recycling alone leaves a gap while a reply's + # chunks arrive and the consumer is a scheduling beat behind, and an + # allocation there reclaims under memory pressure (24 ms, over budget). + self._free.extend(bytearray(self._read_size) for _ in range(FREE_BUFS)) + while not self._closing: + with self._cv: + while self._queued >= MAX_QUEUED and not self._closing: + self._cv.wait(0.1) + if self._closing: + return + prepare_started = time.monotonic() + if not self._configured(): + # Same trap as _ensure_epfiles: the host can drop the configuration + # between two reads, and the readv after that sleeps forever. Treat it + # as the endpoint error it would have been. + if self._wait_for_host_ready(): + continue + self._fail(f"host dropped the gadget configuration{self._udc_note()}") + return + # Reuse a buffer the consumer handed back: a fresh bytearray here was + # measured stalling 20+ ms mid-frame under memory pressure, and the read + # cannot start until it returns. Only this thread pops the free list. + buf = self._free.popleft() if self._free else bytearray(self._read_size) + read_started = time.monotonic() + try: + # Multiples of the packet size only: the OUT endpoint rejects anything + # else, and _read_size is only ever halved from one. + got = os.readv(self.ep_out, [buf]) + except OSError as e: + if self._closing: + return + if e.errno in _NOT_READY and self._wait_for_host_ready(): + continue + if e.errno == errno.ENOMEM and self._shrink('_read_size'): + continue + self._fail(f"gadget read failed: {e}{self._udc_note()}") + return + if got == 0: + self._fail(f"gadget read returned EOF (host disconnected){self._udc_note()}") + return + read_finished = time.monotonic() + self._ready_deadline = None + self._had_host = True + with self._cv: + self._chunks.append((memoryview(buf)[:got], read_finished, + read_started - prepare_started, read_finished - read_started)) + self._queued += got + self._cv.notify_all() + + def _fail(self, why: str) -> None: + with self._cv: + self._reader_error = why + self._cv.notify_all() + + def recv(self, timeout: float | None = None): + # Per-message maxima, in seconds. read_wait covers the peer and kernel IO, + # handoff the consumer's wake; neither is a pure scheduler measurement. + self.last_receive = {'prepare': 0.0, 'read_wait': 0.0, 'handoff': 0.0} + return super().recv(timeout) + + def _read_into(self, dest: memoryview, timeout: float | None) -> int: + self._ensure_epfiles() + with self._cv: + if not self._chunks and self._reader_error is None: + self._cv.wait(timeout) + if self._chunks: + chunk, arrived, prepare, read_wait = self._chunks[0] + self.last_receive['prepare'] = max(self.last_receive['prepare'], prepare) + self.last_receive['read_wait'] = max(self.last_receive['read_wait'], read_wait) + self.last_receive['handoff'] = max(self.last_receive['handoff'], time.monotonic() - arrived) + n = min(chunk.nbytes, dest.nbytes) + dest[:n] = chunk[:n] + if n < chunk.nbytes: + self._chunks[0] = (chunk[n:], arrived, prepare, read_wait) + else: + self._chunks.popleft() + # Copied out, so return the buffer to the pool. chunk.obj survives a + # reslice; skip anything not from this pool at the current size, such + # as a shrunk buffer or a test's bytes. + buf = chunk.obj + if type(buf) is bytearray and len(buf) == self._read_size and len(self._free) < FREE_BUFS: + self._free.append(buf) + self._queued -= n + self._cv.notify_all() + return n + if self._reader_error is not None: + raise LinkError(self._reader_error) + return 0 # timed out with nothing new; _fill owns the deadline + + def _wait_for_host_ready(self) -> bool: + """True while still inside the grace period for the host to claim the link. + + The endpoints return EIO until the host claims the interface. The clock + starts on the first such error, not at open, so it covers a re-enumeration. + """ + if self._host_gone(): + return False + now = time.monotonic() + if self._ready_deadline is None: + self._ready_deadline = now + EP_READY_TIMEOUT + if now < self._ready_deadline: + time.sleep(0.005) + return True + return False + + def _host_gone(self) -> bool: + """A host had the gadget configured and no longer does. + + The endpoints answer ENODEV both before a host configures the gadget and + after it disconnects, and only the first deserves the grace period; giving + it to the second cost 10 s and 201 frames on a mid-drive disconnect. The UDC + state separates them, and it is set in the interrupt that disabled the + endpoint, so it is current when the failed call returns. + """ + if not self._had_host or self.bound_udc is None: + return False + state = self._udc_state() + return state is not None and state != 'configured' + + def close(self) -> None: + self._closing = True + self._write_guard.close() + self._write_guard.thread.join(READER_JOIN_TIMEOUT) + if self._write_guard.thread.is_alive(): + # It is stuck inside its own abort, which is already unbinding the + # gadget. This used to raise, to keep a late abort from unbinding a + # gadget the next owner had bound - but the raise came before a single + # fd was closed, so ep0 stayed open for the life of the process and + # every descriptor write after it answered ESRCH. unbind() is + # idempotent, so a late abort finds nothing left to let go of; the leak + # was the worse half by far. + log.warning("jetlink: gadget watchdog teardown did not finish in %.0f s, " + "closing the endpoints anyway", READER_JOIN_TIMEOUT) + # Unbinding disables the endpoints, which completes the reader's pending + # request with ESHUTDOWN and lets the thread exit before its fd goes away. + self.unbind() + reader, self._reader = self._reader, None + if reader is not None and reader is not threading.current_thread(): + reader.join(READER_JOIN_TIMEOUT) + with self._cv: + self._cv.notify_all() + self._close_fds(('_state_fd', 'ep_in', 'ep_out', 'ep0'), reader) + + def _close_fds(self, names, reader: threading.Thread | None) -> None: + """Close these descriptors, clearing each as it goes. + + Clearing matters because a second close() would otherwise shut whatever + those descriptor numbers had been recycled into. ep_out is closed on its + own thread while a read is still outstanding: the read never came back + (nothing to unbind, or a kernel that will not complete it), and close() + returns regardless on Linux but the caller must not be risked on that. + """ + for name in names: + fd = getattr(self, name, -1) + setattr(self, name, -1) + if fd is None or fd < 0: + continue + if name == 'ep_out' and reader is not None and reader.is_alive(): + threading.Thread(target=_close_quietly, args=(fd,), daemon=True).start() + continue + _close_quietly(fd) + + +def _close_quietly(fd: int) -> None: + # Runs on its own thread when a read never came back; that thread inherits + # the creator's scheduling, which in modeld is the frame loop's. + background_thread() + try: + os.close(fd) + except OSError: + pass diff --git a/jetlink_repo/jetlink/transport/priority.py b/jetlink_repo/jetlink/transport/priority.py new file mode 100644 index 0000000000..cb98b14506 --- /dev/null +++ b/jetlink_repo/jetlink/transport/priority.py @@ -0,0 +1,48 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. +""" +from __future__ import annotations + +import os + + +def all_cpus() -> set[int]: + return set(range(os.cpu_count() or 1)) + + +def widen_affinity() -> None: + """Let the calling thread run on every CPU. + + A thread created inside modeld inherits the frame loop's single-core pin. If + the inherited mask is one core, prefer every other core: simply widening did + not help, because the balancer keeps waking a thread where it last ran. Best + effort, and open-coded because jetlink must not import openpilot. + """ + try: + everything = all_cpus() + inherited = os.sched_getaffinity(0) + if len(inherited) == 1 and everything - inherited: + os.sched_setaffinity(0, everything - inherited) # off the frame-loop core + elif everything - inherited: + os.sched_setaffinity(0, everything) # unpinned already; just widen + except (OSError, AttributeError): + # no affinity call (macOS), or a kernel that refuses + pass + + +def background_thread() -> None: + """Drop the calling thread to SCHED_OTHER 0 on every CPU. + + Threads created after config_realtime_process inherit SCHED_FIFO and the core + pin, so even one that only waits on a condition takes the frame loop's core at + equal priority on every wake. Call it first in any thread started here. Best + effort. + """ + try: + os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0)) + except (OSError, AttributeError, ValueError): + pass + widen_affinity() diff --git a/jetlink_repo/jetlink/transport/tcp.py b/jetlink_repo/jetlink/transport/tcp.py new file mode 100644 index 0000000000..70d30b1418 --- /dev/null +++ b/jetlink_repo/jetlink/transport/tcp.py @@ -0,0 +1,132 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. + +TCP transport, for a phone over the cable, and for testing and benchmarking. + +Over the USB cable it rides the CDC-NCM interface of the comma's composite +gadget: the comma is 192.168.60.1 and a phone dials it. A Jetson or a Mac uses +the vendor interface instead. See docs/transport.md. + +One sendmsg per message (header and body in one segment train, NODELAY set) +and reads straight into the reusable receive buffer, so the steady state does +not allocate on the wire. +""" +from __future__ import annotations + +import socket + +from jetlink.transport.base import LinkError, StreamTransport, udc_speed, usb_link_info + +DEFAULT_PORT = 5599 +# The comma's end of the USB network link a phone dials (jetlink-root.sh gadget --ios). +# A connection whose local address is this one is a cable, not a LAN. +CABLE_ADDRESS = '192.168.60.1' + + +class TcpTransport(StreamTransport): + def __init__(self, sock: socket.socket): + super().__init__() + self.sock = sock + self._timeout: float | None = -1.0 # force the first settimeout + _tune(sock) + + def on_the_cable(self) -> bool: + """Is either end the comma's cable address? Then this is a phone's USB + cable: from the comma the local end, from a server the peer. Swift's + LinkMedium(tcpPeer:) is the same rule.""" + try: + return CABLE_ADDRESS in (self.sock.getsockname()[0], self.sock.getpeername()[0]) + except OSError: + return False + + def link_info(self) -> dict: + return usb_link_info('cable', udc_speed()) if self.on_the_cable() else {'kind': 'tcp'} + + @classmethod + def connect(cls, host: str, port: int = DEFAULT_PORT, timeout: float = 5.0) -> TcpTransport: + return cls(socket.create_connection((host, port), timeout=timeout)) + + @classmethod + def listen(cls, host: str = '0.0.0.0', port: int = DEFAULT_PORT, backlog: int = 1) -> socket.socket: + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind((host, port)) + srv.listen(backlog) + return srv + + @classmethod + def accept(cls, srv: socket.socket) -> tuple[TcpTransport, tuple]: + conn, addr = srv.accept() + return cls(conn), addr + + @classmethod + def listen_once(cls, host: str = '0.0.0.0', port: int = DEFAULT_PORT, + timeout: float | None = None) -> tuple[TcpTransport, tuple]: + """Take exactly one incoming connection and stop listening: a phone that + dials us, for the bench and parity scripts run from a Mac.""" + srv = cls.listen(host, port) + try: + srv.settimeout(timeout) + try: + conn, addr = srv.accept() + except TimeoutError as e: + raise LinkError(f"nobody dialed {host}:{port} in {timeout:.0f}s") from e + except OSError as e: + raise LinkError(f"accept on {host}:{port} failed: {e}") from e + finally: + srv.close() + return cls(conn), addr + + def _set_timeout(self, timeout: float | None) -> None: + if timeout != self._timeout: + self.sock.settimeout(timeout) + self._timeout = timeout + + def _write(self, bufs: list[memoryview]) -> int: + # sendmsg keeps the header and a 393 KB body in one syscall, so with NODELAY + # they go out as one segment train. + self._set_timeout(self._write_timeout()) + try: + return self.sock.sendmsg(bufs) + except OSError as e: + raise LinkError(f"send failed: {e}") from e + + def _read_into(self, dest: memoryview, timeout: float | None) -> int: + self._set_timeout(timeout) + try: + n = self.sock.recv_into(dest, dest.nbytes) + except (TimeoutError, BlockingIOError): + # nothing in time, or nothing at all on a read that may not wait (a + # timeout of 0 makes the socket non-blocking); _fill owns the deadline + return 0 + except OSError as e: + raise LinkError(f"recv failed: {e}") from e + if n == 0: + raise LinkError("peer closed the connection") + return n + + def close(self) -> None: + # shut down first: a close alone keeps the connection up while another + # process holds a copy of the socket (the comma's gadget owner holds the + # phone's dial), and the peer hears nothing until that copy goes too + for let_go in (lambda: self.sock.shutdown(socket.SHUT_RDWR), self.sock.close): + try: + let_go() + except OSError: + pass + + +def _tune(sock: socket.socket) -> None: + # NODELAY is the one that matters: without it the header and the body can be + # split across an RTT. Guarded anyway; a transport must not die in setsockopt. + for level, opt, value in ((socket.IPPROTO_TCP, socket.TCP_NODELAY, 1), + (socket.SOL_SOCKET, socket.SO_SNDBUF, 4 << 20), + (socket.SOL_SOCKET, socket.SO_RCVBUF, 4 << 20), + (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1)): + try: + sock.setsockopt(level, opt, value) + except OSError: + pass diff --git a/jetlink_repo/jetlink/transport/watchdog.py b/jetlink_repo/jetlink/transport/watchdog.py new file mode 100644 index 0000000000..b7cced014b --- /dev/null +++ b/jetlink_repo/jetlink/transport/watchdog.py @@ -0,0 +1,71 @@ +""" +Copyright (c) 2026-, Zeph Leggett. + +This file is part of jetlink and is licensed under the MIT License. +See the LICENSE file in the root directory for more details. +""" +from __future__ import annotations + +import threading +import time + +from jetlink.transport.priority import background_thread + + +class WriteWatchdog: + """One worker per link, with cancellation serialized against expiry. + + Once expiry wins, the link cannot be armed again. The callback runs outside + the lock: unbinding a broken USB controller must not block disarm(). + """ + + def __init__(self, abort): + self._abort = abort + self._cv = threading.Condition() + self._deadline = None + self._expired = False + self._closed = False + self.thread = threading.Thread(target=self._run, name='jetlink-write-guard', daemon=True) + self.thread.start() + + def arm(self, budget: float) -> bool: + with self._cv: + if self._expired or self._closed: + return False + if self._deadline is not None: + raise RuntimeError('concurrent writes on one link') + self._deadline = time.monotonic() + budget + self._cv.notify() + return True + + def disarm(self) -> bool: + with self._cv: + self._deadline = None + self._cv.notify() + return not self._expired and not self._closed + + def close(self) -> None: + with self._cv: + self._closed = True + self._deadline = None + self._cv.notify() + + def _run(self) -> None: + # This inherits the creator's policy and pin, which in modeld is the frame + # loop's SCHED_FIFO 54 on one core. + background_thread() + with self._cv: + while not self._closed: + if self._deadline is None: + self._cv.wait() + continue + remaining = self._deadline - time.monotonic() + if remaining > 0: + self._cv.wait(remaining) + continue + self._expired = True + self._deadline = None + break + else: + return + self._abort() diff --git a/jetlink_repo/pyproject.toml b/jetlink_repo/pyproject.toml new file mode 100644 index 0000000000..31ce4fdcb9 --- /dev/null +++ b/jetlink_repo/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools>=61"] +build-backend = "setuptools.build_meta" + +[project] +name = "jetlink" +# jetlink.__version__: the comma runs from a checkout and reads it there, and +# the Swift server has it from Pinned.swift +dynamic = ["version"] +description = "The comma side of jetlink: openpilot's large driving models on an attached Jetson, Mac or phone" +readme = "README.md" +license = { file = "LICENSE" } +authors = [{ name = "Zeph Leggett" }] +requires-python = ">=3.10" +# numpy only: this is what the comma runs, from a checkout. The server is the +# Swift jetlink-server (JetlinkKit). +dependencies = ["numpy>=1.24"] + +[project.optional-dependencies] +# The tests, and the conformance generators (onnx builds and reads the tiny +# graphs; JetlinkKit/Scripts/fixture-pins.txt names the releases). +dev = ["pytest>=7", "ruff", "onnx>=1.16"] + +[tool.setuptools.packages.find] +include = ["jetlink*"] + +[tool.setuptools.dynamic] +version = { attr = "jetlink.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] + +# The fork's rule set (sunnypilot's pyproject.toml), so code moves between the +# two without reformatting, with two differences the fork does not allow: +# strings may be implicitly concatenated across lines, and E402 stays on so the +# `# noqa: E402` under every importorskip in tests/ means something. +[tool.ruff] +target-version = "py310" +line-length = 160 +indent-width = 2 +lint.select = [ + "E", "F", "W", "PIE", "C4", "ISC", "A", "B", + "NPY", + "UP", + "ASYNC", + "B904", "B905", + "PLC0207", + "TRY203", "TRY400", "TRY401", + "RUF006", "RUF008", "RUF009", "RUF061", "RUF064", "RUF100", "RUF102", "RUF103", "RUF104", + "PLE", "PLR1704", +] +lint.ignore = [ + "E741", + "B027", + # A failure the comma expects (the root script timing out or not running) + # is logged with its message; a traceback there is noise. + "TRY400", +] + +[tool.ruff.format] +quote-style = "preserve" diff --git a/jetlink_repo/scripts/comma/jetlink-root.sh b/jetlink_repo/scripts/comma/jetlink-root.sh new file mode 100755 index 0000000000..736359c830 --- /dev/null +++ b/jetlink_repo/scripts/comma/jetlink-root.sh @@ -0,0 +1,631 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2026-, Zeph Leggett. +# This file is part of jetlink and is licensed under the MIT License. +# +# Everything jetlink does as root on the comma, in one place. The owner runs it +# through jetlink/comma/root.py, which is sudo -n and a timeout: +# +# sudo scripts/comma/jetlink-root.sh gadget # the gadget for a Jetson or a Mac +# sudo scripts/comma/jetlink-root.sh gadget --ios # the gadget for an iPhone +# sudo scripts/comma/jetlink-root.sh net # after a bind, iOS only: address, DHCP +# sudo scripts/comma/jetlink-root.sh check # what is there now +# sudo scripts/comma/jetlink-root.sh teardown +# sudo scripts/comma/jetlink-root.sh port hold|off # the USB-C port held as the device, or let go +# sudo scripts/comma/jetlink-root.sh port device|reset # ask the far end to host over USB PD, or reset PD +# sudo scripts/comma/jetlink-root.sh port source # ask the far end for the source role: the comma charges it +# sudo scripts/comma/jetlink-root.sh udc apply|restore # the USB device controller kept a device, or stock +# sudo scripts/comma/jetlink-root.sh udc start|stop # its device side turned on, or off +# sudo scripts/comma/jetlink-root.sh vm apply|restore # the link's VM tuning, or the stock values +# sudo scripts/comma/jetlink-root.sh draw off|on # no current drawn from the port (iOS), or the charger's own +# +# For the comma four and the comma 3X only. Both are SDM845 on the same AGNOS +# kernel (4.9, dwc3 at a600000.dwc3), with configfs, FunctionFS and NCM built +# in and no modules to load. +# The 3X is assumed from the kernel it shares with the four; only the four has been on the bench. +# +# gadget: the USB gadget, chosen by the comma's Jetlink setting. The +# comma is the USB device and the host runs the model; docs/transport.md says +# why. For a Jetson or a Mac (USB), the FunctionFS vendor interface alone. For +# an iPhone (iOS, with --ios), a composite: the vendor interface first, so it +# stays interface 0, and a CDC-NCM network interface after it, since an iPhone +# app can only use the network: the phone gets 192.168.60.x by DHCP from the +# dnsmasq this script starts on the gadget's netdev, and dials the comma at +# 192.168.60.1:5599. +# +# It does not bind the UDC: a FunctionFS gadget cannot attach to a controller +# until its descriptors are written, and the owner, which opens ep0, writes +# them and binds. The NCM function rides on that bind, and its netdev exists +# only from the first bind on, so the network is set up by net, which the owner +# runs after each bind. +# +# On failure the reason is left in $STATUS_FILE as well as on stderr, so the +# openpilot side can say why the link is unavailable. net reports in +# $NET_STATUS_FILE; it never fails the gadget. +set -euo pipefail + +GADGET=/sys/kernel/config/usb_gadget/jetlink +FFS_MOUNT=/dev/ffs-jetlink +FFS_NAME=jetlink +CONFIGFS=/sys/kernel/config +# who openpilot runs as: the owner opens the endpoints and binds the UDC as it +OPENPILOT_USER=comma +# tmpfs on purpose: per-boot state, and the comma's flash is precious +STATUS_FILE=/dev/shm/jetlink-gadget +NET_STATUS_FILE=/dev/shm/jetlink-net +# pid.codes test allocation; get a real PID before distributing this +VID=0x1209 +PID=0x0001 +NET_IF=usb0 # the function name suffix only; see net_ifname for the netdev +# NCM batches packets, which a 460 KB frame benefits from +NET_FN=ncm.$NET_IF +COMMA_ADDR=192.168.60.1 +COMMA_PREFIX=24 +# The whole subnet and short leases: the comma's 4.9 kernel gives the host a new +# random MAC every bind, so each bind is a new DHCP client. With 8 addresses and +# an hour's lease, eight rebinds in an hour left the next host without one. +DHCP_RANGE=192.168.60.2,192.168.60.254,10m +DNSMASQ_PID=/dev/shm/jetlink-dnsmasq.pid +DNSMASQ_IF=/dev/shm/jetlink-dnsmasq.if +DNSMASQ_LEASES=/dev/shm/jetlink-usb0.leases + +# port: the charger's DISABLE_POWER_ROLE_SWITCH voter on the PMI8998, the one +# role lever that holds across plugs, and the policy engine's own USB PD +# requests; jetlink/comma/port.py says when each is used +POWER_ROLE_VOTER=${JETLINK_POWER_ROLE_VOTER:-/sys/kernel/debug/pmic-votable/DISABLE_POWER_ROLE_SWITCH} +USBPD=${JETLINK_USBPD:-/sys/class/usbpd/usbpd0} +DUAL_ROLE=${JETLINK_DUAL_ROLE:-/sys/class/dual_role_usb/otg_default} + +# udc: the dwc3 glue for the port's controller (usb0, ssusb@a600000) and the +# policy engine's parameters. Both have a usb_compliance_mode, and each gates +# one place only: the policy engine's turns the device side on at every sink +# attach, not just when the charger detection read a USB port (SDP, CDP or +# floating lines); the glue's keeps dwc3 from turning it off again when the +# lines read floating and no host enumerated it in 10 s. An Apple port connects +# its data lines only after USB PD, and at boot no gadget is bound for 10 s. +UDC_GLUE=${JETLINK_UDC_GLUE:-/sys/devices/platform/soc/a600000.ssusb} +PE_PARAMS=${JETLINK_PE_PARAMS:-/sys/module/policy_engine/parameters} +USB_PSY=${JETLINK_USB_PSY:-/sys/class/power_supply/usb} + +# draw: the charger's USB_ICL voter, the input current limit on the port. As +# the sink the charger draws up to 900 mA from whatever powers the port, though +# the comma runs on its own supply. An iPhone powers what it plugs into with +# little current; under that draw its supply sags, the charger's weak-charger +# and reverse-boost handlers cut and restore the input in a loop, and the phone +# never sees a device. Forced to 0 the input is suspended: the port still reads +# present and the roles still settle, and those handlers stand down at 25 mA or +# less. As the source the kernel votes the input to 0 itself. +USB_ICL_VOTER=${JETLINK_USB_ICL_VOTER:-/sys/kernel/debug/pmic-votable/USB_ICL} + +# vm: loggerd's dirty pages pile up until the kernel reclaims them +# synchronously, right while a FunctionFS transfer allocates its buffer: gadget +# reads stalled 200-350 ms and the big model fell back. Capping dirty memory +# keeps that reclaim cheap. System-wide, since the gadget read shares the +# kernel with every writer. +# Not vm.min_free_kbytes: a 128 MB floor takes about three times that out of +# MemAvailable, 360 MB on a 3.6 GB comma, and openpilot's LOW MEMORY alert +# reads MemTotal-MemAvailable against 90 %. Drives at 80 % showed 90 and +# alerted. The caps alone are not yet re-measured against the stall. +# And the socket buffer caps, for the iPhone's TCP link: transport/tcp.py asks +# for 4 MB, which turns autotuning off, and the kernel clamps it to these, 224 +# KB on AGNOS and so 448 KB of buffer. A 393 KB frame in 1448-byte segments does +# not fit, so the send waited on the phone's ACKs: 6 to 50 ms, following its +# round trip. With 4 MB the frame goes in one write and the send took ~4 ms +# (a tester's comma, 2026-10-04). Only a socket that asks gets more; one set up +# before the apply keeps the old cap until the phone reconnects. +VM_SYSCTLS=(vm.dirty_bytes=16777216 vm.dirty_background_bytes=8388608 + net.core.wmem_max=4194304 net.core.rmem_max=4194304) +PROC_SYS=${JETLINK_PROC_SYS:-/proc/sys} +# the stock values to write back, one key=value a line, recorded by the first +# apply and dropped by restore +SYSCTL_PREV=${JETLINK_SYSCTL_PREV:-/dev/shm/jetlink-sysctl-prev} + +usage() { + echo "usage: $0 gadget [--ios] | net | check | teardown | port hold|off|device|reset|source|sink | udc apply|restore|start|stop | vm apply|restore | draw off|on" >&2 + exit 2 +} + +status() { + # best effort: a device with no /dev/shm still gets the stderr line + { echo "$1" > "$STATUS_FILE" && chmod 0644 "$STATUS_FILE"; } 2>/dev/null || true +} + +net_status() { + { echo "$1" > "$NET_STATUS_FILE" && chmod 0644 "$NET_STATUS_FILE"; } 2>/dev/null || true +} + +fail() { + echo "jetlink: $1" >&2 + status "error: $1" + exit 1 +} + +# -- the gadget's network, iOS only ------------------------------------------- + +net_present() { + [[ -d "$GADGET/functions/$NET_FN" ]] +} + +dnsmasq_alive() { + local pid + pid=$(cat "$DNSMASQ_PID" 2>/dev/null || true) + [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null +} + +# The interface the kernel gave the network function. The function is named +# ncm.usb0 but the netdev is not usb0: the modem already holds that name on a +# comma, so ours comes up as usb1 or later, and the number can change from one +# bind to the next. u_ether records the real name in the function's ifname, +# readable only while the gadget is bound. +net_ifname() { + local name + name=$(cat "$GADGET/functions/$NET_FN/ifname" 2>/dev/null || true) + [[ -n "$name" && -d "/sys/class/net/$name" ]] || return 1 + echo "$name" +} + +# The comma's end of the cable network. Idempotent, and never fatal. The owner +# runs it through net once it has bound the UDC and the interface has appeared. +net_up() { + local dev why + if ! dev=$(net_ifname); then + net_status "error: no netdev yet; it appears when the owner binds the UDC (then run net)" + echo "$NET_FN present, its netdev not yet created (appears at bind); run net after binding" >&2 + return 0 + fi + # NetworkManager manages every device on AGNOS and would DHCP on it itself + nmcli dev set "$dev" managed no >/dev/null 2>&1 || true + if ! ip addr replace "$COMMA_ADDR/$COMMA_PREFIX" dev "$dev" 2>/dev/null; then + why="could not set $COMMA_ADDR/$COMMA_PREFIX on $dev" + net_status "error: $why"; echo "jetlink: $why" >&2 + return 0 + fi + if ! ip link set "$dev" up 2>/dev/null; then + why="could not bring $dev up" + net_status "error: $why"; echo "jetlink: $why" >&2 + return 0 + fi + # Steer receive processing onto the big cores: the comma's little cores add + # milliseconds to a 460 KB frame. + if [[ -w "/sys/class/net/$dev/queues/rx-0/rps_cpus" ]]; then + echo f0 > "/sys/class/net/$dev/queues/rx-0/rps_cpus" 2>/dev/null || true + fi + # DHCP for the phone, and only that: no router (option 3) and no DNS (option 6), + # so the phone keeps its default route over Wi-Fi. No DNS service (--port=0). + # dnsmasq binds the interface by name, so a netdev that came back under a new + # name after a rebind needs a fresh dnsmasq: the running one is on a ghost. + if dnsmasq_alive && [[ "$(cat "$DNSMASQ_IF" 2>/dev/null || true)" != "$dev" ]]; then + kill "$(cat "$DNSMASQ_PID")" 2>/dev/null || true + sleep 0.2 + fi + if ! dnsmasq_alive; then + rm -f "$DNSMASQ_PID" 2>/dev/null || true + if ! dnsmasq --conf-file=/dev/null --bind-interfaces --interface="$dev" \ + --except-interface=lo --port=0 --dhcp-range="$DHCP_RANGE" \ + --dhcp-option=3 --dhcp-option=6 --dhcp-leasefile="$DNSMASQ_LEASES" \ + --pid-file="$DNSMASQ_PID" 2>/dev/null; then + why="dnsmasq would not start on $dev" + net_status "error: $why"; echo "jetlink: $why" >&2 + return 0 + fi + echo "$dev" > "$DNSMASQ_IF" + fi + net_status "ok $COMMA_ADDR $dev" + echo "$dev (ncm) at $COMMA_ADDR/$COMMA_PREFIX, DHCP $DHCP_RANGE" + return 0 +} + +net_down() { + local pid + pid=$(cat "$DNSMASQ_PID" 2>/dev/null || true) + if [[ -n "$pid" ]]; then + kill "$pid" 2>/dev/null || true + fi + rm -f "$DNSMASQ_PID" 2>/dev/null || true +} + +# The network function and its DHCP server, gone: for teardown, and for a USB +# gadget left with one by an iOS boot. Unlinking it force-unbinds the UDC. +net_remove() { + local f + net_down + for f in "$GADGET"/configs/c.1/*."$NET_IF"; do + if [[ -L "$f" ]]; then rm -f "$f" 2>/dev/null || true; fi + done + for f in "$GADGET"/functions/*."$NET_IF"; do + if [[ -d "$f" ]]; then rmdir "$f" 2>/dev/null || true; fi + done +} + +# -- subcommands ---------------------------------------------------------------- + +cmd_gadget() { + local ios=0 udcs other owner bound serial uid gid + case "${1:-}" in + "") ;; + --ios) ios=1 ;; + *) usage ;; + esac + [[ $EUID -eq 0 ]] || fail "jetlink-root.sh gadget must run as root" + + # set -e alone exits without going through fail, leaving last boot's "ok" in + # $STATUS_FILE for the openpilot side to read. + trap 'fail "line $LINENO: $BASH_COMMAND failed"' ERR + + mountpoint -q "$CONFIGFS" || mount -t configfs none "$CONFIGFS" 2>/dev/null || true + mountpoint -q "$CONFIGFS" || fail "no configfs at $CONFIGFS; this kernel cannot configure a USB gadget" + + # absent means this AGNOS build has no CONFIG_USB_LIBCOMPOSITE, which nothing + # in userspace can fix + [[ -d "$CONFIGFS/usb_gadget" ]] || fail "kernel has no USB gadget support (CONFIG_USB_LIBCOMPOSITE); jetlink needs an AGNOS build that has it" + + # No FunctionFS preflight on purpose: the kernel registers the functionfs + # filesystem only while some ffs.* function exists, so /proc/filesystems never + # lists it on a cold boot. The mkdir, mount and ep0 checks below test it in use. + + # a gadget needs a device controller + shopt -s nullglob + udcs=("/sys/class/udc"/*) + shopt -u nullglob + [[ ${#udcs[@]} -gt 0 ]] || fail "no USB device controller in /sys/class/udc; this device cannot act as a USB gadget" + + # refuse to fight another gadget for the controller rather than unbinding it + for other in "$CONFIGFS"/usb_gadget/*/UDC; do + if [[ -e "$other" ]]; then + owner=$(basename "$(dirname "$other")") + bound=$(cat "$other" 2>/dev/null || true) + if [[ "$owner" != "jetlink" && -n "$bound" ]]; then + fail "USB gadget '$owner' already holds the device controller ($bound); tear it down first" + fi + fi + done + + mkdir -p "$GADGET" || fail "could not create the gadget at $GADGET" + cd "$GADGET" + + echo "$VID" > idVendor + echo "$PID" > idProduct + echo 0x0320 > bcdUSB # 3.2: advertise SuperSpeed + if [[ $ios -eq 1 ]]; then + # its own bcdDevice, so hosts that cache descriptors by VID/PID/bcdDevice + # (macOS, Windows) fetch the composite ones rather than the plain gadget's + echo 0x0101 > bcdDevice + # Miscellaneous / Common Class / IAD: the composite device class, which tells + # a host to bind a driver per interface association (the vendor interface + # for jetlink, CDC-NCM for the network) rather than one for the whole device + echo 0xEF > bDeviceClass + echo 0x02 > bDeviceSubClass + echo 0x01 > bDeviceProtocol + else + # the plain gadget, as it has always been: class per interface + echo 0x0100 > bcdDevice + echo 0x00 > bDeviceClass + echo 0x00 > bDeviceSubClass + echo 0x00 > bDeviceProtocol + fi + + # The device tree carries a serial on some commas and not others; any stable + # string will do. + serial=$({ tr -d '\0' < /proc/device-tree/serial-number; } 2>/dev/null || cat /etc/machine-id 2>/dev/null || echo 0001) + [[ -n "$serial" ]] || serial=0001 + mkdir -p strings/0x409 + echo "zoompilot" > strings/0x409/manufacturer + echo "jetlink" > strings/0x409/product + echo "$serial" > strings/0x409/serialnumber + + mkdir -p configs/c.1/strings/0x409 + echo "jetlink inference link" > configs/c.1/strings/0x409/configuration + # self-powered, and as little as the spec allows: the Jetson has its own 12 V feed + echo 0xC0 > configs/c.1/bmAttributes + echo 8 > configs/c.1/MaxPower + + # the mkdir instantiates the function and registers functionfs, so a kernel + # genuinely without it fails here + mkdir -p "functions/ffs.$FFS_NAME" || + fail "kernel has no ffs gadget function (CONFIG_USB_CONFIGFS_F_FS); jetlink cannot present its endpoints" + # Link once: this is re-run on every deploy, and unlinking a function from a bound + # config force-unbinds the UDC, so an unconditional ln -sf drops a live link. + # Linked before the network function so it is interface 0: hosts that open the + # vendor interface by number depend on that. + [[ -L "configs/c.1/ffs.$FFS_NAME" ]] || + ln -s "$GADGET/functions/ffs.$FFS_NAME" "configs/c.1/ffs.$FFS_NAME" || + fail "could not link ffs.$FFS_NAME into configs/c.1" + + # The network interface, for an iPhone only. A USB gadget left with one from an + # iOS boot loses it here: unlinking it force-unbinds the UDC, which is why the + # owner switches only while parked. The 4.9 kernel picks the interface's MAC + # addresses itself and refuses them from configfs; DHCP makes that harmless. + if [[ $ios -eq 0 ]]; then + if net_present; then net_remove; fi + else + mkdir -p "functions/$NET_FN" || fail "could not create the $NET_FN gadget function" + # after ffs, so it takes the next interface numbers + [[ -L "configs/c.1/$NET_FN" ]] || + ln -s "$GADGET/functions/$NET_FN" "configs/c.1/$NET_FN" || + fail "could not link $NET_FN into configs/c.1" + fi + + mkdir -p "$FFS_MOUNT" + # Owned by the user openpilot runs as: on a root-only mount the owner and + # modeld cannot open the endpoints, and Path.exists() raises rather than + # returning False. + uid=$(id -u "$OPENPILOT_USER") + gid=$(id -g "$OPENPILOT_USER") + mountpoint -q "$FFS_MOUNT" || mount -t functionfs -o "uid=$uid,gid=$gid" "$FFS_NAME" "$FFS_MOUNT" || + fail "could not mount functionfs at $FFS_MOUNT" + + # the check that proves the chain: ep0 is what the owner opens to write the + # descriptors and bind the controller + [[ -e "$FFS_MOUNT/ep0" ]] || fail "functionfs mounted at $FFS_MOUNT but has no ep0" + + # the owner binds the UDC as the openpilot user, so hand it that one attribute + chown "$OPENPILOT_USER" "$GADGET/UDC" 2>/dev/null || true + + status ok + echo "gadget ready at $GADGET" + echo "functionfs mounted at $FFS_MOUNT" + if [[ $ios -eq 1 ]]; then + echo "network function: $NET_FN (after ffs.$FFS_NAME in configs/c.1)" + # its netdev appears at the owner's bind, and net sets it up then + rm -f "$NET_STATUS_FILE" 2>/dev/null || true + else + net_status "net: off" + fi + echo "available UDCs: ${udcs[*]##*/}" + echo "the owner writes the descriptors and binds the UDC" +} + +cmd_net() { + [[ $EUID -eq 0 ]] || fail "jetlink-root.sh net must run as root" + [[ -d "$GADGET" ]] || { net_status "error: no gadget"; fail "no gadget at $GADGET; run gadget first"; } + net_up +} + +cmd_teardown() { + local f linked=0 + net_remove + if [[ -d "$GADGET" ]]; then + echo "" > "$GADGET/UDC" 2>/dev/null || true + rm -f "$GADGET/configs/c.1/ffs.$FFS_NAME" 2>/dev/null || true + rmdir "$GADGET/configs/c.1/strings/0x409" 2>/dev/null || true + # a config with a function still linked cannot go, and trying is a + # configfs error worth not making + for f in "$GADGET"/configs/c.1/*; do + if [[ -L "$f" ]]; then linked=1; fi + done + if [[ $linked -eq 0 ]]; then + rmdir "$GADGET/configs/c.1" 2>/dev/null || true + else + echo "jetlink: configs/c.1 still has a function linked; left in place" >&2 + fi + rmdir "$GADGET/functions/ffs.$FFS_NAME" 2>/dev/null || true + rmdir "$GADGET/strings/0x409" 2>/dev/null || true + rmdir "$GADGET" 2>/dev/null || true + fi + # A plain umount can block or segfault on a FunctionFS instance whose owner died + # with endpoints open; lazy-detach unhooks it now and lets the kernel finish. + umount -l "$FFS_MOUNT" 2>/dev/null || umount "$FFS_MOUNT" 2>/dev/null || true + rmdir "$FFS_MOUNT" 2>/dev/null || true + status "error: gadget torn down" + net_status "error: gadget torn down" + echo "jetlink gadget torn down" +} + +cmd_check() { + local u g dev udcs + [[ $EUID -eq 0 ]] || { echo "run check as root (sudo)" >&2; exit 1; } + echo "kernel: $(uname -r)" + if mountpoint -q "$CONFIGFS" && [[ -d "$CONFIGFS/usb_gadget" ]]; then + echo "configfs USB gadgets: yes" + else + echo "configfs USB gadgets: NO (no $CONFIGFS/usb_gadget)" + fi + shopt -s nullglob + udcs=(/sys/class/udc/*) + shopt -u nullglob + for u in "${udcs[@]}"; do + # current_speed is the negotiated bus speed: super-speed is USB 3, and a + # 460 KB frame is ~1 ms there against ~11 ms at high-speed (USB 2). The + # owner logs it on every configured edge, since nothing in this script + # runs after enumeration. + echo "device controller: $(basename "$u"), state $(cat "$u/state" 2>/dev/null || echo unknown), speed $(cat "$u/current_speed" 2>/dev/null || echo unknown)" + done + [[ ${#udcs[@]} -gt 0 ]] || echo "device controller: NONE" + for g in "$CONFIGFS"/usb_gadget/*/; do + [[ -d "$g" ]] || continue + echo "gadget $(basename "$g"): bound to '$(cat "$g/UDC" 2>/dev/null)'" + done + echo "gadget status: $(cat "$STATUS_FILE" 2>/dev/null || echo none)" + # A phone plugged straight into the comma negotiates power and the comma may + # end up sourcing it, which reboots the comma. Through a hub the comma sinks. + # On a C-to-C cable the comma can come out the host instead; port hold fixes that. + if [[ -d "$USBPD" ]]; then + echo "USB-C port: power role $(cat "$USBPD/current_pr" 2>/dev/null || echo unknown), data role $(cat "$USBPD/current_dr" 2>/dev/null || echo unknown), USB PD contract $(cat "$USBPD/contract" 2>/dev/null || echo unknown), Type-C $(cat "$USB_PSY/typec_mode" 2>/dev/null || echo unknown)" + fi + if [[ "$(cat "$POWER_ROLE_VOTER/force_active" 2>/dev/null || true)" == 1 ]]; then + echo "USB-C port: held as the device (port hold)" + fi + if [[ "$(cat "$USB_ICL_VOTER/force_active" 2>/dev/null || true)" == 1 ]]; then + echo "USB-C port: no current drawn from it (draw off)" + fi + if [[ -d "$UDC_GLUE" ]]; then + echo "USB device controller: $(cat "$UDC_GLUE/mode" 2>/dev/null || echo unknown), charger detection $(cat "$USB_PSY/real_type" 2>/dev/null || echo unknown), kept a device $(cat "$PE_PARAMS/usb_compliance_mode" 2>/dev/null || echo '?')/$(cat "$UDC_GLUE/usb_compliance_mode" 2>/dev/null || echo '?')" + fi + if ! net_present; then + echo "network function: none (the USB gadget, or no gadget)" + return 0 + fi + echo "network function: $NET_FN" + if dev=$(net_ifname); then + echo "$dev: $(cat "/sys/class/net/$dev/operstate" 2>/dev/null || echo unknown), $(ip -4 -o addr show dev "$dev" 2>/dev/null | awk '{print $4}' | tr '\n' ' ')" + else + echo "netdev: absent (it appears when the owner binds the UDC)" + fi + echo "network status: $(cat "$NET_STATUS_FILE" 2>/dev/null || echo unknown)" + if dnsmasq_alive; then echo "dnsmasq: running"; else echo "dnsmasq: not running"; fi +} + +# The comma's USB-C port, for a link that runs over USB. hold keeps the port at +# sink, which makes the far end the host; off is dual role, as AGNOS boots it. +# device asks the far end over USB PD to take the host role (a DR_Swap): for a +# far end that powers the comma and still came out the device, which hold +# cannot change. reset is a USB PD hard reset, which puts both ends back to +# the roles their power gives them: a sink is the device. source asks a far end +# that powers the comma for the source role (a PR_Swap), which leaves the data +# roles as they are: an iPhone that hosts the comma stays the host and charges +# from it. The voter gates it, so port off comes first. sink is the swap back, +# for a phone whose link went down once it charged. +cmd_port() { + case "${1:-}" in + hold) force_voter "$POWER_ROLE_VOTER" 1 "to hold the port" || exit 1 ;; + # letting go applies the voters' own result, which is dual role + off) release_voter "$POWER_ROLE_VOTER" || exit 1 ;; + # the kernel sends the DR_Swap and waits 100 ms for it; a refusal, or PD not + # ready yet, is a failed write + device) put device "$DUAL_ROLE/data_role" "the far end did not take the host role ($DUAL_ROLE/data_role)" || exit 1 ;; + reset) put 1 "$USBPD/hard_reset" "could not reset USB PD ($USBPD/hard_reset)" || exit 1 ;; + # the kernel sends the PR_Swap and waits up to 2 s for it; only with a USB PD + # contract, so a write before one, or a refusal, fails + source) put source "$DUAL_ROLE/power_role" "the far end did not give up the source role ($DUAL_ROLE/power_role)" || exit 1 ;; + sink) put sink "$DUAL_ROLE/power_role" "the far end did not take the source role back ($DUAL_ROLE/power_role)" || exit 1 ;; + *) usage ;; + esac +} + +# The port's USB device controller, while the link is on. apply keeps it a +# device whatever the charger detection made of the far end's data lines, and +# restore puts the stock behaviour back; see UDC_GLUE. start turns the device +# side on now, for a host that powers the port while it is off, and stop turns +# it off again once that host is gone. +cmd_udc() { + case "${1:-}" in + apply) udc_compliance Y || exit 1 ;; + restore) udc_compliance N || exit 1 ;; + start) put peripheral "$UDC_GLUE/mode" "could not start the USB device controller ($UDC_GLUE/mode)" || exit 1 ;; + stop) put none "$UDC_GLUE/mode" "could not stop the USB device controller ($UDC_GLUE/mode)" || exit 1 ;; + *) usage ;; + esac +} + +# Both knobs, each on its own, so one the kernel lacks does not keep the other. +udc_compliance() { + local failed=0 knob + for knob in "$PE_PARAMS/usb_compliance_mode" "$UDC_GLUE/usb_compliance_mode"; do + put "$1" "$knob" "could not set $knob" || failed=1 + done + return $failed +} + +# One value into one kernel file; a line on stderr, with the kernel's reason, +# and a failure when the kernel refuses it. Not fail(): that is the gadget's record. +put() { + local err + if ! err=$( { echo "$1" > "$2"; } 2>&1 ); then + echo "jetlink: $3${err:+: ${err##*: }}" >&2 + return 1 + fi +} + +# A charger voter under debugfs forced to a value, and let go: force_val first, +# since forcing applies whatever force_val holds at that moment. +force_voter() { + { echo "$2" > "$1/force_val" && echo 1 > "$1/force_active"; } 2>/dev/null || + { echo "jetlink: could not force $1 $3" >&2; return 1; } +} + +release_voter() { + { echo 0 > "$1/force_active" && echo 0 > "$1/force_val"; } 2>/dev/null || + { echo "jetlink: could not release $1" >&2; return 1; } +} + +# One key per write, so a value the kernel rejects does not take the rest with it. +sysctl_write() { + put "$2" "$PROC_SYS/${1//.//}" "could not set $1=$2" +} + +# Applied while the link is on, so a device with the link off runs stock +# values. Only restore undoes them, never an exit: a process that restored on +# exit and stopped at ignition would hand every drive the stock values. A +# reboot resets them. +vm_apply() { + local pair key value rec="" failed=0 + if [[ ! -e "$SYSCTL_PREV" ]]; then + for pair in "${VM_SYSCTLS[@]}"; do + key=${pair%%=*} + value="" + { read -r value < "$PROC_SYS/${key//.//}"; } 2>/dev/null || true + if [[ "$value" == 0 && "$key" == *_bytes ]]; then + # Stock AGNOS runs the dirty limits in ratio mode, so the *_bytes keys + # read 0, and the kernel silently drops a 0 written back to one. + # Writing the ratio key is what zeroes the bytes key, so that is the + # one to put back. + key=${key%_bytes}_ratio + value="" + { read -r value < "$PROC_SYS/${key//.//}"; } 2>/dev/null || true + fi + if [[ -n "$value" ]]; then + rec+="$key=$value"$'\n' + else + echo "jetlink: could not read $key" >&2 + fi + done + if [[ -n "$rec" ]]; then + { printf '%s' "$rec" > "$SYSCTL_PREV" && chmod 0644 "$SYSCTL_PREV"; } 2>/dev/null || + echo "jetlink: could not record the previous sysctls in $SYSCTL_PREV" >&2 + fi + fi + for pair in "${VM_SYSCTLS[@]}"; do + sysctl_write "${pair%%=*}" "${pair#*=}" || failed=1 + done + return $failed +} + +# Writes the record back line by line: a vm or net.core key and a number, +# nothing else. +vm_restore() { + local key value failed=0 + [[ -e "$SYSCTL_PREV" ]] || return 0 + while IFS='=' read -r key value; do + if [[ "$key" =~ ^(vm|net\.core)\.[a-z_]+$ && "$value" =~ ^[0-9]+$ ]]; then + sysctl_write "$key" "$value" || failed=1 + else + echo "jetlink: not restoring '$key=$value' from $SYSCTL_PREV" >&2 + failed=1 + fi + done < "$SYSCTL_PREV" + rm -f "$SYSCTL_PREV" 2>/dev/null || true + return $failed +} + +cmd_vm() { + case "${1:-}" in + apply) vm_apply || exit 1 ;; + restore) vm_restore || exit 1 ;; + *) usage ;; + esac +} + +# The port's input current while the link is iOS; see USB_ICL_VOTER. off forces +# the limit to 0 and on gives the charger its own limit back. Like udc and vm, +# undone only by on, never at an exit; a reboot clears it. +cmd_draw() { + case "${1:-}" in + off) force_voter "$USB_ICL_VOTER" 0 "to stop drawing from the port" || exit 1 ;; + on) release_voter "$USB_ICL_VOTER" || exit 1 ;; + *) usage ;; + esac +} + +cmd=${1:-} +if [[ $# -gt 0 ]]; then shift; fi +case "$cmd" in + gadget) cmd_gadget "$@" ;; + net) cmd_net ;; + check) cmd_check ;; + teardown) cmd_teardown ;; + port) cmd_port "$@" ;; + udc) cmd_udc "$@" ;; + vm) cmd_vm "$@" ;; + draw) cmd_draw "$@" ;; + *) usage ;; +esac diff --git a/models b/models new file mode 100755 index 0000000000..73adfa3827 --- /dev/null +++ b/models @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +set -eo pipefail +DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +source "$DIR/launch_env.sh" +exec python3 "$DIR/scripts/model_compiler.py" "$@" diff --git a/openpilot/starpilot/model_geometry.py b/openpilot/starpilot/model_geometry.py new file mode 100644 index 0000000000..69503f587d --- /dev/null +++ b/openpilot/starpilot/model_geometry.py @@ -0,0 +1,51 @@ +"""Validated shared model geometry; no control authority or mutable state.""" + +import math + +ORIGIN_ROUNDOFF_M = 1e-6 + + +def normalized_origin(distances: tuple) -> tuple: + """Only remove sub-micrometre model-origin roundoff; callers validate all geometry.""" + if distances and _finite(distances[0], low=-ORIGIN_ROUNDOFF_M, high=0.0) is not None and distances[0] < 0: + return (0.0, *distances[1:]) + return distances + + +def _finite(value: object, *, low: float, high: float) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + result = float(value) + except (OverflowError, ValueError): + return None + return result if math.isfinite(result) and low <= result <= high else None + + +def road_curvature(model: object, speed_mps: float) -> tuple[float, float] | None: + """Exact frozen max-|orientationRate.z * velocity.x| selection. + + The original helper divides by max(vEgo, 1)^2 and floors time at 1 s. + Invalid or incomplete current model arrays remain unknown. + """ + speed = _finite(speed_mps, low=0.0, high=80.0) + if speed is None: + return None + try: + rates = tuple(model.orientationRate.z) + times = tuple(model.orientationRate.t) + velocities = tuple(model.velocity.x) + except (AttributeError, TypeError, ValueError, OverflowError): + return None + if not len(rates) == len(times) == len(velocities) == 33: + return None + triples = [] + for rate, timestamp, velocity in zip(rates, times, velocities, strict=True): + r = _finite(rate, low=-10.0, high=10.0) + t = _finite(timestamp, low=0.0, high=20.0) + v = _finite(velocity, low=0.0, high=100.0) + if r is None or t is None or v is None: + return None + triples.append((r * v, t)) + acceleration, timestamp = max(triples, key=lambda pair: abs(pair[0])) + return acceleration / max(speed, 1.0) ** 2, max(timestamp, 1.0) diff --git a/openpilot/starpilot/models/JETLINK.md b/openpilot/starpilot/models/JETLINK.md new file mode 100644 index 0000000000..5de14b4016 --- /dev/null +++ b/openpilot/starpilot/models/JETLINK.md @@ -0,0 +1,91 @@ +# JetLink with StarPilot + +JetLink runs a driving model on a USB-connected computer or phone. The comma +runs **StarPilot**. JetLink is experimental; a prepared host model is not proof +that StarPilot has switched to it. Set up parked, with both devices online. + +## Set up StarPilot + +1. Install or update **StarPilot**, following the [StarPilot repository instructions](../../../README.md). + Use a StarPilot release containing JetLink support. Finish installation and restart the comma. +2. Open **Galaxy → Model Manager → Jetlink**. Choose **USB computer** for a + Jetson, Mac, Linux PC, or Android host; choose **iPhone / iPad** for an Apple phone or tablet. + **Off** is the default. These selections are separate from Small and Chestnut models. +3. Use a USB 3 data cable. Power the comma and host separately. Every cable, + hub, and port in the connection must support USB 3. Chestnut takes precedence + when connected; wireless Android Auto can stay enabled. +4. Install and start the host application below. Leave StarPilot parked while + the host downloads and prepares its model. Watch Galaxy's connection and + preparation status; do not substitute a host icon or benchmark for StarPilot's runtime status. + +## Install the host + +The host program is the upstream **Jetlink** application, developed by Zeph +Leggett / ZoomPilot. Its download identifiers and commands keep their upstream +names; installing the host does not require installing ZoomPilot on the comma. +The bundled client uses Jetlink protocol 3 (package 0.8.3). A different host +release still needs a successful compatible handshake and device validation. + +### Jetson and Linux PC + +On a supported NVIDIA host, review and run the [upstream installer](https://github.com/zoompilot/jetlink/blob/c17cd5c/install.sh). +The upstream Jetson guide specifies an Orin Nano Super 8 GB and JetPack 7.2.1; +the Linux guide specifies an RTX 20-series or newer GPU with driver 580 or newer. +Keep the computer powered, cool, and awake. Connect its USB-A port to the comma's USB-C port. +Use `jetlink status` to check the host and `jetlink logs` to inspect problems. +Host power, sleep, and driver setup details remain in the credited +[Jetson](https://github.com/zoompilot/jetlink/blob/c17cd5c/docs/jetson.md) and +[Linux](https://github.com/zoompilot/jetlink/blob/c17cd5c/docs/linux-pc.md) references; +their comma-side ZoomPilot instructions are replaced by **Set up StarPilot** above. + +### Mac + +Use an Apple silicon Mac with macOS 15 or newer. Get the Jetlink DMG from +[the upstream releases](https://github.com/zoompilot/jetlink/releases), put the app +in Applications, and start it. Keep it awake and connect by USB 3. +Use **USB computer** in StarPilot's Galaxy, rather than following the comma-side +ZoomPilot installation instructions in the credited +[Mac host reference](https://github.com/zoompilot/jetlink/blob/c17cd5c/docs/macos-app.md). + +### iPhone and iPad + +Install the upstream [Jetlink TestFlight beta](https://testflight.apple.com/join/DAsYk5sP). +It requires a USB-C device supporting USB 3 and iOS/iPadOS 26.1 or newer. +Choose **iPhone / iPad** in StarPilot, allow the host app's Local Network access, +connect by USB 3, and keep the app open and the device unlocked. Check sustained +thermal performance with the app's benchmark; that benchmark excludes cable latency. +The [credited Apple host reference](https://github.com/zoompilot/jetlink/blob/c17cd5c/docs/iphone-app.md) +contains hardware details. StarPilot supplies the comma-side instructions above. + +### Android + +Android is experimental and performance is not established. Install the +upstream Jetlink Android APK from [its releases](https://github.com/zoompilot/jetlink/releases). +Use a recent USB 3-capable Android 12+ phone, a powered USB 3 hub, and separate +comma power. Choose **USB computer** in StarPilot. Follow the host's USB permission +prompt and benchmark sustained performance before use. +[Credited Android host reference](https://github.com/zoompilot/jetlink/blob/c17cd5c/docs/android-app.md). + +## Use and recovery + +Before changing the remote model, disengage cruise **and** any independent +Always On Lateral steering. Wait for Galaxy's fresh **Active · Jetlink remote model** +status before re-engaging. If the link stops or stalls, StarPilot falls back to +the local Small model; be ready to take control as behavior changes. Set Jetlink +to **Off** while parked to stop using the remote connection. + +If a connection fails, check separate power, USB 3 support, host sleep, cooling, +and the chosen StarPilot connection mode. An iPhone must stay unlocked with the +app foregrounded. Save host and StarPilot logs when reporting an issue; do not +interpret “prepared” as “active”. Phone charging is a separate Galaxy preference +and defaults off. + +## Attribution and license + +This StarPilot guide adapts the host setup structure from +[ZoomPilot / Jetlink at c17cd5c](https://github.com/zoompilot/jetlink/tree/c17cd5c/docs). +Jetlink is copyright (c) 2026 Zeph Leggett and licensed under the +[MIT license](../../../jetlink_repo/LICENSE), whose full text is preserved. +ZoomPilot source links, project names, authorship, and history are attribution; +they are not instructions to replace StarPilot on the comma. Host binaries and +external installation scripts remain upstream artifacts, with their original identity. diff --git a/openpilot/starpilot/models/README.md b/openpilot/starpilot/models/README.md new file mode 100644 index 0000000000..ad3f9163fe --- /dev/null +++ b/openpilot/starpilot/models/README.md @@ -0,0 +1,34 @@ +# Driving models + +Model Manager installs the versioned catalog into `/data/models/v26`. Downloads use the catalog's declared size and SHA-256, publish atomically, and share progress and cancellation between Galaxy and the native menus. Small and Chestnut selections take effect when modeld next starts. Randomization chooses only installed, verified models for the available hardware and respects exclusions; favorites are a separate browsing preference. + +Catalog behavior versions describe model inputs and outputs. The artifact ABI and compiler revision separately identify runtime compatibility. Existing compiled artifacts can be migrated when their kernels and state can be preserved and verified; changing the version label alone does not establish compatibility. Unavailable catalog entries cannot be downloaded or selected. + +`modeld` records the loaded artifact digest, process identity, variant, and fallback reason in a bounded volatile receipt. Status requires a matching running process and fresh valid model outputs before reporting an active model. A failed Chestnut load falls back to the initialized small runner. Artifact integrity and bench inference do not establish on-road driving quality. + +## Model Laboratory + +Laboratory artifact management uses separate AMD variants of small models. A standard on-device artifact or a Chestnut-class large model cannot substitute for one. The manifest declares each variant under `accelerator_artifacts.amd`, with `execution_device: "AMD"`, the current `compiler_revision`, and the same artifact format, digest, size, and chunk-count fields as ordinary catalog artifacts. Files are named `_driving_amd_tinygrad.pkl` under the model's existing catalog directory. Deleting an AMD variant preserves the ordinary on-device model. + +The Galaxy page and parked configuration owner are available. The standalone pair runner verifies both AMD artifacts, shares compatible camera preprocessing, keeps each model's state separate, and rejects the whole frame if either role fails. It decodes each model's own action units before choosing lateral curvature and longitudinal acceleration/stopping. The compositor assigns lateral geometry and the corresponding plan columns to the lateral model, and leads, scene metadata, and current-frame pose to the longitudinal model. + +Paired inference is not integrated into modeld yet, so production reports `runtimeSupported: false` and rejects enabling a pair. Completing it requires two verified AMD small-model variants, modeld loading/fallback and fresh pair status, then a Chestnut bench run covering real shared-warp execution, memory, timing and driver monitoring coexistence. CPU state-isolation and mocked frame tests do not qualify the AMD pair. + +Jetlink is a separate remote model connection for comma 3X and comma 4. In Galaxy +Model Manager, choose **Off**, **USB computer**, or **iPhone / iPad** while parked. +Off is the default. USB covers supported Jetson, Mac, Linux, and Android hosts; +iPhone uses the direct USB connection. Chestnut takes precedence when fitted. +Wireless Android Auto can remain enabled. + +Use a USB 3 data cable and separate power, and set up both devices online while +parked. Keep the computer awake; keep the iPhone app open and unlocked. Follow +[the StarPilot JetLink setup guide](JETLINK.md) +for the host installation and model preparation. Galaxy shows connection and +preparation progress; **prepared** does not mean the remote model is driving. +Small/Chestnut catalog selections and their defaults remain separate. Phone +charging over USB is optional and starts off. + +For a remote model switch, fully disengage cruise and any lateral control that +stays on without cruise. Wait for Galaxy's fresh **Active · Jetlink remote model** +status before engaging again. A lost or stalled link falls back to the local +Small model; remain ready to take over while its behavior changes. diff --git a/openpilot/starpilot/models/__init__.py b/openpilot/starpilot/models/__init__.py new file mode 100644 index 0000000000..798dbd872c --- /dev/null +++ b/openpilot/starpilot/models/__init__.py @@ -0,0 +1 @@ +"""Driving-model identity and runtime status for StarPilot surfaces.""" diff --git a/openpilot/starpilot/models/catalog-v26.json b/openpilot/starpilot/models/catalog-v26.json new file mode 100644 index 0000000000..60cbe1acb7 --- /dev/null +++ b/openpilot/starpilot/models/catalog-v26.json @@ -0,0 +1,1236 @@ +{ + "generation": "v26", + "compiler_revision": "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf", + "artifact_abi": "tinygrad_single_v1_arena", + "models": [ + { + "id": "tr14223", + "name": "TR14 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-06-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "288980456db5ba083d333c0d08924f896e4141ba41642a9760eeefb497ff9797", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "tr15223", + "name": "TR15 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "551e01170bc6e7f40a32329122a3b07d4e202b45433c7d279243fe5003b00f27", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "tr16223", + "name": "TR16 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-21", + "community_favorite": false, + "uses_external_gpu": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5503a19a84fea7de25887fc3a2ec7148be8f84c5d9ea0f9e118f0df39743da25", + "artifact_size": 89751287, + "artifact_chunk_count": 2 + }, + { + "id": "letr223", + "name": "LeTR 👀📡", + "version": "v8", + "series": "TR Series", + "released": "2025-07-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e55f53c970b011f4dbc5d90b119ba097dec6d390f3eed7cfb55de5233be72ea7", + "artifact_size": 89751287, + "artifact_chunk_count": 2 + }, + { + "id": "spacelab22223", + "name": "Space Lab 2 👀📡", + "version": "v8", + "series": "SL Series", + "released": "2025-07-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "1dae96a137e786bd75b80a8a1e89d215cb7c5b4fbe588c0ac270d4f5f33aa10e", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "vikander223", + "name": "Vikander 👀📡", + "version": "v8", + "series": "Custom Series", + "released": "2025-05-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "3cae48b71fc6db7339e15d0b329a1ee5ebff1e0ea469e1b7d3aac1ffa4db2aa4", + "artifact_size": 89737007, + "artifact_chunk_count": 2 + }, + { + "id": "fof223", + "name": "Filet-O-Fish 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-03-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "06e0967f88f022b3f9fca0164a1aac9ee95b6256a4a457388961d55de0e33ba0", + "artifact_size": 77785913, + "artifact_chunk_count": 2 + }, + { + "id": "vfof223", + "name": "Vegetarian-Filet-O-Fish 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-06-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7a001de39fef29d0ddde3f59e9ba7673d7dce7fbd3145ca2c1e9b715adce0f0c", + "artifact_size": 77800671, + "artifact_chunk_count": 2 + }, + { + "id": "dtr223", + "name": "Down To Ride 👀📡", + "version": "v8", + "series": "DTR Series", + "released": "2025-07-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "747792c151e9c119e138836638731cc7c69782d414805f7e02125ea7c59f4fe5", + "artifact_size": 89750662, + "artifact_chunk_count": 2 + }, + { + "id": "dtr6223", + "name": "Down To Ride v6 👀📡", + "version": "v10", + "series": "DTR Series", + "released": "2025-08-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "af8417340b6f3b70cd0117709af09e85a75b33aee9a66decd69e05936ca48872", + "artifact_size": 89750663, + "artifact_chunk_count": 2 + }, + { + "id": "uvdtr6223", + "name": "UV DTR v6 👀📡", + "version": "v10", + "series": "DTR Series", + "released": "2025-08-13", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "44184497255d0689861caa7719509eb680ce94ea7ac0da6e02afc1b9dece8103", + "artifact_size": 88543399, + "artifact_chunk_count": 2 + }, + { + "id": "sp2223", + "name": "Steam Powered 👀📡", + "version": "v10", + "series": "SP Series", + "released": "2025-08-16", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "93b34adeb2db586072ff179fd11e744fd3a9a39889795538a7933443cc1a4880", + "artifact_size": 83632118, + "artifact_chunk_count": 2 + }, + { + "id": "fp223", + "name": "Falling Phoenix 👀📡", + "version": "v10", + "series": "Custom Series", + "released": "2025-08-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "19b861ae053fd9e5182f1d28ce8875594825c53f3819e226ff57407d861a4a5b", + "artifact_size": 89750883, + "artifact_chunk_count": 2 + }, + { + "id": "kv223", + "name": "Kumar's Vibe 👀📡", + "version": "v10", + "series": "Custom Series", + "released": "2025-08-16", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "f91ce19bbcea90cf5416f01329411fbc8be0a9a467be2be7e62cbd4bfc2d7d6c", + "artifact_size": 89750882, + "artifact_chunk_count": 2 + }, + { + "id": "gwm3223", + "name": "Green Watermelon v3 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-08-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c74cc5e56972469328fb9710794d199aa14d6653412baf19bb6e81bb0d0db917", + "artifact_size": 83632118, + "artifact_chunk_count": 2 + }, + { + "id": "gwm5223", + "name": "Green Watermelon v5 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "2671cfa5daa6d0768ccb81d6eba5d756625ee06cb7c7bfe51f335e831f00ec6f", + "artifact_size": 83632124, + "artifact_chunk_count": 2 + }, + { + "id": "gwm6223", + "name": "Green Watermelon v6 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "gwm8223", + "name": "Green Watermelon v8 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-28", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e0263bb34dba65d4d55d679675dcc3198728f817f2cd98bb872cf3774fa4eabe", + "artifact_size": 83631220, + "artifact_chunk_count": 2 + }, + { + "id": "gwm9223", + "name": "Green Watermelon v9 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-10-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "afc80e282c27c2454da85b1f793fc52dad9d9dd07d88d88630e888317439bd1c", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "cgwm223", + "name": "Cool Green Watermelon 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-10-24", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5e1ecd3c91b7847e855885ab73f1399eda80179667f3c7b310b633479dadc501", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "bd223", + "name": "Firehose 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-08-31", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b1dfa01aa7b2031c4add906445c446844e36c7c11fb47696600fa2ea04ec7cd0", + "artifact_size": 83632124, + "artifact_chunk_count": 2 + }, + { + "id": "nr223", + "name": "Neurips 👀📡", + "version": "v11", + "series": "NR Series", + "released": "2025-11-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "df616dfd649dec7141e4190a3e66c61afd684561d939030c4d6eaa2f709e3143", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "tcp2223", + "name": "The Cool People v2 👀📡", + "version": "v11", + "series": "TCP Series", + "released": "2025-10-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "795b593cb375f299a34c561ad18232690369b251adbde9fd10f26dc3cd6e9ade", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "tcp3223", + "name": "The Cool People v3 👀📡", + "version": "v11", + "series": "TCP Series", + "released": "2025-10-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "85f9ce6a0e88db51ce02d61f594a9ada7de6a8aa2ee206f5f014a7fe9dce59bc", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "fbw223", + "name": "Fly By Wire 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-09-04", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5a58a2bbf05fef21e470efc837d29301ddda5d23b959c9c46d5736c659aaed47", + "artifact_size": 71032111, + "artifact_chunk_count": 2 + }, + { + "id": "nevada223", + "name": "Nevada 👀📡", + "version": "v11", + "series": "NV Series", + "released": "2025-09-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "6f497406e2e914bd29d7140710e3214ffde7ac36cb60a7333e5770517864a1c2", + "artifact_size": 83632120, + "artifact_chunk_count": 2 + }, + { + "id": "wmi4223", + "name": "Watermelon Indigo v4 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7f0983957c87cc280e6b51f9e8ed2f40817da449669abd7410ff3319e287c050", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi523", + "name": "Watermelon Indigo v5 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-29", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c98c2dbe228aad773a2fc85949eb7dd07cb4cb9621a81074276f4aaf06f926da", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi623", + "name": "Watermelon Indigo v6 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2025-12-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c94059abcb0b95cf4a11d912e0aa36eadcb423832dfae1af7c0cbe2f406428b6", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi723", + "name": "Watermelon Indigo v7 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c91bbdb296f56acc084097aafb30d1050abb5213ad2e6fce73e9ddc5a3ab98fe", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi7223", + "name": "Watermelon Indigo v7 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "673ac43ce0f52b0324322ee26b5baad165f71ea08c599385328035620a6cb560", + "artifact_size": 83635845, + "artifact_chunk_count": 2 + }, + { + "id": "wmi823", + "name": "Watermelon Indigo v8 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "eb3f263d370ebd74d5feea288686844901c2e10ab5e941105e7eeed6d6fb06d3", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi923", + "name": "Watermelon Indigo v9 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "04528d7005d5913f35313c346235c5e94111bf8d4baf9765e79a50d093e01c33", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "wmi1023", + "name": "Watermelon Indigo v10 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "8dd02dc16a0ed51978ec55f34875f779e8089757b5153a9216de9010933cf25d", + "artifact_size": 83741673, + "artifact_chunk_count": 2 + }, + { + "id": "wmi1123", + "name": "Watermelon Indigo v11 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-13", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "917894794ddc28ceac39a1f098e875fb3449dbc9b7d4e0f78f0035a7a2651580", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "cd21023", + "name": "CD210 👀📡", + "version": "v11", + "series": "WM Series", + "released": "2026-01-31", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "cb4b9ee7a1a7ffec8883c9e566a051ccf06736195645fa1392cf3fd6af8d5bb8", + "artifact_size": 83728468, + "artifact_chunk_count": 2 + }, + { + "id": "op223", + "name": "Off Policy v2 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "518a4c91cc916eec2d533f68546c2df07222b514bdbcc94bda08f8a581f96c5d", + "artifact_size": 72331687, + "artifact_chunk_count": 2 + }, + { + "id": "op323", + "name": "Off Policy v3 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "791ba9270e6329da9d64c6175e0c0bdbc1a42d84f821bf1c8274e138fbaa074e", + "artifact_size": 60720006, + "artifact_chunk_count": 2 + }, + { + "id": "op423", + "name": "Off Policy v4 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-02-27", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "3334290c8ca613a487a3a267c83a6c4742b8e145e96796a9ff04f8d31c5a3802", + "artifact_size": 67107030, + "artifact_chunk_count": 2 + }, + { + "id": "op523", + "name": "Off Policy v5 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-03-14", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7cf2fb630b3fdab3c68f809f2cb409091bcec90348351fffa126b8378a1aaf6f", + "artifact_size": 69394003, + "artifact_chunk_count": 2 + }, + { + "id": "op623", + "name": "Off Policy v6 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-03-28", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "4dab4296e6e07470ccecfe17555667b7b5894c69e5d01bd70b770fef21208c47", + "artifact_size": 73634626, + "artifact_chunk_count": 2 + }, + { + "id": "opv73", + "name": "Off Policy v7 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e7055bf07e9bdcd223d174545d073e331a6f9a39564b80ffbcb22fafc1a42448", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv83", + "name": "Off Policy v8 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "ab6847f204d935b9cbf51a892d7dd0b7fadf3395c78766da74d06f79e1265c8d", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv93", + "name": "Off Policy v9 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-07", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "d6771747aae786f9bf158ed067ea2c77d1eb1b31cb5cbcb5c754c6ba0df1190d", + "artifact_size": 73612506, + "artifact_chunk_count": 2 + }, + { + "id": "opv103", + "name": "Off Policy v10 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv113", + "name": "Off Policy v11 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-18", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv123", + "name": "Off Policy v12 👀📡", + "version": "v12", + "series": "OP Series", + "released": "2026-04-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "opv133", + "name": "Off Policy v13 👀📡", + "version": "v14", + "series": "OP Series", + "released": "2026-05-05", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op163", + "name": "Off Policy v16 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-19", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op16d3", + "name": "Off Policy v16 Deep 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-20", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "op16dv23", + "name": "Off Policy v16 Deep V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-05-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rlv1dl3", + "name": "RL V1 Lateral Only 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-03", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "deeprl33", + "name": "Deep RL 3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "deeprl333", + "name": "Deep RL 3 V3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-30", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rl343", + "name": "Rebel Legion 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "drl3", + "name": "Divided RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "nopp3", + "name": "No PP 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-26", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tobyrl3", + "name": "Toby RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-28", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "michael-rl3", + "name": "Michael RL 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "michael-rl23", + "name": "Michael RL V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "karnbir3", + "name": "Karnbir 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "karnbir23", + "name": "Karnbir V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-22", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "ms23", + "name": "Macrostiff 👀📡", + "version": "v11", + "series": "MS Series", + "released": "2026-01-01", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "874e4c8a00c483874fc704a449b003ab59bcf0e5d99c8dd591bbc20abf386029", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "pp2223", + "name": "Plan Plus v2 👀📡", + "version": "v11", + "series": "PL Series", + "released": "2025-12-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "ds2223", + "name": "Dark Souls v2 👀📡", + "version": "v11", + "series": "DS Series", + "released": "2025-12-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "7f4a4e9cad0c357f4e66d8e772c7b6748304fc55c440db768671242fccc28431", + "artifact_size": 83737682, + "artifact_chunk_count": 2 + }, + { + "id": "nn2223", + "name": "North Nevada v2 👀📡", + "version": "v11", + "series": "NV Series", + "released": "2025-10-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b86bd737f158a926846f63ce8e2a1e28e0d2c18ad0c3871d43160a566797128f", + "artifact_size": 83632126, + "artifact_chunk_count": 2 + }, + { + "id": "sc23", + "name": "South Carolina 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-01-08", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "66e4837bde0e12bdd90e7eb3e180ea2eab62d3a6af279df366cbf0cc69c469fd", + "artifact_size": 83737646, + "artifact_chunk_count": 2 + }, + { + "id": "pop23", + "name": "Pop Model 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-03-21", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "d0f287497057b47fd91bbfa9eeb6d86f91e6bad84f1c846e4afec1b8b6deb4bb", + "artifact_size": 83728468, + "artifact_chunk_count": 2 + }, + { + "id": "pop223", + "name": "Pop Model V2 👀📡", + "version": "v11", + "series": "SC Series", + "released": "2026-03-24", + "community_favorite": true, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "e147dd4f0d7be4e7931ebc2b07eb28a0eda08a7b26e562188d53e22efc340e58", + "artifact_size": 83737200, + "artifact_chunk_count": 2 + }, + { + "id": "nid223", + "name": "Nuggets in Dijon 👀📡", + "version": "v11", + "series": "NUG Series", + "released": "2025-10-09", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "32cea751d38aeaed25e01882c4dd24e4467d9139ddba4c74ff42c7a2084a9045", + "artifact_size": 83632126, + "artifact_chunk_count": 2 + }, + { + "id": "kerrygold223", + "name": "Kerrygold 👀📡", + "version": "v9", + "series": "FoF Series", + "released": "2025-06-08", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "9379c9a1962d743ea80f7708ae7ab42235f3787dc7f65e1c7b320f73a6e08935", + "artifact_size": 77797177, + "artifact_chunk_count": 2 + }, + { + "id": "deeprl3v23", + "name": "Deep RL 3 V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rh3", + "name": "Rebellious Hope 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-27", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "gyhu3", + "name": "GYHU 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-07-25", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdfv23", + "name": "Regret Driven Framework 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-05", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf23", + "name": "Regret Driven Framework V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf43", + "name": "Regret Driven Framework V4 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf33", + "name": "Regret Driven Framework V3 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-10", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf53", + "name": "Regret Driven Framework V5 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "rdf63", + "name": "Regret Driven Framework V6 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-11", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tsf", + "name": "Terrific Super Fantastic 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-12", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "tsfdo", + "name": "Terrific Super Fantastic Do Over 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-15", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "lebowski3", + "name": "Lebowski 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-15", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "9faae0161a46cce693927da12be5cdac32df5e9855e2e521c213c2de2279e4fe", + "artifact_size": 1773071525, + "artifact_chunk_count": 38 + }, + { + "id": "tt-driving", + "name": "TT Driving 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-17", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "b5ee71cff876bcc5e910c237abac783c8c4d8e51121af78a2c4d94ee13764ba2", + "artifact_size": 1773202548, + "artifact_chunk_count": 38 + }, + { + "id": "tee-time", + "name": "Tee Time 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-08-23", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "608728fb6c53cf39a0d417a2245557c80cdc740ef651ac148cd805ccd2f5a9de", + "artifact_size": 1773210625, + "artifact_chunk_count": 38 + }, + { + "id": "bmrlnapv2", + "name": "BMRLNAP V2 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-26", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "c828394a69d59c101427ffdc55cde6f61bd432abdf15a7e10e03eaa610a03bc0", + "artifact_size": 1776069700, + "artifact_chunk_count": 38 + }, + { + "id": "happybirthday", + "name": "Happy Birthday 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-24", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "f729e4b4711212f314a1fff14058657a86c36405deca5ae92e3ed5b213f3cbdc", + "artifact_size": 1776069700, + "artifact_chunk_count": 38 + }, + { + "id": "berightthere", + "name": "Be Right Here 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-01", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv4", + "name": "Remove AvgPool V4 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-26", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv3", + "name": "Remove AvgPool V3 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-24", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv2", + "name": "Remove AvgPool V2 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-20", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "remove-avgpoolv1", + "name": "Remove AvgPool V1 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-20", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "rdf-driving", + "name": "RDF Driving 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-08-09", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "sad", + "name": "Sad 😢", + "version": "v16", + "series": "OP Series", + "released": "2026-08-28", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "bmrlnapv6", + "name": "BMRLNAP V6 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-01", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "deeprl3v2", + "name": "Deep RL 3 V2 👀📡", + "version": "v15", + "series": "OP Series", + "released": "2026-06-17", + "community_favorite": false, + "model_size": "small", + "model_lab_eligible": true + }, + { + "id": "cinque-driving", + "name": "Cinque-Driving 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-04", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "cdm", + "name": "Cinque Divergence Mask 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-08", + "community_favorite": false, + "uses_external_gpu": true + }, + { + "id": "cinquev3", + "name": "Cinque v3 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-09-18", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "261454e0d170f5282d989d67ea7db08c90d7297369260085fbdf44af4e327a08", + "artifact_size": 779382267, + "artifact_chunk_count": 17 + }, + { + "id": "resfit", + "name": "ResFit", + "version": "v16", + "series": "OP Series", + "released": "2026-10-06", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "5dde72ce3f00bdb4db89fb147b3bef4f684a114a4acd01332b81cde5214baa62", + "artifact_size": 804588697, + "artifact_chunk_count": 18 + }, + { + "id": "mountaindewv1", + "name": "Mountain Dew v1 👀📡", + "version": "v16", + "series": "OP Series", + "released": "2026-10-08", + "community_favorite": false, + "uses_external_gpu": true, + "artifact_format": "tinygrad_single_v1_arena", + "artifact_sha256": "397e0a555c66143b2507b45a55e24a8d638634ad2f40abaa857a857226b20b0c", + "artifact_size": 804589195, + "artifact_chunk_count": 18 + } + ] +} diff --git a/openpilot/starpilot/models/catalog.py b/openpilot/starpilot/models/catalog.py new file mode 100644 index 0000000000..6cc9cc06b8 --- /dev/null +++ b/openpilot/starpilot/models/catalog.py @@ -0,0 +1,89 @@ +"""Reviewed local driving-model catalog. This module never loads model artifacts.""" + +from dataclasses import dataclass, replace +from functools import lru_cache +import json +from pathlib import Path + + +BUNDLED_CURRENT = "bundled-current" +DEFAULT_SMALL = "rdf43" +DEFAULT_SMALL_SHA256 = "e23f65b3790a2092348024603966717506c2ad1fae10b4312bdfe1f2b932df61" +DEFAULT_SMALL_SIZE = 125895084 +GENERATION = "v26" +COMPILER_REVISION = "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf" +ARTIFACT_ABI = "tinygrad_single_v1_arena" +CATALOG_PATH = Path(__file__).with_name("catalog-v26.json") + + +@dataclass(frozen=True) +class ModelEntry: + model_id: str + name: str + source_sha256: str + source_path: str + compiler_revision: str + artifact_abi: str + selectable: bool + version: str = "current" + uses_external_gpu: bool = False + + +BUNDLED = ( + ModelEntry( + model_id=BUNDLED_CURRENT, + name="Regret Driven Framework V4", + source_sha256=DEFAULT_SMALL_SHA256, + source_path="openpilot/selfdrive/modeld/models/rdf43_driving_tinygrad.pkl", + compiler_revision="9d0446a4ba8a532c8b674fb6ad795af015cd9dcf", + artifact_abi=ARTIFACT_ABI, + selectable=True, + version="v15", + ), +) + +CATALOG = BUNDLED + tuple(ModelEntry( + model_id=m["id"], name=m["name"], + source_sha256=DEFAULT_SMALL_SHA256 if m["id"] == DEFAULT_SMALL else "", + source_path="openpilot/selfdrive/modeld/models/rdf43_driving_tinygrad.pkl" if m["id"] == DEFAULT_SMALL else "", + compiler_revision=COMPILER_REVISION, artifact_abi=ARTIFACT_ABI, selectable=True, + version=m["version"], uses_external_gpu=bool(m.get("uses_external_gpu", False)), +) for m in json.loads(CATALOG_PATH.read_text())["models"]) + +BY_ID = {entry.model_id: entry for entry in CATALOG} + + +def _document_identity(path: Path) -> tuple: + try: + stat = path.stat() + return stat.st_dev, stat.st_ino, stat.st_size, stat.st_mtime_ns, stat.st_ctime_ns + except OSError: + return () + + +@lru_cache(maxsize=8) +def _cached_entries(root: Path, identities: tuple) -> tuple[ModelEntry, ...]: + from openpilot.starpilot.models.manager import catalog + entries = dict(BY_ID) + for model_id, row in catalog(root).items(): + entries[model_id] = (replace(BY_ID[model_id], name=row["name"]) if model_id in BY_ID else + ModelEntry(model_id, row["name"], "", "", COMPILER_REVISION, ARTIFACT_ABI, + True, row["version"], row.get("uses_external_gpu", False))) + return tuple(entries.values()) + + +def model_entries(root: Path | None = None) -> dict[str, ModelEntry]: + """Resolve validated cached metadata without mutating the bundled catalog.""" + from openpilot.starpilot.models.manager import ROOT + root = ROOT if root is None else root + identities = (_document_identity(CATALOG_PATH), _document_identity(root / "catalog.json")) + return {entry.model_id: entry for entry in _cached_entries(root, identities)} + + +def resolve_selection(requested_id: str | None, *, root: Path | None = None) -> ModelEntry: + """Resolve a saved request; absence means the bundled model, never an unknown ID.""" + model_id = DEFAULT_SMALL if requested_id in (None, BUNDLED_CURRENT) else requested_id + entries = model_entries(root) + if not isinstance(model_id, str) or model_id not in entries or not entries[model_id].selectable: + raise ValueError("unqualified driving-model selection") + return entries[model_id] diff --git a/openpilot/starpilot/models/compile.py b/openpilot/starpilot/models/compile.py new file mode 100644 index 0000000000..d8347b350a --- /dev/null +++ b/openpilot/starpilot/models/compile.py @@ -0,0 +1,712 @@ +"""Compile StarPilot unified driving artifacts against the vendored tinygrad runtime.""" +import argparse +import base64 +import math +import os +from pathlib import Path +import pickle +import tempfile +import time +from collections import namedtuple +from functools import partial + +import numpy as np +from tinygrad import Device, TinyJit, Context, Tensor +from tinygrad.dtype import _to_np_dtype, dtypes +from tinygrad.device import Buffer, BufferSpec +from tinygrad.uop.ops import UOp +from tinygrad_repo.examples.openpilot.helpers import dump_pickle, load_pickle + +COMPILER_REVISION = "9d0446a4ba8a532c8b674fb6ad795af015cd9dcf" +ARTIFACT_ABI = "tinygrad_single_v1_arena" + + +def make_metadata_dict(path, runner): + from tinygrad.nn.onnx import OnnxPBParser + + parser = OnnxPBParser(path, load_external_data=False) + properties, output_shapes = {}, {} + for field, wire_type in parser._parse_message(parser.reader.len): + if field == 14: + entry = parser._parse_StringStringEntryProto() + properties[entry['key']] = entry['value'] + elif field == 7: + for field, wire_type in parser._parse_message(parser._decode_end_pos()): + if field == 12: + value = parser._parse_ValueInfoProto() + output_shapes[value['name']] = value['parsed_type'].shape if value['parsed_type'] is not None else () + else: + parser.reader.skip_field(wire_type) + else: + parser.reader.skip_field(wire_type) + if "output_slices" not in properties: + raise ValueError(f"Missing output_slices metadata: {path}") + return { + "model_checkpoint": properties.get("model_checkpoint"), + "output_slices": pickle.loads(base64.b64decode(properties["output_slices"])), + "input_shapes": {name: tuple(spec.shape) for name, spec in runner.graph_inputs.items()}, + "output_shapes": output_shapes, + } + + +ARTIFACT_FORMAT_VERSION = 1 +MODEL_TYPES = ("vision_policy", "vision_multi_policy", "supercombo") +NV12Frame = namedtuple("NV12Frame", ["width", "height", "stride", "y_height", "uv_height", "size"]) +IMAGE_HISTORY_IN_WARP = "warp" +IMAGE_HISTORY_IN_POLICY = "policy" +IMAGE_HISTORY_PIPELINES = (IMAGE_HISTORY_IN_WARP, IMAGE_HISTORY_IN_POLICY) +LEGACY_WARP_INPUTS = ("img_q", "big_img_q", "tfm", "big_tfm") +FAST_WARP_INPUTS = ("tfm", "big_tfm") +BASE_POLICY_INPUTS = ("feat_q", "desire_q", "packed_npy_inputs") +FAST_POLICY_INPUTS = ("img_q", "big_img_q", *BASE_POLICY_INPUTS) +WARP_INPUTS = LEGACY_WARP_INPUTS +SPLIT_POLICY_INPUTS = BASE_POLICY_INPUTS +SUPERCOMBO_POLICY_INPUTS = BASE_POLICY_INPUTS +WARP_DEV = os.getenv("WARP_DEV") + + + +def _detect_desire_key(input_shapes): + return next((key for key in input_shapes if key.startswith("desire")), None) + + +def _detect_vision_keys(input_shapes): + image_keys = sorted(key for key in input_shapes if "img" in key) + road_key = next((key for key in image_keys if "big" not in key), None) + wide_key = next((key for key in image_keys if "big" in key), None) + if road_key is None or wide_key is None: + raise ValueError(f"Cannot determine road/wide image keys from {list(input_shapes)}") + return road_key, wide_key + + +def derive_frame_skip(input_shapes): + features_shape = input_shapes.get("features_buffer") + if features_shape is None: + return 1 + return 1 if features_shape[1] >= 99 else 4 + + +def make_random_images(keys, shape, device=None, *, rng=None): + return {key: Tensor.randint(shape, low=0, high=256, dtype="uint8", device=device).realize() for key in keys} + + +def make_random_blob_images(keys, size, device=None): + keepalive: list[np.ndarray] = [] + + def make_inputs(*, rng): + nonlocal keepalive + keepalive = [] + tensors = {} + for key in keys: + frame = (32 * rng.standard_normal(size).astype(np.float32) + 128).clip(0, 255).astype(np.uint8) + keepalive.append(frame) + tensors[key] = Tensor.from_blob(frame.ctypes.data, (size,), dtype="uint8", device=device).realize() + return tensors + + return make_inputs + + +def warp_perspective_tinygrad(src_flat, matrix_inverse, dst_shape, src_shape, stride_pad, border_fill_val=None): + width_dst, height_dst = dst_shape + height_src, width_src = src_shape + + x = Tensor.arange(width_dst).reshape(1, width_dst).expand(height_dst, width_dst).reshape(-1) + y = Tensor.arange(height_dst).reshape(height_dst, 1).expand(height_dst, width_dst).reshape(-1) + + src_x = matrix_inverse[0, 0] * x + matrix_inverse[0, 1] * y + matrix_inverse[0, 2] + src_y = matrix_inverse[1, 0] * x + matrix_inverse[1, 1] * y + matrix_inverse[1, 2] + src_w = matrix_inverse[2, 0] * x + matrix_inverse[2, 1] * y + matrix_inverse[2, 2] + src_x = src_x / src_w + src_y = src_y / src_w + + x_round = Tensor.round(src_x) + y_round = Tensor.round(src_y) + x_nn_clipped = x_round.clip(0, width_src - 1).cast("int") + y_nn_clipped = y_round.clip(0, height_src - 1).cast("int") + sampled = src_flat[y_nn_clipped * (width_src + stride_pad) + x_nn_clipped] + + if border_fill_val is None: + return sampled + + in_bounds = ((x_round >= 0) & (x_round <= width_src - 1) & + (y_round >= 0) & (y_round <= height_src - 1)).cast(sampled.dtype) + return sampled * in_bounds + Tensor(border_fill_val, dtype=sampled.dtype) * (1 - in_bounds) + + +def frames_to_tensor(frames): + height = (frames.shape[0] * 2) // 3 + width = frames.shape[1] + return Tensor.cat( + frames[0:height:2, 0::2], + frames[1:height:2, 0::2], + frames[0:height:2, 1::2], + frames[1:height:2, 1::2], + frames[height:height + height // 4].reshape((height // 2, width // 2)), + frames[height + height // 4:height + height // 2].reshape((height // 2, width // 2)), + dim=0, + ).reshape((6, height // 2, width // 2)) + + +def make_frame_prepare(nv12: NV12Frame, model_w, model_h): + cam_w, cam_h, stride, y_height, uv_height, _ = nv12 + uv_offset = stride * y_height + stride_pad = stride - cam_w + + def frame_prepare(input_frame, matrix_inverse): + matrix_inverse_uv = matrix_inverse * Tensor( + [[1.0, 1.0, 0.5], [1.0, 1.0, 0.5], [2.0, 2.0, 1.0]], + device=WARP_DEV, + ) + uv = input_frame[uv_offset:uv_offset + uv_height * stride].reshape(uv_height, stride) + with Context(SPLIT_REDUCEOP=0): + y = warp_perspective_tinygrad( + input_frame[:cam_h * stride], matrix_inverse, (model_w, model_h), (cam_h, cam_w), stride_pad, + ).realize() + u = warp_perspective_tinygrad( + uv[:cam_h // 2, :cam_w:2].flatten(), matrix_inverse_uv, + (model_w // 2, model_h // 2), (cam_h // 2, cam_w // 2), 0, + ).realize() + v = warp_perspective_tinygrad( + uv[:cam_h // 2, 1:cam_w:2].flatten(), matrix_inverse_uv, + (model_w // 2, model_h // 2), (cam_h // 2, cam_w // 2), 0, + ).realize() + return frames_to_tensor(y.cat(u).cat(v).reshape((model_h * 3 // 2, model_w))) + + return frame_prepare + + +def make_warp_input_queues(vision_input_shapes, frame_skip, device): + road_key, _ = _detect_vision_keys(vision_input_shapes) + image_shape = vision_input_shapes[road_key] + frame_count = image_shape[1] // 6 + image_buffer_shape = (frame_skip * (frame_count - 1) + 1, 6, image_shape[2], image_shape[3]) + + npy = { + "tfm": np.zeros((3, 3), dtype=np.float32), + "big_tfm": np.zeros((3, 3), dtype=np.float32), + } + queues = { + "img_q": Tensor(np.zeros(image_buffer_shape, dtype=np.uint8), device=device).contiguous().realize(), + "big_img_q": Tensor(np.zeros(image_buffer_shape, dtype=np.uint8), device=device).contiguous().realize(), + **{key: Tensor(value, device="NPY").realize() for key, value in npy.items()}, + } + return queues, npy + + +def _packed_policy_shapes(input_shapes, include_prev_feature=False): + desire_key = _detect_desire_key(input_shapes) + if desire_key is None: + raise ValueError(f"No desire input found in {list(input_shapes)}") + + shapes = {"desire": (input_shapes[desire_key][2],)} + for key, shape in input_shapes.items(): + if key in ("features_buffer", desire_key) or "img" in key: + continue + shapes[key] = tuple(shape) + if include_prev_feature: + features_shape = input_shapes["features_buffer"] + shapes["prev_feat"] = (features_shape[0], math.prod(features_shape[2:])) + return shapes, [math.prod(shape) for shape in shapes.values()] + + +def make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device): + queues, npy = make_warp_input_queues(vision_input_shapes, frame_skip, device) + features_shape = policy_input_shapes["features_buffer"] + feature_dim = math.prod(features_shape[2:]) + desire_key = _detect_desire_key(policy_input_shapes) + desire_shape = policy_input_shapes[desire_key] + packed_shapes, packed_sizes = _packed_policy_shapes(policy_input_shapes) + + packed_inputs = np.zeros(sum(packed_sizes), dtype=np.float32) + npy.update({ + key: value.reshape(shape) + for (key, shape), value in zip( + packed_shapes.items(), np.split(packed_inputs, np.cumsum(packed_sizes[:-1])), strict=True, + ) + }) + queues.update({ + "feat_q": Tensor( + np.zeros((frame_skip * (features_shape[1] - 1) + 1, features_shape[0], feature_dim), dtype=np.float32), + device=device, + ).contiguous().realize(), + "desire_q": Tensor( + np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), dtype=np.float32), + device=device, + ).contiguous().realize(), + "packed_npy_inputs": Tensor(packed_inputs, device="NPY").realize(), + }) + return queues, npy + + +def make_supercombo_input_queues(input_shapes, frame_skip, device): + queues, npy = make_warp_input_queues(input_shapes, frame_skip, device) + features_shape = input_shapes["features_buffer"] + feature_dim = math.prod(features_shape[2:]) + desire_key = _detect_desire_key(input_shapes) + desire_shape = input_shapes[desire_key] + packed_shapes, packed_sizes = _packed_policy_shapes(input_shapes, include_prev_feature=True) + + packed_inputs = np.zeros(sum(packed_sizes), dtype=np.float32) + npy.update({ + key: value.reshape(shape) + for (key, shape), value in zip( + packed_shapes.items(), np.split(packed_inputs, np.cumsum(packed_sizes[:-1])), strict=True, + ) + }) + queues.update({ + "feat_q": Tensor( + np.zeros((frame_skip * features_shape[1], features_shape[0], feature_dim), dtype=np.float32), + device=device, + ).contiguous().realize(), + "desire_q": Tensor( + np.zeros((frame_skip * desire_shape[1], desire_shape[0], desire_shape[2]), dtype=np.float32), + device=device, + ).contiguous().realize(), + "packed_npy_inputs": Tensor(packed_inputs, device="NPY").realize(), + }) + return queues, npy + + +def shift_and_sample(buffer, new_value, sample_fn): + buffer.assign(buffer[1:].cat(new_value, dim=0).contiguous()) + return sample_fn(buffer) + + +def sample_skip(buffer, frame_skip): + return buffer[::frame_skip].contiguous().flatten(0, 1).unsqueeze(0) + + +def sample_desire(buffer, frame_skip): + return buffer.reshape(-1, frame_skip, *buffer.shape[1:]).max(1).flatten(0, 1).unsqueeze(0) + + +def make_warp(nv12, model_w, model_h, frame_skip, image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + frame_prepare = make_frame_prepare(nv12, model_w, model_h) + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def warp(tfm, big_tfm, frame, big_frame): + tfm = tfm.to(WARP_DEV) + big_tfm = big_tfm.to(WARP_DEV) + Tensor.realize(tfm, big_tfm) + + return Tensor.cat( + frame_prepare(frame, tfm).unsqueeze(0), + frame_prepare(big_frame, big_tfm).unsqueeze(0), + ) + + return warp + + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + + def warp_enqueue(img_q, big_img_q, tfm, big_tfm, frame, big_frame): + tfm = tfm.to(WARP_DEV) + big_tfm = big_tfm.to(WARP_DEV) + Tensor.realize(tfm, big_tfm) + + warped = Tensor.cat( + frame_prepare(frame, tfm).unsqueeze(0), + frame_prepare(big_frame, big_tfm).unsqueeze(0), + ).to(Device.DEFAULT) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return img, big_img + + return warp_enqueue + + +def make_run_split_policy(vision_runner, policy_runners, metadata, policy_order, frame_skip, + image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + vision_metadata = metadata["vision"] + policy_metadata = metadata[policy_order[0]] + vision_features_slice = vision_metadata["output_slices"]["hidden_state"] + desire_key = _detect_desire_key(policy_metadata["input_shapes"]) + packed_shapes, packed_sizes = _packed_policy_shapes(policy_metadata["input_shapes"]) + road_key, wide_key = _detect_vision_keys(vision_metadata["input_shapes"]) + + def run_model(img, big_img, feat_q, desire_q, packed_npy_inputs): + unpacked = { + key: tensor.reshape(shape) + for (key, shape), tensor in zip( + packed_shapes.items(), packed_npy_inputs.split(packed_sizes), strict=True, + ) + } + desire_buffer = shift_and_sample( + desire_q, unpacked.pop("desire").reshape(1, 1, -1), sample_desire_fn, + ) + vision_output = next(iter(vision_runner({road_key: img, wide_key: big_img}).values())).cast("float32") + new_feature = vision_output[:, vision_features_slice].reshape(1, -1).unsqueeze(0) + features_buffer = shift_and_sample(feat_q, new_feature, sample_skip_fn) + features_buffer = features_buffer.reshape(policy_metadata["input_shapes"]["features_buffer"]) + + policy_inputs = { + "features_buffer": features_buffer, + desire_key: desire_buffer, + **unpacked, + } + policy_outputs = [ + next(iter(policy_runners[key](policy_inputs).values())).cast("float32") + for key in policy_order + ] + return (vision_output, *policy_outputs) + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT) + warped = warped.to(Device.DEFAULT) + Tensor.realize(packed_npy_inputs, warped) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + def run_policy(img, big_img, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT).realize() + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + +def make_run_supercombo(model_runner, metadata, frame_skip, image_history_pipeline=IMAGE_HISTORY_IN_POLICY): + input_shapes = metadata["model"]["input_shapes"] + sample_desire_fn = partial(sample_desire, frame_skip=frame_skip) + sample_skip_fn = partial(sample_skip, frame_skip=frame_skip) + desire_key = _detect_desire_key(input_shapes) + packed_shapes, packed_sizes = _packed_policy_shapes(input_shapes, include_prev_feature=True) + road_key, wide_key = _detect_vision_keys(input_shapes) + + def run_model(img, big_img, feat_q, desire_q, packed_npy_inputs): + unpacked = { + key: tensor.reshape(shape) + for (key, shape), tensor in zip( + packed_shapes.items(), packed_npy_inputs.split(packed_sizes), strict=True, + ) + } + desire_buffer = shift_and_sample( + desire_q, unpacked.pop("desire").reshape(1, 1, -1), sample_desire_fn, + ) + previous_feature = unpacked.pop("prev_feat") + features_buffer = shift_and_sample( + feat_q, previous_feature.reshape(1, 1, -1), sample_skip_fn, + ) + features_buffer = features_buffer.reshape(input_shapes["features_buffer"]) + model_inputs = { + road_key: img, + wide_key: big_img, + "features_buffer": features_buffer, + desire_key: desire_buffer, + **unpacked, + } + model_output = next(iter(model_runner(model_inputs).values())).cast("float32") + return model_output, + + if image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + def run_policy(warped, img_q, big_img_q, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT) + warped = warped.to(Device.DEFAULT) + Tensor.realize(packed_npy_inputs, warped) + img = shift_and_sample(img_q, warped[0:1], sample_skip_fn) + big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn) + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + def run_policy(img, big_img, feat_q, desire_q, packed_npy_inputs): + packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT).realize() + return run_model(img, big_img, feat_q, desire_q, packed_npy_inputs) + + return run_policy + + +def stateful_image_shapes(metadata): + shape = tuple(metadata['input_shapes']['new_img']) + if len(shape) != 4 or shape[:2] != (2, 6): + raise ValueError(f"Unsupported stateful image shape: {shape}") + return dict.fromkeys(('img', 'big_img'), (1, *shape[1:])) + + +def stateful_host_shapes(metadata): + return {name: shape for name, shape in metadata['input_shapes'].items() + if name != 'new_img' and name not in metadata['state_pairs']} + + +def make_stateful_input_queues(metadata, device): + queues, npy = make_warp_input_queues(stateful_image_shapes(metadata), 1, device) + shapes = stateful_host_shapes(metadata) + sizes = [math.prod(shape) for shape in shapes.values()] + packed = np.zeros(sum(sizes), dtype=np.float32) + npy.update({name: value.reshape(shape) for (name, shape), value in + zip(shapes.items(), np.split(packed, np.cumsum(sizes[:-1])), strict=True)}) + queues['packed_npy_inputs'] = Tensor(packed, device='NPY').realize() + if metadata.get('mapped_image_input', False): + image_shape = tuple(metadata['input_shapes']['new_img']) + buffer = Buffer(device, math.prod(image_shape), dtypes.uint8, + options=BufferSpec(cpu_access=True), preallocate=True) + queues['mapped_gpu_image'] = Tensor(UOp.from_buffer(buffer)).reshape(image_shape) + for name in metadata['state_pairs']: + queues[name] = Tensor(np.zeros(metadata['input_shapes'][name], dtype=metadata['input_dtypes'][name]), + device=device).contiguous().realize() + return queues, npy + + +def make_run_stateful_supercombo(model_runner, metadata): + shapes = stateful_host_shapes(metadata) + sizes = [math.prod(shape) for shape in shapes.values()] + + def run_policy(warped, packed_npy_inputs, mapped_gpu_image=None, **state): + packed = packed_npy_inputs.to(Device.DEFAULT).realize() + inputs = {name: value.reshape(shape).cast(model_runner.graph_inputs[name].dtype) + for (name, shape), value in zip(shapes.items(), packed.split(sizes), strict=True)} + if metadata.get('mapped_image_input', False): + if mapped_gpu_image is None: + raise ValueError('Mapped stateful image queue is missing') + image = warped.to('CPU').realize().to(Device.DEFAULT) + image = mapped_gpu_image.assign(image).realize() + else: + image = warped.to(Device.DEFAULT) + inputs['new_img'] = image.cast(model_runner.graph_inputs['new_img'].dtype) + outputs = {name: value.contiguous() for name, value in model_runner(inputs | state).items()} + for name, next_name in metadata['state_pairs'].items(): + if outputs[next_name].dtype != state[name].dtype: + raise ValueError(f'State dtype mismatch: {name} -> {next_name}') + Tensor.realize(*outputs.values()) + Tensor.realize(*(state[name].assign(outputs[next_name]) for name, next_name in metadata['state_pairs'].items())) + return outputs['outputs'].cast('float32'), + + return run_policy + + +def compile_jit(jit, make_random_inputs, input_keys, make_queues, validation_runs=1): + seed = 42 + + def random_inputs_run(fn, current_seed, test_values=None, test_buffers=None, expect_match=True): + input_queues, npy = make_queues(Device.DEFAULT) + rng = np.random.default_rng(current_seed) + Tensor.manual_seed(current_seed) + testing = test_values is not None or test_buffers is not None + run_count = validation_runs if testing else max(3, validation_runs) + values, buffers = [], [] + + for index in range(run_count): + for value in npy.values(): + value[:] = rng.standard_normal(value.shape).astype(value.dtype) + Device.default.synchronize() + random_inputs = make_random_inputs(rng=rng) + start = time.perf_counter() + outputs = fn(**{key: input_queues[key] for key in input_keys}, **random_inputs) + mid = time.perf_counter() + Device.default.synchronize() + end = time.perf_counter() + print(f" [{index + 1}/{run_count}] enqueue {(mid - start) * 1e3:6.2f} ms -- total {(end - start) * 1e3:6.2f} ms") + + if index < validation_runs: + values.extend(np.copy(value.numpy()) for value in outputs) + buffers.extend(np.copy(value.numpy()) for value in input_queues.values()) + if not all(np.isfinite(value).all() for value in values): + raise ValueError("Compiled JIT produced non-finite outputs") + + if test_values is not None: + match = all(np.array_equal(lhs, rhs) for lhs, rhs in zip(values, test_values, strict=True)) + assert match == expect_match, f"outputs {'differ from' if expect_match else 'match'} baseline (seed={current_seed})" + if test_buffers is not None: + match = all(np.array_equal(lhs, rhs) for lhs, rhs in zip(buffers, test_buffers, strict=True)) + assert match == expect_match, f"buffers {'differ from' if expect_match else 'match'} baseline (seed={current_seed})" + return values, buffers + + print("capture + replay") + test_values, test_buffers = random_inputs_run(jit, seed) + print("pickle round trip") + with tempfile.TemporaryDirectory(prefix="starpilot-jit-") as temporary: + path = Path(temporary) / "captured.pkl" + dump_pickle(jit, path) + loaded = load_pickle(path, out_of_band=True) + random_inputs_run(loaded, seed, test_values, test_buffers, expect_match=True) + random_inputs_run(loaded, seed + 1, test_values, test_buffers, expect_match=False) + return jit + + +def _parse_size(value): + width, height = value.lower().split("x") + return int(width), int(height) + + +def validate_metadata(metadata): + output_shapes = metadata.get("output_shapes", {}) + output_shape = output_shapes.get("outputs") + if not output_shape or len(output_shape) < 2: + raise ValueError(f"Invalid model output shape metadata: {output_shapes}") + output_size = output_shape[-1] + for name, output_slice in metadata.get("output_slices", {}).items(): + start, stop, step = output_slice.indices(output_size) + if step != 1 or start < 0 or stop < start or stop > output_size: + raise ValueError(f"Invalid output slice {name}={output_slice} for output size {output_size}") + + +def main(): + from tinygrad.nn.onnx import OnnxRunner + + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + parser = argparse.ArgumentParser() + parser.add_argument("--model-type", choices=MODEL_TYPES, required=True) + parser.add_argument("--model-size", type=_parse_size, required=True) + parser.add_argument("--camera-resolutions", type=_parse_size, nargs="+", required=True) + parser.add_argument("--frame-skip", type=int) + parser.add_argument("--behavior-version") + parser.add_argument("--output", required=True) + parser.add_argument("--out-of-band", action="store_true", help="Compatibility flag; all artifacts use the current buffer arena format.") + parser.add_argument("--vision-onnx") + parser.add_argument("--policy-onnx") + parser.add_argument("--off-policy-onnx") + parser.add_argument("--on-policy-onnx") + parser.add_argument("--supercombo-onnx") + parser.add_argument( + "--image-history-pipeline", + choices=IMAGE_HISTORY_PIPELINES, + default=IMAGE_HISTORY_IN_POLICY, + help="Where img/big_img history queues are updated. 'policy' is the newer faster ABI; 'warp' reproduces legacy v22 artifacts.", + ) + args = parser.parse_args() + if args.frame_skip is not None and args.frame_skip < 1: + parser.error("--frame-skip must be positive") + + output = { + "format_version": ARTIFACT_FORMAT_VERSION, + "artifact_abi": ARTIFACT_ABI, + "compiler_revision": COMPILER_REVISION, + "execution_device": Device.DEFAULT, + "warp_device": WARP_DEV or Device.DEFAULT, + "model_type": args.model_type, + "metadata": {}, + "image_history_pipeline": args.image_history_pipeline, + } + if args.behavior_version: + output["behavior_version"] = args.behavior_version + + if args.model_type == "supercombo": + if not args.supercombo_onnx: + parser.error("--supercombo-onnx is required for supercombo") + model_path = Path(args.supercombo_onnx) + model_runner = OnnxRunner(model_path) + output["metadata"]["model"] = make_metadata_dict(model_path, model_runner) + validate_metadata(output["metadata"]["model"]) + policy_shapes = output["metadata"]["model"]["input_shapes"] + if 'new_img' in policy_shapes: + if args.image_history_pipeline != IMAGE_HISTORY_IN_POLICY: + parser.error('ONNX-managed history requires --image-history-pipeline policy') + metadata = output['metadata']['model'] + metadata['state_pairs'] = {name: f'next_{name}' for name in policy_shapes + if f'next_{name}' in metadata['output_shapes']} + if not metadata['state_pairs']: + raise ValueError('Stateful supercombo is missing next-state outputs') + metadata['input_dtypes'] = {name: np.dtype(_to_np_dtype(spec.dtype)).name for name, spec in model_runner.graph_inputs.items()} + for name, next_name in metadata['state_pairs'].items(): + if policy_shapes[name] != metadata['output_shapes'][next_name]: + raise ValueError(f'State shape mismatch: {name} -> {next_name}') + metadata['mapped_image_input'] = True + frame_skip = 1 + make_policy_queues = partial(make_stateful_input_queues, metadata) + run_policy = make_run_stateful_supercombo(model_runner, metadata) + image_shapes = stateful_image_shapes(metadata) + policy_input_keys = ('packed_npy_inputs', 'mapped_gpu_image', *metadata['state_pairs']) + else: + frame_skip = args.frame_skip or derive_frame_skip(policy_shapes) + make_policy_queues = partial(make_supercombo_input_queues, policy_shapes, frame_skip) + run_policy = make_run_supercombo( + model_runner, output["metadata"], frame_skip, args.image_history_pipeline, + ) + image_shapes = policy_shapes + policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SUPERCOMBO_POLICY_INPUTS + else: + if not args.vision_onnx: + parser.error("--vision-onnx is required for split models") + policy_paths = {} + if args.policy_onnx: + policy_paths["policy"] = args.policy_onnx + if args.off_policy_onnx: + policy_paths["off_policy"] = args.off_policy_onnx + if args.on_policy_onnx: + policy_paths["on_policy"] = args.on_policy_onnx + if args.model_type == "vision_policy" and set(policy_paths) != {"policy"}: + parser.error("vision_policy requires --policy-onnx") + if args.model_type == "vision_multi_policy" and not policy_paths: + parser.error("vision_multi_policy requires at least one policy ONNX") + + vision_path = Path(args.vision_onnx) + resolved_policy_paths = {key: Path(path) for key, path in policy_paths.items()} + vision_runner = OnnxRunner(vision_path) + policy_runners = {key: OnnxRunner(path) for key, path in resolved_policy_paths.items()} + output["metadata"]["vision"] = make_metadata_dict(vision_path, vision_runner) + validate_metadata(output["metadata"]["vision"]) + for key, path in resolved_policy_paths.items(): + output["metadata"][key] = make_metadata_dict(path, policy_runners[key]) + validate_metadata(output["metadata"][key]) + + policy_order = [key for key in ("on_policy", "off_policy", "policy") if key in policy_runners] + output["policy_order"] = policy_order + first_policy_shapes = output["metadata"][policy_order[0]]["input_shapes"] + for key in policy_order[1:]: + if output["metadata"][key]["input_shapes"] != first_policy_shapes: + raise ValueError(f"Policy input shapes differ between {policy_order[0]} and {key}") + frame_skip = args.frame_skip or derive_frame_skip(first_policy_shapes) + make_policy_queues = partial( + make_split_input_queues, + output["metadata"]["vision"]["input_shapes"], + first_policy_shapes, + frame_skip, + ) + run_policy = make_run_split_policy( + vision_runner, policy_runners, output["metadata"], policy_order, frame_skip, + args.image_history_pipeline, + ) + image_shapes = output["metadata"]["vision"]["input_shapes"] + policy_input_keys = FAST_POLICY_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else SPLIT_POLICY_INPUTS + + output["frame_skip"] = frame_skip + output["policy_input_keys"] = policy_input_keys + warp_input_keys = FAST_WARP_INPUTS if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY else LEGACY_WARP_INPUTS + output["warp_input_keys"] = warp_input_keys + run_policy_jit = TinyJit(run_policy, prune=True) + road_key, wide_key = _detect_vision_keys(image_shapes) + if args.image_history_pipeline == IMAGE_HISTORY_IN_POLICY: + make_random_model_inputs = partial( + make_random_images, + keys=["warped"], + shape=(2, 6, *image_shapes[road_key][2:]), + device=WARP_DEV, + ) + else: + make_random_model_inputs = partial( + make_random_images, + keys=[road_key, wide_key], + shape=image_shapes[road_key], + ) + output["run_policy"] = compile_jit( + run_policy_jit, make_random_model_inputs, policy_input_keys, make_policy_queues, + validation_runs=5 if output['metadata'].get('model', {}).get('state_pairs') else 1, + ) + + model_w, model_h = args.model_size + for cam_w, cam_h in args.camera_resolutions: + nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h)) + warp_enqueue = TinyJit( + make_warp(nv12, model_w, model_h, frame_skip, args.image_history_pipeline), + prune=True, + ) + make_random_warp_inputs = make_random_blob_images( + keys=["frame", "big_frame"], size=nv12.size, device=WARP_DEV, + ) + make_warp_queues = partial(make_warp_input_queues, image_shapes, frame_skip) + output[(cam_w, cam_h)] = compile_jit( + warp_enqueue, make_random_warp_inputs, warp_input_keys, make_warp_queues, + ) + + dump_pickle(output, args.output) + print(f"Saved JITs to {args.output} ({os.path.getsize(args.output) / 1e6:.2f} MB)") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/openpilot/starpilot/models/jetlink_warp.py b/openpilot/starpilot/models/jetlink_warp.py new file mode 100644 index 0000000000..1966fdaa99 --- /dev/null +++ b/openpilot/starpilot/models/jetlink_warp.py @@ -0,0 +1,24 @@ +"""Current-frame NV12 warp supplied to Jetlink's build API.""" + +CAMERA_SIZES = ((1928, 1208), (1344, 760)) +MODEL_SIZES = ((512, 256), (1024, 512)) + + +def make_warp(cam_w, cam_h, model_w, model_h): + from examples.openpilot.compile_warp import NV12Frame, make_frame_prepare + from tinygrad import Tensor + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + if (cam_w, cam_h) not in CAMERA_SIZES or (model_w, model_h) not in MODEL_SIZES: + raise ValueError("unsupported Jetlink warp geometry") + layout = get_nv12_info(cam_w, cam_h) + prepare = make_frame_prepare(NV12Frame(cam_w, cam_h, *layout), model_w, model_h) + + def warp(tfm, big_tfm, frame, big_frame): + return Tensor.stack(prepare(frame, tfm), prepare(big_frame, big_tfm)) + return warp, layout[3] + + +def adapter(): + from types import SimpleNamespace + return SimpleNamespace(make_warp=make_warp) diff --git a/openpilot/starpilot/models/laboratory.py b/openpilot/starpilot/models/laboratory.py new file mode 100644 index 0000000000..cdc3e286ec --- /dev/null +++ b/openpilot/starpilot/models/laboratory.py @@ -0,0 +1,134 @@ +from pathlib import Path + +from openpilot.starpilot.models.catalog import model_entries, BUNDLED_CURRENT, GENERATION +from openpilot.starpilot.models.manager import ( + ModelError, ModelManager, artifact_entry, artifact_path, atomic_json, catalog, read_json, state_lock, verified_artifact, +) + + +def configuration(root: Path) -> dict: + default = {"enabled": False, "lateralModel": "", "longitudinalModel": ""} + try: + saved = read_json(root / "laboratory.json", 4096) + validate_configuration(saved, root) + return saved + except (OSError, ValueError, TypeError): + return default + + +def validate_configuration(value: dict, root: Path | None = None) -> None: + entries = model_entries(root) + if (not isinstance(value, dict) or set(value) != {"enabled", "lateralModel", "longitudinalModel"} or + type(value["enabled"]) is not bool): + raise ModelError("Invalid laboratory configuration") + for key in ("lateralModel", "longitudinalModel"): + mid = value[key] + if not isinstance(mid, str) or (mid and (mid not in entries or mid == BUNDLED_CURRENT or entries[mid].uses_external_gpu)): + raise ModelError("Choose small models for the laboratory pair") + if value["enabled"] and (not value["lateralModel"] or not value["longitudinalModel"] or + value["lateralModel"] == value["longitudinalModel"]): + raise ModelError("Choose two different small models") + + +class ModelLaboratory: + def __init__(self, manager: ModelManager, *, runtime_supported: bool = False): + self.manager = manager + self.runtime_supported = runtime_supported + + @property + def runtime_unavailable_reason(self) -> str: + return "" if self.runtime_supported else "Pair inference is not available in this build yet." + + def snapshot(self) -> dict: + owner = self.manager + with owner.lock: + entries = catalog(owner.root) + config = configuration(owner.root) + parked, gpu = owner.parked(), owner.gpu_present() + job = owner._job_status() + models = [] + for mid, row in entries.items(): + small = not row.get("uses_external_gpu", False) + eligible = small and row.get("model_lab_eligible") is True + artifact = artifact_entry(mid, entries, "amd") if eligible else {} + published = "artifact_sha256" in artifact + installed, checking = owner._snapshot_installed(mid, row, "amd") if published else (False, False) + if not small: + state, reason = "unsupported", "Big models cannot be combined in this build." + elif not eligible: + state, reason = "unsupported", "This model is not available for combining." + elif not published: + state, reason = "unpublished", "A Chestnut version for combining has not been published for this model." + elif checking: + state, reason = "checking", "Checking the downloaded Chestnut version." + elif not installed: + state, reason = "missing", "Download the Chestnut version to prepare this model for combining." + elif not self.runtime_supported: + state, reason = "runtime-unavailable", "Download verified. Running two models together is unavailable in this build." + elif not gpu: + state, reason = "gpu-unavailable", "Download verified. Connect a ready Chestnut to use it." + else: + state, reason = "installed", "Download verified. Choose another compatible model to form a pair." + models.append({"value": mid, "label": row["name"], "version": row["version"], "series": row.get("series", ""), + "small": small, "modelSize": "small" if small else "big", "modelLabEligible": eligible, + "modelLabArtifactAvailable": published, "modelLabArtifactInstalled": installed, "checking": checking, + "modelLabStatus": state, "modelLabReason": reason, + "artifactSize": artifact.get("artifact_size", 0)}) + return {"schemaVersion": 1, "chestnutReady": gpu, "isOnroad": not parked, "configuration": config, + "configurationError": self.runtime_unavailable_reason if config["enabled"] else "", + "runtimeSupported": self.runtime_supported, "runtimeUnavailableReason": self.runtime_unavailable_reason, + "runtime": {"requested": config["enabled"], "active": False, "lateralModel": "", "longitudinalModel": "", + "error": "", "health": "unavailable"}, + "download": {"model": job.get("model", ""), "progress": job.get("progress", ""), + "downloading": job["downloading"], "jobId": job.get("jobId", ""), + "variant": job.get("variant", "standard")}, + "summary": {"catalog": len(models), "eligible": sum(row["modelLabEligible"] is True for row in models), + "ready": sum(row["modelLabArtifactInstalled"] is True for row in models), + "published": sum(row["modelLabArtifactAvailable"] is True for row in models), + "declaredSize": sum(int(row["artifactSize"]) for row in models)}, + "models": models, "manifest": {"version": GENERATION}, + "capabilities": {"configure": parked, "download": parked, + "delete": parked, "cancel": True, "refresh": parked}} + + def action(self, action: str, payload: dict) -> dict: + owner = self.manager + if not isinstance(payload, dict): + raise ModelError("Invalid laboratory request") + with owner.lock: + owner._require_parked() + if action == "download": + if set(payload) != {"model"} or not isinstance(payload["model"], str): + raise ModelError("Unknown laboratory model") + return owner.action("download", {"model": payload["model"], "variant": "amd", "allowGpuWithoutGpu": True}) + if action not in ("configure", "delete"): + raise ModelError("Unknown laboratory action") + with state_lock(owner.root): + owner._require_parked() + if owner._job_busy(): + raise ModelError("Wait for the model download to finish") + if action == "configure": + validate_configuration(payload, owner.root) + if payload["enabled"]: + if not self.runtime_supported: + raise ModelError(self.runtime_unavailable_reason) + if not owner.gpu_present(): + raise ModelError("Connect a firmware-ready Chestnut to enable this pair") + entries = catalog(owner.root) + for role in ("lateralModel", "longitudinalModel"): + if verified_artifact(payload[role], entries, owner.root, "amd") is None: + raise ModelError("Download and verify both eGPU variants first") + owner._require_parked() + atomic_json(owner.root / "laboratory.json", payload) + message = "Model Laboratory configuration saved" + else: + if set(payload) != {"model"} or not isinstance(payload["model"], str): + raise ModelError("Unknown laboratory model") + mid = payload["model"] + destination = artifact_path(mid, owner.root, "amd") + config = configuration(owner.root) + if config["enabled"] and mid in (config["lateralModel"], config["longitudinalModel"]): + raise ModelError("Disable the laboratory pair before deleting either eGPU variant") + destination.unlink(missing_ok=True) + owner.checked.pop(f"{mid}:amd", None) + message = "eGPU variant deleted" + return {**self.snapshot(), "message": message} diff --git a/openpilot/starpilot/models/laboratory_outputs.py b/openpilot/starpilot/models/laboratory_outputs.py new file mode 100644 index 0000000000..02e5c0badb --- /dev/null +++ b/openpilot/starpilot/models/laboratory_outputs.py @@ -0,0 +1,60 @@ +from typing import Any + +import numpy as np + + +LATERAL_PLAN_COLUMNS = (1, 4, 7, 11, 14) +LATERAL_OUTPUT_KEYS = ( + "desired_curvature", "desired_curvature_stds", "lat_planner_solution", "lat_planner_solution_stds", + "lane_lines", "lane_lines_stds", "lane_lines_prob", "road_edges", "road_edges_stds", "desire_state", "desire_pred", +) +CURRENT_FRAME_OUTPUT_KEYS = ( + "pose", "pose_stds", "wide_from_device_euler", "wide_from_device_euler_stds", "road_transform", "road_transform_stds", +) + + +def _validate_outputs(outputs: dict[str, np.ndarray]) -> None: + for key, value in outputs.items(): + if not isinstance(value, np.ndarray) or not np.isfinite(value).all(): + raise ValueError(f"Model Laboratory requires finite normalized arrays: {key}") + + +def _merge_plan(lateral: np.ndarray, longitudinal: np.ndarray) -> np.ndarray: + if lateral.shape != longitudinal.shape or lateral.ndim < 2 or lateral.shape[-1] < 15: + raise ValueError(f"Model Laboratory incompatible plan shapes: {lateral.shape}, {longitudinal.shape}") + merged = longitudinal.copy() + merged[..., LATERAL_PLAN_COLUMNS] = lateral[..., LATERAL_PLAN_COLUMNS] + return merged + + +def compose_model_outputs(lateral_output: dict[str, np.ndarray], longitudinal_output: dict[str, np.ndarray], + current_frame_output: dict[str, np.ndarray] | None = None) -> dict[str, np.ndarray]: + """Own lateral plan/geometry and longitudinal extras; decode actions separately per generation.""" + frame_output = longitudinal_output if current_frame_output is None else current_frame_output + for outputs in (lateral_output, longitudinal_output, frame_output): + _validate_outputs(outputs) + if "plan" not in lateral_output or "plan" not in longitudinal_output: + raise ValueError("Model Laboratory requires a plan from both models") + if ("plan_stds" in lateral_output) != ("plan_stds" in longitudinal_output): + raise ValueError("Model Laboratory requires matching plan uncertainty outputs") + + composed = {key: value.copy() for key, value in longitudinal_output.items() if key not in ("action", "action_stds")} + composed["plan"] = _merge_plan(lateral_output["plan"], longitudinal_output["plan"]) + if "plan_stds" in lateral_output: + composed["plan_stds"] = _merge_plan(lateral_output["plan_stds"], longitudinal_output["plan_stds"]) + for keys, outputs in ((LATERAL_OUTPUT_KEYS, lateral_output), (CURRENT_FRAME_OUTPUT_KEYS, frame_output)): + for key in keys: + if key in outputs: + composed[key] = outputs[key].copy() + else: + composed.pop(key, None) + return composed + + +def hybrid_action_values(lateral_action: Any, longitudinal_action: Any) -> dict[str, Any]: + """Accept already decoded actions, never raw generation-dependent action tensors.""" + curvature = float(lateral_action.desiredCurvature) + acceleration = float(longitudinal_action.desiredAcceleration) + if not np.isfinite((curvature, acceleration)).all(): + raise ValueError("Model Laboratory requires finite decoded actions") + return {"desiredCurvature": curvature, "desiredAcceleration": acceleration, "shouldStop": bool(longitudinal_action.shouldStop)} diff --git a/openpilot/starpilot/models/laboratory_runner.py b/openpilot/starpilot/models/laboratory_runner.py new file mode 100644 index 0000000000..fb407bfbb8 --- /dev/null +++ b/openpilot/starpilot/models/laboratory_runner.py @@ -0,0 +1,81 @@ +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +import numpy as np + +from openpilot.cereal import log +from openpilot.starpilot.models.laboratory import validate_configuration +from openpilot.starpilot.models.laboratory_outputs import compose_model_outputs, hybrid_action_values +from openpilot.starpilot.models.manager import ROOT, artifact_entry, artifact_path, catalog +from openpilot.starpilot.models.receipt import PreparedArtifact +from openpilot.starpilot.models.runner import CatalogModelState, action_from_outputs, load_verified_model + + +@dataclass(frozen=True) +class LaboratoryRole: + model_id: str + behavior_version: str + artifact: PreparedArtifact + + +@dataclass(frozen=True) +class LaboratoryFrame: + outputs: dict[str, np.ndarray] + action: log.ModelDataV2.Action + + +class LaboratoryPair: + def __init__(self, lateral: CatalogModelState, longitudinal: CatalogModelState, + lateral_artifact: PreparedArtifact, longitudinal_artifact: PreparedArtifact): + if lateral is longitudinal or lateral.model_id == longitudinal.model_id: + raise ValueError("Model Laboratory requires two distinct models") + if not lateral.chestnut or not longitudinal.chestnut: + raise ValueError("Model Laboratory requires two AMD variants") + if lateral.camera_warp_descriptor is None or lateral.camera_warp_descriptor != longitudinal.camera_warp_descriptor: + raise ValueError("Model Laboratory requires compatible policy-history camera warps") + self.lateral = lateral + self.longitudinal = longitudinal + self.lateral_role = LaboratoryRole(lateral.model_id, lateral.behavior_version, lateral_artifact) + self.longitudinal_role = LaboratoryRole(longitudinal.model_id, longitudinal.behavior_version, longitudinal_artifact) + self.failed = False + + def run_frame(self, bufs: dict, transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], + previous: log.ModelDataV2.Action, lat_action_t: float, long_action_t: float, v_ego: float, + lat_smooth_seconds: float = 0.0, long_smooth_seconds: float = 0.3, + after_enqueue: Callable[[], None] | None = None) -> LaboratoryFrame: + """Run both roles before releasing the camera frame; any failure requires fallback.""" + if self.failed: + raise RuntimeError("Model Laboratory pair failed; load a fallback before continuing") + try: + lateral_output = self.lateral.run(bufs, transforms, inputs) + shared_warp = self.lateral.last_warp + if shared_warp is None: + raise ValueError("Model Laboratory camera warp unavailable") + longitudinal_output = self.longitudinal.run(bufs, transforms, inputs, after_enqueue, shared_warp=shared_warp) + actions = [action_from_outputs(output, model.behavior_version, previous, lat_action_t, long_action_t, v_ego, + lat_smooth_seconds, long_smooth_seconds) + for model, output in ((self.lateral, lateral_output), (self.longitudinal, longitudinal_output))] + action = log.ModelDataV2.Action(**hybrid_action_values(*actions)) + return LaboratoryFrame(compose_model_outputs(lateral_output, longitudinal_output), action) + except Exception: + self.failed = True + self.lateral.last_warp = self.longitudinal.last_warp = None + raise + + +def load_laboratory_pair(cam_w: int, cam_h: int, config: dict, *, root: Path = ROOT) -> LaboratoryPair: + validate_configuration(config) + if not config["enabled"]: + raise ValueError("Model Laboratory is disabled") + entries = catalog(root) + requests = [] + for role in ("lateralModel", "longitudinalModel"): + mid = config[role] + artifact = artifact_entry(mid, entries, "amd") + if "artifact_sha256" not in artifact: + raise ValueError("Model Laboratory requires published AMD variants for both models") + requests.append((mid, entries[mid]["version"], artifact["artifact_sha256"])) + loaded = [load_verified_model(cam_w, cam_h, artifact_path(mid, root, "amd"), version, True, mid, digest, root=root) + for mid, version, digest in requests] + return LaboratoryPair(loaded[0][0], loaded[1][0], loaded[0][1], loaded[1][1]) diff --git a/openpilot/starpilot/models/manager.py b/openpilot/starpilot/models/manager.py new file mode 100644 index 0000000000..cb4c92ec92 --- /dev/null +++ b/openpilot/starpilot/models/manager.py @@ -0,0 +1,642 @@ +"""Versioned model downloads and next-start Small/Chestnut selections.""" + +from __future__ import annotations + +from contextlib import contextmanager +from collections import deque +from dataclasses import dataclass +import fcntl +import hashlib +import json +import os +from pathlib import Path +import re +import random +import shutil +import tempfile +import threading +import uuid +from collections.abc import Callable +from urllib.request import urlopen + +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, BUNDLED_CURRENT, BY_ID, CATALOG_PATH, COMPILER_REVISION, GENERATION + +RESOURCE_URL = "https://huggingface.co/buckets/StarPilot-Driving/StarPilot-Resources/resolve" +ROOT = Path("/data/models") / GENERATION +MAX_MANIFEST = 2 * 1024 * 1024 +MAX_ARTIFACT = 2 * 1024 * 1024 * 1024 +SHA = re.compile(r"[0-9a-f]{64}\Z") +MODEL_RUNNER_REVISION = 2 +RUNTIME_ARTIFACT_KEYS = {"min_runner_revision", "artifact_format", "artifact_sha256", "artifact_size", + "artifact_chunk_count", "conversion_provenance", "validation"} + + +class ModelError(ValueError): + pass + + +def read_json(path: Path, limit: int = MAX_MANIFEST) -> dict: + with path.open("rb") as stream: + raw = stream.read(limit + 1) + if len(raw) > limit: + raise ModelError("Model document is too large") + data = json.loads(raw) + if not isinstance(data, dict): + raise ModelError("Invalid model document") + return data + + +def atomic_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + fd, name = tempfile.mkstemp(prefix=".model-", dir=path.parent) + try: + with os.fdopen(fd, "w") as out: + json.dump(value, out, ensure_ascii=False, allow_nan=False, sort_keys=True) + out.flush() + os.fsync(out.fileno()) + os.replace(name, path) + finally: + Path(name).unlink(missing_ok=True) + + +@contextmanager +def state_lock(root: Path): + root.mkdir(parents=True, exist_ok=True) + with (root / ".state.lock").open("a") as lock: + fcntl.flock(lock, fcntl.LOCK_EX) + yield + + +def validate_manifest(data: dict) -> dict[str, dict]: + if (data.get("generation") != GENERATION or data.get("compiler_revision") != COMPILER_REVISION or + data.get("artifact_abi") != ARTIFACT_ABI or not isinstance(data.get("models"), list)): + raise ModelError("The model catalog targets a different runtime") + rows = {} + for entry in data["models"]: + if not isinstance(entry, dict): + raise ModelError("Invalid model entry") + model_id = entry.get("id") + if not isinstance(model_id, str) or model_id not in BY_ID or model_id == BUNDLED_CURRENT or model_id in rows: + raise ModelError("Unknown or duplicate model") + known = BY_ID[model_id] + if entry.get("version") != known.version or bool(entry.get("uses_external_gpu", False)) != known.uses_external_gpu: + raise ModelError("Model behavior or hardware changed") + variants = entry.get("accelerator_artifacts", {}) + if not isinstance(variants, dict) or set(variants) - {"amd"}: + raise ModelError("Unknown model accelerator") + amd = variants.get("amd") + if amd is not None and (not isinstance(amd, dict) or known.uses_external_gpu or + entry.get("model_lab_eligible") is not True or amd.get("execution_device") != "AMD" or + amd.get("compiler_revision") != COMPILER_REVISION): + raise ModelError("Invalid laboratory artifact hardware or runtime") + runtime_artifacts = entry.get("runtime_artifacts", []) + if not isinstance(runtime_artifacts, list): + raise ModelError("Invalid runtime artifacts") + revisions = set() + for artifact in runtime_artifacts: + if (not isinstance(artifact, dict) or set(artifact) - RUNTIME_ARTIFACT_KEYS or + set(artifact) & {"artifact_format", "artifact_sha256", "artifact_size", "artifact_chunk_count"} != + {"artifact_format", "artifact_sha256", "artifact_size", "artifact_chunk_count"} or + type(artifact.get("min_runner_revision")) is not int or artifact["min_runner_revision"] < 2 or + artifact["min_runner_revision"] in revisions): + raise ModelError("Invalid runtime artifact variant") + revisions.add(artifact["min_runner_revision"]) + for artifact in (entry, *variants.values(), *runtime_artifacts): + if "artifact_sha256" not in artifact: + continue + size, chunks = artifact.get("artifact_size"), artifact.get("artifact_chunk_count") + if (not isinstance(artifact["artifact_sha256"], str) or not SHA.fullmatch(artifact["artifact_sha256"]) or + type(size) is not int or not 0 < size <= MAX_ARTIFACT or type(chunks) is not int or not 0 <= chunks <= 99 or + artifact.get("artifact_format") != ARTIFACT_ABI): + raise ModelError("Invalid compiled artifact metadata") + row = dict(entry) + compatible = [artifact for artifact in runtime_artifacts if artifact["min_runner_revision"] <= MODEL_RUNNER_REVISION] + if compatible: + selected = max(compatible, key=lambda artifact: artifact["min_runner_revision"]) + row.update({key: value for key, value in selected.items() if key != "min_runner_revision"}) + rows[model_id] = row + return rows + + +def catalog(root: Path = ROOT) -> dict[str, dict]: + base = validate_manifest(read_json(CATALOG_PATH)) + try: + base.update(validate_manifest(read_json(root / "catalog.json"))) + except (OSError, ValueError, TypeError): + pass + return base + + +def preferences(root: Path = ROOT) -> dict: + default = {"small": BUNDLED_CURRENT, "big": BUNDLED_CURRENT, "userFavorites": [], "sortMode": "name", + "randomizer": False, "blacklistedModels": []} + try: + saved = read_json(root / "preferences.json", 16384) + for profile in ("small", "big"): + mid = saved.get(profile) + if isinstance(mid, str) and (mid == BUNDLED_CURRENT or (profile == "big" and mid == "") or ( + mid in BY_ID and BY_ID[mid].uses_external_gpu == (profile == "big"))): + default[profile] = mid + for key in ("userFavorites", "blacklistedModels"): + if isinstance(saved.get(key), list): + default[key] = list(dict.fromkeys(x for x in saved[key] if isinstance(x, str) and x in BY_ID)) + if type(saved.get("randomizer")) is bool: + default["randomizer"] = saved["randomizer"] + if saved.get("sortMode") in ("name", "date", "date_oldest", "community", "series", "favorites", "released"): + default["sortMode"] = saved["sortMode"] + except (OSError, ValueError, TypeError): + pass + return default + + +def artifact_path(model_id: str, root: Path = ROOT, variant: str = "standard") -> Path: + if not isinstance(model_id, str) or model_id not in BY_ID or model_id == BUNDLED_CURRENT: + raise ModelError("Unknown downloadable model") + if variant not in ("standard", "amd") or (variant == "amd" and BY_ID[model_id].uses_external_gpu): + raise ModelError("Unknown model artifact variant") + suffix = "amd_" if variant == "amd" else "" + return root / model_id / f"{model_id}_driving_{suffix}tinygrad.pkl" + + +def artifact_entry(model_id: str, entries: dict[str, dict], variant: str = "standard") -> dict: + row = entries.get(model_id, {}) + if variant == "standard": + return row + if variant == "amd" and not row.get("uses_external_gpu") and row.get("model_lab_eligible") is True: + return row.get("accelerator_artifacts", {}).get("amd", {}) + return {} + + +def verified_artifact(model_id: str, entries: dict[str, dict], root: Path = ROOT, variant: str = "standard") -> Path | None: + row = artifact_entry(model_id, entries, variant) + if "artifact_sha256" not in row: + return None + path = artifact_path(model_id, root, variant) + try: + if path.is_symlink() or not path.is_file() or path.stat().st_size != row["artifact_size"]: + return None + with path.open("rb") as stream: + digest = hashlib.file_digest(stream, "sha256").hexdigest() + return path if digest == row["artifact_sha256"] else None + except OSError: + return None + + +@dataclass(frozen=True) +class RuntimeSelection: + small_id: str = BUNDLED_CURRENT + small_path: Path | None = None + small_version: str = "current" + big_id: str = BUNDLED_CURRENT + big_path: Path | None = None + big_version: str = "current" + allow_big: bool = True + small_sha256: str | None = None + big_sha256: str | None = None + + +def requested_runtime_id(chestnut_available: bool, *, root: Path = ROOT) -> str: + prefs = preferences(root) + return prefs["big"] if chestnut_available and prefs["big"] else prefs["small"] + + +def randomize_next_start(chestnut_available: bool, *, root: Path = ROOT, chooser=random.choice) -> dict: + with state_lock(root): + prefs = preferences(root) + if not prefs["randomizer"]: + return prefs + profile = "big" if chestnut_available and prefs["big"] else "small" + entries = catalog(root) + blocked = set(prefs["blacklistedModels"]) + choices = [mid for mid in entries if mid not in blocked and + BY_ID[mid].uses_external_gpu == (profile == "big") and verified_artifact(mid, entries, root) is not None] + if profile == "small" and BUNDLED_CURRENT not in blocked: + choices.append(BUNDLED_CURRENT) + alternatives = [mid for mid in choices if mid != prefs[profile]] + prefs[profile] = chooser(alternatives or choices) if choices else ("" if profile == "big" else BUNDLED_CURRENT) + atomic_json(root / "preferences.json", prefs) + return prefs + + +def resolve_runtime(chestnut_available: bool, *, root: Path = ROOT, randomize: bool = False) -> RuntimeSelection: + prefs = randomize_next_start(chestnut_available, root=root) if randomize else preferences(root) + entries = catalog(root) + small, big = prefs["small"], prefs["big"] + small_path = verified_artifact(small, entries, root) if small != BUNDLED_CURRENT else None + if small_path is None: + small = BUNDLED_CURRENT + big_path = verified_artifact(big, entries, root) if big not in ("", BUNDLED_CURRENT) and chestnut_available else None + allow_big = chestnut_available and (big == BUNDLED_CURRENT or big_path is not None) + return RuntimeSelection(small, small_path, BY_ID[small].version, big or BUNDLED_CURRENT, big_path, + BY_ID[big].version if big in BY_ID else "current", allow_big, + entries[small]["artifact_sha256"] if small_path is not None else None, + entries[big]["artifact_sha256"] if big_path is not None else None) + + +class ModelManager: + def __init__(self, *, root: Path = ROOT, parked: Callable[[], bool] | None = None, + gpu_present: Callable[[], bool] | None = None, opener=urlopen): + self.root, self.opener = root, opener + self.parked = parked or (lambda: False) + if gpu_present is None: + from openpilot.selfdrive.modeld.helpers import chestnut_present + gpu_present = chestnut_present + self.gpu_present = gpu_present + self.lock = threading.RLock() + self.cancelled = threading.Event() + self.worker: threading.Thread | None = None + self.progress = "" + self.downloading = "" + self.download_all = False + self.job_id = "" + self.checked: dict[str, tuple[tuple, bool]] = {} + self.verify_queue: deque[tuple[str, dict, tuple, str]] = deque() + self.verifying: dict[str, tuple] = {} + self.verify_worker: threading.Thread | None = None + self.verify_closed = False + + def _read_job(self) -> dict: + try: + job = read_json(self.root / ".download-job.json", 16384) + if (job.get("schemaVersion") == 1 and isinstance(job.get("jobId"), str) and + re.fullmatch(r"[0-9a-f]{32}", job["jobId"]) and job.get("state") in ("running", "completed", "failed", "interrupted") and + job.get("model") in ("", *BY_ID) and isinstance(job.get("progress"), str) and + type(job.get("downloadAll")) is bool and type(job.get("cancelRequested")) is bool and + job.get("variant", "standard") in ("standard", "amd")): + return job + except (OSError, ValueError, TypeError): + pass + return {} + + def _job_busy(self) -> bool: + with (self.root / ".download.lock").open("a") as lock: + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError: + return True + return False + + def _job_status(self) -> dict: + with state_lock(self.root): + job, busy = self._read_job(), self._job_busy() + if not busy and job.get("state") == "running": + job.update(state="interrupted", progress="Previous model download was interrupted", model="", downloadAll=False, cancelRequested=False) + atomic_json(self.root / ".download-job.json", job) + return {**job, "downloading": busy} + + def _publish_job(self, *, progress: str | None = None, model: str | None = None) -> None: + with state_lock(self.root): + job = self._read_job() + if job.get("jobId") != self.job_id or job.get("state") != "running": + raise ModelError("Download job state is unavailable") + if progress is not None: + self.progress = progress[:1000] + job["progress"] = self.progress + if model is not None: + self.downloading = model + job["model"] = model + atomic_json(self.root / ".download-job.json", job) + + def _check_cancelled(self) -> None: + if self.cancelled.is_set(): + raise ModelError("Download cancelled") + job = self._read_job() + if job.get("jobId") != self.job_id or job.get("state") != "running": + raise ModelError("Download job state is unavailable") + if job["cancelRequested"]: + raise ModelError("Download cancelled") + + def _finish_job(self, lock, failed: bool) -> None: + self.downloading, self.download_all = "", False + try: + with state_lock(self.root): + try: + job = self._read_job() + if job.get("jobId") == self.job_id: + job.update(state="failed" if failed else "completed", progress=self.progress[:1000], model="", downloadAll=False) + atomic_json(self.root / ".download-job.json", job) + finally: + lock.close() + finally: + if not lock.closed: + lock.close() + + def _signature(self, mid: str, row: dict, variant: str = "standard") -> tuple | None: + path = artifact_path(mid, self.root, variant) + try: + st = path.stat() + return (st.st_dev, st.st_ino, st.st_size, st.st_mtime_ns, st.st_ctime_ns, + row["artifact_sha256"], row["artifact_size"], path.is_symlink()) + except OSError: + return None + + def _installed(self, mid: str, entries: dict, variant: str = "standard") -> bool: + if mid == BUNDLED_CURRENT: + return True + row = artifact_entry(mid, entries, variant) + if "artifact_sha256" not in row: + return False + signature = self._signature(mid, row, variant) + if signature is None: + return False + key = f"{mid}:{variant}" + if self.checked.get(key, (None,))[0] != signature: + self.checked[key] = (signature, verified_artifact(mid, entries, self.root, variant) is not None) + return self.checked[key][1] + + def _snapshot_installed(self, mid: str, row: dict, variant: str = "standard") -> tuple[bool, bool]: + artifact = artifact_entry(mid, {mid: row}, variant) + if "artifact_sha256" not in artifact: + return False, False + signature = self._signature(mid, artifact, variant) + if signature is None or signature[-1] or signature[2] != artifact["artifact_size"]: + return False, False + key = f"{mid}:{variant}" + checked = self.checked.get(key) + if checked is not None and checked[0] == signature: + return checked[1], False + if not self.verify_closed and self.verifying.get(key) != signature: + self.verifying[key] = signature + self.verify_queue.append((mid, row, signature, variant)) + if self.verify_worker is None: + self.verify_worker = threading.Thread(target=self._verify_pending, daemon=True) + self.verify_worker.start() + return False, True + + def _verify_pending(self) -> None: + while True: + with self.lock: + if self.verify_closed or not self.verify_queue: + self.verify_worker = None + return + mid, row, signature, variant = self.verify_queue.popleft() + key = f"{mid}:{variant}" + if self.verifying.get(key) != signature: + continue + valid = verified_artifact(mid, {mid: row}, self.root, variant) is not None + with self.lock: + if self.verifying.get(key) == signature: + self.verifying.pop(key) + if self._signature(mid, artifact_entry(mid, {mid: row}, variant), variant) == signature: + self.checked[key] = (signature, valid) + + def snapshot(self) -> dict: + with self.lock: + entries, prefs = catalog(self.root), preferences(self.root) + parked, gpu = self.parked(), self.gpu_present() + job = self._job_status() + bundled_dir = Path(__file__).resolve().parents[2] / "selfdrive/modeld/models" + bundled_big = all((bundled_dir / name).is_file() for name in + ("big_driving_tinygrad.pkl", "big_driving_warp_1344x760_tinygrad.pkl", "big_driving_warp_1928x1208_tinygrad.pkl")) + models = [{"value": BUNDLED_CURRENT, "label": "Bundled driving model", "series": "openpilot", "version": "current", + "installed": True, "builtin": True, "selectable": True, "requiresGpu": False, "gpuAvailable": True, + "profiles": ["small", "big"] if bundled_big else ["small"], "bundledBigAvailable": bundled_big, + "communityFavorite": False, "userFavorite": BUNDLED_CURRENT in prefs["userFavorites"], + "blacklisted": BUNDLED_CURRENT in prefs["blacklistedModels"], "unavailableReason": ""}] + for mid, row in entries.items(): + installed, checking = self._snapshot_installed(mid, row) + requires_gpu = bool(row.get("uses_external_gpu")) + published = "artifact_sha256" in row + models.append({"value": mid, "label": row["name"], "version": row["version"], "series": row.get("series", ""), + "released": row.get("released", ""), "communityFavorite": bool(row.get("community_favorite")), + "userFavorite": mid in prefs["userFavorites"], "blacklisted": mid in prefs["blacklistedModels"], + "installed": installed, "checking": checking, + "builtin": False, + "requiresGpu": requires_gpu, "gpuAvailable": gpu or not requires_gpu, "small": not requires_gpu, + "selectable": installed, "downloadAvailable": published, "artifactSize": row.get("artifact_size", 0), + "unavailableReason": "" if installed else "Checking artifact" if checking else + "Download required" if published else f"{GENERATION} download not published"}) + count = sum(bool(row["installed"]) for row in models) + return {"schemaVersion": 1, "models": models, "activeSmallModel": prefs["small"], "activeBigModel": prefs["big"], + "currentModel": prefs["big"] if gpu and prefs["big"] else prefs["small"], "userFavorites": prefs["userFavorites"], + "sortMode": prefs["sortMode"], "randomizer": prefs["randomizer"], "blacklistedModels": prefs["blacklistedModels"], + "summary": {"total": len(models), "installed": count, "missing": len(models)-count}, + "jobId": job.get("jobId", ""), "cancelRequested": job.get("cancelRequested", False), + "downloadVariant": job.get("variant", "standard"), + "modelToDownload": job.get("model", ""), "downloadAll": job.get("downloadAll", False), + "downloading": job["downloading"], "progress": job.get("progress", ""), + "isOnroad": not parked, "gpuAvailable": gpu, + "capabilities": {"select": parked, "download": parked, "downloadAll": parked, "cancel": True, + "refresh": parked, "favorites": True, "delete": parked, "randomizer": parked, "exclusions": parked}} + + def _require_parked(self) -> None: + if not self.parked(): + raise ModelError("Turn the vehicle off before changing driving models") + + def action(self, action: str, payload: dict) -> dict: + if not isinstance(payload, dict): + raise ModelError("Invalid model request") + with self.lock: + if action == "cancel": + if set(payload) - {"jobId"} or ("jobId" in payload and not isinstance(payload["jobId"], str)): + raise ModelError("Invalid cancel request") + with state_lock(self.root): + job, busy = self._read_job(), self._job_busy() + if "jobId" in payload and payload["jobId"] != job.get("jobId"): + raise ModelError("This download job is no longer running") + if not busy: + return {"message": "No model download is running"} + if job.get("state") != "running": + raise ModelError("Download job state is unavailable") + job["cancelRequested"] = True + atomic_json(self.root / ".download-job.json", job) + if job["jobId"] == self.job_id: + self.cancelled.set() + return {"message": "Download cancellation requested"} + if action == "preferences": + if not payload or set(payload) - {"userFavorites", "sortMode", "randomizer", "blacklistedModels"}: + raise ModelError("Invalid model preferences") + for key in ("userFavorites", "blacklistedModels"): + values = payload.get(key, []) + if not isinstance(values, list) or len(values) > len(BY_ID) or any(not isinstance(x, str) or x not in BY_ID for x in values): + raise ModelError("Unknown model preference") + changes_selection = bool(set(payload) & {"randomizer", "blacklistedModels"}) + if "randomizer" in payload and type(payload["randomizer"]) is not bool: + raise ModelError("Invalid randomizer preference") + if changes_selection: + self._require_parked() + if "sortMode" in payload and payload["sortMode"] not in ("name", "date", "date_oldest", "community", "series", "favorites", "released"): + raise ModelError("Invalid sort mode") + with state_lock(self.root): + if changes_selection: + self._require_parked() + saved = preferences(self.root) + saved.update(payload) + atomic_json(self.root / "preferences.json", saved) + return {"message": "Model preferences saved"} + self._require_parked() + if action in ("download", "download_all", "refresh_manifest"): + if self.worker is not None and self.worker.is_alive(): + raise ModelError("A model download is already running") + if action == "download" and (set(payload) - {"model", "allowGpuWithoutGpu", "variant"} or + not isinstance(payload.get("model"), str) or payload["model"] not in BY_ID): + raise ModelError("Unknown model download") + if action != "download" and set(payload) - {"allowGpuWithoutGpu"}: + raise ModelError("Invalid download request") + if "allowGpuWithoutGpu" in payload and type(payload["allowGpuWithoutGpu"]) is not bool: + raise ModelError("Invalid GPU download preference") + variant = payload.get("variant", "standard") + if variant not in ("standard", "amd"): + raise ModelError("Unknown model artifact variant") + if variant == "amd" and not artifact_entry(payload["model"], catalog(self.root), "amd").get("artifact_sha256"): + raise ModelError("This model has no eGPU variant for this software version yet") + if (action == "download" and BY_ID[payload["model"]].uses_external_gpu and + not self.gpu_present() and not payload.get("allowGpuWithoutGpu")): + raise ModelError("Connect Chestnut or choose to download for later") + with state_lock(self.root): + self._require_parked() + lock = (self.root / ".download.lock").open("a") + try: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + except BlockingIOError as error: + lock.close() + raise ModelError("Another model manager is downloading") from error + except OSError: + lock.close() + raise + self.cancelled.clear() + self.job_id = uuid.uuid4().hex + self.progress = "Refreshing catalog" if action == "refresh_manifest" else "Starting download" + self.downloading = payload.get("model", "") + self.download_all = action == "download_all" + try: + atomic_json(self.root / ".download-job.json", { + "schemaVersion": 1, "jobId": self.job_id, "state": "running", "model": self.downloading, + "progress": self.progress, "downloadAll": self.download_all, "cancelRequested": False, "variant": variant, + }) + except BaseException: + lock.close() + raise + message = self.progress + self.worker = threading.Thread(target=self._work, args=(action, dict(payload), lock), daemon=True) + try: + self.worker.start() + except BaseException: + self.progress = "Model download could not start" + self._finish_job(lock, True) + raise + return {"message": message} + if action == "active": + if self._job_status()["downloading"]: + raise ModelError("Wait for the model download to finish before selecting a model") + if (set(payload) != {"profile", "model"} or payload["profile"] not in ("small", "big") or + not isinstance(payload["model"], str)): + raise ModelError("Invalid model selection") + if preferences(self.root)["randomizer"]: + raise ModelError("Turn Model Randomizer off before choosing a model") + mid, profile = payload["model"], payload["profile"] + if mid != BUNDLED_CURRENT and not (profile == "big" and mid == ""): + if mid not in BY_ID or BY_ID[mid].uses_external_gpu != (profile == "big"): + raise ModelError("Model does not match the selected hardware profile") + if verified_artifact(mid, catalog(self.root), self.root) is None: + raise ModelError("Download and verify this model first") + with state_lock(self.root): + self._require_parked() + saved = preferences(self.root) + if saved["randomizer"]: + raise ModelError("Turn Model Randomizer off before choosing a model") + if self._job_busy(): + raise ModelError("Wait for the model download to finish before selecting a model") + saved[profile] = mid + atomic_json(self.root / "preferences.json", saved) + return {"message": "Model selected for the next drive"} + if action == "delete": + if set(payload) != {"model"}: + raise ModelError("Invalid delete request") + mid = payload["model"] + path = artifact_path(mid, self.root) + with state_lock(self.root): + saved = preferences(self.root) + if self._job_busy(): + raise ModelError("Wait for the model download to finish before deleting a model") + if mid in (saved["small"], saved["big"]): + raise ModelError("Select another model before deleting this one") + self._require_parked() + path.unlink(missing_ok=True) + self.checked.pop(f"{mid}:standard", None) + return {"message": "Downloaded model deleted"} + raise ModelError("Unknown model action") + + def _refresh(self) -> None: + url = f"{RESOURCE_URL}/manifests/model_names_{GENERATION}.json" + with self.opener(url, timeout=30) as response: + raw = response.read(MAX_MANIFEST + 1) + if len(raw) > MAX_MANIFEST: + raise ModelError("Model catalog is too large") + data = json.loads(raw) + validate_manifest(data) + atomic_json(self.root / "catalog.json", data) + + def _work(self, action: str, payload: dict, lock) -> None: + failed = True + try: + self._check_cancelled() + self._refresh() + self._check_cancelled() + entries = catalog(self.root) + variant = payload.get("variant", "standard") + ids = [payload["model"]] if action == "download" else [mid for mid, row in entries.items() + if "artifact_sha256" in row and (not row.get("uses_external_gpu") or self.gpu_present() or payload.get("allowGpuWithoutGpu"))] + if action != "refresh_manifest": + for mid in ids: + self._check_cancelled() + if not self._installed(mid, entries, variant): + self._publish_job(model=mid) + self._download(mid, artifact_entry(mid, entries, variant), variant) + self._check_cancelled() + self.progress = "Catalog updated" if action == "refresh_manifest" else "Downloaded!" + failed = False + except (OSError, ValueError, KeyError, TimeoutError) as error: + failed = True + self.progress = str(error) or "Model download failed" + finally: + self._finish_job(lock, failed) + + def _download(self, mid: str, entry: dict, variant: str = "standard") -> None: + if "artifact_sha256" not in entry: + raise ModelError("This model is not available for this software version yet") + destination = artifact_path(mid, self.root, variant) + destination.parent.mkdir(parents=True, exist_ok=True) + size, chunks = entry["artifact_size"], entry["artifact_chunk_count"] + if shutil.disk_usage(self.root).free < size + 256 * 1024 * 1024: + raise ModelError("Not enough storage for this model") + filename = destination.name + pieces = [filename] if chunks == 0 else [f"{filename}.chunk{i:02d}of{chunks:02d}" for i in range(1, chunks+1)] + fd, temp = tempfile.mkstemp(prefix=".download-", dir=destination.parent) + try: + digest, received = hashlib.sha256(), 0 + with os.fdopen(fd, "wb") as out: + for piece in pieces: + with self.opener(f"{RESOURCE_URL}/models/{GENERATION}/{mid}/{piece}", timeout=30) as response: + while data := response.read(1024 * 1024): + self._check_cancelled() + received += len(data) + if received > size: + raise ModelError("Model exceeds its declared size") + out.write(data) + digest.update(data) + progress = f"{BY_ID[mid].name}: {received * 100 // size}%" + if progress != self.progress: + self._publish_job(progress=progress) + out.flush() + os.fsync(out.fileno()) + if received != size or digest.hexdigest() != entry["artifact_sha256"]: + raise ModelError("Model checksum did not match; download discarded") + with state_lock(self.root): + self._require_parked() + self._check_cancelled() + os.replace(temp, destination) + self.checked.pop(f"{mid}:{variant}", None) + finally: + Path(temp).unlink(missing_ok=True) + + def close(self) -> None: + self.cancelled.set() + with self.lock: + self.verify_closed = True + self.verify_queue.clear() + verify_worker = self.verify_worker + if verify_worker is not None: + verify_worker.join(timeout=2) + if self.worker is not None: + self.worker.join(timeout=2) diff --git a/openpilot/starpilot/models/parser.py b/openpilot/starpilot/models/parser.py new file mode 100644 index 0000000000..b2131b8b45 --- /dev/null +++ b/openpilot/starpilot/models/parser.py @@ -0,0 +1,87 @@ +"""Output parsing for the v8–v16 model catalog; bundled parsing stays upstream.""" + +import numpy as np +from openpilot.selfdrive.modeld.constants import ModelConstants +from openpilot.selfdrive.modeld.parse_model_outputs import Parser as CurrentParser + + +class Parser(CurrentParser): + def split_outputs(self, outs: dict[str, np.ndarray]) -> None: + if 'lead' in outs: + if outs['lead'].shape[1] == 2 * ModelConstants.LEAD_MHP_SELECTION * ModelConstants.LEAD_TRAJ_LEN * ModelConstants.LEAD_WIDTH: + self.parse_mdn('lead', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.LEAD_MHP_SELECTION, ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH)) + else: + self.parse_mdn('lead', outs, in_N=ModelConstants.LEAD_MHP_N, out_N=ModelConstants.LEAD_MHP_SELECTION, + out_shape=(ModelConstants.LEAD_TRAJ_LEN, ModelConstants.LEAD_WIDTH)) + if 'plan' in outs: + plan_mhp = outs['plan'].shape[1] != 2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + plan_in_N, plan_out_N = (ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) if plan_mhp else (0, 0) + self.parse_mdn('plan', outs, in_N=plan_in_N, out_N=plan_out_N, + out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + if 'planplus' in outs: + self.parse_mdn('planplus', outs, in_N=plan_in_N, out_N=plan_out_N, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + elif 'planplus' in outs: + planplus_mhp = outs['planplus'].shape[1] != 2 * ModelConstants.IDX_N * ModelConstants.PLAN_WIDTH + planplus_in_N, planplus_out_N = (ModelConstants.PLAN_MHP_N, ModelConstants.PLAN_MHP_SELECTION) if planplus_mhp else (0, 0) + self.parse_mdn('planplus', outs, in_N=planplus_in_N, out_N=planplus_out_N, out_shape=(ModelConstants.IDX_N, ModelConstants.PLAN_WIDTH)) + if 'lane_lines' in outs: + self.parse_mdn('lane_lines', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.NUM_LANE_LINES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + if 'road_edges' in outs: + self.parse_mdn('road_edges', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.NUM_ROAD_EDGES, ModelConstants.IDX_N, ModelConstants.LANE_LINES_WIDTH)) + if 'sim_pose' in outs: + self.parse_mdn('sim_pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + if 'lane_lines_prob' in outs: + self.parse_binary_crossentropy('lane_lines_prob', outs) + if 'lead_prob' in outs: + self.parse_binary_crossentropy('lead_prob', outs) + + def parse_off_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.split_outputs(outs) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs + + def parse_vision_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) + self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.split_outputs(outs) + self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN, ModelConstants.DESIRE_PRED_WIDTH)) + self.parse_binary_crossentropy('meta', outs) + return outs + + def parse_policy_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + self.split_outputs(outs) + if 'action' in outs: + self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(ModelConstants.ACTION_WIDTH,)) + if 'lat_planner_solution' in outs: + self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N, ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) + if 'desired_curvature' in outs: + self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs + + def parse_outputs(self, outs: dict[str, np.ndarray]) -> dict[str, np.ndarray]: + # Combined supercombo outputs contain both vision and policy slices in the + # same dictionary. Parse shared MDN outputs once; the split runtime keeps + # using parse_vision_outputs/parse_policy_outputs independently. + self.parse_mdn('pose', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.parse_mdn('wide_from_device_euler', outs, in_N=0, out_N=0, out_shape=(ModelConstants.WIDE_FROM_DEVICE_WIDTH,)) + self.parse_mdn('road_transform', outs, in_N=0, out_N=0, out_shape=(ModelConstants.POSE_WIDTH,)) + self.split_outputs(outs) + self.parse_categorical_crossentropy('desire_pred', outs, out_shape=(ModelConstants.DESIRE_PRED_LEN, ModelConstants.DESIRE_PRED_WIDTH)) + self.parse_binary_crossentropy('meta', outs) + if 'action' in outs: + self.parse_mdn('action', outs, in_N=0, out_N=0, out_shape=(ModelConstants.ACTION_WIDTH,)) + if 'lat_planner_solution' in outs: + self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, + out_shape=(ModelConstants.IDX_N, ModelConstants.LAT_PLANNER_SOLUTION_WIDTH)) + if 'desired_curvature' in outs: + self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,)) + if 'desire_state' in outs: + self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,)) + return outs diff --git a/openpilot/starpilot/models/receipt.py b/openpilot/starpilot/models/receipt.py new file mode 100644 index 0000000000..1eb2838e26 --- /dev/null +++ b/openpilot/starpilot/models/receipt.py @@ -0,0 +1,364 @@ +"""Small volatile modeld load receipt; no model artifact is deserialized here.""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import stat +import time + +from openpilot.common.hardware.hw import Paths +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, model_entries +from openpilot.starpilot.models.status import ModelLoad, ModelVariant + + +MAX_RECEIPT_BYTES = 1024 +MAX_ARTIFACT_BYTES = 2 * 1024 * 1024 * 1024 +_PREFIX = re.compile(r"[A-Za-z0-9_-]{1,32}\Z") +_DIGEST = re.compile(r"[0-9a-f]{64}\Z") +_REASONS = {None, "chestnut-load-failed", "chestnut-run-stalled", "selected-load-failed"} + + +class ReceiptUnavailable(ValueError): + pass + + +def receipt_path() -> Path: + prefix = os.environ.get("OPENPILOT_PREFIX", "d") + if _PREFIX.fullmatch(prefix) is None: + raise ReceiptUnavailable("invalid IPC namespace") + return Path(Paths.shm_path()) / f"starpilot_modeld_{prefix}" / "load-v1.json" + + +def _root_fd(path: Path, *, create: bool) -> int: + if create: + try: + os.mkdir(path.parent, 0o700) + except FileExistsError: + pass + fd = os.open(path.parent, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW | os.O_CLOEXEC) + try: + info = os.fstat(fd) + if not stat.S_ISDIR(info.st_mode) or info.st_uid != os.getuid() or stat.S_IMODE(info.st_mode) != 0o700: + raise ReceiptUnavailable("model receipt directory unavailable") + return fd + except BaseException: + os.close(fd) + raise + + +def process_start_ticks(pid: int, *, proc_root: Path = Path("/proc")) -> int: + """Return Linux's immutable per-process start ticks; zero means unavailable.""" + if not isinstance(pid, int) or isinstance(pid, bool) or pid <= 0: + return 0 + try: + raw = (proc_root / str(pid) / "stat").read_bytes() + tail = raw[raw.rfind(b")") + 2:].split() + ticks = int(tail[19]) + return ticks if ticks > 0 else 0 + except (OSError, ValueError, IndexError): + return 0 + + +def _identity(info: os.stat_result) -> tuple[int, int, int, int, int]: + return info.st_dev, info.st_ino, info.st_size, info.st_mtime_ns, info.st_ctime_ns + + +@dataclass(frozen=True) +class PreparedArtifact: + identity: tuple[int, int, int, int, int] + sha256: str + + +def artifact_identity(path: Path) -> tuple[int, int, int, int, int]: + """Capture the installed file identity immediately before ModelState opens it.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + try: + info = os.fstat(fd) + if not stat.S_ISREG(info.st_mode) or info.st_size <= 0 or info.st_size > MAX_ARTIFACT_BYTES: + raise ReceiptUnavailable("compiled artifact is not a bounded regular file") + return _identity(info) + finally: + os.close(fd) + + +def artifact_digest(path: Path, expected_identity: tuple[int, int, int, int, int]) -> str: + """Hash a bounded installed compiled artifact once, without following a link or FIFO.""" + try: + fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC) + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + try: + before = os.fstat(fd) + if (not stat.S_ISREG(before.st_mode) or before.st_size <= 0 or before.st_size > MAX_ARTIFACT_BYTES or + _identity(before) != expected_identity): + raise ReceiptUnavailable("compiled artifact is not a bounded regular file") + digest = hashlib.sha256() + remaining = before.st_size + while remaining: + chunk = os.read(fd, min(1024 * 1024, remaining)) + if not chunk: + raise ReceiptUnavailable("compiled artifact changed during hash") + digest.update(chunk) + remaining -= len(chunk) + after = os.fstat(fd) + named = os.stat(path, follow_symlinks=False) + if _identity(before) != _identity(after) or _identity(before) != _identity(named): + raise ReceiptUnavailable("compiled artifact changed during hash") + return digest.hexdigest() + except OSError as exc: + raise ReceiptUnavailable("compiled artifact unavailable") from exc + finally: + os.close(fd) + + +def prepare_artifact(path: Path, expected_identity: tuple[int, int, int, int, int]) -> PreparedArtifact: + """Complete bounded installed-file hashing before modeld's live loop starts.""" + return PreparedArtifact(expected_identity, artifact_digest(path, expected_identity)) + + +def _pairs(items: list[tuple[str, object]]) -> dict[str, object]: + result: dict[str, object] = {} + for key, value in items: + if key in result: + raise ReceiptUnavailable("duplicate model receipt field") + result[key] = value + return result + + +def _positive_int(value: object) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and 0 < value <= (1 << 64) - 1 + + +def _decode(raw: bytes) -> ModelLoad: + entries = model_entries() + try: + record = json.loads(raw.decode("utf-8"), object_pairs_hook=_pairs) + except (UnicodeDecodeError, json.JSONDecodeError, RecursionError) as exc: + raise ReceiptUnavailable("malformed model receipt") from exc + if not isinstance(record, dict) or set(record) != {"version", "pid", "processStartTicks", "loadedMonoNs", + "modelId", "variant", "artifactSha256", "fallbackReason"}: + raise ReceiptUnavailable("invalid model receipt fields") + if record["version"] != 1 or type(record["version"]) is not int: + raise ReceiptUnavailable("unsupported model receipt") + if not all(_positive_int(record[key]) for key in ("pid", "processStartTicks", "loadedMonoNs")): + raise ReceiptUnavailable("invalid model process identity") + if not isinstance(record["modelId"], str) or record["modelId"] not in entries: + raise ReceiptUnavailable("unknown loaded model") + try: + variant = ModelVariant(record["variant"]) + except (ValueError, TypeError) as exc: + raise ReceiptUnavailable("unknown model variant") from exc + if record["modelId"] != BUNDLED_CURRENT and entries[record["modelId"]].uses_external_gpu != (variant is ModelVariant.CHESTNUT): + raise ReceiptUnavailable("model receipt hardware mismatch") + digest = record["artifactSha256"] + if not isinstance(digest, str) or _DIGEST.fullmatch(digest) is None or record["fallbackReason"] not in _REASONS: + raise ReceiptUnavailable("invalid model artifact identity") + return ModelLoad(record["pid"], record["processStartTicks"], record["loadedMonoNs"], + record["modelId"], variant, digest, record["fallbackReason"]) + + +def read_receipt(path: Path | None = None) -> ModelLoad | None: + """Missing or malformed volatile state is unavailable, never a default identity.""" + try: + path = receipt_path() if path is None else path + root_fd = _root_fd(path, create=False) + except (OSError, ReceiptUnavailable): + return None + try: + fd = os.open(path.name, os.O_RDONLY | os.O_NONBLOCK | os.O_NOFOLLOW | os.O_CLOEXEC, dir_fd=root_fd) + except (OSError, ReceiptUnavailable): + os.close(root_fd) + return None + try: + before = os.fstat(fd) + if not stat.S_ISREG(before.st_mode) or before.st_size > MAX_RECEIPT_BYTES: + return None + raw = os.read(fd, MAX_RECEIPT_BYTES + 1) + after = os.fstat(fd) + named = os.stat(path.name, dir_fd=root_fd, follow_symlinks=False) + if len(raw) > MAX_RECEIPT_BYTES or _identity(before) != _identity(after) or _identity(before) != _identity(named): + return None + return _decode(raw) + except (OSError, ReceiptUnavailable, TypeError): + return None + finally: + os.close(fd) + os.close(root_fd) + + +def write_receipt(load: ModelLoad, path: Path | None = None) -> None: + """Replace one volatile record after the selected model has warmed successfully.""" + path = receipt_path() if path is None else path + raw = json.dumps({"version": 1, "pid": load.pid, "processStartTicks": load.process_start_ticks, + "loadedMonoNs": load.loaded_mono_ns, "modelId": load.model_id, + "variant": load.variant.value, "artifactSha256": load.artifact_sha256, + "fallbackReason": load.fallback_reason}, separators=(",", ":"), sort_keys=True).encode() + if len(raw) > MAX_RECEIPT_BYTES or _decode(raw) != load: + raise ReceiptUnavailable("invalid model receipt") + root_fd = _root_fd(path, create=True) + temporary = f".load-{secrets.token_hex(8)}" + try: + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW | os.O_CLOEXEC, + 0o600, dir_fd=root_fd) + try: + with os.fdopen(descriptor, "wb") as stream: + stream.write(raw) + except BaseException: + os.unlink(temporary, dir_fd=root_fd) + raise + os.replace(temporary, path.name, src_dir_fd=root_fd, dst_dir_fd=root_fd) + finally: + try: + os.unlink(temporary, dir_fd=root_fd) + except FileNotFoundError: + pass + os.close(root_fd) + + +def recorded_model_message(load: ModelLoad) -> str: + return json.dumps({'process': load.pid, 'msg': {'event': 'modeld.loaded', 'version': 1, + 'pid': load.pid, 'processStartTicks': load.process_start_ticks, + 'loadedMonoNs': load.loaded_mono_ns, 'modelId': load.model_id, + 'variant': load.variant.value, 'artifactSha256': load.artifact_sha256, + 'fallbackReason': load.fallback_reason}}, separators=(',', ':')) + + +def log_loaded_model(load: ModelLoad) -> None: + """Record the actual initialized runner once per load or fallback in the rlog.""" + from openpilot.common.swaglog import cloudlog + cloudlog.event("modeld.loaded", version=1, pid=load.pid, processStartTicks=load.process_start_ticks, + loadedMonoNs=load.loaded_mono_ns, modelId=load.model_id, variant=load.variant.value, + artifactSha256=load.artifact_sha256, fallbackReason=load.fallback_reason) + + +def logged_model_load(raw: str) -> ModelLoad | None: + """Read our load event, never a requested selection or a free-form load message.""" + if len(raw) > 8192: + return None + try: + record = json.loads(raw, object_pairs_hook=_pairs) + payload = record.get("msg") if isinstance(record, dict) else None + if not isinstance(payload, dict) or payload.get("event") != "modeld.loaded": + return None + payload = {key: value for key, value in payload.items() if key != "event"} + load = _decode(json.dumps(payload).encode()) + return load if type(record.get("process")) is int and record["process"] == load.pid else None + except (ReceiptUnavailable, ValueError, TypeError, RecursionError): + return None + + +def clear_receipt(path: Path | None = None) -> None: + """Invalidate a prior run without following the previous file or directory.""" + path = receipt_path() if path is None else path + try: + root_fd = _root_fd(path, create=False) + except (OSError, ReceiptUnavailable): + return + try: + os.unlink(path.name, dir_fd=root_fd) + except FileNotFoundError: + pass + finally: + os.close(root_fd) + + +def record_prepared(prepared: PreparedArtifact, variant: ModelVariant, fallback_reason: str | None = None, + *, model_id: str = BUNDLED_CURRENT, receipt_file: Path | None = None, pid: int | None = None, + start_ticks: int | None = None, now_mono_ns: int | None = None) -> ModelLoad: + """Write a tiny receipt from a startup-verified artifact; never hash on fallback.""" + pid = os.getpid() if pid is None else pid + start_ticks = process_start_ticks(pid) if start_ticks is None else start_ticks + now_mono_ns = time.monotonic_ns() if now_mono_ns is None else now_mono_ns + if not _positive_int(start_ticks) or not _positive_int(now_mono_ns): + raise ReceiptUnavailable("process identity unavailable") + load = ModelLoad(pid, start_ticks, now_mono_ns, model_id, variant, + prepared.sha256, fallback_reason) + write_receipt(load, receipt_file) + return load + + +class ModelReceiptOwner: + """Best-effort modeld hook: diagnostic failures never stop inference.""" + + def __init__(self, receipt_file: Path | None = None, report: Callable[[str], None] | None = None, + record: Callable[[ModelLoad], None] = log_loaded_model): + self.receipt_file = receipt_file + self.report = report if report is not None else lambda _reason: None + self.record = record + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + + def start(self) -> None: + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + try: + clear_receipt(self.receipt_file) + except Exception: + self.report("clear-failed") + + def capture(self, path: Path) -> tuple[int, int, int, int, int] | None: + try: + return artifact_identity(path) + except Exception: + self.report("artifact-unavailable") + return None + + def prepare(self, path: Path, expected_identity: tuple[int, int, int, int, int] | None) -> PreparedArtifact | None: + if expected_identity is None: + self.report("artifact-identity-unavailable") + return None + try: + return prepare_artifact(path, expected_identity) + except Exception: + self.report("artifact-digest-unavailable") + return None + + def publish(self, publisher, now_ns: int) -> bool: + """Repeat actual load identity so routes started after warm-up retain it.""" + if self._last_publication_ns and now_ns - self._last_publication_ns < 5_000_000_000: + return False + try: + from openpilot.cereal import messaging + message = messaging.new_message(None, valid=True, logMonoTime=now_ns) + message.logMessage = (recorded_model_message(self.load) if self.load is not None else + json.dumps({'msg': {'event': 'modeld.identity-unavailable', + 'sinceMonoNs': self._identity_since_ns}})) + publisher.send('modelIdentity', message) + self._last_publication_ns = now_ns + return True + except Exception: + self.report('publication-failed') + return False + + def loaded(self, prepared: PreparedArtifact | None, variant: ModelVariant, + fallback_reason: str | None = None, *, model_id: str = BUNDLED_CURRENT) -> bool: + self.load = None + self._identity_since_ns = time.monotonic_ns() + self._last_publication_ns = 0 + if prepared is None: + self.report("prepared-artifact-unavailable") + return False + try: + load = record_prepared(prepared, variant, fallback_reason, model_id=model_id, receipt_file=self.receipt_file) + except Exception: + self.report("record-failed") + return False + self.load = load + self._last_publication_ns = 0 + try: + self.record(load) + except Exception: + self.report("log-failed") + return True diff --git a/openpilot/starpilot/models/recovery.py b/openpilot/starpilot/models/recovery.py new file mode 100644 index 0000000000..a8a6dec64e --- /dev/null +++ b/openpilot/starpilot/models/recovery.py @@ -0,0 +1,122 @@ +"""Drive-scoped process recovery after an established Big publisher stops. + +This owner never touches GPU buffers or retained model selection. Small may start +only after the exact failed modeld process has exited. +""" +from dataclasses import dataclass + +ARM_SPAN_NS = 1_000_000_000 +ARM_MAX_GAP_NS = 1_000_000_000 +STALL_NS = 2_000_000_000 +CAMERA_MAX_AGE_NS = 200_000_000 +INT_GRACE_NS = 500_000_000 +KILL_GRACE_NS = 1_000_000_000 + + +@dataclass(frozen=True) +class RecoveryEvidence: + now_ns: int + drive_id: int + identity: tuple[int, int] | None + alive: bool + big_receipt: bool + loaded_ns: int + model_ns: int + driving_ns: int + output_valid: bool + output_big: bool + cameras: tuple[tuple[int, int], ...] # producer timestamp, frameId + + +class ModelRecoveryOwner: + def __init__(self): + self.drive_id = 0 + self.small_only = False + self.phase = 'monitor' + self.identity = None + self.first_output_ns = 0 + self.last_output = (0, 0) + self.last_progress_ns = 0 + self.last_cameras = () + self.camera_progress_since = 0 + self.deadline_ns = 0 + self.armed = False + + def _withdraw(self): + self.identity = None + self.first_output_ns = 0 + self.last_output = (0, 0) + self.last_progress_ns = 0 + self.last_cameras = () + self.camera_progress_since = 0 + self.armed = False + + def step(self, e: RecoveryEvidence) -> str | None: + if self.phase in ('sigint', 'sigkill', 'failed'): + if not e.alive: + if self.phase == 'failed' and e.drive_id == self.drive_id: + return None + if e.drive_id == self.drive_id and e.drive_id > 0: + self.phase = 'small' + return 'small' + self.__init__() + self.drive_id = e.drive_id + return 'exit' + if e.identity != self.identity: + self.phase = 'failed' + return None + if self.phase != 'failed' and e.now_ns >= self.deadline_ns: + if self.phase == 'sigint': + self.phase = 'sigkill' + self.deadline_ns = e.now_ns + KILL_GRACE_NS + return 'kill' + self.phase = 'failed' + return 'failed' + return None + if e.drive_id <= 0: + self.__init__() + return None + if e.drive_id != self.drive_id: + self.__init__() + self.drive_id = e.drive_id + if self.phase != 'monitor' or self.small_only: + return None + if (not e.alive or e.identity is None or min(e.identity) <= 0 or not e.big_receipt or + not self.drive_id <= e.loaded_ns <= e.now_ns): + self._withdraw() + return None + if e.identity != self.identity: + self._withdraw() + self.identity = e.identity + cameras_fresh = (len(e.cameras) == 2 and all(self.drive_id <= t <= e.now_ns and + e.now_ns - t <= CAMERA_MAX_AGE_NS and f >= 0 for t, f in e.cameras)) + if not cameras_fresh: + self.camera_progress_since = 0 + self.last_cameras = () + return None + cameras_advance = bool(self.last_cameras and all(t > pt and f > pf for (t, f), (pt, pf) in zip(e.cameras, self.last_cameras, strict=True))) + if cameras_advance: + if not self.camera_progress_since: + self.camera_progress_since = e.now_ns + else: + self.camera_progress_since = 0 + self.last_cameras = e.cameras + output = (e.model_ns, e.driving_ns) + valid_output = (e.output_valid and e.output_big and all(e.loaded_ns <= t <= e.now_ns for t in output) and + e.now_ns - min(output) <= CAMERA_MAX_AGE_NS) + if valid_output and all(t > old for t, old in zip(output, self.last_output, strict=True)): + if not self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns > ARM_MAX_GAP_NS: + self.first_output_ns = 0 + if not self.first_output_ns: + self.first_output_ns = min(output) + self.last_output = output + self.last_progress_ns = e.now_ns + self.armed = min(output) - self.first_output_ns >= ARM_SPAN_NS + return None + if (self.armed and self.last_progress_ns and e.now_ns - self.last_progress_ns >= STALL_NS and + self.camera_progress_since and e.now_ns - self.camera_progress_since >= STALL_NS): + self.small_only = True + self.phase = 'sigint' + self.deadline_ns = e.now_ns + INT_GRACE_NS + return 'interrupt' + return None diff --git a/openpilot/starpilot/models/recovery_process.py b/openpilot/starpilot/models/recovery_process.py new file mode 100644 index 0000000000..7514a71d9e --- /dev/null +++ b/openpilot/starpilot/models/recovery_process.py @@ -0,0 +1,110 @@ +"""Manager adapter for process-isolated modeld recovery; no GPU context here.""" +import os +import signal +import time +from multiprocessing import Process + +from openpilot.common.swaglog import cloudlog +from openpilot.starpilot.models.receipt import process_start_ticks, read_receipt +from openpilot.starpilot.models.recovery import KILL_GRACE_NS, ModelRecoveryOwner, RecoveryEvidence +from openpilot.starpilot.models.status import ModelVariant +from openpilot.system.manager.process import PythonProcess, join_process, launcher + +SMALL_ONLY_ENV = 'STARPILOT_MODELD_RECOVERY_SMALL_ONLY' + + +def modeld_launcher(module, name, small_only): + # Set in this fresh child before importing modeld; never alter manager's env. + if small_only: + os.environ[SMALL_ONLY_ENV] = '1' + else: + os.environ.pop(SMALL_ONLY_ENV, None) + launcher(module, name) + + +class ModeldProcess(PythonProcess): + def __init__(self, name, module, should_run, enabled=True): + super().__init__(name, module, should_run, enabled=enabled) + self.recovery = ModelRecoveryOwner() + self.context_ready = False + + def observe(self, sm, started): + now = time.monotonic_ns() + alive = self.proc is not None and self.proc.is_alive() + pid = self.proc.pid if self.proc is not None else 0 + ticks = process_start_ticks(pid or 0) if alive else 0 + identity = (pid, ticks) if pid and ticks else None + load = read_receipt() if alive else None + matches = load is not None and identity == (load.pid, load.process_start_ticks) + cameras = [] + for service in ('narrowRoadCameraState', 'wideRoadCameraState'): + if sm.seen[service] and sm.valid[service] and sm.alive[service]: + cameras.append((int(sm.logMonoTime[service]), int(sm[service].frameId))) + evidence = RecoveryEvidence( + now, int(sm['deviceState'].startedMonoTime) if started else 0, identity, alive, + bool(matches and load.variant == ModelVariant.CHESTNUT), load.loaded_mono_ns if matches else 0, + int(sm.logMonoTime['modelV2']), int(sm.logMonoTime['drivingModelData']), + bool(sm.seen['modelV2'] and sm.seen['drivingModelData'] and sm.valid['modelV2'] and sm.valid['drivingModelData']), + bool(sm['modelV2'].big), tuple(cameras), + ) + device_fresh = (sm.seen['deviceState'] and sm.valid['deviceState'] and sm.alive['deviceState'] and + 0 < int(sm.logMonoTime['deviceState']) <= now <= int(sm.logMonoTime['deviceState']) + 1_000_000_000) + self.context_ready = bool(device_fresh and started) + if not device_fresh and self.recovery.phase == 'monitor': + self.recovery._withdraw() + return + action = self.recovery.step(evidence) + if action in ('interrupt', 'kill'): + # Owner identity is checked above; SIGKILL gets a bounded exit observation, + # never an unbounded join and never a concurrent Small publisher. + try: + self.signal(signal.SIGINT if action == 'interrupt' else signal.SIGKILL) + except ProcessLookupError: + pass # Exit is confirmed on the next owner observation. + cloudlog.event('modeld.recovery', action=action, pid=pid, drive=evidence.drive_id) + elif action == 'failed': + cloudlog.error('modeld recovery could not confirm process exit; Small restart blocked') + elif action in ('small', 'exit'): + assert self.proc is not None and not self.proc.is_alive() + self.proc.join(timeout=0) + self.proc = None + self.shutting_down = False + cloudlog.event('modeld.recovery', action='small-after-exit', pid=pid, drive=evidence.drive_id) + + def stop(self, retry=True, block=True, sig=None): + if self.recovery.phase in ('sigint', 'sigkill', 'failed'): + # Recovery owns teardown even when manager transitions offroad. Never + # enter the generic five-second/unbounded join for this known stalled PID. + if self.proc is None: + return None + if self.proc.is_alive() and block: + # manager_cleanup has no future observation tick. Kill this owned child + # now and bound the exit wait instead of abandoning a stalled process. + try: + self.signal(signal.SIGKILL) + except ProcessLookupError: + pass + join_process(self.proc, KILL_GRACE_NS / 1e9) + if not self.proc.is_alive(): + self.proc.join(timeout=0) + ret = self.proc.exitcode + self.proc = None + self.shutting_down = False + return ret + if block: + self.recovery.phase = 'failed' + cloudlog.error('modeld shutdown could not confirm stalled process exit') + return None + return super().stop(retry=retry, block=block, sig=sig) + + def start(self): + if not self.context_ready or self.recovery.phase in ('sigint', 'sigkill', 'failed'): + return + if self.shutting_down: + self.stop() + if self.proc is not None: + return + self.proc = Process(name=self.name, target=modeld_launcher, + args=(self.module, self.name, self.recovery.small_only)) + self.proc.start() + self.shutting_down = False diff --git a/openpilot/starpilot/models/runner.py b/openpilot/starpilot/models/runner.py new file mode 100644 index 0000000000..32ed204752 --- /dev/null +++ b/openpilot/starpilot/models/runner.py @@ -0,0 +1,294 @@ +"""Run catalog models using their declared history and output contracts.""" + +from collections.abc import Callable +from dataclasses import dataclass +import os +from pathlib import Path +import weakref + +import numpy as np +from tinygrad import Tensor, dtypes + +from openpilot.cereal import log +from openpilot.selfdrive.controls.lib.drive_helpers import MIN_SPEED, smooth_value +from openpilot.selfdrive.modeld.constants import ModelConstants, Plan +from openpilot.selfdrive.modeld.helpers import load_oob +from openpilot.starpilot.models.parser import Parser +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, COMPILER_REVISION +from openpilot.starpilot.models.receipt import PreparedArtifact, artifact_identity, prepare_artifact +from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + +SUPPORTED_VERSIONS = frozenset(f"v{version}" for version in range(8, 17)) +REQUIRED_OUTPUTS = frozenset(("plan", "lane_lines", "lane_lines_prob", "road_edges", "lead", "lead_prob", + "meta", "desire_pred", "pose", "wide_from_device_euler", "road_transform")) + + +@dataclass(frozen=True) +class CameraWarpDescriptor: + camera_size: tuple[int, int] + nv12_layout: tuple[int, int, int, int] + image_layouts: tuple[tuple[int, ...], tuple[int, ...]] + output_shape: tuple[int, ...] + dtype: object + device: str + + +@dataclass(frozen=True) +class SharedCameraWarp: + tensor: Tensor + descriptor: CameraWarpDescriptor + frame_pointers: tuple[int, int] + transforms: tuple[bytes, bytes] + producer: weakref.ReferenceType + + +def load_verified_model(cam_w: int, cam_h: int, artifact_path: Path, behavior_version: str, chestnut: bool, + model_id: str, expected_sha256: str | None, *, root: Path | None = None) -> tuple["CatalogModelState", PreparedArtifact]: + from openpilot.starpilot.models.manager import ROOT, state_lock + + if expected_sha256 is None: + raise ValueError("selected catalog model has no verified digest") + with state_lock(ROOT if root is None else root): + identity = artifact_identity(artifact_path) + prepared = prepare_artifact(artifact_path, identity) + if prepared.sha256 != expected_sha256: + raise ValueError("selected catalog model changed before loading") + model = CatalogModelState(cam_w, cam_h, artifact_path, behavior_version, chestnut) + model.warmup() + if artifact_identity(artifact_path) != identity: + raise ValueError("selected catalog model changed while loading") + model.model_id = model_id + return model, prepared + + +def validate_artifact(artifact: dict, version: str, camera_size: tuple[int, int]) -> None: + if (not isinstance(artifact, dict) or artifact.get("artifact_abi") != ARTIFACT_ABI or + artifact.get("compiler_revision") != COMPILER_REVISION or artifact.get("format_version") != 1): + raise ValueError("catalog model compiler/ABI mismatch") + if version not in SUPPORTED_VERSIONS or artifact.get("behavior_version") != version: + raise ValueError("catalog model behavior version mismatch") + if artifact.get("model_type") not in ("supercombo", "vision_policy", "vision_multi_policy"): + raise ValueError("unsupported catalog model layout") + if artifact.get("image_history_pipeline") not in ("policy", "warp"): + raise ValueError("unsupported catalog model history") + if type(artifact.get("frame_skip")) is not int or not 1 <= artifact["frame_skip"] <= 20: + raise ValueError("unsupported catalog model frame interval") + if camera_size not in artifact or not callable(artifact.get("run_policy")) or not callable(artifact[camera_size]): + raise ValueError("catalog model camera/runner unavailable") + metadata = artifact.get("metadata") + if not isinstance(metadata, dict): + raise ValueError("missing catalog model metadata") + keys = ["model"] if artifact["model_type"] == "supercombo" else ["vision", *artifact.get("policy_order", [])] + if not keys or (keys[0] == "vision" and not any(key in keys for key in ("on_policy", "policy"))): + raise ValueError("catalog model has no primary policy") + names = set() + for key in keys: + entry = metadata.get(key) + if not isinstance(entry, dict) or not isinstance(entry.get("input_shapes"), dict) or not isinstance(entry.get("output_slices"), dict): + raise ValueError("invalid catalog model metadata") + shape = entry.get("output_shapes", {}).get("outputs") + if (not isinstance(shape, (tuple, list)) or len(shape) != 2 or shape[0] != 1 or + type(shape[1]) is not int or shape[1] <= 0): + raise ValueError("catalog model output width unavailable") + width = shape[1] + for name, section in entry["output_slices"].items(): + if (not isinstance(section, slice) or section.step not in (None, 1) or + any(bound is not None and (type(bound) is not int or not -width <= bound <= width) + for bound in (section.start, section.stop))): + raise ValueError("catalog model output slice exceeds output") + # Metadata padding aliases use ordinary Python negative/open endpoints. + # Check explicit bounds before indices() so its clipping cannot hide a bad slice. + start, stop, _ = section.indices(width) + if not 0 <= start < stop <= width: + raise ValueError("catalog model output slice is empty or reversed") + names.add(name) + if not REQUIRED_OUTPUTS <= names or (version in ("v14", "v15", "v16") and "action" not in names): + raise ValueError("catalog model is missing required driving outputs") + + +def action_from_outputs(outputs: dict[str, np.ndarray], version: str, previous: log.ModelDataV2.Action, + lat_action_t: float, long_action_t: float, v_ego: float, + lat_smooth_seconds: float = 0.0, long_smooth_seconds: float = 0.3) -> log.ModelDataV2.Action: + """Preserve the catalog's generation-specific action units and v9 curvature.""" + if version not in SUPPORTED_VERSIONS or min(lat_action_t, long_action_t) <= 0: + raise ValueError("invalid catalog model action contract") + if version in ("v14", "v15", "v16"): + curvature, acceleration = outputs["action"][0] + curvature = curvature / (100.0 if version == "v14" else max(1.0, v_ego) ** 2) + stop = v_ego < 0.3 and acceleration < 0.1 + else: + plan = outputs["plan"][0] + if "planplus" in outputs: + plan = plan + outputs["planplus"][0] + speeds, accelerations = plan[:, Plan.VELOCITY][:, 0], plan[:, Plan.ACCELERATION][:, 0] + acceleration = 2 * (np.interp(long_action_t, ModelConstants.T_IDXS, speeds) - speeds[0]) / long_action_t - accelerations[0] + stop = speeds[0] < 0.3 and acceleration < 0.1 + if version == "v9": + curvature = float(outputs["desired_curvature"][0, 0]) if "desired_curvature" in outputs else previous.desiredCurvature + else: + yaw = np.interp(lat_action_t, ModelConstants.T_IDXS, plan[:, Plan.T_FROM_CURRENT_EULER][:, 2]) + yaw_rate = plan[0, Plan.ORIENTATION_RATE][2] + curvature = 2 * yaw / (max(MIN_SPEED, v_ego) * lat_action_t) - yaw_rate / max(MIN_SPEED, v_ego) + acceleration = smooth_value(float(acceleration), previous.desiredAcceleration, long_smooth_seconds) + curvature = (smooth_value(float(curvature), previous.desiredCurvature, lat_smooth_seconds) + if v_ego > 0.3 else previous.desiredCurvature) + if not np.isfinite((acceleration, curvature)).all(): + raise ValueError("non-finite catalog model action") + return log.ModelDataV2.Action(desiredCurvature=float(curvature), desiredAcceleration=float(acceleration), shouldStop=bool(stop)) + + +class CatalogModelState: + """Current modeld interface around a current-pin unified catalog artifact.""" + + vision_input_names = ("img", "big_img") + + def __init__(self, cam_w: int, cam_h: int, artifact_path: Path, behavior_version: str, chestnut: bool): + from openpilot.starpilot.models.compile import _detect_vision_keys, stateful_image_shapes, stateful_host_shapes + + artifact = load_oob(artifact_path, chestnut) + validate_artifact(artifact, behavior_version, (cam_w, cam_h)) + self.chestnut = chestnut + self.behavior_version = behavior_version + self.model_type = artifact["model_type"] + self.metadata = artifact["metadata"] + self.policy_order = artifact.get("policy_order", []) + self.frame_skip = artifact["frame_skip"] + self.image_history_pipeline = artifact["image_history_pipeline"] + self.warp_input_keys = tuple(artifact["warp_input_keys"]) + self.policy_input_keys = tuple(artifact["policy_input_keys"]) + self.run_policy = artifact["run_policy"] + self.warp_enqueue = artifact[(cam_w, cam_h)] + self.onnx_history = self.model_type == "supercombo" and bool(self.metadata["model"].get("state_pairs")) + self.queue_device = artifact["execution_device"] + self.warp_device = artifact["warp_device"] + if (not isinstance(self.queue_device, str) or not isinstance(self.warp_device, str) or + chestnut != (self.queue_device.rsplit("+", 1)[-1].split(":", 1)[0] == "AMD")): + raise ValueError("catalog model execution hardware mismatch") + if self.onnx_history: + metadata = self.metadata["model"] + image_shapes = stateful_image_shapes(metadata) + self.policy_input_shapes = stateful_host_shapes(metadata) + self.output_slices = metadata["output_slices"] + elif self.model_type == "supercombo": + image_shapes = self.policy_input_shapes = self.metadata["model"]["input_shapes"] + self.output_slices = self.metadata["model"]["output_slices"] + else: + image_shapes = self.metadata["vision"]["input_shapes"] + primary = "on_policy" if "on_policy" in self.policy_order else "policy" + self.policy_input_shapes = self.metadata[primary]["input_shapes"] + self.road_key, self.wide_key = _detect_vision_keys(image_shapes) + self.desire_key = next(key for key in self.policy_input_shapes if key.startswith("desire")) + self.parser = Parser() + self.aux_parser = Parser(ignore_missing=True) + self.frame_buf_size = get_nv12_info(cam_w, cam_h)[3] + image_layouts = tuple(tuple(image_shapes[key][-2:]) for key in (self.road_key, self.wide_key)) + self.camera_warp_descriptor = (CameraWarpDescriptor((cam_w, cam_h), get_nv12_info(cam_w, cam_h), image_layouts, + (2, 6, *image_layouts[0]), dtypes.uint8, self.warp_device) + if self.image_history_pipeline == "policy" else None) + self._reset_state() + + def _reset_state(self) -> None: + from openpilot.starpilot.models.compile import make_stateful_input_queues, make_supercombo_input_queues, make_split_input_queues + + if self.onnx_history: + self.input_queues, self.npy = make_stateful_input_queues(self.metadata["model"], self.queue_device) + elif self.model_type == "supercombo": + self.input_queues, self.npy = make_supercombo_input_queues(self.policy_input_shapes, self.frame_skip, self.queue_device) + else: + self.input_queues, self.npy = make_split_input_queues(self.metadata["vision"]["input_shapes"], self.policy_input_shapes, + self.frame_skip, self.queue_device) + self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32) + self._blob_cache: dict[tuple[str, int], Tensor] = {} + self.last_warp: SharedCameraWarp | None = None + self._last_shared_warp: SharedCameraWarp | None = None + + def _validate_shared_warp(self, shared: SharedCameraWarp, pointers: tuple[int, int], transforms: tuple[bytes, bytes]) -> None: + if (not isinstance(shared, SharedCameraWarp) or self.camera_warp_descriptor is None or + shared.descriptor != self.camera_warp_descriptor): + raise ValueError("catalog models have incompatible camera warp descriptors/history") + tensor = shared.tensor + if (tuple(tensor.shape) != shared.descriptor.output_shape or tensor.dtype != shared.descriptor.dtype or + tensor.device != shared.descriptor.device): + raise ValueError("catalog shared camera warp tensor contract mismatch") + producer = shared.producer() + if producer is None or producer.last_warp is not shared or shared is self._last_shared_warp: + raise ValueError("catalog shared camera warp is stale or already consumed") + if shared.frame_pointers != pointers or shared.transforms != transforms: + raise ValueError("catalog shared camera warp frame/transforms mismatch") + + @staticmethod + def slice_outputs(output: np.ndarray, sections: dict[str, slice]) -> dict[str, np.ndarray]: + return {key: output[np.newaxis, section].copy() for key, section in sections.items()} + + def _parse(self, outputs: list[np.ndarray]) -> dict[str, np.ndarray]: + if self.model_type == "supercombo": + parsed = self.parser.parse_outputs(self.slice_outputs(outputs[0], self.output_slices)) + if "prev_feat" in self.npy and "hidden_state" in self.output_slices: + self.npy["prev_feat"][:] = outputs[0][self.output_slices["hidden_state"]] + return parsed + vision_output, *policies = outputs + parsed = self.parser.parse_vision_outputs(self.slice_outputs(vision_output, self.metadata["vision"]["output_slices"])) + results = {} + for key, output in zip(self.policy_order, policies, strict=True): + slices = self.slice_outputs(output, self.metadata[key]["output_slices"]) + results[key] = self.aux_parser.parse_off_policy_outputs(slices) if key == "off_policy" else self.parser.parse_policy_outputs(slices) + for key in self.policy_order: + if key not in ("on_policy", "policy"): + parsed.update(results[key]) + parsed.update(results["on_policy" if "on_policy" in results else "policy"]) + return parsed + + def warmup(self) -> None: + frames = {name: np.zeros(self.frame_buf_size, dtype=np.uint8) for name in self.vision_input_names} + self._blob_cache.update({(name, frame.ctypes.data): Tensor(frame, device=self.warp_device).realize() + for name, frame in frames.items()}) + inputs = {"desire_pulse": np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)} + self.run(frames, dict.fromkeys(self.vision_input_names, np.eye(3, dtype=np.float32)), inputs) + self._reset_state() + + def run(self, bufs: dict, transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], + after_enqueue: Callable[[], None] | None = None, *, shared_warp: SharedCameraWarp | None = None) -> dict[str, np.ndarray]: + self.last_warp = None + pointers = tuple(np.frombuffer(bufs[name].data, dtype=np.uint8).ctypes.data for name in self.vision_input_names) + transform_signature = tuple(np.asarray(transforms[name], dtype=np.float32).tobytes() for name in self.vision_input_names) + if shared_warp is not None: + self._validate_shared_warp(shared_warp, pointers, transform_signature) + self._last_shared_warp = shared_warp + desire = inputs["desire_pulse"].copy() + desire[0] = 0 + self.npy["desire"][:] = np.where(desire - self.prev_desire > 0.99, desire, 0) + self.prev_desire[:] = desire + for name, value in self.npy.items(): + if name not in ("desire", "tfm", "big_tfm", "prev_feat") and name in inputs: + value[:] = inputs[name] + self.npy["tfm"][:] = transforms["img"] + self.npy["big_tfm"][:] = transforms["big_img"] + if shared_warp is None: + frames = {} + for name, pointer in zip(self.vision_input_names, pointers, strict=True): + key = (name, pointer) + if key not in self._blob_cache: + self._blob_cache[key] = Tensor.from_blob(pointer, (self.frame_buf_size,), dtype="uint8", device=self.warp_device) + frames[name] = self._blob_cache[key] + warped = self.warp_enqueue(**{key: self.input_queues[key] for key in self.warp_input_keys}, + frame=frames["img"], big_frame=frames["big_img"]) + else: + warped = shared_warp.tensor + policy_inputs = {key: self.input_queues[key] for key in self.policy_input_keys} + output_tensors = (self.run_policy(**policy_inputs, warped=warped) if self.image_history_pipeline == "policy" else + self.run_policy(**policy_inputs, img=warped[0], big_img=warped[1])) + if after_enqueue is not None: + after_enqueue() + outputs = [output.numpy().flatten() for output in output_tensors] + if not all(np.isfinite(output).all() for output in outputs): + raise ValueError("catalog model output not finite") + parsed = self._parse(outputs) + if not REQUIRED_OUTPUTS <= parsed.keys() or not all(np.isfinite(value).all() for value in parsed.values()): + raise ValueError("catalog model parsed outputs invalid") + if os.getenv("SEND_RAW_PRED"): + parsed["raw_pred"] = np.concatenate(outputs) + if self.camera_warp_descriptor is not None: + self.last_warp = (shared_warp if shared_warp is not None else + SharedCameraWarp(warped, self.camera_warp_descriptor, pointers, transform_signature, weakref.ref(self))) + return parsed diff --git a/openpilot/starpilot/models/runtime.py b/openpilot/starpilot/models/runtime.py new file mode 100644 index 0000000000..7ed03bfdca --- /dev/null +++ b/openpilot/starpilot/models/runtime.py @@ -0,0 +1,95 @@ +"""One read-only model snapshot adapter for native UI and authenticated Galaxy.""" + +from collections.abc import Callable +from pathlib import Path +import time +from typing import Any + +from openpilot.cereal.services import SERVICE_LIST +from openpilot.starpilot.models.catalog import CATALOG +from openpilot.starpilot.models.receipt import process_start_ticks, read_receipt +from openpilot.starpilot.models.status import ModelLoad, ModelOutput, ModelProcess, ModelStatus, project_status + + +MODEL_SERVICES = ("managerState", "modelV2", "drivingModelData") + + +def _stamp(sm: Any, service: str, now_mono_ns: int) -> tuple[int, bool]: + try: + stamp = sm.logMonoTime[service] + valid = (type(stamp) is int and 0 < stamp <= now_mono_ns and bool(sm.valid[service]) and + bool(sm.alive[service]) and now_mono_ns - stamp <= int(2e9 / SERVICE_LIST[service].frequency)) + return stamp if type(stamp) is int else 0, valid + except (AttributeError, KeyError, TypeError, ValueError, ZeroDivisionError): + return 0, False + + +def _process(sm: Any, now_mono_ns: int, start_ticks: Callable[[int], int]) -> ModelProcess | None: + if not _stamp(sm, "managerState", now_mono_ns)[1]: + return None + try: + processes = [state for state in sm["managerState"].processes if state.name == "modeld"] + if len(processes) != 1: + return None + state = processes[0] + pid = int(state.pid) + if not state.running or pid <= 0: + return None + return ModelProcess(pid, start_ticks(pid), True) + except (AttributeError, KeyError, TypeError, ValueError, OSError): + return None + + +def _output(sm: Any, params: Any, now_mono_ns: int) -> ModelOutput | None: + model_stamp, model_valid = _stamp(sm, "modelV2", now_mono_ns) + driving_stamp, driving_valid = _stamp(sm, "drivingModelData", now_mono_ns) + if model_stamp == 0 and driving_stamp == 0: + return None + try: + raw_chestnut = params.get("ChestnutActive") + chestnut = (raw_chestnut if type(raw_chestnut) is bool else + None if raw_chestnut not in (None, b"0", b"1") else raw_chestnut == b"1") + big = bool(sm["modelV2"].big) if model_stamp else False + return ModelOutput(model_stamp, model_valid, driving_stamp, driving_valid, big, chestnut) + except (AttributeError, KeyError, TypeError, ValueError, OSError): + return None + + +def snapshot(sm: Any, params: Any, now_mono_ns: int, *, requested_id: str | None = None, + receipt_file: Path | None = None, start_ticks: Callable[[int], int] = process_start_ticks, + read_load: Callable[[Path | None], ModelLoad | None] = read_receipt) -> ModelStatus: + """Sample one manager process and two live model outputs; never infer a load from Params.""" + process = _process(sm, now_mono_ns, start_ticks) + output = _output(sm, params, now_mono_ns) if process is not None else None + load = read_load(receipt_file) if process is not None else None + return project_status(requested_id, process, load, output, now_mono_ns) + + +class ModelStatusSource: + """Independent optional subscriber; it does not affect UI/global service health.""" + + def __init__(self, params: Any, sm: Any | None = None, clock: Callable[[], int] = time.monotonic_ns): + if sm is None: + from openpilot.cereal import messaging + sm = messaging.SubMaster(list(MODEL_SERVICES)) + self.sm = sm + self.params = params + self.clock = clock + + def snapshot(self) -> ModelStatus: + self.sm.update(0) + from openpilot.selfdrive.modeld.helpers import chestnut_present + from openpilot.starpilot.models.manager import requested_runtime_id + return snapshot(self.sm, self.params, self.clock(), requested_id=requested_runtime_id(chestnut_present())) + + def json(self) -> dict[str, object]: + status = self.snapshot() + return {"schemaVersion": 1, "catalog": [{"id": entry.model_id, "name": entry.name, + "selectable": entry.selectable} for entry in CATALOG], + "requestedId": status.requested_id, "loadedId": status.loaded_id, + "variant": status.variant.value if status.variant is not None else None, + "health": status.health.value, "fallbackReason": status.fallback_reason, + "artifactSha256": status.artifact_sha256, "pendingNextStart": status.pending_next_start} + + def close(self) -> None: + self.sm = None diff --git a/openpilot/starpilot/models/startup.py b/openpilot/starpilot/models/startup.py new file mode 100644 index 0000000000..050af0619c --- /dev/null +++ b/openpilot/starpilot/models/startup.py @@ -0,0 +1,90 @@ +"""Wait for vehicle and external GPU power before opening the GPU runtime.""" + +from dataclasses import dataclass +import time + +from openpilot.cereal import messaging +from openpilot.common.swaglog import cloudlog +from openpilot.common.hardware import HARDWARE + + +POWER_MIN_MV = 10_000 +POWER_STABLE_NS = 3_000_000_000 +POWER_MAX_AGE_NS = 500_000_000 +VEHICLE_MAX_AGE_NS = 250_000_000 + + +@dataclass +class PowerReadiness: + stable_since_ns: int = 0 + previous_ns: int = 0 + sample_max_age_ns: int = POWER_MAX_AGE_NS + + def update(self, now_ns: int, sample_ns: int, voltage_mv: int, vehicle_ready: bool) -> bool: + continuous = self.previous_ns == 0 or 0 <= now_ns - self.previous_ns <= POWER_MAX_AGE_NS + self.previous_ns = now_ns + healthy = (0 < sample_ns <= now_ns and now_ns - sample_ns <= self.sample_max_age_ns and + voltage_mv >= POWER_MIN_MV and vehicle_ready) + if not continuous or not healthy: + self.stable_since_ns = 0 + if not healthy: + return False + if self.stable_since_ns == 0: + self.stable_since_ns = now_ns + return now_ns - self.stable_since_ns >= POWER_STABLE_NS + + +def paired_clocks_ns() -> tuple[int, int] | None: + before = time.monotonic_ns() + clock = getattr(time, 'CLOCK_BOOTTIME', None) + boot = time.clock_gettime_ns(clock) if clock is not None else before + after = time.monotonic_ns() + return ((before + after) // 2, boot) if 0 <= after - before <= 1_000_000 else None + + +def current_drive(sm, now_ns: int) -> bool: + device = sm['deviceState'] + stamp = sm.logMonoTime['deviceState'] + return (sm.seen['deviceState'] and sm.valid['deviceState'] and device.started and + 0 < device.startedMonoTime <= stamp <= now_ns <= stamp + 2_000_000_000) + + +def supply_voltage(states) -> int: + known = [state for state in states if str(state.pandaType) != 'unknown'] + return known[0].voltage if len(known) == 1 else 0 + + +def wait_for_chestnut_power(CP, timeout: float = 30.) -> None: + ready_bus = None + if CP.brand == "hyundai": + from opendbc.car.hyundai.model_startup import model_power_ready_bus, model_power_ready + ready_bus = model_power_ready_bus(CP) + + power_service = "peripheralState" if HARDWARE.get_device_type() == "tici" else "pandaStates" + sm = messaging.SubMaster([power_service, "deviceState"] + (["can"] if ready_bus is not None else [])) + readiness = PowerReadiness(sample_max_age_ns=1_000_000_000 if power_service == "peripheralState" else POWER_MAX_AGE_NS) + vehicle_ready, vehicle_sample_ns = ready_bus is None, 0 + started = time.monotonic() + cloudlog.info("waiting for Chestnut power stability") + while time.monotonic() - started < timeout: + sm.update(100) + clocks = paired_clocks_ns() + if clocks is None: + readiness.stable_since_ns = 0 + continue + now_mono_ns, now_ns = clocks + drive_ready = current_drive(sm, now_mono_ns) + if sm.seen["deviceState"] and not drive_ready: + raise RuntimeError("drive ended while waiting for Chestnut power") + if ready_bus is not None and sm.updated["can"]: + ready = model_power_ready(sm["can"], ready_bus) if sm.valid["can"] else False + if ready is not None: + vehicle_ready, vehicle_sample_ns = ready, sm.logMonoTime["can"] + fresh_vehicle = ready_bus is None or 0 < vehicle_sample_ns <= now_ns and now_ns - vehicle_sample_ns <= VEHICLE_MAX_AGE_NS + states = [sm[power_service]] if power_service == "peripheralState" else sm[power_service] + voltage = supply_voltage(states) + if readiness.update(now_ns, sm.logMonoTime[power_service] if sm.valid[power_service] else 0, + voltage, drive_ready and vehicle_ready and fresh_vehicle): + cloudlog.info("Chestnut power stable; starting GPU load") + return + raise TimeoutError("Chestnut power or vehicle READY did not stabilize") diff --git a/openpilot/starpilot/models/status.py b/openpilot/starpilot/models/status.py new file mode 100644 index 0000000000..bbc2d02b9c --- /dev/null +++ b/openpilot/starpilot/models/status.py @@ -0,0 +1,113 @@ +"""Pure projection of requested model, loaded identity, and fresh modeld output.""" + +from dataclasses import dataclass +from enum import StrEnum + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, model_entries, resolve_selection + + +OUTPUT_MAX_AGE_NS = 250_000_000 + + +class ModelHealth(StrEnum): + UNAVAILABLE = "unavailable" + LOADING = "loading" + IDENTITY_UNAVAILABLE = "identity-unavailable" + ACTIVE = "active" + STALE = "stale" + FAILED = "failed" + + +class ModelVariant(StrEnum): + SMALL = "small" + CHESTNUT = "chestnut" + + +@dataclass(frozen=True) +class ModelProcess: + pid: int + start_ticks: int + running: bool + + +@dataclass(frozen=True) +class ModelLoad: + """Identity recorded only after ModelState has loaded and initialized a runner.""" + pid: int + process_start_ticks: int + loaded_mono_ns: int + model_id: str + variant: ModelVariant + artifact_sha256: str + fallback_reason: str | None = None + + +@dataclass(frozen=True) +class ModelOutput: + model_v2_mono_ns: int + model_v2_valid: bool + driving_data_mono_ns: int + driving_data_valid: bool + model_v2_big: bool + chestnut_active: bool | None + + +@dataclass(frozen=True) +class ModelStatus: + requested_id: str + loaded_id: str | None + variant: ModelVariant | None + health: ModelHealth + pending_next_start: bool + fallback_reason: str | None + artifact_sha256: str | None + + +def project_status(requested_id: str | None, process: ModelProcess | None, load: ModelLoad | None, + output: ModelOutput | None, now_mono_ns: int) -> ModelStatus: + """Keep saved intent distinct from actual modeld identity and output health. + + The caller must obtain process and load records from the same owner. A stale + load receipt cannot make a new modeld process appear active. An output older + than the load boundary cannot establish health after a restart. + """ + entries = model_entries() + requested = resolve_selection(requested_id).model_id + if not isinstance(now_mono_ns, int) or isinstance(now_mono_ns, bool) or now_mono_ns <= 0: + raise ValueError("invalid model-status clock") + + if process is None or not process.running or process.pid <= 0: + return ModelStatus(requested, None, None, ModelHealth.UNAVAILABLE, False, None, None) + if process.start_ticks <= 0: + return ModelStatus(requested, None, None, ModelHealth.IDENTITY_UNAVAILABLE, False, None, None) + output_is_fresh = (output is not None and output.model_v2_valid and output.driving_data_valid and + all(isinstance(stamp, int) and not isinstance(stamp, bool) and 0 < stamp <= now_mono_ns and + now_mono_ns - stamp <= OUTPUT_MAX_AGE_NS + for stamp in (output.model_v2_mono_ns, output.driving_data_mono_ns))) + if load is None: + health = ModelHealth.IDENTITY_UNAVAILABLE if output_is_fresh else ModelHealth.LOADING + return ModelStatus(requested, None, None, health, False, None, None) + if (load.pid != process.pid or load.process_start_ticks != process.start_ticks or + load.loaded_mono_ns <= 0 or load.loaded_mono_ns > now_mono_ns or + load.model_id not in entries or not isinstance(load.variant, ModelVariant) or + not isinstance(load.artifact_sha256, str) or len(load.artifact_sha256) != 64 or + any(c not in "0123456789abcdef" for c in load.artifact_sha256)): + health = ModelHealth.IDENTITY_UNAVAILABLE if output_is_fresh else ModelHealth.LOADING + return ModelStatus(requested, None, None, health, False, None, None) + + if load.model_id != BUNDLED_CURRENT and entries[load.model_id].uses_external_gpu != (load.variant is ModelVariant.CHESTNUT): + return ModelStatus(requested, None, None, ModelHealth.IDENTITY_UNAVAILABLE, False, None, None) + pending = requested != load.model_id + common = (requested, load.model_id, load.variant, pending, load.fallback_reason, load.artifact_sha256) + if output is None: + return ModelStatus(common[0], common[1], common[2], ModelHealth.LOADING, *common[3:]) + stamps = (output.model_v2_mono_ns, output.driving_data_mono_ns) + if (any(not isinstance(stamp, int) or isinstance(stamp, bool) or stamp < load.loaded_mono_ns or stamp > now_mono_ns + for stamp in stamps) or + not output.model_v2_valid or not output.driving_data_valid): + return ModelStatus(common[0], common[1], common[2], ModelHealth.STALE, *common[3:]) + if any(now_mono_ns - stamp > OUTPUT_MAX_AGE_NS for stamp in stamps): + return ModelStatus(common[0], common[1], common[2], ModelHealth.STALE, *common[3:]) + if output.model_v2_big != (load.variant is ModelVariant.CHESTNUT): + return ModelStatus(common[0], common[1], common[2], ModelHealth.FAILED, *common[3:]) + return ModelStatus(common[0], common[1], common[2], ModelHealth.ACTIVE, *common[3:]) diff --git a/openpilot/starpilot/models/tests/__init__.py b/openpilot/starpilot/models/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json b/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json new file mode 100644 index 0000000000..7e0fa7b4f1 --- /dev/null +++ b/openpilot/starpilot/models/tests/fixtures/laboratory_outputs_original.json @@ -0,0 +1,111 @@ +{ + "original_source_sha256": "f0210b2ca26a1b9c4adbcf5825d0d01851254dbfbdbc54595c16d9d77d71731f", + "generator": "Original AST-selected pure composition; explicit longitudinal current-frame source; normalized float32 plan (1,33,15), action decoded separately.", + "cases": [ + { + "versions": [ + "v8", + "v9" + ], + "lateral_offset": 10, + "longitudinal_offset": 20, + "expected": { + "desired_curvature": "c2600e1170a6c8870bfa1db230512017ce5569d52a67906d2bd8c70edf57d4ed", + "desired_curvature_stds": "00c86d4ebf27bafc31acbbcd4aacae39e1642e5726d1b27858cfe6b127a65022", + "lat_planner_solution": "9d10c66e47cb0fc52a36e7e0be9497177f8df9d82fc6605a00fe73e147434dec", + "lat_planner_solution_stds": "d2046121607eb00f881f35e99e14a5d3164e550160b3f9eae5ef74d4d184cc48", + "lane_lines": "b1fa5fcbd1577014f692d8e633d7fb70f1dcd8ec95b3c4a6b734bd6e17f90d5c", + "lane_lines_stds": "4c0dcf842694d757c8200ba201d437506e54c4643cfa32f2511f0d9dfa3960ef", + "lane_lines_prob": "0374f5e2a29b996ebdb7fe2bd452ccd3b55290dc71fb5b3843d577f44655f9f3", + "road_edges": "a1177c7c6de49027ff21ad2fd78edbeef618a3000e1fff4c231074853b50180f", + "road_edges_stds": "d41173b5c2be5672a0abca35f34e0aac8cf2a6ff8ccdc472c88e00b45d271230", + "desire_state": "54a684f78cccd3f5bf7cdcfba9a25d2e1e979edf2a900f9064371bbe7ff12eaf", + "desire_pred": "93253864fb1cc58f3670d912c108b2c75165dd41f71deeead539c22b5bbafca9", + "pose": "db3ee5a345d0834277ff12d708791943e1ab9b33f829c6491789db5dd176fc9e", + "pose_stds": "e72a0a1259c6dfaa33587bbbb131a9ed8a5b1d8a7892a35973dfb6aa50dc341c", + "wide_from_device_euler": "e369af615af989f296ef62a9582e7c4bbb80e4a568d0b69ed93dfee0b7fb03b9", + "wide_from_device_euler_stds": "9dd8a25b398d69f4a2f5a14d16f2224918ab1a329d0c0b7fae3355a26247dd9e", + "road_transform": "45af797502186a93eb4632248488571f4c7d8e9d3a09febb4d96deee39a1e65e", + "road_transform_stds": "39d3270cfe99fb33d907c7a37451cdde0e2fa3f38683ceecd1a46ff1f447cd1b", + "lead": "66bfc2d5b98423fe06d351f687bef88be82f7119c409db82da3c87105c465fc6", + "lead_prob": "e45bb2ec3e63950b3e464a45bb8171c1209393c458e4991028ea42620f88b607", + "meta": "2befc666fccb379e084e3feed7c461eb095c2277b91733ae62333052a0be7196", + "planplus": "26d4b5717d6a9ed1a987c23e9cf779cfffee11e996be50688d90de9de49bace7", + "raw_pred": "51127f644b8fcf6b45f756098619960e20674e6dc726856e4c20da834253575a", + "unknown_extra": "19b7e0870c270281002ac440b752780a1ceba740b30bb62effca8d01b7b7ecbb", + "plan": "ca530c67c1613fe1cf9224ea8ed125fcdfe180da62341a181f84547ec17c3862", + "plan_stds": "f55f9fca10f6f0223c16d207242748ee3ac8bee75900f6703dae824e3387186b" + } + }, + { + "versions": [ + "v14", + "v15" + ], + "lateral_offset": 1000, + "longitudinal_offset": 2000, + "expected": { + "desired_curvature": "48270c20e131431ad33afbf76bae7286fbf4f0ad55373a4624f6957c0a7e5bd3", + "desired_curvature_stds": "7f54928b9e681540bf154d5e9c14f57da0d3e5b9654d5c3d573243d5ab168ac0", + "lat_planner_solution": "40b20dd85498461a7eb26e813eafc09ef8b784dd26f485f7a481e304d7f0614f", + "lat_planner_solution_stds": "acf086207faa6273da611964f94906db6696d2fd463a0bcfa4c650f9d80d2c6d", + "lane_lines": "a4eb06d857b4da55875b9f3455244d443abff2262da5bff5fb89ae056ba42214", + "lane_lines_stds": "41e3303c4a9f97ec24c23921c01adab75b0ad7ad3b71ba8d1412c289a87d5b75", + "lane_lines_prob": "6f8db4b04211c6c31bca3665e64193083078f673a0e3cbe6c3a8c473a2afe7a6", + "road_edges": "e43108080f9f9fab9d7729b018a50b0a60d1d1aff6a68d69320a91c30d29ac7a", + "road_edges_stds": "afa058f368a910c1a0124182a74f757ae28e55ef1ec1c3deffd6cba058070283", + "desire_state": "255106e4dfc96b87fe4f9f4738785df0d7ace598f89ee235222cf425f46cddd6", + "desire_pred": "30d67c7cc4f201e7b297b17f5f5d40ac9d538fc86003c1ef9939f2da43cad756", + "pose": "44478d7cfb3996843e47d12865b343435dceea321e4de85127c03e80143434c0", + "pose_stds": "9603324367b63793a975a24ab3db4356d339a0b47baea3544b3c4d760e2afcf3", + "wide_from_device_euler": "374258046cf62cce54400b1d0b2fc7fc4b454d54edb75a6767d31ee791981788", + "wide_from_device_euler_stds": "0e90893b1e358b79b82cb35ac987603190272e19a556ae84d56f4856f1744219", + "road_transform": "5bb150e05e927f66f67c1a45674521f9fb835e87f51a81c3367ef77820d337ce", + "road_transform_stds": "1f5a36d32fc56a69d52b66b3d341351c9786b61eda6002eb7a5f0cd049f99ee0", + "lead": "41ef548226148429978482c46219c6ead8b7f64eec5b4e0f26ab99cb79a7c524", + "lead_prob": "91ac1f3d1d11cc5e9a0cd33fcad703d177dd539cdfb2ba649241e1c2496d5ce0", + "meta": "fac9d42caf83ae32a0091f914b7b7cfa9be839f96675b64c5dd916698c5a6c19", + "planplus": "35eb89634463c1d99b1f6deaae2cb67b33512fe2798145c52e834c3c7932f51f", + "raw_pred": "6725fcf159c53c6e7896eeaa2a8cf1cb04d5e09e01438338678cbae35ddcfb6a", + "unknown_extra": "55f8e7ccecf5c95d544a093099e53cbd8e409942b6b73470d196beafd1f7b9a5", + "plan": "6d71aaf15c9646d847ddb04b2fb2718455ff998107ec111c54bc5575735a1ec3", + "plan_stds": "fa7d0ef0cad07421cb47b3d4c470238bda21673a59f3ed4653ab5e2ddd224118" + } + }, + { + "versions": [ + "v15", + "v16" + ], + "lateral_offset": -1000, + "longitudinal_offset": 1000, + "expected": { + "desired_curvature": "8e296dfc73dec27c9ae57442a6ef354b212d45a924187b11468316dae3984967", + "desired_curvature_stds": "d4c04057be63e1a687d71501cc123f7250e26cbbc7005dc181c05230c9cd089a", + "lat_planner_solution": "494e83454e5201015d521620cde3ff32e1d684963ae210ee995e4ea1d7ade207", + "lat_planner_solution_stds": "76ddcd6935a84860444f86c0e581153ed943f7b604791876ceee12fed6d43c3f", + "lane_lines": "316774c6fd093921c6862f6c0ca17ee1e97d2d061c061f9a424c7d72742bf4d7", + "lane_lines_stds": "cf2d8052f3a6fc55c9adc939b34207d1fab0a9b4906bb61dbeb74a2dcd22cb85", + "lane_lines_prob": "50f3cf87d50de3015734b51878f1c2e3025716fc5f763dc6ff9c06e028cee335", + "road_edges": "085b0e6d88f419906362cd94347b8d7c410d1af45a3b833a3860b299616c44df", + "road_edges_stds": "7df3713221372e1b5c4608b259e4127ee96d221d1aea7611e82b823ce0b11e8d", + "desire_state": "4ca63b15b26aedeafdd39ec2c2c00a3b8ff0a56b139ea7a8d07c7cd75544ff4a", + "desire_pred": "fa02d18f4158f326c6621437a79b95b7db666fedcaf7bbdca0ad8327ee57cdeb", + "pose": "45a99de0255b714a798c4d63f607c4d562a273d2a6587414a97c5fa3ed127b8c", + "pose_stds": "ddbc8c0795480fab91d495a81558ddcf9ce5852e9911edfeaa2fbd1a6cc0395f", + "wide_from_device_euler": "7e4a1fc996517d862e43dab9d5b38b32283726bc5cf1b88263fc6afd79cce0ba", + "wide_from_device_euler_stds": "c40c85d37bb3c50b863a25ad66e2342fbaab420e83a004ca1477f1052fec7d30", + "road_transform": "e234d71da99b30e8c1ca0c8ef8d4887785ecc8cffdd54aa13863de2051305799", + "road_transform_stds": "e5aecc961bf2b500458cae3023aa5d59aecefac735fc72b1ea98fae332383ff8", + "lead": "3db9b14a5f25b121c02137faff7b00d8263b99a267e017a4e35da2a8e9a728cb", + "lead_prob": "f9ee34aa05c8ee301ab80eb5303717023e4e1727992df63a81510d6e8e9b9a20", + "meta": "8e2bcca09d503d7472c56bf9f0c5345a40cbe055edde30db17f6eb705e245cd6", + "planplus": "24dcdc5ff3867ed237ad56915c7e9d1cf56db727a5787af327b0d7742d4629dd", + "raw_pred": "7bb224457c483a72c156f4b2705fabdaa305da839428baa00900a1f2df911388", + "unknown_extra": "9c9f0ed46eb94c2c6f896f0838eee44891aa4c8a938df4c544c80de400921ce2", + "plan": "efe95619a7b0efc4ab4e8975c49cf9cf883b8ee1994219f3e30b1d2d3b844b8f", + "plan_stds": "159b92b6f0d62985827f59be459a8c4a32fc828b705cade66fe14180fe4e633e" + } + } + ] +} diff --git a/openpilot/starpilot/models/tests/test_artifact_isolation.py b/openpilot/starpilot/models/tests/test_artifact_isolation.py new file mode 100644 index 0000000000..1df2cd2fb5 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_artifact_isolation.py @@ -0,0 +1,81 @@ +from itertools import combinations +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +import numpy as np +from tinygrad import Context, Tensor, TinyJit +from tinygrad.uop.ops import Ops +from tinygrad_repo.examples.openpilot.helpers import dump_pickle + +from openpilot.selfdrive.modeld.helpers import load_oob + + +def stateful_artifact(path, seed): + state = Tensor([seed, seed + 1], device="CPU").realize() + weight = Tensor([2., 3.], device="CPU").realize() + + @TinyJit + def step(x): + state.assign(state + x * weight).realize() + return state + + x = Tensor([1., 1.], device="CPU").realize() + for _ in range(3): + step(x) + artifact = {"state": state, "weight": weight, "step": step} + dump_pickle(artifact, path) + return artifact + + +def captured_buffers(artifact): + return {u.arg.buffer for u in artifact["step"].captured._linear.toposort() if u.op is Ops.BUFFER and u.arg.buffer is not None} + + +class TestArtifactIsolation(unittest.TestCase): + def setUp(self): + self.context = Context(DEV="CPU:LLVM") + self.context.__enter__() + self.addCleanup(self.context.__exit__, None, None, None) + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.paths = [Path(self.directory.name) / name for name in ("lateral.pkl", "longitudinal.pkl")] + self.originals = [stateful_artifact(path, seed) for path, seed in zip(self.paths, (1., 100.), strict=True)] + self.loads = [self.load(self.paths[0]), self.load(self.paths[1]), self.load(self.paths[0])] + + @staticmethod + def load(path): + # Exercise the production loader's CPU branch without opening Metal or any external device. + with patch("openpilot.selfdrive.modeld.helpers.sys.platform", "linux"), patch("openpilot.selfdrive.modeld.helpers.AGNOS", False): + return load_oob(path) + + def test_separate_load_arenas_and_captured_buffers_with_originals_alive(self): + artifacts = self.originals + self.loads + for artifact in self.loads: + self.assertIs(artifact["state"].uop.buffer.base, artifact["weight"].uop.buffer.base) + self.assertIsNot(artifact["state"].uop.buffer, artifact["weight"].uop.buffer) + for first, second in combinations(artifacts, 2): + self.assertIsNot(first["state"].uop.buffer.base, second["state"].uop.buffer.base) + self.assertTrue(captured_buffers(first).isdisjoint(captured_buffers(second))) + np.testing.assert_array_equal(self.loads[0]["state"].numpy(), self.loads[2]["state"].numpy()) + + def test_interleaved_execution_reset_and_repeat_load_do_not_change_other_state(self): + expected = [artifact["state"].numpy().copy() for artifact in self.loads] + originals = [artifact["state"].numpy().copy() for artifact in self.originals] + for index, scale in ((0, 1.), (1, 2.), (0, 3.), (2, 4.), (1, 5.)): + result = self.loads[index]["step"](Tensor([scale, scale], device="CPU").realize()) + expected[index] += np.array([2., 3.], dtype=np.float32) * scale + np.testing.assert_array_equal(result.numpy(), expected[index]) + for artifact, state in zip(self.loads, expected, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) + self.loads[0]["state"].uop.buffer.copy_from(Tensor(np.zeros(2, dtype=np.float32), device="CPU").realize().uop.buffer) + expected[0][:] = 0 + self.loads[0]["step"](Tensor([1., 1.], device="CPU").realize()) + expected[0] += [2., 3.] + fresh = self.load(self.paths[0]) + np.testing.assert_array_equal(fresh["state"].numpy(), originals[0]) + for artifact, state in zip(self.loads, expected, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) + for artifact, state in zip(self.originals, originals, strict=True): + np.testing.assert_array_equal(artifact["state"].numpy(), state) diff --git a/openpilot/starpilot/models/tests/test_catalog_status.py b/openpilot/starpilot/models/tests/test_catalog_status.py new file mode 100644 index 0000000000..1e7cefff85 --- /dev/null +++ b/openpilot/starpilot/models/tests/test_catalog_status.py @@ -0,0 +1,67 @@ +import hashlib +from pathlib import Path +import unittest + +from openpilot.starpilot.models.catalog import BUNDLED_CURRENT, CATALOG, resolve_selection +from openpilot.starpilot.models.status import (ModelHealth, ModelLoad, ModelOutput, ModelProcess, + ModelVariant, project_status) + + +class TestModelCatalogStatus(unittest.TestCase): + def test_catalog_pins_current_source_and_lists_model_metadata(self): + self.assertEqual(len(CATALOG), 100) + entry = resolve_selection(None) + self.assertEqual(entry.model_id, BUNDLED_CURRENT) + source = Path(__file__).resolve().parents[4] / entry.source_path + self.assertEqual(hashlib.sha256(source.read_bytes()).hexdigest(), entry.source_sha256) + self.assertEqual(resolve_selection("rdf43").version, "v15") + with self.assertRaises(ValueError): + resolve_selection("unknown-model") + with self.assertRaises(ValueError): + resolve_selection("") + + def test_loaded_identity_requires_same_process_and_fresh_both_outputs(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + self.assertEqual(project_status(None, process, load, output, 1_200_000_000).health, ModelHealth.ACTIVE) + self.assertEqual(project_status(None, ModelProcess(422, 5001, True), load, output, + 1_200_000_000).health, ModelHealth.IDENTITY_UNAVAILABLE) + self.assertEqual(project_status(None, process, load, output, 1_500_000_001).health, ModelHealth.STALE) + self.assertEqual(project_status(None, process, load, ModelOutput(1_050_000_000, True, 1_185_000_000, + True, False, False), 1_200_000_000).health, + ModelHealth.STALE) + self.assertEqual(project_status(None, process, load, ModelOutput(1_180_000_000, True, 1_185_000_000, + True, True, True), 1_200_000_000).health, + ModelHealth.FAILED) + + def test_runtime_stall_preserves_requested_big_and_active_small_identity(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, 'sc23', ModelVariant.SMALL, 'a' * 64, 'chestnut-run-stalled') + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + status = project_status('cinquev3', process, load, output, 1_200_000_000) + self.assertEqual((status.requested_id, status.loaded_id, status.health), ('cinquev3', 'sc23', ModelHealth.ACTIVE)) + self.assertTrue(status.pending_next_start) + self.assertEqual(status.fallback_reason, 'chestnut-run-stalled') + + def test_catalog_selection_preserves_actual_fallback_identity(self): + process = ModelProcess(421, 5000, True) + load = ModelLoad(421, 5000, 1_100_000_000, "sc23", ModelVariant.SMALL, "a" * 64, "chestnut-load-failed") + output = ModelOutput(1_180_000_000, True, 1_185_000_000, True, False, False) + status = project_status("cinquev3", process, load, output, 1_200_000_000) + self.assertEqual((status.requested_id, status.loaded_id, status.health), ("cinquev3", "sc23", ModelHealth.ACTIVE)) + self.assertTrue(status.pending_next_start) + self.assertEqual(status.fallback_reason, "chestnut-load-failed") + impossible = ModelLoad(421, 5000, 1_100_000_000, "cinquev3", ModelVariant.SMALL, "a" * 64) + self.assertEqual(project_status("cinquev3", process, impossible, output, 1_200_000_000).health, ModelHealth.IDENTITY_UNAVAILABLE) + + def test_no_saved_choice_or_stale_receipt_claims_active(self): + process = ModelProcess(421, 5000, True) + self.assertEqual(project_status(None, None, None, None, 1_200_000_000).health, ModelHealth.UNAVAILABLE) + self.assertEqual(project_status(None, process, None, None, 1_200_000_000).health, ModelHealth.LOADING) + stale_load = ModelLoad(421, 4999, 1_100_000_000, BUNDLED_CURRENT, ModelVariant.SMALL, "a" * 64) + self.assertEqual(project_status(None, process, stale_load, None, 1_200_000_000).health, ModelHealth.LOADING) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/models/tests/test_compile.py b/openpilot/starpilot/models/tests/test_compile.py new file mode 100644 index 0000000000..3f888f8eda --- /dev/null +++ b/openpilot/starpilot/models/tests/test_compile.py @@ -0,0 +1,247 @@ +import importlib.util +import base64 +import json +import os +from pathlib import Path +import pickle +import subprocess +import sys +import tempfile +import unittest +from functools import partial +from unittest.mock import patch + +import numpy as np + +_script_path = Path(__file__).resolve().parents[4] / "scripts" / "model_compiler.py" +_script_spec = importlib.util.spec_from_file_location("starpilot_test_model_compiler", _script_path) +assert _script_spec is not None and _script_spec.loader is not None +model_compiler = importlib.util.module_from_spec(_script_spec) +_script_spec.loader.exec_module(model_compiler) + + +def varint(value): + result = bytearray() + while value > 127: + result.append((value & 127) | 128) + value >>= 7 + return bytes(result) + bytes([value]) + + +def field(number, value): + if isinstance(value, int): + return varint(number << 3) + varint(value) + if isinstance(value, str): + value = value.encode() + return varint((number << 3) | 2) + varint(len(value)) + value + + +def value_info(name, shape): + dimensions = b"".join(field(1, field(1, dimension)) for dimension in shape) + return field(1, name) + field(2, field(1, field(1, 1) + field(2, dimensions))) + + +def stateful_onnx(): + inputs = {"new_img": (2, 6, 2, 2), "state": (1, 2), "desire": (1, 2)} + add = field(1, "state") + field(1, "desire") + field(2, "next_state") + field(4, "Add") + identity = field(1, "next_state") + field(2, "outputs") + field(4, "Identity") + graph = field(1, add) + field(1, identity) + field(2, "stateful-fixture") + graph += b"".join(field(11, value_info(name, shape)) for name, shape in inputs.items()) + graph += field(12, value_info("outputs", (1, 2))) + field(12, value_info("next_state", (1, 2))) + metadata = field(1, "output_slices") + field(2, base64.b64encode(pickle.dumps({"fixture": slice(0, 2)}))) + return field(1, 8) + field(7, graph) + field(8, field(2, 13)) + field(14, metadata) + + +class CompileTest(unittest.TestCase): + def test_legacy_split_and_supercombo_history_survive_arena_round_trip(self): + from tinygrad import Context, TinyJit + from openpilot.starpilot.models.compile import (compile_jit, make_random_images, make_run_split_policy, make_run_supercombo, + make_split_input_queues, make_supercombo_input_queues, FAST_POLICY_INPUTS) + image_shapes = {"input_imgs": (1, 12, 2, 2), "big_input_imgs": (1, 12, 2, 2)} + policy_shapes = {"features_buffer": (1, 3, 2), "desire": (1, 2, 2), "traffic_convention": (1, 2)} + def vision(inputs): + values = [inputs[name].cast("float32").mean().reshape(1, 1) for name in image_shapes] + return {"outputs": values[0].cat(values[1], dim=1)} + def policy(inputs): + return {"outputs": inputs["features_buffer"].sum(axis=1) + inputs["desire"].sum(axis=1) + inputs["traffic_convention"]} + def supercombo(inputs): + return {"outputs": policy(inputs)["outputs"] + vision(inputs)["outputs"]} + with Context(DEV="CPU:LLVM", IMAGE=0, FLOAT16=0, JIT_BATCH_SIZE=0): + for kind in ("split", "supercombo"): + with self.subTest(kind=kind): + if kind == "split": + metadata = {"vision": {"input_shapes": image_shapes, "output_slices": {"hidden_state": slice(0, 2)}}, + "policy": {"input_shapes": policy_shapes}} + run = make_run_split_policy(vision, {"policy": policy}, metadata, ["policy"], 2) + queues = partial(make_split_input_queues, image_shapes, policy_shapes, 2) + else: + metadata = {"model": {"input_shapes": image_shapes | policy_shapes}} + run = make_run_supercombo(supercombo, metadata, 2) + queues = partial(make_supercombo_input_queues, image_shapes | policy_shapes, 2) + compile_jit(TinyJit(run, prune=True), partial(make_random_images, ["warped"], (2, 6, 2, 2)), + FAST_POLICY_INPUTS, queues, validation_runs=5) + + def test_mapped_stateful_image_queue_survives_jit_reload_and_reset(self): + from types import SimpleNamespace + from tinygrad import Context, Device, Tensor, TinyJit + from tinygrad.dtype import dtypes + from openpilot.starpilot.models.compile import compile_jit, make_run_stateful_supercombo, make_stateful_input_queues + + metadata = {"input_shapes": {"new_img": (2, 6, 4, 8), "action_t": (1, 2), "history": (1, 2)}, + "state_pairs": {"history": "next_history"}, "input_dtypes": {"history": "float32"}, + "mapped_image_input": True} + + class Model: + graph_inputs = {"new_img": SimpleNamespace(dtype=dtypes.uint8), "action_t": SimpleNamespace(dtype=dtypes.float32)} + + def __call__(self, inputs): + state = inputs["history"] + inputs["new_img"].float().mean() + inputs["action_t"] + return {"outputs": state, "next_history": state} + + with Context(DEV="CPU:LLVM", IMAGE=0, FLOAT16=0, JIT_BATCH_SIZE=0): + queues, _ = make_stateful_input_queues(metadata, Device.DEFAULT) + image_buffer = queues["mapped_gpu_image"].uop.base.buffer + self.assertTrue(image_buffer.options.cpu_access) + self.assertIsNotNone(image_buffer.get_storage().host) + run = make_run_stateful_supercombo(Model(), metadata) + with self.assertRaisesRegex(ValueError, "Mapped stateful image queue is missing"): + run(Tensor(np.zeros((2, 6, 4, 8), dtype=np.uint8)), queues["packed_npy_inputs"], history=queues["history"]) + compile_jit(TinyJit(run, prune=True), + lambda rng: {"warped": Tensor(rng.integers(0, 256, (2, 6, 4, 8), dtype=np.uint8), device="NPY").realize()}, + ("packed_npy_inputs", "mapped_gpu_image", "history"), + partial(make_stateful_input_queues, metadata), validation_runs=3) + + def test_legacy_stateful_queue_layout_and_values_remain_unchanged(self): + from tinygrad import Context, Device + from openpilot.starpilot.models.compile import make_stateful_input_queues + metadata = {"input_shapes": {"new_img": (2, 6, 4, 8), "action_t": (1, 2), "history": (1, 2)}, + "state_pairs": {"history": "next_history"}, "input_dtypes": {"history": "float32"}} + with Context(DEV="CPU:LLVM"): + queues, host = make_stateful_input_queues(metadata, Device.DEFAULT) + self.assertEqual(tuple(queues), ("img_q", "big_img_q", "tfm", "big_tfm", "packed_npy_inputs", "history")) + self.assertEqual(tuple(host), ("tfm", "big_tfm", "action_t")) + expected_shapes = {"img_q": (1, 6, 4, 8), "big_img_q": (1, 6, 4, 8), "tfm": (3, 3), "big_tfm": (3, 3), + "packed_npy_inputs": (2,), "history": (1, 2)} + for name, value in queues.items(): + with self.subTest(queue=name): + self.assertEqual(value.shape, expected_shapes[name]) + self.assertFalse(value.uop.base.buffer.options.cpu_access) + np.testing.assert_array_equal(value.numpy(), np.zeros(expected_shapes[name])) + for value in host.values(): + np.testing.assert_array_equal(value, np.zeros(value.shape)) + + def test_out_of_band_pickle_without_default_device_buffers(self): + import io + import struct + from tinygrad import Context, Tensor + from tinygrad_repo.examples.openpilot.helpers import dump_pickle, load_pickle + + with Context(DEV="CPU:LLVM"), tempfile.TemporaryDirectory() as temporary: + path = Path(temporary) / "fixture.pkl" + dump_pickle({"metadata": (1, "warp")}, path) + self.assertEqual(load_pickle(path, out_of_band=True), {"metadata": (1, "warp")}) + values = np.arange(12, dtype=np.float32).reshape(2, 6) + dump_pickle({"other_device": Tensor(values, device="NPY").realize()}, path) + np.testing.assert_array_equal(load_pickle(path, out_of_band=True)["other_device"].numpy(), values) + dump_pickle({"default_device": Tensor(values).realize()}, path) + np.testing.assert_array_equal(load_pickle(path, out_of_band=True)["default_device"].numpy(), values) + + opcodes = io.BytesIO() + marker = object() + class ReferencingPickler(pickle.Pickler): + def persistent_id(self, value): + return (1, "invalid", 0) if value is marker else None + pickler = ReferencingPickler(opcodes) + pickler.dump(marker) + path.write_bytes(struct.pack("= 16) & (y[:height, :width] <= 235))) + self.assertTrue(np.all(y[height:] == 16)) + self.assertTrue(np.all(uv[:height // 2, :width:2] == 147)) + self.assertTrue(np.all(uv[:height // 2, 1:width:2] == 159)) + + def test_missing_nonfinite_wrong_shape_and_frozen_outputs_fail(self): + for field, value in (("plan", np.full((1, 33, 15), np.nan)), ("lead", np.zeros((1, 2, 6, 4)))): + with self.subTest(field=field), self.assertRaises(ValueError): + validator.check_outputs(outputs() | {field: value}, "v15") + missing_action = outputs() + del missing_action["action"] + with self.assertRaisesRegex(ValueError, "Missing"): + validator.check_outputs(missing_action, "v15") + validator.check_outputs(missing_action, "v11") + runner = FixtureRunner(8, 8) + runner.run = Mock(return_value=outputs()) + with self.assertRaisesRegex(ValueError, "unchanged"): + validator.validate_camera(Path("fixture.pkl"), "v15", (8, 8), 3, 42, False, + runner_factory=lambda *_: runner, action_update=update_action, + initial_action=SimpleNamespace(desiredCurvature=0., desiredAcceleration=0.)) + + def test_parent_binds_artifact_sources_and_runs_both_camera_processes(self): + with tempfile.TemporaryDirectory() as temporary: + root = Path(temporary) + artifact, source, report = root / "test.pkl", root / "driving_supercombo.onnx", root / "report.json" + artifact.write_bytes(b"compiled") + source.write_bytes(b"onnx") + receipt = {"artifact_sha256": validator.sha256_file(artifact), "behavior_version": "v15", + "source_sha256": {source.name: validator.sha256_file(source)}} + artifact.with_suffix(".build.json").write_text(json.dumps(receipt)) + def child(command, **_kwargs): + camera = list(map(int, command[command.index("--camera") + 1].split("x"))) + Path(command[command.index("--report") + 1]).write_text(json.dumps({"passed": True, "camera": camera})) + return SimpleNamespace(returncode=0) + with patch.object(validator.subprocess, "run", side_effect=child) as child_run: + self.assertEqual(validator.main([str(artifact), "--version", "v15", "--source-dir", str(root), "--report", str(report)]), 0) + self.assertEqual(child_run.call_count, 2) + result = json.loads(report.read_text()) + self.assertTrue(result["sources_verified"]) + self.assertEqual([camera["camera"] for camera in result["cameras"]], [list(camera) for camera in validator.CAMERAS]) + source.write_bytes(b"changed") + with patch.object(validator.subprocess, "run") as child_run: + self.assertEqual(validator.main([str(artifact), "--version", "v15", "--source-dir", str(root), "--report", str(report)]), 1) + child_run.assert_not_called() + self.assertFalse(json.loads(report.read_text())["passed"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/openpilot/starpilot/system/jetlink/__init__.py b/openpilot/starpilot/system/jetlink/__init__.py new file mode 100644 index 0000000000..fc3ccdc0ae --- /dev/null +++ b/openpilot/starpilot/system/jetlink/__init__.py @@ -0,0 +1 @@ +"""Managed external inference transport.""" diff --git a/openpilot/starpilot/system/jetlink/process.py b/openpilot/starpilot/system/jetlink/process.py new file mode 100644 index 0000000000..b63a893029 --- /dev/null +++ b/openpilot/starpilot/system/jetlink/process.py @@ -0,0 +1,48 @@ +"""Jetlink's child teardown grace without blocking normal manager ticks.""" +import signal +import time + +from openpilot.common.swaglog import cloudlog +from openpilot.system.manager.process import PythonProcess, join_process + + +class JetlinkProcess(PythonProcess): + STOP_GRACE = 15.0 + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.stop_requested_at = None + + def stop(self, retry=True, block=True, sig=None): + if self.proc is None: + return None + if self.proc.exitcode is None: + if not self.shutting_down: + self.signal(sig or signal.SIGINT) + self.shutting_down = True + self.stop_requested_at = time.monotonic() + requested = self.stop_requested_at + assert requested is not None + remaining = max(0.0, self.STOP_GRACE - (time.monotonic() - requested)) + if block: + join_process(self.proc, remaining) + if self.proc.exitcode is None and retry and (block or remaining == 0): + cloudlog.warning(f'Jetlink owner did not stop within {self.STOP_GRACE}s; killing') + self.signal(signal.SIGKILL) + if block: + join_process(self.proc, 2.0) + result = self.proc.exitcode + if result is not None: + self.proc = None + self.shutting_down = False + self.stop_requested_at = None + self.restart_failures = 0 + self.restart_at = 0.0 + return result + + def start(self): + if self.shutting_down: + self.stop(block=False) + if self.proc is not None: + return + super().start() diff --git a/scripts/model_compiler.py b/scripts/model_compiler.py new file mode 100755 index 0000000000..32f0a82eb6 --- /dev/null +++ b/scripts/model_compiler.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Compile staged ONNX sources without installing or selecting the result.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile + +REPO_ROOT = Path(__file__).resolve().parents[1] +COMPONENTS = { + "supercombo": ("driving_supercombo", "supercombo"), + "off-policy": ("driving_off_policy", "off_policy", "offpolicy"), + "on-policy": ("driving_on_policy", "on_policy", "onpolicy"), + "policy": ("driving_policy", "policy"), + "vision": ("driving_vision", "vision"), +} + + +def source_files(directory: Path, model: str) -> dict[str, Path]: + selected = directory / model if (directory / model).is_dir() else directory + found = {} + for path in sorted(selected.glob("*.onnx")): + stem = path.stem.lower() + for component, aliases in COMPONENTS.items(): + if any(stem == alias or stem == f"{model}_{alias}" for alias in aliases): + if component in found: + raise ValueError(f"Ambiguous {component} source: {found[component]} and {path}") + found[component] = path.resolve() + break + return found + + +def driving_compile_args(files: dict[str, Path], requested: str) -> tuple[str, list[str]]: + selected = "supercombo" if requested == "auto" and "supercombo" in files else requested + if selected == "supercombo": + if "supercombo" not in files: + raise ValueError("Supercombo input requires driving_supercombo.onnx") + return "supercombo", ["--supercombo-onnx", str(files["supercombo"])] + if "vision" not in files or not ({"policy", "on-policy"} & files.keys()): + raise ValueError("Split input requires driving_vision.onnx and driving_policy.onnx or driving_on_policy.onnx") + result = ["--vision-onnx", str(files["vision"])] + primary = files.get("on-policy", files.get("policy")) + if "off-policy" in files: + return "vision_multi_policy", [*result, "--on-policy-onnx", str(primary), "--off-policy-onnx", str(files["off-policy"])] + return "vision_policy", [*result, "--policy-onnx", str(primary)] + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for block in iter(lambda: source.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def compile_env(gpu: bool = False) -> dict[str, str]: + env = dict(os.environ) + env["PYTHONPATH"] = os.pathsep.join((str(REPO_ROOT), str(REPO_ROOT / "tinygrad_repo"), env.get("PYTHONPATH", ""))) + defaults = {"DEV": "QCOM", "IMAGE": "1", "FLOAT16": "1", "NOLOCALS": "1", "JIT_BATCH_SIZE": "0", "OPENPILOT_HACKS": "1"} + for key, value in defaults.items(): + env.setdefault(key, value) + if gpu: + for key in ("IMAGE", "NOLOCALS", "OPENPILOT_HACKS"): + env.pop(key, None) + env.update(DEV="USB+AMD:LLVM", WARP_DEV="QCOM", GMMU="0", TC_OPT="2", TC_MIN_GLOBALS="32") + return env + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--model", help="Stable output model ID") + parser.add_argument("--version", required=True, choices=tuple(f"v{value}" for value in range(8, 17))) + parser.add_argument("--input-dir", type=Path, default=Path("/data/openpilot/uncompiledmodels")) + parser.add_argument("--output-dir", type=Path, default=Path("/data/openpilot/compiledmodels")) + parser.add_argument("--input-format", choices=("auto", "supercombo", "split"), default="auto") + parser.add_argument("--gpu", action="store_true") + parser.add_argument("--image-history-pipeline", choices=("policy", "warp"), default="policy") + parser.add_argument("--frame-skip", type=int) + args, unknown = parser.parse_known_args(argv) + if unknown: + if len(unknown) != 1 or not unknown[0].startswith("--") or args.model: + parser.error("Use --model ID or one --ID flag") + args.model = unknown[0][2:] + if not args.model or not re.fullmatch(r"[a-z0-9][a-z0-9_-]{0,63}", args.model): + parser.error("A lowercase model ID containing letters, digits, hyphens or underscores is required") + if args.frame_skip is not None and args.frame_skip < 1: + parser.error("--frame-skip must be positive") + return args + + +def main(argv=None): + args = parse_args(argv) + model_type, source_args = driving_compile_args(source_files(args.input_dir, args.model), args.input_format) + sources = [Path(source_args[index]) for index in range(1, len(source_args), 2)] + for source in sources: + with source.open("rb") as file: + prefix = file.read(128) + if not prefix or prefix.startswith(b"version https://git-lfs.github.com/spec/v1"): + raise ValueError(f"Expected ONNX model bytes, not an empty file or Git LFS pointer: {source}") + before = {path.name: sha256_file(path) for path in sources} + output_dir = args.output_dir.resolve() + output_dir.mkdir(parents=True, exist_ok=True) + output = output_dir / f"{args.model}_driving_tinygrad.pkl" + env = compile_env(args.gpu) + with tempfile.TemporaryDirectory(prefix=f".{args.model}-compile-", dir=output_dir) as temporary: + env["TMPDIR"] = temporary + staged = Path(temporary) / output.name + command = [sys.executable, "-m", "openpilot.starpilot.models.compile", "--model-type", model_type, + "--model-size", "512x256", "--camera-resolutions", "1928x1208", "1344x760", + "--behavior-version", args.version, "--output", str(staged), "--out-of-band", + "--image-history-pipeline", args.image_history_pipeline, *source_args] + if args.frame_skip is not None: + command += ["--frame-skip", str(args.frame_skip)] + subprocess.run(command, cwd=REPO_ROOT, env=env, check=True) + if {path.name: sha256_file(path) for path in sources} != before: + raise ValueError("ONNX source changed during compilation") + revision = next(item["commit"] for item in json.loads((REPO_ROOT / "upstream-sync.json").read_text())["dependencies"] + if item["path"] == "tinygrad_repo") + receipt = {"model_id": args.model, "behavior_version": args.version, "model_type": model_type, + "artifact_abi": "tinygrad_single_v1_arena", "compiler_revision": revision, + "source_sha256": before, "artifact_sha256": sha256_file(staged), "artifact_size": staged.stat().st_size, + "execution_device": env["DEV"], "warp_device": env.get("WARP_DEV", env["DEV"]), + "image_history_pipeline": args.image_history_pipeline, "camera_resolutions": [[1928, 1208], [1344, 760]]} + staged.replace(output) + receipt_path = output.with_suffix(".build.json") + temporary_receipt = Path(temporary) / receipt_path.name + temporary_receipt.write_text(json.dumps(receipt, indent=2) + "\n") + temporary_receipt.replace(receipt_path) + print(f"Compiled {output}; build receipt {receipt_path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/model_rebuild.py b/scripts/model_rebuild.py new file mode 100644 index 0000000000..1cdf979603 --- /dev/null +++ b/scripts/model_rebuild.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Rebuild a model generation on an explicitly selected compiler device.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path +import shlex +import subprocess +import sys +import time +from urllib.request import urlopen +from urllib.error import HTTPError + +REPO = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(REPO)) +from openpilot.starpilot.models.catalog import ARTIFACT_ABI, CATALOG_PATH, COMPILER_REVISION, GENERATION +from openpilot.starpilot.models.manager import atomic_json, validate_manifest + +BUCKET = "StarPilot-Driving/StarPilot-Resources" +REMOTE_WORK = "/data/starpilot-v26-rebuild" +CHUNK_BYTES = 45 * 1024 * 1024 + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def run(args: list[str], *, output=None) -> None: + subprocess.run(args, check=True, stdout=output, stderr=subprocess.STDOUT if output is not None else None) + + +def remote(device: str, command: list[str], *, output=None) -> None: + run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=10", "-o", "ServerAliveInterval=30", + device, shlex.join(command)], output=output) + + +def inventory() -> list[dict]: + result = [] + url = f"https://huggingface.co/api/buckets/{BUCKET}/tree?recursive=true&limit=1000" + while url: + with urlopen(url, timeout=40) as response: + result.extend(json.load(response)) + links = response.headers.get("Link", "") + url = next((p.split(">")[0].strip("< ") for p in links.split(",") if 'rel="next"' in p), "") + return result + + +def source_plan(model_id: str, source_map: dict, objects: list[dict]) -> list[dict]: + source_id = source_map[model_id]["source_id"] + files = [obj for obj in objects if obj["type"] == "file" and obj["path"].startswith(f"onnx/{source_id}/") and + obj["path"].endswith(".onnx")] + if not files: + raise ValueError(f"No archived ONNX source for {model_id} ({source_id})") + return files + + +def prepare_sources(model_id: str, files: list[dict], workspace: Path, hf: str) -> Path: + directory = workspace / "onnx" / model_id + directory.mkdir(parents=True, exist_ok=True) + for obj in files: + name = Path(obj["path"]).name + if "driving_" not in name: + raise ValueError(f"Unrecognized model source filename: {name}") + name = model_id + "_" + name[name.index("driving_"):] + destination = directory / name + if not destination.is_file() or destination.stat().st_size != obj["size"]: + run([hf, "buckets", "cp", f"hf://buckets/{BUCKET}/{obj['path']}", str(destination), "--format", "quiet"]) + if destination.stat().st_size != obj["size"]: + raise ValueError("Source download size mismatch") + return directory + + +def split_artifact(artifact: Path, directory: Path) -> tuple[list[Path], int]: + directory.mkdir(parents=True, exist_ok=True) + count = (artifact.stat().st_size + CHUNK_BYTES - 1) // CHUNK_BYTES + if count == 1: + return [artifact], 0 + parts = [] + with artifact.open("rb") as stream: + for index in range(1, count + 1): + part = directory / f"{artifact.name}.chunk{index:02d}of{count:02d}" + part.write_bytes(stream.read(CHUNK_BYTES)) + parts.append(part) + return parts, count + + +def publish_artifact(model_id: str, artifact: Path, receipt: dict, workspace: Path, hf: str) -> dict: + if receipt["compiler_revision"] != COMPILER_REVISION or receipt["artifact_abi"] != ARTIFACT_ABI or digest(artifact) != receipt["artifact_sha256"]: + raise ValueError("Compiled artifact identity changed") + parts, count = split_artifact(artifact, workspace / "release" / model_id) + for part in parts: + run([hf, "buckets", "cp", str(part), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{part.name}", "--format", "quiet"]) + if count: + marker = workspace / "release" / model_id / f"{artifact.name}.chunkmanifest" + marker.write_text(str(count)) + run([hf, "buckets", "cp", str(marker), f"hf://buckets/{BUCKET}/models/{GENERATION}/{model_id}/{marker.name}", "--format", "quiet"]) + return {"artifact_format": ARTIFACT_ABI, "artifact_size": artifact.stat().st_size, + "artifact_sha256": receipt["artifact_sha256"], "artifact_chunk_count": count, + "source_sha256": receipt["source_sha256"]} + + +def compile_one(model: dict, args, source_map: dict, objects: list[dict]) -> dict: + mid = model["id"] + work = args.workspace + result_path = work / "results" / f"{mid}.json" + output_dir = work / "compiled" / mid + artifact = output_dir / f"{mid}_driving_tinygrad.pkl" + if result_path.is_file(): + previous = json.loads(result_path.read_text()) + if (previous.get("status") == "passed" and previous.get("compiler_revision") == COMPILER_REVISION and artifact.is_file() and + digest(artifact) == previous["artifact_sha256"] and (not args.publish or previous.get("published"))): + return previous + sources = prepare_sources(mid, source_plan(mid, source_map, objects), work, args.hf) + remote_dir = f"{REMOTE_WORK}/work/{mid}" + remote(args.device, ["mkdir", "-p", f"{remote_dir}/onnx", f"{remote_dir}/compiled"]) + run(["rsync", "-az", str(sources) + "/", f"{args.device}:{remote_dir}/onnx/"]) + command = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/model_compiler.py", "--model", mid, + "--version", model["version"], "--input-format", source_map[mid]["input_format"], + "--input-dir", f"{remote_dir}/onnx", "--output-dir", f"{remote_dir}/compiled"] + if model.get("uses_external_gpu"): + command.append("--gpu") + with (work / "logs" / f"{mid}.log").open("ab") as log: + remote(args.device, ["env", "PYTHONUNBUFFERED=1", *command], output=log) + validate = ["/usr/local/venv/bin/python", "/data/openpilot/scripts/validate_driving_model.py", + f"{remote_dir}/compiled/{artifact.name}", "--version", model["version"], + "--source-dir", f"{remote_dir}/onnx", "--report", f"{remote_dir}/compiled/validation.json", "--max-p95-ms", "50"] + if model.get("uses_external_gpu"): + validate.append("--gpu") + remote(args.device, validate, output=log) + output_dir.mkdir(parents=True, exist_ok=True) + run(["rsync", "-az", f"{args.device}:{remote_dir}/compiled/", str(output_dir) + "/"]) + receipt = json.loads(artifact.with_suffix(".build.json").read_text()) + validation = json.loads((output_dir / "validation.json").read_text()) + if (digest(artifact) != receipt["artifact_sha256"] or not validation.get("passed") or + validation.get("artifact_sha256") != receipt["artifact_sha256"]): + raise ValueError("Compiled model failed runtime validation") + result = {**receipt, "status": "passed", "published": False, "validation": validation} + if args.publish: + result["release"] = publish_artifact(mid, artifact, receipt, work, args.hf) + result["published"] = True + atomic_json(result_path, result) + # Only this invocation's sources and output are removed after the verified host copy exists. + remote(args.device, ["/usr/local/venv/bin/python", "-c", "import shutil,sys; shutil.rmtree(sys.argv[1])", remote_dir]) + return result + + +def publish_manifest(workspace: Path, hf: str) -> None: + manifest = json.loads(CATALOG_PATH.read_text()) + try: + with urlopen(f"https://huggingface.co/buckets/{BUCKET}/resolve/manifests/model_names_{GENERATION}.json", timeout=30) as response: + live = json.load(response) + existing = validate_manifest(live) + for row in manifest["models"]: + if row["id"] in existing: + row.update(existing[row["id"]]) + except HTTPError as error: + if error.code != 404: + raise + for model in manifest["models"]: + path = workspace / "results" / f"{model['id']}.json" + if path.is_file(): + result = json.loads(path.read_text()) + if result.get("status") == "passed" and result.get("published") and result.get("compiler_revision") == COMPILER_REVISION: + model.update(result["release"]) + path = workspace / "manifests" / f"model_names_{GENERATION}.json" + atomic_json(path, manifest) + run([hf, "buckets", "cp", str(path), f"hf://buckets/{BUCKET}/manifests/{path.name}", "--format", "quiet"]) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--device", required=True, help="Explicit SSH target, for example comma@comma-device.local") + parser.add_argument("--workspace", required=True, type=Path) + parser.add_argument("--models", nargs="+", help="Stable model IDs; defaults to the full catalog, favorites first") + parser.add_argument("--hf", default=str(Path.home()/".local/bin/hf")) + parser.add_argument("--publish", action="store_true", help="Upload validated v26 artifacts and manifest to Hugging Face") + args = parser.parse_args() + args.workspace = args.workspace.resolve() + for directory in ("results", "logs", "manifests"): + (args.workspace / directory).mkdir(parents=True, exist_ok=True) + rows = json.loads(CATALOG_PATH.read_text())["models"] + source_map = json.loads((REPO / "scripts/model_source_map_v26.json").read_text()) + if args.models: + by_id = {row["id"]: row for row in rows} + rows = [by_id[mid] for mid in args.models] + else: + rows.sort(key=lambda row: (not row.get("community_favorite", False), row["id"] != "cinquev3")) + objects = inventory() + atomic_json(args.workspace / "source-inventory.json", {"objects": objects}) + failures = [] + for model in rows: + mid = model["id"] + print(f"START {mid}", flush=True) + started = time.monotonic() + try: + compile_one(model, args, source_map, objects) + if args.publish: + publish_manifest(args.workspace, args.hf) + print(f"PASS {mid} {time.monotonic()-started:.0f}s", flush=True) + except (OSError, ValueError, subprocess.CalledProcessError) as error: + failures.append(mid) + atomic_json(args.workspace / "results" / f"{mid}.failure.json", {"model_id": mid, "error": str(error)}) + print(f"FAIL {mid}: {error}", flush=True) + return int(bool(failures)) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/model_source_map_v26.json b/scripts/model_source_map_v26.json new file mode 100644 index 0000000000..186143b25c --- /dev/null +++ b/scripts/model_source_map_v26.json @@ -0,0 +1,495 @@ +{ + "tr14223": { + "source_id": "tr1422", + "input_format": "split", + "ref": "64fd3f986081ba6cb488d02e799845367add29e2" + }, + "tr15223": { + "source_id": "tr1522", + "input_format": "split", + "ref": "ed38ca8cc998f0a2b47cde29da11b782e4551e8b" + }, + "tr16223": { + "source_id": "tr16", + "input_format": "split", + "ref": "e4b6a2736580dc427473bcf8a6e3b08eb352edd4" + }, + "letr223": { + "source_id": "letr22", + "input_format": "split", + "ref": "c2fa07b82c2e09964e4b6dccb3e4465a2e4ad138" + }, + "spacelab22223": { + "source_id": "spacelab2222", + "input_format": "split", + "ref": "d3a0378ae601f61307a53d1173c251ee36c56317" + }, + "vikander223": { + "source_id": "vikander22", + "input_format": "split", + "ref": "d134fdd7d51ba84848866c0265e37508c38f9366" + }, + "fof223": { + "source_id": "fof22", + "input_format": "split", + "ref": "d807b5c476658dba7760d30949baa619074c4a4e" + }, + "vfof223": { + "source_id": "vfof22", + "input_format": "split", + "ref": "c790d3f58ad32cf15f384e6018d942b2348f0c5b" + }, + "dtr223": { + "source_id": "dtr22", + "input_format": "split", + "ref": "5b405920ec9511e146432de08171d6f93295248b" + }, + "dtr6223": { + "source_id": "dtr622", + "input_format": "split", + "ref": "a6cf39c07a67dbc346af03010093fe4788c402d0" + }, + "uvdtr6223": { + "source_id": "uvdtr622", + "input_format": "split", + "ref": "079d0461604632719e73dc5b6a5eb9897c624da2" + }, + "sp2223": { + "source_id": "sp222", + "input_format": "split", + "ref": "f1b8f510b2c554191ca15cebe35f2c5b304cf23c" + }, + "fp223": { + "source_id": "fp22", + "input_format": "split", + "ref": "95aff72e351e935221ec965aa0384dfd502d383d" + }, + "kv223": { + "source_id": "kv22", + "input_format": "split", + "ref": "16e87d4c72e5efec18032f6b4a5f6ec35d0df4e4" + }, + "gwm3223": { + "source_id": "gwm322", + "input_format": "split", + "ref": "93b26a91a62b69b809162ad2bf65e36be2158dc1" + }, + "gwm5223": { + "source_id": "gwm522", + "input_format": "split", + "ref": "e3ee440f7d5ac12c50db6381e292a55a69b745a7" + }, + "gwm6223": { + "source_id": "gwm622", + "input_format": "split", + "ref": "91a1cea814f76d605d5b15de9634a3c8f79df509" + }, + "gwm8223": { + "source_id": "gwm822", + "input_format": "split", + "ref": "96e7b310b164b55fb73abdd505b66fb36630718e" + }, + "gwm9223": { + "source_id": "gwm922", + "input_format": "split", + "ref": "f2413040a8560c0c17c18353a3c75124a2f09d17" + }, + "cgwm223": { + "source_id": "cgwm22", + "input_format": "split", + "ref": "94dd3bd9107cd413ed2418f9e6c0cf805a9ca495" + }, + "bd223": { + "source_id": "bd22", + "input_format": "split", + "ref": "b74f5189a74446015c0cf78a4a9f0134a347ae3b" + }, + "nr223": { + "source_id": "nr22", + "input_format": "split", + "ref": "266b642180f0e9278e1db46ccf8c1a2a8dae4bb3" + }, + "tcp2223": { + "source_id": "tcp222", + "input_format": "split", + "ref": "19084132cd3956413a0fdfcdc18971a55df5e0ff" + }, + "tcp3223": { + "source_id": "tcp322", + "input_format": "split", + "ref": "5eb912c025a2207c7d428deee74ded49e5905527" + }, + "fbw223": { + "source_id": "fbw22", + "input_format": "split", + "ref": "c4488e3411285f6fc3d008bca885e05731fd75c2" + }, + "nevada223": { + "source_id": "nevada22", + "input_format": "split", + "ref": "3193eac5e385aa010694a8ac192ff38ffe000193" + }, + "wmi4223": { + "source_id": "wmi422", + "input_format": "split", + "ref": "2d8e596a0208dafe4ed2d945b07d6a7d72f70fb8" + }, + "wmi523": { + "source_id": "wmi52", + "input_format": "split", + "ref": "545e0ed13f85e4c744fa1f69095863ea9dbe6d6b" + }, + "wmi623": { + "source_id": "wmi62", + "input_format": "split", + "ref": "54c6c5776a37dc56fde626314f53ec3316d88a48" + }, + "wmi723": { + "source_id": "wmi72", + "input_format": "split", + "ref": "aa3d90c92a7bd7a4e7db1b3fdee3d3e522358823" + }, + "wmi7223": { + "source_id": "wmi722", + "input_format": "split", + "ref": "a2ace1ed6b84d7c7a2dbf990c1b566f9a9df167f" + }, + "wmi823": { + "source_id": "wmi82", + "input_format": "split", + "ref": "4c438f59e7dcf5fddc769032882b62d0cb805e9b" + }, + "wmi923": { + "source_id": "wmi92", + "input_format": "split", + "ref": "8950897d7e4d2ba2426b2b31cf29f25e87a3d4ba" + }, + "wmi1023": { + "source_id": "wmi102", + "input_format": "split", + "ref": "855f5e4ddefd69a20cc4e9da004eb53f3e00d950" + }, + "wmi1123": { + "source_id": "wmi112", + "input_format": "split", + "ref": "7e3fd7a63c5a09c3fabe108b1b62bba3cf684878" + }, + "cd21023": { + "source_id": "cd2102", + "input_format": "split", + "ref": "55f66e2246359c6593605399a0199d94d13ad90d" + }, + "op223": { + "source_id": "op22", + "input_format": "split", + "ref": "ae34a24c59d85df7efad5fde2b760fb6d0b9dd6e" + }, + "op323": { + "source_id": "op32", + "input_format": "split", + "ref": "7e548dd765873bed301c0a19cfe10c3ca6be2bbe" + }, + "op423": { + "source_id": "op42", + "input_format": "split", + "ref": "25abcc49fce2f6268d8cabf759e31c626edbf584" + }, + "op523": { + "source_id": "op52", + "input_format": "split", + "ref": "b390b98c5a359c293fdc2501af79b19867fcdac4" + }, + "op623": { + "source_id": "op62", + "input_format": "split", + "ref": "831f13da61d937bb40ea0f50590cdc3b5380f313" + }, + "opv73": { + "source_id": "opv7", + "input_format": "split", + "ref": "cb327933002bc1a00bf60a3c20af2eb7a5f653e5" + }, + "opv83": { + "source_id": "opv8", + "input_format": "split", + "ref": "052692b25d63c5ddda276b5c2271383b6aff129f" + }, + "opv93": { + "source_id": "opv9", + "input_format": "split", + "ref": "eae2a73e0ac600d7f0342fc4397568e0733fe6e2" + }, + "opv103": { + "source_id": "opv10", + "input_format": "split", + "ref": "5faf14e04e4038ec4673d119c950b46051630205" + }, + "opv113": { + "source_id": "opv11", + "input_format": "split", + "ref": "bbde5ddb5a0ee35c98a962029355f9a95403a825" + }, + "opv123": { + "source_id": "opv12", + "input_format": "split", + "ref": "7d4c295c27bc3f72727c49ec73b98e45745b588d" + }, + "opv133": { + "source_id": "opv13", + "input_format": "split", + "ref": "faeeaad3a5841694b4c3bd6d37e41e3db4b4873e" + }, + "op163": { + "source_id": "op16", + "input_format": "split", + "ref": "72101c6e50ea594f24e75d3b605abf8b5cab1d6b" + }, + "op16d3": { + "source_id": "op16d", + "input_format": "split", + "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4" + }, + "op16dv23": { + "source_id": "op16dv2", + "input_format": "split", + "ref": "0a628146d1a9a314e329e2c0b5de20e6223211e4" + }, + "rlv1dl3": { + "source_id": "rlv1dl", + "input_format": "split", + "ref": "38c0e3c95de5b63b56ecbbe507abc49b9b0091a3" + }, + "deeprl33": { + "source_id": "deeprl3", + "input_format": "split", + "ref": "a1060427b9b3ac1d11cbad05380929ea8235840a" + }, + "deeprl333": { + "source_id": "deeprl33", + "input_format": "supercombo", + "ref": "4604c98f21c9a22230e417d17d03c402b9b5f869" + }, + "rl343": { + "source_id": "rl34", + "input_format": "supercombo", + "ref": "4e6b072404110056ccf9bcc232c718ac71d45478" + }, + "drl3": { + "source_id": "drl", + "input_format": "supercombo", + "ref": "a881acfbd59cb9ed18c54535b835721e2bc96def" + }, + "nopp3": { + "source_id": "nopp", + "input_format": "supercombo", + "ref": "c740fe5f58faefcb9184c6f9f1fb45130b83cfe8" + }, + "tobyrl3": { + "source_id": "tobyrl", + "input_format": "supercombo", + "ref": "2d4acf10e34c4c8e68043d60b9437f42d89ca746" + }, + "michael-rl3": { + "source_id": "michael-rl", + "input_format": "supercombo", + "ref": "968fd3c989b993cf03187aa0e05cd2088e275551" + }, + "michael-rl23": { + "source_id": "michael-rl2", + "input_format": "supercombo", + "ref": "37b38bf738edfb1daa6875255f581e5fc9b0a258" + }, + "karnbir3": { + "source_id": "karnbir", + "input_format": "supercombo", + "ref": "c0fbd2b13030df95ad249b6e253156a4d7ddb7bb" + }, + "karnbir23": { + "source_id": "karnbir2", + "input_format": "supercombo", + "ref": "682d33d0414d7cef5a7286c24298003503ea13f4" + }, + "ms23": { + "source_id": "ms2", + "input_format": "split", + "ref": "d70b13931793bc0e4d8efd36128dd66f227a3f81" + }, + "pp2223": { + "source_id": "pp222", + "input_format": "split", + "ref": "50c78a9dd670a305ca898b96245cadbc2ecdc1a6" + }, + "ds2223": { + "source_id": "ds222", + "input_format": "split", + "ref": "5ff0e48f90d662d8c9d5061adc63e90c198c9b81" + }, + "nn2223": { + "source_id": "nn222", + "input_format": "split", + "ref": "83b81b83cdf76a577946a1567f4644e1a018443c" + }, + "sc23": { + "source_id": "sc2", + "input_format": "split", + "ref": "e4a4b4b1adf2d19fedab4d195faa382b061fa754" + }, + "pop23": { + "source_id": "pop2", + "input_format": "split", + "ref": "6f71783a8a8faa07ddaeef5bbb6809b4f4f44a15" + }, + "pop223": { + "source_id": "pop22", + "input_format": "split", + "ref": "62bf6fb072880905a4c490f0f4f4a6b3c23346ec" + }, + "nid223": { + "source_id": "nid22", + "input_format": "split", + "ref": "13e79e9fad60c19e751e4f9ab0538d39f1bb54dd" + }, + "kerrygold223": { + "source_id": "kerrygold22", + "input_format": "split", + "ref": "dc7a92ea630e4f6053082fc25eca83938927b91e" + }, + "deeprl3v23": { + "source_id": "deeprl3v2", + "input_format": "supercombo", + "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9" + }, + "rh3": { + "source_id": "rh", + "input_format": "supercombo", + "ref": "93f5aa469a72b7621aef7da7901c100e0113e4d9" + }, + "gyhu3": { + "source_id": "gyhu", + "input_format": "supercombo", + "ref": "574735edc6e1aafdc2a69395f9a32e7f5cc4b62b" + }, + "rdfv23": { + "source_id": "rdf", + "input_format": "supercombo", + "ref": "a95e2c25cae5fbf1afba7628bfb7acc4af59e0cc" + }, + "rdf23": { + "source_id": "rdf2", + "input_format": "supercombo", + "ref": "0f8b4248a2e8bdd63b6ea4c6e1bbb32119cb2620" + }, + "rdf43": { + "source_id": "rdf4", + "input_format": "supercombo", + "ref": "a5a6412d08474cffb49a69afb910756afdee123e" + }, + "rdf33": { + "source_id": "rdf3", + "input_format": "supercombo", + "ref": "ea2151ba4b82854277f37f03b949f15fe2733dc8" + }, + "rdf53": { + "source_id": "rdf5", + "input_format": "supercombo", + "ref": "7fb03ca474f03e95e59ec0c8a6c5fba831bd5fd1" + }, + "rdf63": { + "source_id": "rdf6", + "input_format": "supercombo", + "ref": "35703097905a122c9f3ddf0d12889b4873d7e2a2" + }, + "tsf": { + "source_id": "tsf", + "input_format": "supercombo", + "ref": "4d911346cde4e0d2978a625f31679808284cc19d" + }, + "tsfdo": { + "source_id": "tsfdo", + "input_format": "supercombo", + "ref": "0394d7284cfc229f772ee4fa73ff724ba8245110" + }, + "lebowski3": { + "source_id": "lebowski", + "input_format": "supercombo", + "ref": "fa0c6876d3cf070e91e25e5353ceadc68a5b3285" + }, + "tt-driving": { + "source_id": "tt-driving", + "input_format": "supercombo" + }, + "tee-time": { + "source_id": "tee-time", + "input_format": "supercombo" + }, + "bmrlnapv2": { + "source_id": "bmrlnapv2", + "input_format": "supercombo", + "ref": "ba37c02c7c64225b8aec79411d1a8c2f5c1aa343" + }, + "happybirthday": { + "source_id": "happybirthday", + "input_format": "supercombo", + "ref": "db06f5f36caa379e112bafd38423e37b66fa4149" + }, + "berightthere": { + "source_id": "berightthere", + "input_format": "supercombo", + "ref": "bf74ce5448306088b4e9bc843472c39402930bc7" + }, + "remove-avgpoolv4": { + "source_id": "remove-avgpoolv4", + "input_format": "supercombo" + }, + "remove-avgpoolv3": { + "source_id": "remove-avgpoolv3", + "input_format": "supercombo" + }, + "remove-avgpoolv2": { + "source_id": "remove-avgpoolv2", + "input_format": "supercombo" + }, + "remove-avgpoolv1": { + "source_id": "remove-avgpoolv1", + "input_format": "supercombo" + }, + "rdf-driving": { + "source_id": "rdf-driving", + "input_format": "supercombo" + }, + "sad": { + "source_id": "sad", + "input_format": "supercombo", + "ref": "30de303d5ffb63957f8acbdb256fd4c7d360455a" + }, + "bmrlnapv6": { + "source_id": "bmrlnapv6", + "input_format": "supercombo", + "ref": "9d683c06518c0358fb402f38468a7030700c38ac" + }, + "deeprl3v2": { + "source_id": "deeprl3v2", + "input_format": "supercombo", + "ref": "702fa71ad4dd8de08425eb11a1a42aaeb64892c9" + }, + "cinque-driving": { + "source_id": "cinque-driving", + "input_format": "supercombo" + }, + "cdm": { + "source_id": "cdm", + "input_format": "supercombo" + }, + "cinquev3": { + "source_id": "cinquev3", + "input_format": "supercombo" + }, + "resfit": { + "source_id": "resfit", + "input_format": "supercombo" + }, + "mountaindewv1": { + "source_id": "mountaindewv1", + "input_format": "supercombo" + } +} diff --git a/scripts/validate_driving_model.py b/scripts/validate_driving_model.py new file mode 100755 index 0000000000..e62edba548 --- /dev/null +++ b/scripts/validate_driving_model.py @@ -0,0 +1,233 @@ +#!/usr/bin/env python3 +"""Exercise a compiled catalog model on synthetic NV12 frames without starting modeld.""" +import argparse +import gc +import hashlib +import json +import math +from pathlib import Path +import subprocess +import sys +import tempfile +import time + +import numpy as np + +REPO_ROOT = Path(__file__).resolve().parents[1] +if str(REPO_ROOT) not in sys.path: + sys.path.insert(0, str(REPO_ROOT)) + +CAMERAS = ((1928, 1208), (1344, 760)) +REQUIRED_SHAPES = {"plan": (1, 33, 15), "lane_lines": (1, 4, 33, 2), "road_edges": (1, 2, 33, 2), + "lead": (1, 3, 6, 4), "pose": (1, 6)} +REQUIRED_KEYS = frozenset((*REQUIRED_SHAPES, "lane_lines_prob", "lead_prob", "meta", "desire_state", "desire_pred", + "wide_from_device_euler", "road_transform")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as file: + for block in iter(lambda: file.read(1024 * 1024), b""): + digest.update(block) + return digest.hexdigest() + + +def check_outputs(outputs: dict[str, np.ndarray], version: str) -> dict[str, str]: + required = REQUIRED_KEYS | ({"action"} if version in ("v14", "v15", "v16") else set()) + if missing := required - outputs.keys(): + raise ValueError(f"Missing parsed outputs: {sorted(missing)}") + shapes = REQUIRED_SHAPES | ({"action": (1, 2)} if "action" in outputs else {}) + hashes = {} + for name, value in outputs.items(): + array = np.asarray(value) + if not array.size or not np.isfinite(array).all(): + raise ValueError(f"Empty or non-finite parsed output: {name}") + if name in shapes and array.shape != shapes[name]: + raise ValueError(f"Incorrect parsed {name} shape: {array.shape}, expected {shapes[name]}") + hashes[name] = hashlib.sha256(array.tobytes()).hexdigest() + return hashes + + +def fill_nv12(frame: np.ndarray, width: int, height: int, step: int, seed: int) -> None: + from openpilot.system.camerad.cameras.nv12_info import get_nv12_info + + stride, y_height, uv_height, _ = get_nv12_info(width, height) + frame.fill(16) + y = frame[:stride * y_height].reshape(y_height, stride) + x_coordinate = np.arange(width, dtype=np.uint32)[None, :] + y_coordinate = np.arange(height, dtype=np.uint32)[:, None] + y[:height, :width] = 16 + ((x_coordinate * 3 + y_coordinate * 5 + step * 17 + seed) % 220) + uv = frame[stride * y_height:stride * (y_height + uv_height)].reshape(uv_height, stride) + uv.fill(128) + uv[:height // 2, :width:2] = 96 + ((step * 3 + seed) % 64) + uv[:height // 2, 1:width:2] = 96 + ((step * 7 + seed) % 64) + + +def frame_transform(width: int, height: int, step: int) -> np.ndarray: + return np.array([[0.8 * width / 512, 0, 0.1 * width + math.sin(step) * 2], + [0, 0.8 * height / 256, 0.1 * height + math.cos(step) * 2], [0, 0, 1]], dtype=np.float32) + + +def upload_frames(runner, frames: dict[str, np.ndarray]) -> None: + from tinygrad import Device, Tensor + + for name, frame in frames.items(): + runner._blob_cache[(name, frame.ctypes.data)] = Tensor(frame, device=runner.warp_device).realize() + Device[runner.warp_device].synchronize() + if runner.queue_device != runner.warp_device: + Device[runner.queue_device].synchronize() + + +def validate_camera(artifact: Path, version: str, camera: tuple[int, int], steps: int, seed: int, + gpu: bool, max_p95_ms: float | None = None, runner_factory=None, action_update=None, initial_action=None) -> dict: + device_frames = runner_factory is None + if runner_factory is None: + from openpilot.starpilot.models.runner import CatalogModelState, action_from_outputs + from openpilot.cereal import log + runner_factory, action_update = CatalogModelState, action_from_outputs + initial_action = log.ModelDataV2.Action() + width, height = camera + start = time.perf_counter() + runner = runner_factory(width, height, artifact, version, gpu) + load_ms = (time.perf_counter() - start) * 1000 + start = time.perf_counter() + runner.warmup() + warmup_ms = (time.perf_counter() - start) * 1000 + frames = {name: np.zeros(runner.frame_buf_size, dtype=np.uint8) for name in runner.vision_input_names} + previous_action = initial_action + latencies, changed, baseline = [], set(), None + output_shapes = {} + actions = [] + for step in range(steps): + for index, frame in enumerate(frames.values()): + fill_nv12(frame, width, height, step, seed + index * 97) + transforms = {name: frame_transform(width, height, step + index) for index, name in enumerate(frames)} + desire = np.zeros(8, dtype=np.float32) + if step % 4 == 0: + desire[1 + (step // 4) % 2] = 1 + speed = 5.0 + step % 20 + inputs = {"desire_pulse": desire, "traffic_convention": np.array([step % 2, 1 - step % 2], dtype=np.float32), + "action_t": np.array([.2, .4], dtype=np.float32), + "prev_action": np.array([previous_action.desiredCurvature * max(1., speed) ** 2, + previous_action.desiredAcceleration], dtype=np.float32), + "lateral_control_params": np.array([speed, .2], dtype=np.float32)} + if device_frames: + upload_frames(runner, frames) + start = time.perf_counter() + outputs = runner.run(frames, transforms, inputs) + latencies.append((time.perf_counter() - start) * 1000) + hashes = check_outputs(outputs, version) + if baseline is None: + baseline = hashes + output_shapes = {name: list(np.asarray(value).shape) for name, value in outputs.items()} + else: + if hashes.keys() != baseline.keys(): + raise ValueError("Parsed output names changed between frames") + changed.update(name for name, value in hashes.items() if value != baseline[name]) + previous_action = action_update(outputs, version, previous_action, .2, .4, speed) + action = [float(previous_action.desiredCurvature), float(previous_action.desiredAcceleration)] + if not np.isfinite(action).all(): + raise ValueError("Non-finite derived driving action") + actions.append(action) + if not changed.intersection(REQUIRED_KEYS | {"action"}): + raise ValueError("Driving outputs stayed unchanged across changing frames and inputs") + timing = {"min": float(np.min(latencies)), "median": float(np.median(latencies)), + "p95": float(np.percentile(latencies, 95)), "max": float(np.max(latencies))} + if max_p95_ms is not None and timing["p95"] > max_p95_ms: + raise ValueError(f"p95 latency {timing['p95']:.2f} ms exceeds {max_p95_ms:.2f} ms") + result = {"passed": True, "camera": [width, height], "steps": steps, "seed": seed, + "load_ms": load_ms, "warmup_ms": warmup_ms, "inference_ms": timing, + "changed_outputs": sorted(changed), "output_shapes": output_shapes, "last_output_sha256": hashes, + "last_action": {"desired_curvature": actions[-1][0], "desired_acceleration": actions[-1][1]}, + "execution_device": getattr(runner, "queue_device", None), "warp_device": getattr(runner, "warp_device", None)} + del runner + gc.collect() + return result + + +def parse_args(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("artifact", type=Path) + parser.add_argument("--version", required=True, choices=tuple(f"v{number}" for number in range(8, 17))) + parser.add_argument("--steps", type=int, default=24) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--gpu", action="store_true") + parser.add_argument("--max-p95-ms", type=float) + parser.add_argument("--build-receipt", type=Path) + parser.add_argument("--source-dir", type=Path, help="Also verify the build receipt's ONNX source hashes") + parser.add_argument("--report", type=Path) + parser.add_argument("--camera", choices=("1928x1208", "1344x760"), help=argparse.SUPPRESS) + args = parser.parse_args(argv) + if args.steps < 2: + parser.error("--steps must be at least 2") + if args.max_p95_ms is not None and (not math.isfinite(args.max_p95_ms) or args.max_p95_ms <= 0): + parser.error("--max-p95-ms must be positive and finite") + if args.seed < 0: + parser.error("--seed must be nonnegative") + return args + + +def main(argv=None): + args = parse_args(argv) + artifact = args.artifact.resolve() + report_path = args.report or artifact.with_suffix(".validation.json") + receipt_path = args.build_receipt or artifact.with_suffix(".build.json") + protected = {artifact, receipt_path.resolve()} + report = {"passed": False, "validation": "synthetic_nv12_catalog_runner", "artifact": str(artifact), + "behavior_version": args.version, "steps": args.steps, "seed": args.seed, "cameras": []} + try: + if args.camera: + camera = tuple(map(int, args.camera.split("x"))) + report.update(validate_camera(artifact, args.version, camera, args.steps, args.seed, args.gpu, args.max_p95_ms)) + else: + before = sha256_file(artifact) + report.update(artifact_sha256=before, artifact_size=artifact.stat().st_size, sources_verified=False) + if receipt_path.is_file(): + receipt = json.loads(receipt_path.read_text()) + if receipt.get("artifact_sha256") != before or receipt.get("behavior_version") != args.version: + raise ValueError("Build receipt does not match artifact hash/behavior version") + source_hashes = receipt.get("source_sha256", {}) + report.update(build_receipt_sha256=sha256_file(receipt_path), source_sha256=source_hashes, + compiler_revision=receipt.get("compiler_revision"), artifact_abi=receipt.get("artifact_abi")) + if args.source_dir: + if not source_hashes or any(Path(name).name != name for name in source_hashes): + raise ValueError("Build receipt has no bounded source filenames") + protected.update((args.source_dir / name).resolve() for name in source_hashes) + if {name: sha256_file(args.source_dir / name) for name in source_hashes} != source_hashes: + raise ValueError("Source hashes do not match build receipt") + report["sources_verified"] = True + elif args.build_receipt or args.source_dir: + raise ValueError("Build receipt required for requested source verification") + # Separate processes release all device allocations between camera formats. + with tempfile.TemporaryDirectory(prefix="starpilot-model-validation-") as temporary: + for width, height in CAMERAS: + child_report = Path(temporary) / f"{width}x{height}.json" + command = [sys.executable, str(Path(__file__).resolve()), str(artifact), "--version", args.version, + "--steps", str(args.steps), "--seed", str(args.seed), "--camera", f"{width}x{height}", + "--report", str(child_report)] + if args.gpu: + command.append("--gpu") + if args.max_p95_ms is not None: + command += ["--max-p95-ms", str(args.max_p95_ms)] + completed = subprocess.run(command, cwd=REPO_ROOT, check=False) + result = json.loads(child_report.read_text()) if child_report.is_file() else { + "passed": False, "camera": [width, height], "error": "Validation process exited without a report"} + if completed.returncode: + result["passed"] = False + report["cameras"].append(result) + if sha256_file(artifact) != before: + raise ValueError("Artifact changed during validation") + report["passed"] = all(camera["passed"] for camera in report["cameras"]) + except Exception as error: + report.update(passed=False, error=f"{type(error).__name__}: {error}") + if report_path.resolve() in protected: + print("Refusing to overwrite a validation input with its report", file=sys.stderr) + return 1 + report_path.parent.mkdir(parents=True, exist_ok=True) + report_path.write_text(json.dumps(report, indent=2, allow_nan=False) + "\n") + print(f"{'PASS' if report['passed'] else 'FAIL'}: {report_path}") + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main())