diff --git a/openpilot/starpilot/system/starport/bugsink_report.py b/openpilot/starpilot/system/starport/bugsink_report.py index dfc3aebbd8..c739af77e6 100644 --- a/openpilot/starpilot/system/starport/bugsink_report.py +++ b/openpilot/starpilot/system/starport/bugsink_report.py @@ -103,6 +103,7 @@ def video_summary(events: list[dict]) -> dict: "screen": f"{mode['width'] - (mode.get('margin_width') or 0)}x{mode['height'] - (mode.get('margin_height') or 0)}" if mode.get("width") and mode.get("height") else None, "density_dpi": mode.get("density") or None, + "codec": mode.get("codec"), # present only for H.265; H.264 reports are unchanged "target_fps": pipeline.get("target_fps"), "encoder": pipeline.get("encoder"), "nv12": pipeline.get("nv12"), diff --git a/openpilot/starpilot/system/starport/hw/libstarport_encoder.so b/openpilot/starpilot/system/starport/hw/libstarport_encoder.so index 1b8e37108c..791e7ae260 100755 Binary files a/openpilot/starpilot/system/starport/hw/libstarport_encoder.so and b/openpilot/starpilot/system/starport/hw/libstarport_encoder.so differ diff --git a/openpilot/starpilot/system/starport/hw/starport_encoder.cc b/openpilot/starpilot/system/starport/hw/starport_encoder.cc index 2001e43a0c..e28c322f66 100644 --- a/openpilot/starpilot/system/starport/hw/starport_encoder.cc +++ b/openpilot/starpilot/system/starport/hw/starport_encoder.cc @@ -3,11 +3,12 @@ // Altering them can degrade video smoothness or introduce lag on Android Auto. // Do not change them autonomously; require an explicit user request specifically // for these video optimizations. -// Isolated Qualcomm H.264 session for Android Auto projection. No cereal publisher, +// Isolated Qualcomm H.264 (or, with AA_ENCODER_HEVC, H.265) session for Android Auto projection. No cereal publisher, // camera buffers, realtime priority, or background thread. Exactly one input in flight. // Input is RGBA (converted here with NEON) or NV12 already converted on the GPU by the // car renderer (copied into the encoder's padded layout). ABI 3 adds the NV12 entry -// point and create() options (AA_ENCODER_CBR). +// point and create() options (AA_ENCODER_CBR). ABI 4 adds AA_ENCODER_HEVC for a head unit that offers only H.265; +// the H.264 path is untouched. HEVC controls follow system/loggerd/encoder/v4l_encoder.cc. // Buffer/ioctl conventions follow system/loggerd/encoder/v4l_encoder.cc and // msgq/visionipc/visionbuf_ion.cc. Built for comma_arm64 by this directory's SConscript. // @@ -38,6 +39,7 @@ namespace { using Clock = std::chrono::steady_clock; constexpr unsigned CODECCONFIG = 0x00020000; constexpr int AA_ENCODER_CBR = 1; // hold the configured bitrate instead of letting it float with scene complexity +constexpr int AA_ENCODER_HEVC = 2; // encode H.265 (Main profile) instead of H.264 baseline constexpr size_t MAX_PACKET = 2 * 1024 * 1024 - 10; void check(bool ok, const char *what) { @@ -142,8 +144,9 @@ struct Encoder { strcmp(reinterpret_cast(cap.card), "msm_vidc_venc")) throw std::runtime_error("Unsupported encoder driver"); v4l2_format out = {}; out.type = V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE; out.fmt.pix_mp.width = width; out.fmt.pix_mp.height = height; - out.fmt.pix_mp.pixelformat = V4L2_PIX_FMT_H264; - call(fd, VIDIOC_S_FMT, &out, "H264 format"); + const bool hevc = options & AA_ENCODER_HEVC; + out.fmt.pix_mp.pixelformat = hevc ? V4L2_PIX_FMT_HEVC : V4L2_PIX_FMT_H264; + call(fd, VIDIOC_S_FMT, &out, hevc ? "HEVC format" : "H264 format"); v4l2_streamparm parm = {}; parm.type = V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE; parm.parm.output.timeperframe.numerator = 1; parm.parm.output.timeperframe.denominator = fps; call(fd, VIDIOC_S_PARM, &parm, "frame rate"); @@ -167,10 +170,15 @@ struct Encoder { : V4L2_CID_MPEG_VIDC_VIDEO_RATE_CONTROL_VBR_CFR); control(V4L2_CID_MPEG_VIDC_VIDEO_PRIORITY, V4L2_MPEG_VIDC_VIDEO_PRIORITY_REALTIME_DISABLE); control(V4L2_CID_MPEG_VIDC_VIDEO_IDR_PERIOD, 1); - control(V4L2_CID_MPEG_VIDEO_H264_PROFILE, V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE); - control(V4L2_CID_MPEG_VIDEO_H264_LEVEL, V4L2_MPEG_VIDEO_H264_LEVEL_3_1); - control(V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE, V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC); - control(V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE, V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED); + if (hevc) { + control(V4L2_CID_MPEG_VIDC_VIDEO_HEVC_PROFILE, V4L2_MPEG_VIDC_VIDEO_HEVC_PROFILE_MAIN); + control(V4L2_CID_MPEG_VIDC_VIDEO_HEVC_TIER_LEVEL, V4L2_MPEG_VIDC_VIDEO_HEVC_LEVEL_HIGH_TIER_LEVEL_5); + } else { + control(V4L2_CID_MPEG_VIDEO_H264_PROFILE, V4L2_MPEG_VIDEO_H264_PROFILE_BASELINE); + control(V4L2_CID_MPEG_VIDEO_H264_LEVEL, V4L2_MPEG_VIDEO_H264_LEVEL_3_1); + control(V4L2_CID_MPEG_VIDEO_H264_ENTROPY_MODE, V4L2_MPEG_VIDEO_H264_ENTROPY_MODE_CAVLC); + control(V4L2_CID_MPEG_VIDEO_H264_LOOP_FILTER_MODE, V4L2_MPEG_VIDEO_H264_LOOP_FILTER_MODE_ENABLED); + } control(V4L2_CID_MPEG_VIDEO_MULTI_SLICE_MODE, 0); unsigned count = buffers(V4L2_BUF_TYPE_VIDEO_CAPTURE_MPLANE, 6); buffers(V4L2_BUF_TYPE_VIDEO_OUTPUT_MPLANE, 9); @@ -257,7 +265,7 @@ struct Encoder { auto &buffer = *outputs[b.index]; buffer.sync(true); auto *bytes = static_cast(buffer.addr); if (b.flags & CODECCONFIG) { - if (plane.bytesused > 65536) throw std::runtime_error("Oversized H264 header"); + if (plane.bytesused > 65536) throw std::runtime_error("Oversized codec header"); header.assign(bytes, bytes + plane.bytesused); } else { if (b.timestamp.tv_sec * 1000000LL + b.timestamp.tv_usec != timestamp || frame_done) @@ -281,15 +289,20 @@ struct Encoder { } extern "C" { -int starport_encoder_abi() { return 3; } +int starport_encoder_abi() { return 4; } void starport_encoder_timing(void *ptr, double *output) { if (ptr && output) std::copy_n(static_cast(ptr)->timing, 3, output); } void *starport_encoder_create(int width, int height, int fps, int bitrate, int margin_height, int options, char *error, size_t size) { try { - if (width < 2 || width > 1280 || height < 2 || height > 720 || width % 2 || height % 2 || + // H.264 baseline level 3.1 is a 720p30 session; HEVC Main at high-tier level 5 carries 1080p30 for a head unit + // that offers nothing else. + const bool hevc = options & AA_ENCODER_HEVC; + const int max_width = hevc ? 1920 : 1280, max_height = hevc ? 1080 : 720; + if (width < 2 || width > max_width || height < 2 || height > max_height || width % 2 || height % 2 || fps != 30 || bitrate < 1000000 || bitrate > 20000000 || margin_height < 0 || - margin_height >= height || margin_height % 4 || (options & ~AA_ENCODER_CBR)) throw std::runtime_error("Unsupported encoder configuration"); + margin_height >= height || margin_height % 4 || (options & ~(AA_ENCODER_CBR | AA_ENCODER_HEVC))) + throw std::runtime_error("Unsupported encoder configuration"); auto encoder = std::make_unique(width, height, fps, margin_height, options); encoder->init(bitrate); return encoder.release(); } catch (const std::exception &e) { error_text(error, size, e); return nullptr; } diff --git a/openpilot/starpilot/system/starport/hw_encoder.py b/openpilot/starpilot/system/starport/hw_encoder.py index ad79cc4b3c..279fd56d1d 100644 --- a/openpilot/starpilot/system/starport/hw_encoder.py +++ b/openpilot/starpilot/system/starport/hw_encoder.py @@ -23,11 +23,14 @@ from pathlib import Path from openpilot.starpilot.system.starport.encoder import MAX_ACCESS_UNIT, START_CODE, H264Encoder, nal_types LIBRARY = Path(__file__).with_name("hw") / "libstarport_encoder.so" -ABI = 3 -SUPPORTED_ABIS = (2, 3) +ABI = 4 +SUPPORTED_ABIS = (2, 3, 4) OPTION_CBR = 1 +OPTION_HEVC = 2 # ABI 4: H.265 Main profile instead of H.264 baseline MAX_WIDTH, MAX_HEIGHT, HW_FPS = 1280, 720, 30 +HEVC_MAX_WIDTH, HEVC_MAX_HEIGHT = 1920, 1080 AUD = b"\x00\x00\x00\x01\x09\xf0" +HEVC_AUD = b"\x00\x00\x00\x01\x46\x01\x50" def normalize_access_unit(data: bytes, *, keyframe: bool) -> bytes: @@ -52,16 +55,51 @@ def normalize_access_unit(data: bytes, *, keyframe: bool) -> bytes: return result +def hevc_nal_kinds(data: bytes) -> list[int]: + return [(data[match.end()] >> 1) & 0x3F for match in START_CODE.finditer(data) if match.end() < len(data)] + + +def normalize_access_unit_hevc(data: bytes, *, keyframe: bool) -> bytes: + """H.265 counterpart of normalize_access_unit: exactly one leading AUD, one picture, VPS/SPS/PPS on keyframes. + + A picture NAL is a VCL unit (types 0-9 and 16-21); an IDR is type 19 or 20. The comma's encoder splits every H.265 + picture into two slices (measured on the device: [19, 19] and [1, 1]), so one or more VCL units make a picture. + Separate from the H.264 function so that path is unchanged.""" + if not data.startswith((b"\x00\x00\x01", b"\x00\x00\x00\x01")) or len(data) > MAX_ACCESS_UNIT: + raise RuntimeError("Expected a bounded Annex B hardware frame") + starts = list(START_CODE.finditer(data)) + units = [] + for index, match in enumerate(starts): + end = starts[index + 1].start() if index + 1 < len(starts) else len(data) + if match.end() + 1 >= end: + raise RuntimeError("Empty hardware NAL unit") + units.append(((data[match.end()] >> 1) & 0x3F, data[match.start():end])) + kinds = [kind for kind, _ in units] + pictures = [kind for kind in kinds if kind <= 9 or 16 <= kind <= 21] + if not pictures or len({kind in (19, 20) for kind in pictures}) != 1: # slices of one picture are all IDR or none are + raise RuntimeError("Hardware encoder did not produce one picture") + if keyframe and not ({19, 20} & set(kinds) and {32, 33, 34}.issubset(kinds)): + raise RuntimeError("Hardware encoder did not produce an independently decodable keyframe") + result = HEVC_AUD + b"".join(unit for kind, unit in units if kind != 35) + if len(result) > MAX_ACCESS_UNIT: + raise RuntimeError("Hardware access unit too large") + return result + + # ANDROID AUTO VIDEO PERFORMANCE GUARDRAIL: Hardware encoding and bounded frame submission avoid CPU stalls and encoder backlog. # Altering this can degrade video smoothness or introduce lag on Android Auto. # Do not change it autonomously; require an explicit user request specifically for this video optimization. class HardwareH264Encoder: backend = "qcom-v4l2" + codec = "h264" + max_width, max_height = MAX_WIDTH, MAX_HEIGHT # level 3.1 baseline is a 720p30 session + codec_options = 0 + required_abi = 2 def __init__(self, width: int, height: int, fps: int = HW_FPS, bitrate_kbps: int = 6000, margin_height: int = 0, library: Path = LIBRARY, rate_control: str = "cbr"): - if not (0 < width <= MAX_WIDTH and 0 < height <= MAX_HEIGHT) or width % 2 or height % 2 or fps != HW_FPS: - raise ValueError("Hardware encoder supports even sizes up to 1280x720 at 30 fps") + if not (0 < width <= self.max_width and 0 < height <= self.max_height) or width % 2 or height % 2 or fps != HW_FPS: + raise ValueError(f"Hardware encoder supports even sizes up to {self.max_width}x{self.max_height} at 30 fps") if margin_height % 4: margin_height = 0 # only chroma-aligned black margins may be skipped self.width, self.height, self.fps = width, height, fps @@ -71,7 +109,7 @@ class HardwareH264Encoder: self.lib = ctypes.CDLL(str(library)) self.lib.starport_encoder_abi.restype = ctypes.c_int self.abi = self.lib.starport_encoder_abi() - if self.abi not in SUPPORTED_ABIS: + if self.abi not in SUPPORTED_ABIS or self.abi < self.required_abi: raise RuntimeError("libstarport_encoder.so ABI mismatch; rebuild") pointer = ctypes.c_void_p encode_args = [pointer, pointer, ctypes.c_size_t, ctypes.c_int, pointer, ctypes.c_size_t, pointer, ctypes.c_size_t] @@ -88,7 +126,7 @@ class HardwareH264Encoder: self.lib.starport_encoder_destroy.restype = None self.error = ctypes.create_string_buffer(512) self.output = ctypes.create_string_buffer(MAX_ACCESS_UNIT) - options = (OPTION_CBR if self.rate_control == "cbr" else 0,) if self.abi >= 3 else () + options = ((OPTION_CBR if self.rate_control == "cbr" else 0) | self.codec_options,) if self.abi >= 3 else () self._create_args = (width, height, fps, int(bitrate_kbps) * 1000, margin_height, *options) self._open() @@ -127,10 +165,16 @@ class HardwareH264Encoder: if size > len(self.output): raise RuntimeError("Hardware access unit too large") # .raw copies the entire 2 MiB capacity, even for a small inter frame. - data = normalize_access_unit(ctypes.string_at(self.output, size), keyframe=force) + data = self.normalize(ctypes.string_at(self.output, size), keyframe=force) self.frame_index += 1 self.last_encode_ms = (time.monotonic() - started) * 1000 - return data, {5, 7, 8}.issubset(nal_types(data)) + return data, self.is_keyframe(data) + + def normalize(self, data: bytes, *, keyframe: bool) -> bytes: + return normalize_access_unit(data, keyframe=keyframe) + + def is_keyframe(self, data: bytes) -> bool: + return {5, 7, 8}.issubset(nal_types(data)) def close(self) -> None: handle, self.handle = self.handle, None @@ -138,6 +182,42 @@ class HardwareH264Encoder: self.lib.starport_encoder_destroy(handle) +class HardwareHevcEncoder(HardwareH264Encoder): + """H.265 for a head unit that offers nothing else (Allow H.265 video). Hardware only: there is no software fallback.""" + backend = "qcom-v4l2-hevc" + codec = "h265" + max_width, max_height = HEVC_MAX_WIDTH, HEVC_MAX_HEIGHT + codec_options = OPTION_HEVC + required_abi = 4 + + def normalize(self, data: bytes, *, keyframe: bool) -> bytes: + return normalize_access_unit_hevc(data, keyframe=keyframe) + + def is_keyframe(self, data: bytes) -> bool: + kinds = set(hevc_nal_kinds(data)) + return bool({19, 20} & kinds) and {32, 33, 34}.issubset(kinds) + + +def create_hevc_encoder(width: int, height: int, *, bitrate_kbps: int, margin_height: int, log, + rate_control: str = "cbr") -> tuple[object, int]: + """Return ``(encoder, fps)`` for an H.265 session. The car was told H.265, so a stand-in codec would send it a stream it + cannot decode: if the hardware encoder cannot open, the session fails with the reason.""" + encoder = None + try: + if not LIBRARY.is_file(): + raise RuntimeError(f"{LIBRARY.name} not built") + encoder = HardwareHevcEncoder(width, height, bitrate_kbps=max(bitrate_kbps, 4000), margin_height=margin_height, + rate_control=rate_control) + encoder.encode_rgba(bytes(width * height * 4), keyframe=True) # exercise the driver and the keyframe contract + except Exception as error: + if encoder is not None: + encoder.close() + raise RuntimeError(f"The car needs H.265 and the hardware encoder could not start it: {error}") from error + log("encoder", backend=encoder.backend, codec=encoder.codec, fps=HW_FPS, abi=encoder.abi, rate_control=encoder.rate_control, + nv12=encoder.supports_nv12) + return encoder, HW_FPS + + # ANDROID AUTO VIDEO PERFORMANCE GUARDRAIL: Backend selection and frame-rate budgets are part of the low-latency video pipeline. # Altering this can degrade video smoothness or introduce lag on Android Auto. # Do not change it autonomously; require an explicit user request specifically for this video optimization. diff --git a/openpilot/starpilot/system/starport/session.py b/openpilot/starpilot/system/starport/session.py index c1002d2e49..9042b765cb 100644 --- a/openpilot/starpilot/system/starport/session.py +++ b/openpilot/starpilot/system/starport/session.py @@ -80,6 +80,7 @@ BLUETOOTH_AUTH_DATA = 0x8003 SENSOR_NIGHT, SENSOR_DRIVING_STATUS = 10, 13 CODEC_H264_BP = 3 +CODEC_H265 = 7 # MediaCodecType VIDEO_H265, offered by some head units instead of H.264 SETUP_STATUS_READY = 2 FOCUS_PROJECTED = 1 FOCUS_NATIVE = 2 @@ -141,6 +142,39 @@ def wire_frame(access_unit: bytes, *, drop_config: bool) -> bytes: return b"".join(units) or access_unit +# H.265 handling is kept apart from the H.264 functions above so the H.264 path stays byte-for-byte what it was. +# A NAL header is two bytes; the type is bits 1-6 of the first (ITU-T H.265 7.3.1.2). +HEVC_PARAMETER_SETS = (32, 33, 34) # VPS, SPS, PPS +HEVC_AUD = 35 + + +def hevc_nal_type(first_byte: int) -> int: + return (first_byte >> 1) & 0x3F + + +def codec_config_hevc(access_unit: bytes) -> bytes: + """The VPS, SPS and PPS NAL units of an H.265 keyframe, as the codec-config buffer carries them.""" + starts = list(START_CODE.finditer(access_unit)) + units = [] + for index, match in enumerate(starts): + end = starts[index + 1].start() if index + 1 < len(starts) else len(access_unit) + if match.end() < end and hevc_nal_type(access_unit[match.end()]) in HEVC_PARAMETER_SETS: + units.append(b"\x00\x00\x00\x01" + access_unit[match.end():end]) + return b"".join(units) + + +def wire_frame_hevc(access_unit: bytes, *, drop_config: bool) -> bytes: + """The H.265 picture NAL units without the access unit delimiter, and without parameter sets once they went separately.""" + starts = list(START_CODE.finditer(access_unit)) + drop = {HEVC_AUD, *HEVC_PARAMETER_SETS} if drop_config else {HEVC_AUD} + units = [] + for index, match in enumerate(starts): + end = starts[index + 1].start() if index + 1 < len(starts) else len(access_unit) + if match.end() < end and hevc_nal_type(access_unit[match.end()]) not in drop: + units.append(b"\x00\x00\x00\x01" + access_unit[match.end():end]) + return b"".join(units) or access_unit + + @dataclass(frozen=True) class VideoMode: channel: int @@ -153,6 +187,7 @@ class VideoMode: content_left: int | None = None content_top: int | None = None density: int = 0 # dpi the head unit reports; 0 when it does not say + codec: int = CODEC_H264_BP @property def content_width(self) -> int: @@ -166,6 +201,7 @@ class VideoMode: return {"channel": self.channel, "config_index": self.config_index, "width": self.width, "height": self.height, "fps": self.fps, "margin_width": self.margin_width, "margin_height": self.margin_height, "density": self.density, + **({"codec": CODEC_NAMES[self.codec]} if self.codec != CODEC_H264_BP else {}), **({"content_left": self.content_left, "content_top": self.content_top} if self.content_left is not None else {})} @@ -188,14 +224,18 @@ def describe_video_configs(channels: list[dict]) -> str: return ", ".join(offered) or "no video" -def choose_video_mode(channels: list[dict]) -> VideoMode: - """Pick a negotiated H.264 mode the software encoder can sustain.""" +def choose_video_mode(channels: list[dict], allow_hevc: bool = False) -> VideoMode: + """Pick a negotiated H.264 mode the software encoder can sustain. + + H.265 is considered only when the owner allows it and the car offered no usable H.264 mode, so a car that works on + H.264 is never changed.""" candidates = [] + hevc_candidates = [] for channel in channels: for index, config in enumerate(channel.get("video_configs", [])): resolution = one(config, 1) codec = one(config, 10, CODEC_H264_BP) - if resolution not in RESOLUTIONS or codec != CODEC_H264_BP: + if resolution not in RESOLUTIONS or codec not in (CODEC_H264_BP, CODEC_H265): continue width, height = RESOLUTIONS[resolution] margin_width, margin_height = int(one(config, 3, 0) or 0), int(one(config, 4, 0) or 0) @@ -215,10 +255,13 @@ def choose_video_mode(channels: list[dict]) -> VideoMode: top_edge + bottom == margin_height and left_edge + right == margin_width): left, top = left_edge, top_edge mode = VideoMode(int(channel["id"]), index, width, height, fps, margin_width, margin_height, left, top, - int(one(config, 5, 0) or 0)) - candidates.append((RESOLUTION_PREFERENCE.index(resolution), fps != 30, index, mode)) + int(one(config, 5, 0) or 0), codec) + (candidates if codec == CODEC_H264_BP else hevc_candidates).append((RESOLUTION_PREFERENCE.index(resolution), fps != 30, index, mode)) + if not candidates and allow_hevc: + candidates = hevc_candidates if not candidates: - raise ValueError(f"Head unit did not advertise a supported H.264 video mode (offered {describe_video_configs(channels)})") + hint = "; this car offers H.265, which can be turned on in Developer > StarPort" if hevc_candidates else "" + raise ValueError(f"Head unit did not advertise a supported H.264 video mode (offered {describe_video_configs(channels)}){hint}") return min(candidates, key=lambda item: item[:3])[3] @@ -631,6 +674,7 @@ class ProjectionSession(Session): self.video_confirmed = False self.input_channel: int | None = None self.max_window = self.MAX_WINDOW # raised by the supervisor for adapters that need the car's full window + self.allow_hevc = False # the owner's Allow H.265 video setting; used only when the car offers no H.264 mode self.phone_services = False # open the car-data and Bluetooth channels like a phone (adapter profile) self.phone_frames = False # send picture NAL units only (adapter profile) self.phone_address = "" # this comma's Bluetooth address, for the pairing request @@ -652,7 +696,7 @@ class ProjectionSession(Session): def start(self, device_name: str, device_brand: str) -> VideoMode: self.channels = self.discover(device_name, device_brand) - self.mode = choose_video_mode(self.channels) + self.mode = choose_video_mode(self.channels, self.allow_hevc) self.open_video() self.open_input() if self.phone_services: @@ -667,7 +711,7 @@ class ProjectionSession(Session): opened = parse_fields(self.wait_for(mode.channel, MSG_CHANNEL_OPEN_RESPONSE)) if signed(one(opened, 1)) != 0: raise ValueError(f"Head unit rejected opening the video channel: {json_fields(opened)}") - self.send(mode.channel, AV_SETUP_REQUEST, field(1, CODEC_H264_BP)) + self.send(mode.channel, AV_SETUP_REQUEST, field(1, mode.codec)) setup = parse_fields(self.wait_for(mode.channel, AV_SETUP_RESPONSE)) window = int(one(setup, 2, 0) or 0) if one(setup, 1) != SETUP_STATUS_READY or window < 1: # the window is capped to MAX_WINDOW below; MIB3 offers 100 @@ -944,13 +988,13 @@ class ProjectionSession(Session): fragments = 0 config = b"" if self.needs_keyframe: - config = codec_config(data) + config = codec_config_hevc(data) if self.mode.codec == CODEC_H265 else codec_config(data) if config: fragments += self.send(self.mode.channel, AV_MEDIA_CODEC_CONFIG, config) if not self.fragment_acks: self.config_ack_slack += 1 if self.phone_frames: - data = wire_frame(data, drop_config=bool(config)) + data = (wire_frame_hevc if self.mode.codec == CODEC_H265 else wire_frame)(data, drop_config=bool(config)) fragments += self.send(self.mode.channel, AV_MEDIA_WITH_TIMESTAMP, struct.pack(">Q", timestamp_us) + data) if self.fragment_acks: self.frame_fragments.append(fragments) diff --git a/openpilot/starpilot/system/starport/supervisor.py b/openpilot/starpilot/system/starport/supervisor.py index 74d286563f..6e7d94c581 100644 --- a/openpilot/starpilot/system/starport/supervisor.py +++ b/openpilot/starpilot/system/starport/supervisor.py @@ -42,7 +42,7 @@ from openpilot.starpilot.system.starport.frame_source import (DEFAULT_PATH as DE from openpilot.starpilot.system.starport.projection_control import DEFAULT_CONTROL_SOCKET, NATIVE_FOCUS from openpilot.starpilot.system.starport.touch import DEFAULT_TOUCH_SOCKET from openpilot.starpilot.system.starport.view import CAR_FRAME_PATH, ViewSource, renderer_available -from openpilot.starpilot.system.starport.session import AuthenticationRejected, PeerRequestedStop, ProjectionSession +from openpilot.starpilot.system.starport.session import CODEC_H265, AuthenticationRejected, PeerRequestedStop, ProjectionSession BACKOFF_SECONDS = (2.0, 4.0, 8.0, 15.0, 30.0) STABLE_SESSION_SECONDS = 30.0 @@ -769,7 +769,7 @@ class Supervisor: paired in Bluetooth settings) becomes the chosen car when it is the only one. Never during a pairing window or session.""" config = self.config if (config["receiver_address"] or config["connection"] != "wireless" or self.car_type() == "adapter" - or self._pairing_active() or self._session_alive() or now - getattr(self, "_adopt_at", -ADOPT_INTERVAL) < ADOPT_INTERVAL): + or not self._projection_enabled() or self._pairing_active() or self._session_alive() or now - getattr(self, "_adopt_at", -ADOPT_INTERVAL) < ADOPT_INTERVAL): if config["receiver_address"]: self._candidates = [] return @@ -1568,6 +1568,7 @@ class Supervisor: tuning = car_profiles.tuning(config) session.max_window = tuning["max_window"] session.phone_frames = tuning["phone_frames"] + session.allow_hevc = bool(config["allow_hevc"]) if tuning["phone_services"]: session.phone_services = True session.phone_address = self._adapter_address() @@ -1583,16 +1584,20 @@ class Supervisor: except (OSError, ValueError, TypeError, KeyError) as error: self.log("screen_profile_error", error=str(error)) - from openpilot.starpilot.system.starport.hw_encoder import create_encoder + from openpilot.starpilot.system.starport.hw_encoder import create_encoder, create_hevc_encoder # ANDROID AUTO VIDEO PERFORMANCE GUARDRAIL: Encoder cadence and pipeline flags are tuned together for Android Auto video. # Altering this can degrade video smoothness or introduce lag on Android Auto. # Do not change it autonomously; require an explicit user request specifically for this video optimization. software_fps = min(SOFTWARE_FPS, config["fps"]) if config["fps"] else SOFTWARE_FPS - encoder, fps = create_encoder(mode.width, mode.height, preference=encoder_preference(config["encoder"]), - bitrate_kbps=config["bitrate_kbps"], - margin_height=mode.margin_height if mode.content_top in (None, mode.margin_height // 2) else 0, - software_fps=software_fps, log=self.log, - rate_control=config["rate_control"]) + margin_height = mode.margin_height if mode.content_top in (None, mode.margin_height // 2) else 0 + if mode.codec == CODEC_H265: # only a car with no H.264 mode, and only with Allow H.265 video on + encoder, fps = create_hevc_encoder(mode.width, mode.height, bitrate_kbps=config["bitrate_kbps"], margin_height=margin_height, + log=self.log, rate_control=config["rate_control"]) + else: + encoder, fps = create_encoder(mode.width, mode.height, preference=encoder_preference(config["encoder"]), + bitrate_kbps=config["bitrate_kbps"], margin_height=margin_height, + software_fps=software_fps, log=self.log, + rate_control=config["rate_control"]) fps = min(fps, mode.fps, config["fps"] or fps) interval = 1.0 / fps flags = (FLAG_NV12 if config["gpu_nv12"] and getattr(encoder, "supports_nv12", False) else 0) | \ diff --git a/openpilot/starpilot/system/starport/tests/test_hevc.py b/openpilot/starpilot/system/starport/tests/test_hevc.py new file mode 100644 index 0000000000..8e4e916447 --- /dev/null +++ b/openpilot/starpilot/system/starport/tests/test_hevc.py @@ -0,0 +1,113 @@ +"""H.265 is an opt-in for a car that offers nothing else; every car with an H.264 mode is chosen exactly as before.""" +import itertools + +import pytest + +from openpilot.starpilot.system.starport import hw_encoder +from openpilot.starpilot.system.starport.hw_encoder import HEVC_AUD, normalize_access_unit_hevc +from openpilot.starpilot.system.starport.session import (CODEC_H264_BP, CODEC_H265, START_CODE, VideoMode, choose_video_mode, + codec_config, codec_config_hevc, wire_frame, wire_frame_hevc) + +SC = b"\x00\x00\x00\x01" + + +def config(resolution, codec, margin_w=0, margin_h=0, fps=2): + return {1: [resolution], 2: [fps], 3: [margin_w], 4: [margin_h], 10: [codec]} + + +def channels(*configs): + return [{"id": 3, "video_configs": list(configs), "display_type": 0}] + + +# What Bugsink showed one "AndroidAuto" unit offering: every mode H.265, none H.264 (enums 8, 7, 6 are portrait). +H265_ONLY = channels(config(8, 7, 288, 1378), config(7, 7, 0, 811), config(6, 7, 0, 541), config(3, 7, 0, 360)) +H264_OFFERS = [ + channels(config(2, 3, 0, 240)), + channels(config(1, 3), config(2, 3, 0, 240), config(3, 3, 0, 360)), + channels(config(3, 3, 0, 360), config(2, 3, 0, 240, fps=1)), + channels(config(1, 3)), +] + + +def test_without_the_setting_an_h265_only_car_is_refused_with_a_hint(): + with pytest.raises(ValueError, match="supported H.264 video mode.*Developer > StarPort"): + choose_video_mode(H265_ONLY) + with pytest.raises(ValueError, match="supported H.264 video mode.*Developer > StarPort"): + choose_video_mode(H265_ONLY, False) + + +def test_with_the_setting_an_h265_only_car_gets_the_1080p_h265_mode_inside_the_known_resolutions(): + mode = choose_video_mode(H265_ONLY, True) + assert (mode.codec, mode.width, mode.height, mode.margin_height) == (CODEC_H265, 1920, 1080, 360) + assert mode.as_dict()["codec"] == "H.265" + + +@pytest.mark.parametrize("offer", H264_OFFERS) +def test_a_car_with_an_h264_mode_is_chosen_identically_whatever_the_setting(offer): + without, with_setting = choose_video_mode(offer), choose_video_mode(offer, True) + assert without == with_setting and without.codec == CODEC_H264_BP + assert "codec" not in without.as_dict() # H.264 reports and screen files are unchanged + + +def test_mixed_offers_never_prefer_h265_over_h264(): + for h264, h265 in itertools.product(H264_OFFERS, [H265_ONLY, channels(config(2, 7, 0, 240))]): + mixed = [{"id": 3, "video_configs": h264[0]["video_configs"] + h265[0]["video_configs"]}] + assert choose_video_mode(mixed, True) == choose_video_mode(h264) + + +def test_h265_prefers_the_smaller_known_resolution_like_h264(): + mode = choose_video_mode(channels(config(3, 7, 0, 360), config(2, 7, 0, 240)), True) + assert (mode.codec, mode.width, mode.height) == (CODEC_H265, 1280, 720) + + +def au(*kinds): + """An Annex B H.265 access unit with the given NAL types.""" + return b"".join(SC + bytes([kind << 1, 1]) + b"\x42" * 6 for kind in kinds) + + +def test_hevc_codec_config_and_wire_frame_use_the_two_byte_header(): + frame = au(35, 32, 33, 34, 19) + assert START_CODE.findall(codec_config_hevc(frame)) == [SC] * 3 + assert b"".join(bytes([kind << 1, 1]) + b"\x42" * 6 for kind in (32, 33, 34)) == codec_config_hevc(frame).replace(SC, b"") + assert wire_frame_hevc(frame, drop_config=True) == SC + bytes([19 << 1, 1]) + b"\x42" * 6 + assert wire_frame_hevc(frame, drop_config=False) == au(32, 33, 34, 19) # the AUD is always dropped + assert codec_config_hevc(au(35, 1)) == b"" + + +def test_h264_functions_are_unchanged_by_the_h265_ones(): + h264 = SC + b"\x09\xf0" + SC + b"\x67" + b"\x01" * 5 + SC + b"\x68" + b"\x02" * 3 + SC + b"\x65" + b"\x03" * 9 + assert codec_config(h264) == SC + b"\x67" + b"\x01" * 5 + SC + b"\x68" + b"\x02" * 3 + assert wire_frame(h264, drop_config=True) == SC + b"\x65" + b"\x03" * 9 + + +def test_hevc_access_units_are_checked_like_the_h264_ones(): + keyframe = normalize_access_unit_hevc(au(32, 33, 34, 20), keyframe=True) + assert keyframe.startswith(HEVC_AUD) and keyframe.count(SC) == 5 + assert normalize_access_unit_hevc(au(35, 1), keyframe=False).startswith(HEVC_AUD) + # The comma's encoder emits two slices per picture; both belong to the same picture. + assert normalize_access_unit_hevc(au(32, 33, 34, 19, 19), keyframe=True).count(SC) == 6 + assert normalize_access_unit_hevc(au(1, 1), keyframe=False).count(SC) == 3 + for bad, key in ((au(32, 33, 34), True), (au(35), False), (au(33, 34, 19), True), (au(32, 33, 34, 1), True), (au(19, 1), False)): + with pytest.raises(RuntimeError): + normalize_access_unit_hevc(bad, keyframe=key) + with pytest.raises(RuntimeError): + normalize_access_unit_hevc(b"not annex b", keyframe=False) + + +def test_the_hevc_encoder_class_only_changes_what_h265_needs(): + h264, hevc = hw_encoder.HardwareH264Encoder, hw_encoder.HardwareHevcEncoder + assert (h264.max_width, h264.max_height, h264.codec_options, h264.required_abi) == (1280, 720, 0, 2) + assert (hevc.max_width, hevc.max_height, hevc.codec_options, hevc.required_abi) == (1920, 1080, hw_encoder.OPTION_HEVC, 4) + assert hw_encoder.OPTION_HEVC & hw_encoder.OPTION_CBR == 0 + + +def test_h265_never_falls_back_to_another_codec(monkeypatch, tmp_path): + monkeypatch.setattr(hw_encoder, "LIBRARY", tmp_path / "missing.so") + events = [] + with pytest.raises(RuntimeError, match="needs H.265 and the hardware encoder could not start it"): + hw_encoder.create_hevc_encoder(1920, 1080, bitrate_kbps=6000, margin_height=360, log=lambda *a, **k: events.append(a)) + assert events == [] # nothing was announced as started + + +def test_video_mode_defaults_to_h264(): + assert VideoMode(3, 0, 1280, 720, 30, 0, 240).codec == CODEC_H264_BP