Sync: commaai/openpilot:master → sunnypilot/sunnypilot:master (#1875)

# Conflicts:
#	.github/workflows/prebuilt.yaml
#	common/api.py
#	common/params_keys.h
#	opendbc_repo
#	selfdrive/controls/controlsd.py
#	selfdrive/controls/lib/desire_helper.py
#	selfdrive/modeld/modeld.py
#	selfdrive/ui/mici/layouts/settings/device.py
#	selfdrive/ui/soundd.py
#	system/athena/athenad.py
#	system/athena/registration.py
#	system/hardware/hardwared.py
#	system/loggerd/deleter.py
#	system/manager/manager.py
#	system/manager/process_config.py
#	system/sentry.py
#	system/updated/updated.py
#	tinygrad_repo
#	tools/scripts/uiview.py

.dockerignore

@@ -1,18 +0,0 @@
-**/.git
-.DS_Store
-*.dylib
-*.DSYM
-*.d
-*.pyc
-*.pyo
-.*.swp
-.*.swo
-.*.un~
-*.tmp
-*.o
-*.o-*
-*.os
-*.os-*
-
-venv/
-.venv/

.gitattributes

@@ -11,4 +11,4 @@
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+common/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git

Dockerfile.openpilot

@@ -1,38 +0,0 @@
-FROM ubuntu:24.04
-
-ENV PYTHONUNBUFFERED=1
-
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
-RUN mkdir -p ${OPENPILOT_PATH}
-WORKDIR ${OPENPILOT_PATH}
-
-COPY --chown=$USER . ${OPENPILOT_PATH}/
-
-ENV UV_BIN="/home/$USER/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'

Jenkinsfile

@@ -12,7 +12,7 @@ def retryWithDelay(int maxRetries, int delay, Closure body) {
 def device(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
-ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
+ssh -o ControlMaster=auto -o ControlPath=/tmp/ssh_control_%C -o ControlPersist=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
 
 set -e
 
@@ -210,7 +210,7 @@ node {
       'HW + Unit Tests': {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
+          step("test power draw", "pytest -s selfdrive/test//test_power_draw.py"),
           step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
@@ -246,7 +246,7 @@ node {
           step("build openpilot", "cd system/manager && ./build.py"),
           step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
-          step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
+          step("test amp", "pytest common/hardware/tici/tests/test_amplifier.py"),
         ])
       },
 

SConstruct

@@ -276,6 +276,8 @@ def count_scons_nodes(nodes):
     if node in seen:
       continue
     seen.add(node)
+    if hasattr(node, 'has_builder') and node.has_builder():
+      build_product_nodes.add(node)
     executor = node.get_executor()
     if executor is not None:
       stack += executor.get_all_prerequisites() + executor.get_all_children()
@@ -284,6 +286,7 @@ def count_scons_nodes(nodes):
 
 progress_interval = 5
 progress_count = 0
+build_product_nodes = set()
 progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
 
 def progress_function(node):
@@ -299,3 +302,11 @@ def progress_function(node):
 
 Progress(progress_function, interval=progress_interval)
 AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
+
+def check_build_product_size(target, source, env):
+  limit = 50 * 1024 * 1024  # GitHub max size
+  for t in target:
+    if hasattr(t, 'isfile') and t.isfile() and (size := os.path.getsize(t.abspath)) > limit:
+      raise SCons.Errors.UserError(f"{t} is {size / (1024 * 1024):.1f} MiB, exceeding the {limit / (1024 * 1024):.1f} MiB limit")
+if not GetOption('extras'):
+  AddPostAction(list(build_product_nodes), Action(check_build_product_size, None))

cereal/deprecated.capnp

@@ -760,18 +760,6 @@ struct LateralLQRState @0x9024e2d790c82ade {
   steeringAngleDesiredDeg @6 :Float32;
 }
 
-struct LateralCurvatureState @0xad9d8095c06f7c61 {
-  active @0 :Bool;
-  actualCurvature @1 :Float32;
-  desiredCurvature @2 :Float32;
-  error @3 :Float32;
-  p @4 :Float32;
-  i @5 :Float32;
-  f @6 :Float32;
-  output @7 :Float32;
-  saturated @8 :Bool;
-}
-
 struct LateralPlannerSolution @0x84caeca5a6b4acfe {
   x @0 :List(Float32);
   y @1 :List(Float32);

cereal/log.capnp

@@ -818,7 +818,7 @@ struct ControlsState @0x97ff69c53601abf1 {
     debugState @59 :LateralDebugState;
     torqueState @60 :LateralTorqueState;
 
-    curvatureStateDEPRECATED @65 :Deprecated.LateralCurvatureState;
+    curvatureState @65 :LateralCurvatureState;
     lqrStateDEPRECATED @55 :Deprecated.LateralLQRState;
     indiStateDEPRECATED @52 :Deprecated.LateralINDIState;
   }
@@ -867,6 +867,18 @@ struct ControlsState @0x97ff69c53601abf1 {
     saturated @3 :Bool;
   }
 
+  struct LateralCurvatureState @0xad9d8095c06f7c61 {
+    active @0 :Bool;
+    actualCurvature @1 :Float32;
+    desiredCurvature @2 :Float32;
+    error @3 :Float32;
+    p @4 :Float32;
+    i @5 :Float32;
+    f @6 :Float32;
+    output @7 :Float32;
+    saturated @8 :Bool;
+  }
+
   deprecated :group {
     vEgo @0 :Float32;
     vEgoRaw @32 :Float32;
@@ -2247,6 +2259,7 @@ struct LiveDelayData {
   lateralDelayEstimateStd @5 :Float32;
   points @4 :List(Float32);
   calPerc @6 :Int8;
+  version @7 :Int32;
 
   enum Status {
     unestimated @0;

common/api/base.py

@@ -3,7 +3,7 @@ import os
 import requests
 import unicodedata
 from datetime import datetime, timedelta, UTC
-from openpilot.system.hardware.hw import Paths
+from openpilot.common.hardware.hw import Paths
 from openpilot.system.version import get_version
 
 # name: jwt signature algorithm

common/esim/__init__.py

@@ -0,0 +1,3 @@
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+
+__all__ = ["LPABase", "LPAError", "LPAProfileNotFoundError", "Profile"]

common/esim/base.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+
+class LPAError(RuntimeError):
+  pass
+
+
+class LPAProfileNotFoundError(LPAError):
+  pass
+
+
+@dataclass
+class Profile:
+  iccid: str
+  nickname: str
+  enabled: bool
+  provider: str
+
+  @property
+  def is_comma(self) -> bool:
+    return self.provider == 'Webbing' and self.iccid.startswith('8985235')
+
+
+class LPABase(ABC):
+  @abstractmethod
+  def list_profiles(self) -> list[Profile]:
+    pass
+
+  @abstractmethod
+  def get_active_profile(self) -> Profile | None:
+    pass
+
+  @abstractmethod
+  def delete_profile(self, iccid: str) -> None:
+    pass
+
+  @abstractmethod
+  def download_profile(self, qr: str, nickname: str | None = None) -> None:
+    pass
+
+  @abstractmethod
+  def nickname_profile(self, iccid: str, nickname: str) -> None:
+    pass
+
+  @abstractmethod
+  def switch_profile(self, iccid: str) -> None:
+    pass
+
+  def process_notifications(self) -> None:
+    pass
+
+  @abstractmethod
+  def is_euicc(self) -> bool:
+    pass

common/esim/esim.py

@@ -1,8 +1,8 @@
 #!/usr/bin/env python3
 
 import argparse
-from openpilot.system.hardware import HARDWARE
-from openpilot.system.hardware.base import LPABase, Profile
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.esim.base import LPABase, Profile
 
 
 def sorted_profiles(lpa: LPABase) -> list[Profile]:

common/esim/lpa.py

@@ -19,7 +19,7 @@ from typing import Any
 from pathlib import Path
 
 from openpilot.common.time_helpers import system_time_valid
-from openpilot.system.hardware.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
+from openpilot.common.esim.base import LPABase, LPAError, LPAProfileNotFoundError, Profile
 
 GSMA_CI_BUNDLE = str(Path(__file__).parent / "gsma_ci_bundle.pem")
 

common/hardware/__init__.py

@@ -1,9 +1,9 @@
 import os
 from typing import cast
 
-from openpilot.system.hardware.base import HardwareBase
-from openpilot.system.hardware.tici.hardware import Tici
-from openpilot.system.hardware.pc.hardware import Pc
+from openpilot.common.hardware.base import HardwareBase
+from openpilot.common.hardware.tici.hardware import Tici
+from openpilot.common.hardware.pc.hardware import Pc
 
 TICI = os.path.isfile('/TICI')
 AGNOS = os.path.isfile('/AGNOS')

common/hardware/base.h

@@ -12,8 +12,6 @@ class HardwareNone {
 public:
   static std::string get_name() { return ""; }
   static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
-  static int get_voltage() { return 0; }
-  static int get_current() { return 0; }
 
   static std::string get_serial() { return "cccccc"; }
 
@@ -24,6 +22,4 @@ public:
   static void set_ir_power(int percentage) {}
 
   static bool PC() { return false; }
-  static bool TICI() { return false; }
-  static bool AGNOS() { return false; }
 };

common/hardware/base.py

@@ -3,27 +3,11 @@ from abc import abstractmethod, ABC
 from dataclasses import dataclass, fields
 
 from cereal import log
+from openpilot.common.esim.base import LPABase
 
 NetworkType = log.DeviceState.NetworkType
 NetworkStrength = log.DeviceState.NetworkStrength
 
-class LPAError(RuntimeError):
-  pass
-
-class LPAProfileNotFoundError(LPAError):
-  pass
-
-@dataclass
-class Profile:
-  iccid: str
-  nickname: str
-  enabled: bool
-  provider: str
-
-  @property
-  def is_comma(self) -> bool:
-    return self.provider == 'Webbing' and self.iccid.startswith('8985235')
-
 @dataclass
 class ThermalZone:
   # a zone from /sys/class/thermal/thermal_zone*
@@ -70,38 +54,6 @@ class ThermalConfig:
           ret[f.name + "TempC"] = v.read()
     return ret
 
-class LPABase(ABC):
-  @abstractmethod
-  def list_profiles(self) -> list[Profile]:
-    pass
-
-  @abstractmethod
-  def get_active_profile(self) -> Profile | None:
-    pass
-
-  @abstractmethod
-  def delete_profile(self, iccid: str) -> None:
-    pass
-
-  @abstractmethod
-  def download_profile(self, qr: str, nickname: str | None = None) -> None:
-    pass
-
-  @abstractmethod
-  def nickname_profile(self, iccid: str, nickname: str) -> None:
-    pass
-
-  @abstractmethod
-  def switch_profile(self, iccid: str) -> None:
-    pass
-
-  def process_notifications(self) -> None:
-    pass
-
-  @abstractmethod
-  def is_euicc(self) -> bool:
-    pass
-
 class HardwareBase(ABC):
   @staticmethod
   def get_cmdline() -> dict[str, str]:
@@ -133,7 +85,7 @@ class HardwareBase(ABC):
   def get_device_type(self):
     pass
 
-  def get_imei(self, slot) -> str:
+  def get_imei(self) -> str:
     return ""
 
   def get_serial(self):
@@ -190,21 +142,12 @@ class HardwareBase(ABC):
   def get_gpu_usage_percent(self):
     return 0
 
-  def get_modem_version(self):
-    return None
-
   def get_modem_temperatures(self):
     return []
 
   def initialize_hardware(self):
     pass
 
-  def get_networks(self):
-    return None
-
-  def has_internal_panda(self) -> bool:
-    return False
-
   def reset_internal_panda(self):
     pass
 
@@ -214,11 +157,5 @@ class HardwareBase(ABC):
   def get_modem_data_usage(self):
     return -1, -1
 
-  def get_voltage(self) -> float:
-    return 0.
-
-  def get_current(self) -> float:
-    return 0.
-
   def set_ir_power(self, percent: int):
     pass

common/hardware/hw.h

@@ -2,14 +2,14 @@
 
 #include <string>
 
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 #include "common/util.h"
 
 #if __TICI__
-#include "system/hardware/tici/hardware.h"
+#include "common/hardware/tici/hardware.h"
 #define Hardware HardwareTici
 #else
-#include "system/hardware/pc/hardware.h"
+#include "common/hardware/pc/hardware.h"
 #define Hardware HardwarePC
 #endif
 

common/hardware/hw.py

@@ -2,7 +2,7 @@ import os
 import platform
 from pathlib import Path
 
-from openpilot.system.hardware import PC
+from openpilot.common.hardware import PC
 
 DEFAULT_DOWNLOAD_CACHE_ROOT = "/tmp/comma_download_cache"
 

common/hardware/pc/hardware.h

@@ -2,13 +2,11 @@
 
 #include <string>
 
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 
 class HardwarePC : public HardwareNone {
 public:
   static std::string get_name() { return "pc"; }
   static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
   static bool PC() { return true; }
-  static bool TICI() { return util::getenv("TICI", 0) == 1; }
-  static bool AGNOS() { return util::getenv("TICI", 0) == 1; }
 };

common/hardware/pc/hardware.py

@@ -0,0 +1,10 @@
+from cereal import log
+from openpilot.common.hardware.base import HardwareBase
+
+class Pc(HardwareBase):
+  def get_device_type(self):
+    return "pc"
+
+  def get_network_type(self):
+    # some stuff is gated on wifi, so just assume for now
+    return log.DeviceState.NetworkType.wifi

common/hardware/tici/__init__.py

@@ -0,0 +1 @@
+

common/hardware/tici/agnos.json

@@ -0,0 +1,84 @@
+[
+  {
+    "name": "xbl",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl-e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb.img.xz",
+    "hash": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "hash_raw": "e8acf2a9cc7f0ce84cb803bfea9477f765c0d7b4daf26048e59651b9e6a7bfbb",
+    "size": 3282256,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "bea7f1a24428c3ededf672fa4fc78baf180cfbd8aafb77c974655b38517283e3"
+  },
+  {
+    "name": "xbl_config",
+    "url": "https://commadist.azureedge.net/agnosupdate/xbl_config-758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85.img.xz",
+    "hash": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "hash_raw": "758552ecf92b5569677197783bf0ccb73d7f961685308e45d3276ac9dd974f85",
+    "size": 98124,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "fb18cde08a98a168961ecd357e92474823046752b94e112f59fe51a6acd7197d"
+  },
+  {
+    "name": "abl",
+    "url": "https://commadist.azureedge.net/agnosupdate/abl-b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c.img.xz",
+    "hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "hash_raw": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c",
+    "size": 274432,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "b6fba807b9bcd66a31f2afb0eba5163ec239693ad32e2e4200f6c356adfe098c"
+  },
+  {
+    "name": "aop",
+    "url": "https://commadist.azureedge.net/agnosupdate/aop-78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1.img.xz",
+    "hash": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "hash_raw": "78b2287ca219a0811b3004c523fa0f4749e4d1fd92be3aba61699305b7943ad1",
+    "size": 184364,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "6c9135446bd3fc075fcee59b887a12e49029ab1f98ed8d6d1e32c73569d47de3"
+  },
+  {
+    "name": "devcfg",
+    "url": "https://commadist.azureedge.net/agnosupdate/devcfg-f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc.img.xz",
+    "hash": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "hash_raw": "f71df3a86958c093ba3969254c4db025187eef9385427f1ade946742939b43cc",
+    "size": 40336,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "2a67971602012c1b43544964709da13c322786b456a8e78568b117e8b1540ce3"
+  },
+  {
+    "name": "boot",
+    "url": "https://commadist.azureedge.net/agnosupdate/boot-8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8.img.xz",
+    "hash": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "hash_raw": "8806802b195a5b1396a3ae8dd92a8b7711dc522f6aceafd820e871bae5c8a6d8",
+    "size": 17487872,
+    "sparse": false,
+    "full_check": true,
+    "has_ab": true,
+    "ondevice_hash": "edca8bee1531e66953d107eeceeed2dc7b3ca46417e49d55508f94e58bf95db8"
+  },
+  {
+    "name": "system",
+    "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img.xz",
+    "hash": "78acfe16a7b62a3a91fc7a81f40a693e4468cec1c69df7d0b1e550aacc646113",
+    "hash_raw": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+    "size": 4718592000,
+    "sparse": true,
+    "full_check": false,
+    "has_ab": true,
+    "ondevice_hash": "743142c5a898f27b2a1029cca42c8a5d5d1fc0096414422b850fe84c8d0b8342",
+    "alt": {
+      "hash": "ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-ef0d879302cb29e72110e9c8d3f947c830fd7d37c8192744fc9dbea1af78501f.img",
+      "size": 4718592000
+    }
+  }
+]

common/hardware/tici/agnos.py

@@ -10,10 +10,7 @@ from collections.abc import Generator
 
 import requests
 
-import openpilot.system.updated.casync.casync as casync
-
 SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
-CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"
 
 AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json"
 
@@ -187,50 +184,6 @@ def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog)
     os.sync()
 
 
-def extract_casync_image(target_slot_number: int, partition: dict, cloudlog):
-  path = get_partition_path(target_slot_number, partition)
-  seed_path = path[:-1] + ('b' if path[-1] == 'a' else 'a')
-
-  target = casync.parse_caibx(partition['casync_caibx'])
-
-  sources: list[tuple[str, casync.ChunkReader, casync.ChunkDict]] = []
-
-  # First source is the current partition.
-  try:
-    raw_hash = get_raw_hash(seed_path, partition['size'])
-    caibx_url = f"{CAIBX_URL}{partition['name']}-{raw_hash}.caibx"
-
-    try:
-      cloudlog.info(f"casync fetching {caibx_url}")
-      sources += [('seed', casync.FileChunkReader(seed_path), casync.build_chunk_dict(casync.parse_caibx(caibx_url)))]
-    except requests.RequestException:
-      cloudlog.error(f"casync failed to load {caibx_url}")
-  except Exception:
-    cloudlog.exception("casync failed to hash seed partition")
-
-  # Second source is the target partition, this allows for resuming
-  sources += [('target', casync.FileChunkReader(path), casync.build_chunk_dict(target))]
-
-  # Finally we add the remote source to download any missing chunks
-  sources += [('remote', casync.RemoteChunkReader(partition['casync_store']), casync.build_chunk_dict(target))]
-
-  last_p = 0
-
-  def progress(cur):
-    nonlocal last_p
-    p = int(cur / partition['size'] * 100)
-    if p != last_p:
-      last_p = p
-      print(f"Installing {partition['name']}: {p}", flush=True)
-
-  stats = casync.extract(target, sources, path, progress)
-  cloudlog.error(f'casync done {json.dumps(stats)}')
-
-  os.sync()
-  if not verify_partition(target_slot_number, partition, force_full_check=True):
-    raise Exception(f"Raw hash mismatch '{partition['hash_raw'].lower()}'")
-
-
 def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False):
   cloudlog.info(f"Downloading and writing {partition['name']}")
 
@@ -245,10 +198,7 @@ def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalo
 
   path = get_partition_path(target_slot_number, partition)
 
-  if ('casync_caibx' in partition) and not standalone:
-    extract_casync_image(target_slot_number, partition, cloudlog)
-  else:
-    extract_compressed_image(target_slot_number, partition, cloudlog)
+  extract_compressed_image(target_slot_number, partition, cloudlog)
 
   # Write hash after successful flash
   if not full_check:

common/hardware/tici/hardware.h

@@ -1,6 +1,5 @@
 #pragma once
 
-#include <cstdlib>
 #include <cassert>
 #include <fstream>
 #include <map>
@@ -8,7 +7,7 @@
 #include <algorithm>  // for std::clamp
 
 #include "common/util.h"
-#include "system/hardware/base.h"
+#include "common/hardware/base.h"
 
 class HardwareTici : public HardwareNone {
 public:
@@ -22,7 +21,6 @@ public:
 
   static cereal::InitData::DeviceType get_device_type() {
     static const std::map<std::string, cereal::InitData::DeviceType> device_map = {
-      {"tici", cereal::InitData::DeviceType::TICI},
       {"tizi", cereal::InitData::DeviceType::TIZI},
       {"mici", cereal::InitData::DeviceType::MICI}
     };
@@ -31,9 +29,6 @@ public:
     return it->second;
   }
 
-  static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); }
-  static int get_current() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/curr1_input").c_str()); }
-
   static std::string get_serial() {
     static std::string serial("");
     if (serial.empty()) {
@@ -54,8 +49,7 @@ public:
 
   static void set_ir_power(int percent) {
     auto device = get_device_type();
-    if (device == cereal::InitData::DeviceType::TICI ||
-        device == cereal::InitData::DeviceType::TIZI) {
+    if (device == cereal::InitData::DeviceType::TIZI) {
       return;
     }
 

common/hardware/tici/hardware.py

@@ -10,11 +10,11 @@ from pathlib import Path
 from cereal import log
 from openpilot.common.utils import sudo_read, sudo_write
 from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
-from openpilot.system.hardware.base import HardwareBase, LPABase, ThermalConfig, ThermalZone
-from openpilot.system.hardware.tici import iwlist
-from openpilot.system.hardware.tici.lpa import TiciLPA
-from openpilot.system.hardware.tici.pins import GPIO
-from openpilot.system.hardware.tici.amplifier import Amplifier
+from openpilot.common.esim.base import LPABase
+from openpilot.common.hardware.base import HardwareBase, ThermalConfig, ThermalZone
+from openpilot.common.esim.lpa import TiciLPA
+from openpilot.common.hardware.tici.pins import GPIO
+from openpilot.common.hardware.tici.amplifier import Amplifier
 
 MODEM_STATE_PATH = "/dev/shm/modem"
 
@@ -147,9 +147,7 @@ class Tici(HardwareBase):
   def get_sim_lpa(self) -> LPABase:
     return TiciLPA()
 
-  def get_imei(self, slot):
-    if slot != 0:
-      return ""
+  def get_imei(self):
     return self.get_modem_state().get('imei', '')
 
   def get_network_info(self):
@@ -233,9 +231,6 @@ class Tici(HardwareBase):
 
     return super().get_network_metered(network_type)
 
-  def get_modem_version(self):
-    return self.get_modem_state().get('modem_version') or None
-
   def get_modem_temperatures(self):
     return self.get_modem_state().get('temperatures', [])
 
@@ -313,6 +308,9 @@ class Tici(HardwareBase):
         continue
       gov = 'ondemand' if powersave_enabled else 'performance'
       sudo_write(gov, f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_governor')
+      if not powersave_enabled:
+        # cap max core freq to 1689 Mhz
+        sudo_write('1689600', f'/sys/devices/system/cpu/cpufreq/policy{n}/scaling_max_freq')
 
     # *** IRQ config ***
 
@@ -380,43 +378,13 @@ class Tici(HardwareBase):
       pid = subprocess.check_output(["pgrep", "-f", "spi0"], encoding='utf8').strip()
       subprocess.call(["sudo", "chrt", "-f", "-p", "1", pid])
       subprocess.call(["sudo", "taskset", "-pc", "3", pid])
-    except subprocess.CalledProcessException as e:
+    except subprocess.CalledProcessError as e:
       print(str(e))
 
-  def get_networks(self):
-    r = {}
-
-    wlan = iwlist.scan()
-    if wlan is not None:
-      r['wlan'] = wlan
-
-    lte_info = self.get_network_info()
-    if lte_info is not None:
-      extra = lte_info['extra']
-
-      # <state>,"LTE",<is_tdd>,<mcc>,<mnc>,<cellid>,<pcid>,<earfcn>,<freq_band_ind>,
-      # <ul_bandwidth>,<dl_bandwidth>,<tac>,<rsrp>,<rsrq>,<rssi>,<sinr>,<srxlev>
-      if 'LTE' in extra:
-        extra = extra.split(',')
-        try:
-          r['lte'] = [{
-            "mcc": int(extra[3]),
-            "mnc": int(extra[4]),
-            "cid": int(extra[5], 16),
-            "nmr": [{"pci": int(extra[6]), "earfcn": int(extra[7])}],
-          }]
-        except (ValueError, IndexError):
-          pass
-
-    return r
-
   def get_modem_data_usage(self):
     ms = self.get_modem_state()
     return ms.get('tx_bytes', -1), ms.get('rx_bytes', -1)
 
-  def has_internal_panda(self):
-    return True
-
   def reset_internal_panda(self):
     gpio_init(GPIO.STM_RST_N, True)
     gpio_init(GPIO.STM_BOOT0, True)

common/hardware/tici/tests/__init__.py

@@ -0,0 +1 @@
+

common/hardware/tici/tests/test_amplifier.py

@@ -4,8 +4,8 @@ import random
 import subprocess
 
 from panda import Panda
-from openpilot.system.hardware import TICI, HARDWARE
-from openpilot.system.hardware.tici.amplifier import Amplifier
+from openpilot.common.hardware import TICI, HARDWARE
+from openpilot.common.hardware.tici.amplifier import Amplifier
 
 
 class TestAmplifier:

common/params.cc

@@ -12,7 +12,7 @@
 #include "common/queue.h"
 #include "common/swaglog.h"
 #include "common/util.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 
 namespace {
 

common/params_keys.h

@@ -58,6 +58,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsEngaged", {PERSISTENT, BOOL}},
     {"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
+    {"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsMetric", {PERSISTENT | BACKUP, BOOL}},
     {"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
     {"IsOnroad", {PERSISTENT, BOOL}},
@@ -81,6 +82,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
     {"LiveParameters", {PERSISTENT, JSON}},
     {"LiveParametersV2", {PERSISTENT, BYTES}},
     {"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
+    {"LivestreamRequestKeyframe", {CLEAR_ON_MANAGER_START | DONT_LOG, BOOL}},
     {"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
     {"LocationFilterInitialState", {PERSISTENT, BYTES}},
     {"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},

common/prefix.h

@@ -5,7 +5,7 @@
 
 #include "common/params.h"
 #include "common/util.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 
 class OpenpilotPrefix {
 public:

common/prefix.py

@@ -5,9 +5,9 @@ import uuid
 
 
 from openpilot.common.params import Params
-from openpilot.system.hardware import PC
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+from openpilot.common.hardware import PC
+from openpilot.common.hardware.hw import Paths
+from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 
 class OpenpilotPrefix:
   def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):

common/realtime.py

@@ -7,7 +7,7 @@ import time
 from setproctitle import getproctitle
 
 from openpilot.common.utils import MovingAverage
-from openpilot.system.hardware import PC
+from openpilot.common.hardware import PC
 
 
 # time step for each process

common/swaglog.cc

@@ -13,7 +13,7 @@
 #include <stdarg.h>
 #include "json11/json11.hpp"
 #include "common/version.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 
 #include "sunnypilot/common/version.h"
 

common/swaglog.py

@@ -8,7 +8,7 @@ from logging.handlers import BaseRotatingHandler
 import zmq
 
 from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from openpilot.system.hardware.hw import Paths
+from openpilot.common.hardware.hw import Paths
 
 
 def get_file_handler():

common/tests/test_swaglog.cc

@@ -6,7 +6,7 @@
 #include "common/swaglog.h"
 #include "common/util.h"
 #include "common/version.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 #include "json11/json11.hpp"
 
 #include "sunnypilot/common/version.h"

conftest.py

@@ -5,7 +5,7 @@ import pytest
 
 from openpilot.common.prefix import OpenpilotPrefix
 from openpilot.system.manager import manager
-from openpilot.system.hardware import TICI, HARDWARE
+from openpilot.common.hardware import TICI, HARDWARE
 
 # these are heavy CI-only tests, invoked explicitly in .github/workflows/tests.yaml
 collect_ignore = [
@@ -15,9 +15,12 @@ collect_ignore = [
   # which corrupts JIT captures for test_warp.py in the same process. Run separately in CI.
   "sunnypilot/modeld_v2/tests/test_warp.py",
 ]
-collect_ignore_glob = [
-  "selfdrive/debug/*.py",
-]
+
+
+def pytest_sessionstart(session):
+  # TODO: fix tests and enable test order randomization
+  if session.config.pluginmanager.hasplugin('randomly'):
+    session.config.option.randomly_reorganize = False
 
 
 @pytest.hookimpl(hookwrapper=True, trylast=True)

launch_chffrplus.sh

@@ -19,12 +19,12 @@ function agnos_init {
 
   # Check if AGNOS update is required
   if [ $(< /VERSION) != "$AGNOS_VERSION" ]; then
-    AGNOS_PY="$DIR/system/hardware/tici/agnos.py"
+    AGNOS_PY="$DIR/common/hardware/tici/agnos.py"
     MANIFEST="$DIR/system/hardware/tici/agnos.json"
     if $AGNOS_PY --verify $MANIFEST; then
       sudo reboot
     fi
-    $DIR/system/hardware/tici/updater $AGNOS_PY $MANIFEST
+    $DIR/common/hardware/tici/updater $AGNOS_PY $MANIFEST
   fi
 }
 

pyproject.toml

@@ -68,7 +68,7 @@ dependencies = [
 
   # these should be removed
   "psutil",
-  "pycryptodome", # used in updated/casync, panda, body, and a test
+  "pycryptodome", # used in panda, body, and a test
   "setproctitle",
 
   # logreader
@@ -127,7 +127,7 @@ allow-direct-references = true
 
 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--ignore=openpilot/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/ -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
+addopts = "--ignore=openpilot/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/ --ignore=selfdrive/modeld -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
 cpp_files = "test_*"
 cpp_harness = "selfdrive/test/cpp_harness.py"
 python_files = "test_*.py"

scripts/cell.sh

@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-nmcli connection modify --temporary esim ipv4.route-metric 1 ipv6.route-metric 1
-nmcli con up esim

scripts/disable-powersave.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 
 if __name__ == "__main__":
   HARDWARE.set_power_save(False)

scripts/manage-powersave.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import multiprocessing
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 
 
 def main():

scripts/reporter.py

@@ -25,7 +25,9 @@ class MetadataOnnxPBParser(OnnxPBParser):
 def get_checkpoint(f):
   model = MetadataOnnxPBParser(f).parse()
   metadata = {prop["key"]: prop["value"] for prop in model["metadata_props"]}
-  return metadata['model_checkpoint'].split('/')[0]
+  # "<uuid>" or ".../<run_uuid>/<step>"; combined models list vision then policy
+  parts = metadata['model_checkpoint'].split('/')
+  return parts[-2] if len(parts) > 1 else parts[0]
 
 
 if __name__ == "__main__":
@@ -37,8 +39,8 @@ if __name__ == "__main__":
     master_path = MASTER_PATH + MODEL_PATH + fn
     if os.path.exists(master_path):
       master = get_checkpoint(master_path)
-      master_col = f"[{master}](https://reporter.comma.life/experiment/{master})"
+      master_col = f"[{master}](https://reporter.comma.life/{master})"
     else:
       master_col = "N/A (new model)"
     pr = get_checkpoint(BASEDIR + MODEL_PATH + fn)
-    print("|", fn, "|", master_col, "|", f"[{pr}](https://reporter.comma.life/experiment/{pr})", "|")
+    print("|", fn, "|", master_col, "|", f"[{pr}](https://reporter.comma.life/{pr})", "|")

scripts/stop_updater.sh

@@ -1,7 +0,0 @@
-#!/usr/bin/env sh
-
-# Stop updater
-pkill -2 -f system.updated.updated
-
-# Remove pending update
-rm -f /data/safe_staging/finalized/.overlay_consistent

scripts/usb.sh

@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-# testing the GPU box
-
-export XDG_CACHE_HOME=/data/tinycache
-mkdir -p $XDG_CACHE_HOME
-
-cd /data/openpilot/tinygrad_repo/examples
-while true; do
-  AMD=1 AMD_IFACE=usb python ./beautiful_cartpole.py
-  sleep 1
-done

scripts/usbgpu/benchmark.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
-cd $DIR/../../tinygrad_repo
-
-GREEN='\033[0;32m'
-NC='\033[0m'
-
-
-#export DEBUG=2
-export PYTHONPATH=.
-export AM_RESET=1
-export AMD=1
-export AMD_IFACE=USB
-export AMD_LLVM=1
-
-python3 -m unittest -q --buffer test.test_tiny.TestTiny.test_plus \
-  > /tmp/test_tiny.log 2>&1 || (cat /tmp/test_tiny.log; exit 1)
-printf "${GREEN}Booted in ${SECONDS}s${NC}\n"
-printf "${GREEN}=============${NC}\n"
-
-printf "\n\n"
-printf "${GREEN}Transfer speeds:${NC}\n"
-printf "${GREEN}================${NC}\n"
-python3 test/external/external_test_usb_asm24.py TestDevCopySpeeds

selfdrive/assets/icons_mici/settings/software.png

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c4c38772e6080aa4b8bf5212d3619e949775468c64f3edb88a1a426d767c38d2
+size 1579

selfdrive/car/tests/test_docs.py

@@ -1,9 +1,5 @@
-import os
 
-from openpilot.common.basedir import BASEDIR
 from opendbc.car.docs import generate_cars_md, get_all_car_docs
-from openpilot.selfdrive.debug.dump_car_docs import dump_car_docs
-from openpilot.selfdrive.debug.print_docs_diff import print_car_docs_diff
 from openpilot.selfdrive.car.docs import CARS_MD_TEMPLATE
 
 
@@ -14,9 +10,3 @@ class TestCarDocs:
 
   def test_generator(self):
     generate_cars_md(self.all_cars, CARS_MD_TEMPLATE)
-
-  def test_docs_diff(self):
-    dump_path = os.path.join(BASEDIR, "selfdrive", "car", "tests", "cars_dump")
-    dump_car_docs(dump_path)
-    print_car_docs_diff(dump_path)
-    os.remove(dump_path)

selfdrive/car/tests/test_models.py

@@ -21,7 +21,7 @@ from opendbc.safety.tests.libsafety import libsafety_py
 from openpilot.common.basedir import BASEDIR
 from openpilot.selfdrive.pandad import can_capnp_to_list
 from openpilot.selfdrive.test.helpers import read_segment_list
-from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
 from openpilot.tools.lib.logreader import LogReader, LogsUnavailable, openpilotci_source, internal_source, comma_api_source
 from openpilot.tools.lib.route import SegmentName
 
@@ -192,7 +192,7 @@ class TestCarModelBase(unittest.TestCase):
     # make sure car params are within a valid range
     self.assertGreater(self.CP.mass, 1)
 
-    if self.CP.steerControlType != SteerControlType.angle:
+    if self.CP.steerControlType not in (SteerControlType.angle, SteerControlType.curvature):
       tuning = self.CP.lateralTuning.which()
       if tuning == 'pid':
         self.assertTrue(len(self.CP.lateralTuning.pid.kpV))
@@ -442,8 +442,9 @@ class TestCarModelBase(unittest.TestCase):
                               v_ego_raw < (self.safety.get_vehicle_speed_min() - 1e-3))
 
       # check steering angle for angle control cars (panda stores angle_meas in CAN units)
-      # ford excluded since it tracks curvature, not steering angle
-      if self.CP.steerControlType == SteerControlType.angle and not self.CP.notCar and self.CP.brand != "ford":
+      # ford and VW MEB excluded since they track curvature, not steering angle
+      # TODO: add curvature check, standardize CAN units to rm brand specific ANGLE_DEG_TO_CAN
+      if self.CP.steerControlType == SteerControlType.angle and not self.CP.notCar and self.CP.brand not in ("ford", "volkswagen"):
         angle_can = (CS.steeringAngleDeg + CS.steeringAngleOffsetDeg) * ANGLE_DEG_TO_CAN[self.CP.brand]
         checks['steeringAngleDeg'] += (angle_can > (self.safety.get_angle_meas_max() + 1) or
                                        angle_can < (self.safety.get_angle_meas_min() - 1))

selfdrive/controls/controlsd.py

@@ -15,6 +15,7 @@ from openpilot.selfdrive.controls.lib.drive_helpers import clip_curvature
 from openpilot.selfdrive.controls.lib.latcontrol import LatControl
 from openpilot.selfdrive.controls.lib.latcontrol_pid import LatControlPID
 from openpilot.selfdrive.controls.lib.latcontrol_angle import LatControlAngle, STEER_ANGLE_SATURATION_THRESHOLD
+from openpilot.selfdrive.controls.lib.latcontrol_curvature import LatControlCurvature
 from openpilot.selfdrive.controls.lib.latcontrol_torque import LatControlTorque
 from openpilot.selfdrive.controls.lib.longcontrol import LongControl
 from openpilot.selfdrive.modeld.modeld import LAT_SMOOTH_SECONDS
@@ -59,6 +60,8 @@ class Controls(ControlsExt):
     self.LaC: LatControl
     if self.CP.steerControlType == car.CarParams.SteerControlType.angle:
       self.LaC = LatControlAngle(self.CP, self.CP_SP, self.CI, DT_CTRL)
+    elif self.CP.steerControlType == car.CarParams.SteerControlType.curvature:
+      self.LaC = LatControlCurvature(self.CP, self.CP_SP, self.CI, DT_CTRL)
     elif self.CP.lateralTuning.which() == 'pid':
       self.LaC = LatControlPID(self.CP, self.CP_SP, self.CI, DT_CTRL)
     elif self.CP.lateralTuning.which() == 'torque':
@@ -143,11 +146,14 @@ class Controls(ControlsExt):
     lat_delay = self.sm["liveDelay"].lateralDelay + LAT_SMOOTH_SECONDS
 
     actuators.curvature = self.desired_curvature
-    steer, steeringAngleDeg, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
-                                                       self.steer_limited_by_safety, self.desired_curvature,
-                                                       self.calibrated_pose, curvature_limited, lat_delay)
+    steer, lateral_output, lac_log = self.LaC.update(CC.latActive, CS, self.VM, lp,
+                                                     self.steer_limited_by_safety, self.desired_curvature,
+                                                     self.calibrated_pose, curvature_limited, lat_delay)
     actuators.torque = float(steer)
-    actuators.steeringAngleDeg = float(steeringAngleDeg)
+    if self.CP.steerControlType == car.CarParams.SteerControlType.curvature:
+      actuators.curvature = float(lateral_output)
+    else:
+      actuators.steeringAngleDeg = float(lateral_output)
     # Ensure no NaNs/Infs
     for p in ACTUATOR_FIELDS:
       attr = getattr(actuators, p)
@@ -215,9 +221,14 @@ class Controls(ControlsExt):
     cs.forceDecel = bool((self.sm['driverMonitoringState'].alertLevel == log.DriverMonitoringState.AlertLevel.three) or
                          (self.sm['selfdriveState'].state == State.softDisabling))
 
+    # trigger the car's stock driver monitoring escalation
+    CC.driverMonitoringEscalation = cs.forceDecel
+
     lat_tuning = self.CP.lateralTuning.which()
     if self.CP.steerControlType == car.CarParams.SteerControlType.angle:
       cs.lateralControlState.angleState = lac_log
+    elif self.CP.steerControlType == car.CarParams.SteerControlType.curvature:
+      cs.lateralControlState.curvatureState = lac_log
     elif lat_tuning == 'pid':
       cs.lateralControlState.pidState = lac_log
     elif lat_tuning == 'torque':

selfdrive/controls/lib/desire_helper.py

@@ -45,7 +45,6 @@ class DesireHelper:
     self.lane_change_direction = LaneChangeDirection.none
     self.lane_change_timer = 0.0
     self.lane_change_ll_prob = 1.0
-    self.keep_pulse_timer = 0.0
     self.prev_one_blinker = False
     self.desire = log.Desire.none
     self.alc = AutoLaneChangeController(self)
@@ -132,14 +131,4 @@ class DesireHelper:
     else:
       self.desire = DESIRES[self.lane_change_direction][self.lane_change_state]
 
-    # Send keep pulse once per second during LaneChangeStart.preLaneChange
-    if self.lane_change_state in (LaneChangeState.off, LaneChangeState.laneChangeStarting):
-      self.keep_pulse_timer = 0.0
-    elif self.lane_change_state == LaneChangeState.preLaneChange:
-      self.keep_pulse_timer += DT_MDL
-      if self.keep_pulse_timer > 1.0:
-        self.keep_pulse_timer = 0.0
-      elif self.desire in (log.Desire.keepLeft, log.Desire.keepRight):
-        self.desire = log.Desire.none
-
     self.alc.update_state()

selfdrive/controls/lib/latcontrol_curvature.py

@@ -0,0 +1,58 @@
+import math
+
+from cereal import log
+from openpilot.common.pid import PIDController
+from openpilot.selfdrive.controls.lib.latcontrol import LatControl
+from openpilot.selfdrive.controls.lib.drive_helpers import MAX_CURVATURE
+
+CURVATURE_SATURATION_THRESHOLD = 1e-3  # 1/m
+
+
+class LatControlCurvature(LatControl):
+  def __init__(self, CP, CP_SP, CI, dt):
+    super().__init__(CP, CP_SP, CI, dt)
+    self.sat_check_min_speed = 5.
+    if CP.lateralTuning.which() == 'pid':
+      ct = CP.lateralTuning.pid
+      self.pid = PIDController((ct.kpBP, ct.kpV), (ct.kiBP, ct.kiV),
+                               pos_limit=MAX_CURVATURE, neg_limit=-MAX_CURVATURE, rate=1 / dt)
+      self.kf = ct.kf
+    else:
+      self.pid = None
+      self.kf = 1.
+
+  def reset(self):
+    super().reset()
+    if self.pid is not None:
+      self.pid.reset()
+
+  def update(self, active, CS, VM, params, steer_limited_by_safety, desired_curvature, calibrated_pose, curvature_limited, lat_delay):
+    curvature_log = log.ControlsState.LateralCurvatureState.new_message()
+    actual_curvature = -VM.calc_curvature(math.radians(CS.steeringAngleDeg - params.angleOffsetDeg), CS.vEgo, params.roll)
+    error = desired_curvature - actual_curvature
+
+    if not active:
+      output_curvature = 0.0
+      curvature_log.active = False
+      if self.pid is not None:
+        self.pid.reset()
+    elif self.pid is None or CS.steeringPressed:
+      # no PID or override: feedforward only
+      if self.pid is not None:
+        self.pid.reset()
+      output_curvature = self.kf * desired_curvature
+      curvature_log.active = True
+    else:
+      output_curvature = self.pid.update(error, speed=CS.vEgo, feedforward=self.kf * desired_curvature)
+      curvature_log.p = float(self.pid.p)
+      curvature_log.i = float(self.pid.i)
+      curvature_log.f = float(self.pid.f)
+      curvature_log.active = True
+
+    curvature_log.error = float(error)
+    curvature_log.actualCurvature = float(actual_curvature)
+    curvature_log.desiredCurvature = float(desired_curvature)
+    curvature_log.output = float(output_curvature)
+    curvature_log.saturated = bool(self._check_saturation(abs(error) > CURVATURE_SATURATION_THRESHOLD, CS,
+                                                          False, curvature_limited))
+    return 0.0, float(output_curvature), curvature_log

selfdrive/debug/README.md

@@ -1,59 +0,0 @@
-# debug scripts
-
-## [can_printer.py](can_printer.py)
-
-```
-usage: can_printer.py [-h] [--bus BUS] [--max_msg MAX_MSG] [--addr ADDR]
-
-simple CAN data viewer
-
-optional arguments:
-  -h, --help         show this help message and exit
-  --bus BUS          CAN bus to print out (default: 0)
-  --max_msg MAX_MSG  max addr (default: None)
-  --addr ADDR
-```
-
-## [dump.py](dump.py)
-
-```
-usage: dump.py [-h] [--pipe] [--raw] [--json] [--dump-json] [--no-print] [--addr ADDR] [--values VALUES] [socket [socket ...]]
-
-Dump communication sockets. See cereal/services.py for a complete list of available sockets.
-
-positional arguments:
-  socket           socket names to dump. defaults to all services defined in cereal
-
-optional arguments:
-  -h, --help       show this help message and exit
-  --pipe
-  --raw
-  --json
-  --dump-json
-  --no-print
-  --addr ADDR
-  --values VALUES  values to monitor (instead of entire event)
-```
-
-## [vw_mqb_config.py](vw_mqb_config.py)
-
-```
-usage: vw_mqb_config.py [-h] [--debug] {enable,show,disable}
-
-Shows Volkswagen EPS software and coding info, and enables or disables Heading Control
-Assist (Lane Assist). Useful for enabling HCA on cars without factory Lane Assist that want
-to use openpilot integrated at the CAN gateway (J533).
-
-positional arguments:
-  {enable,show,disable}
-                        show or modify current EPS HCA config
-
-optional arguments:
-  -h, --help            show this help message and exit
-  --debug               enable ISO-TP/UDS stack debugging output
-
-This tool is meant to run directly on a vehicle-installed comma three, with
-the openpilot/tmux processes stopped. It should also work on a separate PC with a USB-
-attached comma panda. Vehicle ignition must be on. Recommend engine not be running when
-making changes. Must turn ignition off and on again for any changes to take effect.
-```

selfdrive/debug/analyze-msg-size.py

@@ -1,82 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-from tqdm import tqdm
-
-from cereal.services import SERVICE_LIST, QueueSize
-from openpilot.tools.lib.logreader import LogReader
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="Analyze message sizes from a log route")
-  parser.add_argument("route", nargs="?", default="98395b7c5b27882e/000000a8--f87e7cd255",
-                      help="Log route to analyze (default: 98395b7c5b27882e/000000a8--f87e7cd255)")
-  args = parser.parse_args()
-
-  lr = LogReader(args.route)
-
-  szs = {}
-  for msg in tqdm(lr):
-    sz = len(msg.as_builder().to_bytes())
-    msg_type = msg.which()
-    if msg_type not in szs:
-      szs[msg_type] = {'min': sz, 'max': sz, 'sum': sz, 'count': 1}
-    else:
-      szs[msg_type]['min'] = min(szs[msg_type]['min'], sz)
-      szs[msg_type]['max'] = max(szs[msg_type]['max'], sz)
-      szs[msg_type]['sum'] += sz
-      szs[msg_type]['count'] += 1
-
-  print()
-  print(f"{'Service':<36} {'Min (KB)':>12} {'Max (KB)':>12} {'Avg (KB)':>12} {'KB/min':>12} {'KB/sec':>12} {'Minutes in 10MB':>18} {'Seconds in Queue':>18}")
-  print("-" * 132)
-  def sort_key(x):
-    k, v = x
-    avg = v['sum'] / v['count']
-    freq = SERVICE_LIST.get(k, None)
-    freq_val = freq.frequency if freq else 0.0
-    kb_per_min = (avg * freq_val * 60) / 1024 if freq_val > 0 else 0.0
-    return kb_per_min
-  total_kb_per_min = 0.0
-  RINGBUFFER_SIZE_KB = 10 * 1024  # 10MB old default
-  for k, v in sorted(szs.items(), key=sort_key, reverse=True):
-    avg = v['sum'] / v['count']
-    service = SERVICE_LIST.get(k, None)
-    freq_val = service.frequency if service else 0.0
-    queue_size_kb = (service.queue_size / 1024) if service else 250  # default to SMALL
-    kb_per_min = (avg * freq_val * 60) / 1024 if freq_val > 0 else 0.0
-    kb_per_sec = kb_per_min / 60
-    minutes_in_buffer = RINGBUFFER_SIZE_KB / kb_per_min if kb_per_min > 0 else float('inf')
-    seconds_in_queue = (queue_size_kb / kb_per_sec) if kb_per_sec > 0 else float('inf')
-    total_kb_per_min += kb_per_min
-    min_str = f"{minutes_in_buffer:.2f}" if minutes_in_buffer != float('inf') else "inf"
-    sec_queue_str = f"{seconds_in_queue:.2f}" if seconds_in_queue != float('inf') else "inf"
-    print(f"{k:<36} {v['min']/1024:>12.2f} {v['max']/1024:>12.2f} {avg/1024:>12.2f} {kb_per_min:>12.2f} {kb_per_sec:>12.2f} {min_str:>18} {sec_queue_str:>18}")
-
-  # Summary section
-  print()
-  print(f"Total usage: {total_kb_per_min / 1024:.2f} MB/min")
-
-  # Calculate memory usage: old (10MB for all) vs new (from services.py)
-  OLD_SIZE = 10 * 1024 * 1024  # 10MB was the old default
-  old_total = len(SERVICE_LIST) * OLD_SIZE
-
-  new_total = sum(s.queue_size for s in SERVICE_LIST.values())
-
-  # Count by queue size
-  size_counts = {QueueSize.BIG: 0, QueueSize.MEDIUM: 0, QueueSize.SMALL: 0}
-  for s in SERVICE_LIST.values():
-    size_counts[s.queue_size] += 1
-
-  savings_pct = (1 - new_total / old_total) * 100
-
-  print()
-  print(f"{'Queue Size Comparison':<40}")
-  print("-" * 60)
-  print(f"{'Old (10MB default):':<30} {old_total / 1024 / 1024:>10.2f} MB")
-  print(f"{'New (from services.py):':<30} {new_total / 1024 / 1024:>10.2f} MB")
-  print(f"{'Savings:':<30} {savings_pct:>10.1f}%")
-  print()
-  print(f"{'Breakdown:':<30}")
-  print(f"  BIG (10MB):    {size_counts[QueueSize.BIG]:>3} services")
-  print(f"  MEDIUM (2MB):  {size_counts[QueueSize.MEDIUM]:>3} services")
-  print(f"  SMALL (250KB): {size_counts[QueueSize.SMALL]:>3} services")

selfdrive/debug/check_can_parser_performance.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-import numpy as np
-import time
-from tqdm import tqdm
-
-from cereal import car
-from opendbc.car.tests.routes import CarTestRoute
-from openpilot.selfdrive.car.tests.test_models import TestCarModelBase
-from openpilot.tools.plotjuggler.juggle import DEMO_ROUTE
-
-N_RUNS = 10
-
-
-class CarModelTestCase(TestCarModelBase):
-  test_route = CarTestRoute(DEMO_ROUTE, None)
-
-
-if __name__ == '__main__':
-  # Get CAN messages and parsers
-  tm = CarModelTestCase()
-  tm.setUpClass()
-  tm.setUp()
-
-  CC = car.CarControl.new_message()
-  ets = []
-  for _ in tqdm(range(N_RUNS)):
-    start_t = time.process_time_ns()
-    for msg in tm.can_msgs:
-      for cp in tm.CI.can_parsers.values():
-        if cp is not None:
-          cp.update_strings(msg)
-    ets.append((time.process_time_ns() - start_t) * 1e-6)
-
-  print(f'{len(tm.can_msgs)} CAN packets, {N_RUNS} runs')
-  print(f'{np.mean(ets):.2f} mean ms, {max(ets):.2f} max ms, {min(ets):.2f} min ms, {np.std(ets):.2f} std ms')
-  print(f'{np.mean(ets) / len(tm.can_msgs):.4f} mean ms / CAN packet')

selfdrive/debug/check_freq.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import numpy as np
-import time
-from collections import defaultdict, deque
-from collections.abc import MutableSequence
-
-import cereal.messaging as messaging
-
-
-if __name__ == "__main__":
-  context = messaging.Context()
-  poller = messaging.Poller()
-
-  parser = argparse.ArgumentParser()
-  parser.add_argument("socket", type=str, nargs='*', help="socket name")
-  args = parser.parse_args()
-
-  socket_names = args.socket
-  sockets = {}
-
-  rcv_times: defaultdict[str, MutableSequence[float]] = defaultdict(lambda: deque(maxlen=100))
-  valids: defaultdict[str, deque[bool]] = defaultdict(lambda: deque(maxlen=100))
-
-  t = time.monotonic()
-  for name in socket_names:
-    sock = messaging.sub_sock(name, poller=poller)
-    sockets[sock] = name
-
-  prev_print = t
-  while True:
-    for socket in poller.poll(100):
-      msg = messaging.recv_one(socket)
-      if msg is None:
-        continue
-
-      name = msg.which()
-
-      t = time.monotonic()
-      rcv_times[name].append(msg.logMonoTime / 1e9)
-      valids[name].append(msg.valid)
-
-    if t - prev_print > 1:
-      print()
-      for name in socket_names:
-        dts = np.diff(rcv_times[name])
-        mean = np.mean(dts)
-        print(f"{name}: Freq {1.0 / mean:.2f} Hz, Min {np.min(dts) / mean * 100:.2f}%, Max {np.max(dts) / mean * 100:.2f}%, valid ", all(valids[name]))
-
-      prev_print = t

selfdrive/debug/check_lag.py

@@ -1,27 +0,0 @@
-#!/usr/bin/env python3
-
-import cereal.messaging as messaging
-from cereal.services import SERVICE_LIST
-
-TO_CHECK = ['carState']
-
-
-if __name__ == "__main__":
-  sm = messaging.SubMaster(TO_CHECK)
-
-  prev_t: dict[str, float] = {}
-
-  while True:
-    sm.update()
-
-    for s in TO_CHECK:
-      if sm.updated[s]:
-        t = sm.logMonoTime[s] / 1e9
-
-        if s in prev_t:
-          expected = 1.0 / (SERVICE_LIST[s].frequency)
-          dt = t - prev_t[s]
-          if dt > 10 * expected:
-            print(t, s, dt)
-
-        prev_t[s] = t

selfdrive/debug/check_timings.py

@@ -1,36 +0,0 @@
-#!/usr/bin/env python3
-import sys
-import time
-import numpy as np
-import datetime
-from collections.abc import MutableSequence
-from collections import defaultdict
-
-import cereal.messaging as messaging
-
-
-if __name__ == "__main__":
-  ts: defaultdict[str, MutableSequence[float]] = defaultdict(list)
-  socks = {s: messaging.sub_sock(s, conflate=False) for s in sys.argv[1:]}
-  try:
-    st = time.monotonic()
-    while True:
-      print()
-      for s, sock in socks.items():
-        msgs = messaging.drain_sock(sock)
-        for m in msgs:
-          ts[s].append(m.logMonoTime / 1e6)
-
-        if len(ts[s]) > 2:
-          d = np.diff(ts[s])[-100:]
-          print(f"{s:25} {np.mean(d):7.2f} {np.std(d):7.2f} {np.max(d):7.2f} {np.min(d):7.2f}")
-      time.sleep(1)
-  except KeyboardInterrupt:
-    print("\n")
-    print("="*5, "timing summary", "="*5)
-    for s, sock in socks.items():
-      msgs = messaging.drain_sock(sock)
-      if len(ts[s]) > 2:
-        d = np.diff(ts[s])
-        print(f"{s:25} {np.mean(d):7.2f} {np.std(d):7.2f} {np.max(d):7.2f} {np.min(d):7.2f}")
-    print("="*5, datetime.timedelta(seconds=time.monotonic()-st), "="*5)

selfdrive/debug/dump_car_docs.py

@@ -1,18 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-import pickle
-
-from opendbc.car.docs import get_all_car_docs
-
-
-def dump_car_docs(path):
-  with open(path, 'wb') as f:
-    pickle.dump(get_all_car_docs(), f)
-  print(f'Dumping car info to {path}')
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument("--path", required=True)
-  args = parser.parse_args()
-  dump_car_docs(args.path)

selfdrive/debug/print_docs_diff.py

@@ -1,120 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-from collections import defaultdict
-import difflib
-import pickle
-
-from opendbc.car.docs import get_all_car_docs
-from opendbc.car.docs_definitions import Column
-
-FOOTNOTE_TAG = "<sup>{}</sup>"
-STAR_ICON = '<a href="##"><img valign="top" ' + \
-            'src="https://media.githubusercontent.com/media/commaai/openpilot/master/docs/assets/icon-star-{}.svg" width="22" /></a>'
-VIDEO_ICON = '<a href="{}" target="_blank">' + \
-             '<img height="18px" src="https://media.githubusercontent.com/media/commaai/openpilot/master/docs/assets/icon-youtube.svg" /></a>'
-COLUMNS = "|" + "|".join([column.value for column in Column]) + "|"
-COLUMN_HEADER = "|---|---|---|{}|".format("|".join([":---:"] * (len(Column) - 3)))
-ARROW_SYMBOL = "➡️"
-
-
-def load_base_car_docs(path):
-  with open(path, "rb") as f:
-    return pickle.load(f)
-
-
-def match_cars(base_cars, new_cars):
-  changes = []
-  additions = []
-  for new in new_cars:
-    # Addition if no close matches or close match already used
-    # Change if close match and not already used
-    matches = difflib.get_close_matches(new.name, [b.name for b in base_cars], cutoff=0.)
-    if not len(matches) or matches[0] in [c[1].name for c in changes]:
-      additions.append(new)
-    else:
-      changes.append((new, next(car for car in base_cars if car.name == matches[0])))
-
-  # Removal if base car not in changes
-  removals = [b for b in base_cars if b.name not in [c[1].name for c in changes]]
-  return changes, additions, removals
-
-
-def build_column_diff(base_car, new_car):
-  row_builder = []
-  for column in Column:
-    base_column = base_car.get_column(column, STAR_ICON, VIDEO_ICON, FOOTNOTE_TAG)
-    new_column = new_car.get_column(column, STAR_ICON, VIDEO_ICON, FOOTNOTE_TAG)
-
-    if base_column != new_column:
-      row_builder.append(f"{base_column} {ARROW_SYMBOL} {new_column}")
-    else:
-      row_builder.append(new_column)
-
-  return format_row(row_builder)
-
-
-def format_row(builder):
-  return "|" + "|".join(builder) + "|"
-
-
-def print_car_docs_diff(path):
-  base_car_docs = defaultdict(list)
-  new_car_docs = defaultdict(list)
-
-  for car in load_base_car_docs(path):
-    base_car_docs[car.car_fingerprint].append(car)
-  for car in get_all_car_docs():
-    new_car_docs[car.car_fingerprint].append(car)
-
-  # Add new platforms to base cars so we can detect additions and removals in one pass
-  base_car_docs.update({car: [] for car in new_car_docs if car not in base_car_docs})
-
-  changes = defaultdict(list)
-  for base_car_model, base_cars in base_car_docs.items():
-    # Match car info changes, and get additions and removals
-    new_cars = new_car_docs[base_car_model]
-    car_changes, car_additions, car_removals = match_cars(base_cars, new_cars)
-
-    # Removals
-    for car_docs in car_removals:
-      changes["removals"].append(format_row([car_docs.get_column(column, STAR_ICON, VIDEO_ICON, FOOTNOTE_TAG) for column in Column]))
-
-    # Additions
-    for car_docs in car_additions:
-      changes["additions"].append(format_row([car_docs.get_column(column, STAR_ICON, VIDEO_ICON, FOOTNOTE_TAG) for column in Column]))
-
-    for new_car, base_car in car_changes:
-      # Column changes
-      row_diff = build_column_diff(base_car, new_car)
-      if ARROW_SYMBOL in row_diff:
-        changes["column"].append(row_diff)
-
-      # Detail sentence changes
-      if base_car.detail_sentence != new_car.detail_sentence:
-        changes["detail"].append(f"- Sentence for {base_car.name} changed!\n" +
-                                 "  ```diff\n" +
-                                 f"  - {base_car.detail_sentence}\n" +
-                                 f"  + {new_car.detail_sentence}\n" +
-                                 "  ```")
-
-  # Print diff
-  if any(len(c) for c in changes.values()):
-    markdown_builder = ["### ⚠️ This PR makes changes to [CARS.md](../blob/master/docs/CARS.md) ⚠️"]
-
-    for title, category in (("## 🔀 Column Changes", "column"), ("## ❌ Removed", "removals"),
-                            ("## ➕ Added", "additions"), ("## 📖 Detail Sentence Changes", "detail")):
-      if len(changes[category]):
-        markdown_builder.append(title)
-        if category not in ("detail",):
-          markdown_builder.append(COLUMNS)
-          markdown_builder.append(COLUMN_HEADER)
-        markdown_builder.extend(changes[category])
-
-    print("\n".join(markdown_builder))
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument("--path", required=True)
-  args = parser.parse_args()
-  print_car_docs_diff(args.path)

selfdrive/debug/touch_replay.py

@@ -1,54 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-
-import numpy as np
-import matplotlib.pyplot as plt
-
-from openpilot.tools.lib.logreader import LogReader
-
-if __name__ == '__main__':
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--width', default=2160, type=int)
-  parser.add_argument('--height', default=1080, type=int)
-  parser.add_argument('--route', default='rlog', type=str)
-  args = parser.parse_args()
-
-  w = args.width
-  h = args.height
-  route = args.route
-
-  fingers = [[-1, -1]] * 5
-  touch_points = []
-  current_slot = 0
-
-  lr = list(LogReader(route))
-  for msg in lr:
-    if msg.which() == 'touch':
-      for event in msg.touch:
-        if event.type == 3 and event.code == 47:
-          current_slot = event.value
-        elif event.type == 3 and event.code == 57 and event.value == -1:
-          fingers[current_slot] = [-1, -1]
-        elif event.type == 3 and event.code == 53:
-          fingers[current_slot][1] = event.value
-          if fingers[current_slot][0] != -1:
-            touch_points.append(fingers[current_slot].copy())
-        elif event.type == 3 and event.code == 54:
-          fingers[current_slot][0] = w - event.value
-          if fingers[current_slot][1] != -1:
-            touch_points.append(fingers[current_slot].copy())
-
-  if not touch_points:
-    print(f'No touch events found for {route}')
-    quit()
-
-  unique_points, counts = np.unique(touch_points, axis=0, return_counts=True)
-
-  plt.figure(figsize=(10, 3))
-  plt.scatter(unique_points[:, 0], unique_points[:, 1], c=counts, s=counts * 20, edgecolors='red')
-  plt.colorbar()
-  plt.title(f'Touches for {route}')
-  plt.xlim(0, w)
-  plt.ylim(0, h)
-  plt.grid(True)
-  plt.show()

selfdrive/locationd/calibrationd.py

@@ -13,7 +13,7 @@ from typing import NoReturn
 
 from cereal import log, car
 import cereal.messaging as messaging
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 from openpilot.common.constants import CV
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process

selfdrive/locationd/lagd.py

@@ -8,6 +8,7 @@ from functools import partial
 import cereal.messaging as messaging
 from cereal import car, log
 from cereal.services import SERVICE_LIST
+from openpilot.common.constants import CV
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process
 from openpilot.common.swaglog import cloudlog
@@ -20,7 +21,7 @@ BLOCK_NUM_NEEDED = 5
 MOVING_WINDOW_SEC = 60.0
 MIN_OKAY_WINDOW_SEC = 25.0
 MIN_RECOVERY_BUFFER_SEC = 2.0
-MIN_VEGO = 15.0
+MIN_VEGO = 50.0 * CV.MPH_TO_MS
 MIN_ABS_YAW_RATE = 0.0
 MAX_YAW_RATE_SANITY_CHECK = 1.0
 MIN_NCC = 0.95
@@ -36,6 +37,8 @@ LAG_CANDIDATE_CORR_THRESHOLD = 0.9
 SMOOTH_K = 5
 SMOOTH_SIGMA = 1.0
 
+VERSION = 1  # bump this to invalidate old parameter caches
+
 
 def masked_symmetric_moving_average(x: np.ndarray, mask: np.ndarray, k: int, sigma: float) -> np.ndarray:
   assert k >= 1 and k % 2 == 1, "k must be positive and odd"
@@ -248,6 +251,7 @@ class LateralLagEstimator:
                             (self.min_valid_block_count * self.block_size), 100)
     if debug:
       liveDelay.points = self.block_avg.values.flatten().tolist()
+    liveDelay.version = VERSION
 
     return msg
 
@@ -368,9 +372,10 @@ def retrieve_initial_lag(params: Params, CP: car.CarParams):
         if last_CP.carFingerprint != CP.carFingerprint:
           raise Exception("Car model mismatch")
 
-        lag, valid_blocks, status = ld.lateralDelayEstimate, ld.validBlocks, ld.status
+        lag, valid_blocks, status, version = ld.lateralDelayEstimate, ld.validBlocks, ld.status, ld.version
         assert valid_blocks <= BLOCK_NUM, "Invalid number of valid blocks"
         assert status != log.LiveDelayData.Status.invalid, "Lag estimate is invalid"
+        assert version == VERSION, f"Lag estimate is from a different version (got {version}, expected {VERSION})"
         return lag, valid_blocks
     except Exception as e:
       cloudlog.error(f"Failed to retrieve initial lag: {e}")

selfdrive/locationd/locationd.py

@@ -206,9 +206,11 @@ class LocationEstimator:
       self._finite_check(t, new_x, new_P)
     return HandleLogResult.SUCCESS
 
-  def get_msg(self, sensors_valid: bool, inputs_valid: bool, filter_valid: bool):
+  def get_msg(self, sensors_valid: bool, inputs_valid: bool, filter_initialized: bool):
     state, cov = self.kf.x, self.kf.P
     std = np.sqrt(np.diag(cov))
+    filter_time_valid = bool(np.isfinite(self.kf.t))
+    filter_valid = filter_initialized and filter_time_valid
 
     orientation_ned, orientation_ned_std = state[States.NED_ORIENTATION], std[States.NED_ORIENTATION]
     velocity_device, velocity_device_std = state[States.DEVICE_VELOCITY], std[States.DEVICE_VELOCITY]

selfdrive/locationd/test/test_lagd.py

@@ -5,18 +5,18 @@ import pytest
 
 from cereal import messaging, log, car
 from openpilot.selfdrive.locationd.lagd import LateralLagEstimator, retrieve_initial_lag, masked_normalized_cross_correlation, \
-                                               BLOCK_NUM_NEEDED, BLOCK_SIZE, MIN_OKAY_WINDOW_SEC
+                                               BLOCK_NUM_NEEDED, BLOCK_SIZE, MIN_OKAY_WINDOW_SEC, VERSION
 from openpilot.selfdrive.test.process_replay.migration import migrate, migrate_carParams
 from openpilot.selfdrive.locationd.test.test_locationd_scenarios import TEST_ROUTE
 from openpilot.common.params import Params
 from openpilot.tools.lib.logreader import LogReader
-from openpilot.system.hardware import PC
+from openpilot.common.hardware import PC
 
 MAX_ERR_FRAMES = 1
 DT = 0.05
 
 
-def process_messages(estimator, lag_frames, n_frames, vego=20.0, rejection_threshold=0.0):
+def process_messages(estimator, lag_frames, n_frames, vego=25.0, rejection_threshold=0.0):
   for i in range(n_frames):
     t = i * estimator.dt
     desired_la = np.cos(10 * t) * 0.3
@@ -53,6 +53,7 @@ class TestLagd:
     msg = messaging.new_message('liveDelay')
     msg.liveDelay.lateralDelayEstimate = random.random()
     msg.liveDelay.validBlocks = random.randint(1, 10)
+    msg.liveDelay.version = VERSION
     params.put("LiveDelay", msg.to_bytes(), block=True)
     params.put("CarParamsPrevRoute", CP.as_builder().to_bytes(), block=True)
 
@@ -63,6 +64,20 @@ class TestLagd:
     assert lag == msg.liveDelay.lateralDelayEstimate
     assert valid_blocks == msg.liveDelay.validBlocks
 
+  def test_read_invalid_saved_params(self, subtests):
+    params = Params()
+
+    lr = migrate(LogReader(TEST_ROUTE), [migrate_carParams])
+    CP = next(m for m in lr if m.which() == "carParams").carParams
+
+    for msg_dict in [{'version': 0}, {'status': 'invalid'}, {'validBlocks': 100}]:
+      with subtests.test(msg=f"liveDelay={msg_dict}"):
+        msg = messaging.new_message('liveDelay')
+        msg.liveDelay = msg_dict
+        params.put("LiveDelay", msg.to_bytes(), block=True)
+        params.put("CarParamsPrevRoute", CP.as_builder().to_bytes(), block=True)
+        assert retrieve_initial_lag(params, CP) is None
+
   def test_ncc(self):
     lag_frames = random.randint(1, 19)
 

selfdrive/modeld/SConscript

@@ -79,22 +79,21 @@ compile_modeld_script = [
   File(f"{modeld_dir}/compile_modeld.py"),
   File(f"{modeld_dir}/get_model_metadata.py"),
   File("#system/camerad/cameras/nv12_info.py"),
-  File("#system/hardware/hw.py"),
+  File("#common/hardware/hw.py"),
 ]
 model_w, model_h = MEDMODEL_INPUT_SIZE
 frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ
 
 for usbgpu in [False, True] if USBGPU else [False]:
   target_pkl_path = File(modeld_pkl_path(usbgpu)).abspath
-  file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('', tg_flags)
-  driving_onnx_deps = [p for m in [f'{file_prefix}driving_vision', f'{file_prefix}driving_on_policy']
-                       for p in get_existing_chunks(File(f"models/{m}.onnx").abspath)]
+  # BIG_INTO_SMALL=1 builds the default target from the big model, e.g. to test it without a USB GPU
+  file_prefix, cmd_flags = ('big_', usbgpu_tg_flags) if usbgpu else ('big_' if os.getenv('BIG_INTO_SMALL') else '', tg_flags)
+  driving_onnx_deps = get_existing_chunks(File(f"models/{file_prefix}driving_supercombo.onnx").abspath)
   camera_res_args = ' '.join(f'{cw}x{ch}' for cw, ch in CAMERA_CONFIGS)
   cmd = (f'{cmd_flags} {mac_brew_string} python3 {modeld_dir}/compile_modeld.py '
         f'--model-size {model_w}x{model_h} '
         f'--camera-resolutions {camera_res_args} '
-        f'--vision-onnx {File(f"models/{file_prefix}driving_vision.onnx").abspath} '
-        f'--on-policy-onnx {File(f"models/{file_prefix}driving_on_policy.onnx").abspath} '
+        f'--onnx {File(f"models/{file_prefix}driving_supercombo.onnx").abspath} '
         f'--output {target_pkl_path} --frame-skip {frame_skip}')
   onnx_sizes_sum = sum(os.path.getsize(f) for f in driving_onnx_deps)
   chunk_targets = get_chunk_targets(target_pkl_path, estimate_pickle_max_size(onnx_sizes_sum))

selfdrive/modeld/compile_modeld.py

@@ -1,11 +1,13 @@
 #!/usr/bin/env python3
 import argparse
 import atexit
+import math
 import os
 import pickle
+import tempfile
 import time
 from functools import partial
-from collections import namedtuple, defaultdict
+from collections import namedtuple
 
 import numpy as np
 
@@ -33,7 +35,7 @@ from tinygrad.engine.jit import TinyJit
 
 NV12Frame = namedtuple("NV12Frame", ['width', 'height', 'stride', 'y_height', 'uv_height', 'size'])
 WARP_INPUTS = ['img_q', 'big_img_q', 'tfm', 'big_tfm']
-POLICY_INPUTS = ['feat_q', 'desire_q', 'desire', 'traffic_convention', 'action_t']
+POLICY_INPUTS = ['feat_q', 'desire_q', 'packed_npy_inputs']
 
 UV_SCALE_MATRIX = np.array([[0.5, 0, 0], [0, 0.5, 0], [0, 0, 1]], dtype=np.float32)
 UV_SCALE_MATRIX_INV = np.linalg.inv(UV_SCALE_MATRIX)
@@ -49,8 +51,8 @@ def warp_perspective_tinygrad(src_flat, M_inv, dst_shape, src_shape, stride_pad,
   w_dst, h_dst = dst_shape
   h_src, w_src = src_shape
 
-  x = Tensor.arange(w_dst, device=WARP_DEV).reshape(1, w_dst).expand(h_dst, w_dst).reshape(-1)
-  y = Tensor.arange(h_dst, device=WARP_DEV).reshape(h_dst, 1).expand(h_dst, w_dst).reshape(-1)
+  x = Tensor.arange(w_dst).reshape(1, w_dst).expand(h_dst, w_dst).reshape(-1)
+  y = Tensor.arange(h_dst).reshape(h_dst, 1).expand(h_dst, w_dst).reshape(-1)
 
   # inline 3x3 matmul as elementwise to avoid reduce op (enables fusion with gather)
   src_x = M_inv[0, 0] * x + M_inv[0, 1] * y + M_inv[0, 2]
@@ -113,34 +115,51 @@ def make_frame_prepare(nv12: NV12Frame, model_w, model_h):
   return frame_prepare_tinygrad
 
 
-def make_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device):
+def make_warp_input_queues(vision_input_shapes, frame_skip, device):
   img = vision_input_shapes['img']  # (1, 12, 128, 256)
   n_frames = img[1] // 6
   img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img[2], img[3])
 
-  fb = policy_input_shapes['features_buffer']  # (1, 25, 512)
-  dp = policy_input_shapes['desire_pulse']  # (1, 25, 8)
-  tc = policy_input_shapes['traffic_convention']  # (1, 2)
-  #TODO action_t is hardcoded to match tc for future compatibility
-  at = tc
-
   npy = {
-    'desire': np.zeros(dp[2], dtype=np.float32),
-    'traffic_convention': np.zeros(tc, dtype=np.float32),
     'tfm': np.zeros((3, 3), dtype=np.float32),
     'big_tfm': np.zeros((3, 3), dtype=np.float32),
-    'action_t': np.zeros(at, dtype=np.float32),
   }
   input_queues = {
     'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
     'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
-    'feat_q': Tensor(np.zeros((frame_skip * (fb[1] - 1) + 1, fb[0], fb[2]), dtype=np.float32), device=device).contiguous().realize(),
-    'desire_q': Tensor(np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32), device=device).contiguous().realize(),
     **{k: Tensor(v, device='NPY').realize() for k, v in npy.items()},
   }
   return input_queues, npy
 
 
+def get_policy_npy_shapes(input_shapes):
+  dp = input_shapes['desire_pulse']  # (1, 25, 8)
+  tc = input_shapes['traffic_convention']  # (1, 2)
+  at = input_shapes['action_t']  # (1, 2)
+  fb = input_shapes['features_buffer']  # (1, 24, 512)
+  # TODO prev_feat shouldn't exist and be handled inside the JIT, but corrupt on QCOM for now
+  shapes = {'desire': (dp[2],), 'traffic_convention': tuple(tc), 'action_t': tuple(at), 'prev_feat': (fb[0], fb[2])}
+  return shapes, [math.prod(s) for s in shapes.values()]
+
+
+def make_input_queues(input_shapes, frame_skip, device):
+  input_queues, npy = make_warp_input_queues(input_shapes, frame_skip, device)
+
+  fb = input_shapes['features_buffer']  # (1, 24, 512), past features only; the model appends the current frame's feature
+  dp = input_shapes['desire_pulse']  # (1, 25, 8)
+
+  shapes, sizes = get_policy_npy_shapes(input_shapes)
+  packed_npy_inputs = np.zeros(sum(sizes), dtype=np.float32)
+  # views into the packed inputs, to be refilled at runtime
+  npy.update({k: v.reshape(s) for (k, s), v in zip(shapes.items(), np.split(packed_npy_inputs, np.cumsum(sizes[:-1])), strict=True)})
+  input_queues.update({
+    'feat_q': Tensor(np.zeros((frame_skip * fb[1], fb[0], fb[2]), dtype=np.float32), device=device).contiguous().realize(),
+    'desire_q': Tensor(np.zeros((frame_skip * dp[1], dp[0], dp[2]), dtype=np.float32), device=device).contiguous().realize(),
+    'packed_npy_inputs': Tensor(packed_npy_inputs, device='NPY').realize(),
+  })
+  return input_queues, npy
+
+
 def shift_and_sample(buf, new_val, sample_fn):
   buf.assign(buf[1:].cat(new_val, dim=0).contiguous())
   return sample_fn(buf)
@@ -163,49 +182,43 @@ def make_warp(nv12, model_w, model_h, frame_skip):
     big_tfm = big_tfm.to(WARP_DEV)
     Tensor.realize(tfm, big_tfm)
 
-    warped_frame = frame_prepare(frame, tfm).unsqueeze(0).to(Device.DEFAULT)
-    warped_big_frame = frame_prepare(big_frame, big_tfm).unsqueeze(0).to(Device.DEFAULT)
-    img = shift_and_sample(img_q, warped_frame, sample_skip_fn)
-    big_img = shift_and_sample(big_img_q, warped_big_frame, sample_skip_fn)
+    warped_frame = frame_prepare(frame, tfm).unsqueeze(0)
+    warped_big_frame = frame_prepare(big_frame, big_tfm).unsqueeze(0)
+    warped = Tensor.cat(warped_frame, warped_big_frame).to(Device.DEFAULT)
+    img = shift_and_sample(img_q, warped[0:1], sample_skip_fn)
+    big_img = shift_and_sample(big_img_q, warped[1:2], sample_skip_fn)
     return img, big_img
   return warp_enqueue
 
 
-def make_run_policy(vision_runner, on_policy_runner, vision_features_slice, frame_skip):
+def make_run_policy(model_runner, model_metadata, frame_skip):
   sample_desire_fn = partial(sample_desire, frame_skip=frame_skip)
   sample_skip_fn = partial(sample_skip, frame_skip=frame_skip)
+  npy_shapes, npy_sizes = get_policy_npy_shapes(model_metadata['input_shapes'])
 
-  def run_policy(img, big_img, feat_q, desire_q, desire, traffic_convention, action_t):
-    desire = desire.to(Device.DEFAULT)
-    traffic_convention = traffic_convention.to(Device.DEFAULT)
-    action_t = action_t.to(Device.DEFAULT)
-    Tensor.realize(desire, traffic_convention, action_t)
+  def run_policy(img, big_img, feat_q, desire_q, packed_npy_inputs):
+    packed_npy_inputs = packed_npy_inputs.to(Device.DEFAULT).realize()
+    desire, traffic_convention, action_t, prev_feat = (t.reshape(s) for t, s in zip(packed_npy_inputs.split(npy_sizes), npy_shapes.values(), strict=True))
     desire_buf = shift_and_sample(desire_q, desire.reshape(1, 1, -1), sample_desire_fn)
-    vision_out = next(iter(vision_runner({'img': img, 'big_img': big_img}).values())).cast('float32')
-
-    new_feat = vision_out[:, vision_features_slice].reshape(1, -1).unsqueeze(0)
-    feat_buf = shift_and_sample(feat_q, new_feat, sample_skip_fn)
+    feat_buf = shift_and_sample(feat_q, prev_feat.reshape(1, 1, -1), sample_skip_fn)
 
     inputs = {
+      'img': img,
+      'big_img': big_img,
       'features_buffer': feat_buf,
       'desire_pulse': desire_buf,
       'traffic_convention': traffic_convention,
       'action_t': action_t,
     }
-    on_policy_out = next(iter(on_policy_runner(inputs).values())).cast('float32')
-    #off_policy_out = next(iter(off_policy_runner(inputs).values())).cast('float32')
-    return vision_out, on_policy_out
-
+    out = next(iter(model_runner(inputs).values())).cast('float32')
+    return out,
   return run_policy
 
 
-def compile_jit(jit, make_random_inputs, input_keys, frame_skip, vision_metadata, policy_metadata):
-  vision_input_shapes = vision_metadata['input_shapes']
-  policy_input_shapes = policy_metadata['input_shapes']
-
+def compile_jit(jit, make_random_inputs, input_keys, make_queues):
   SEED = 42
   def random_inputs_run(fn, seed, test_val=None, test_buffers=None, expect_match=True):
-    input_queues, npy = make_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, Device.DEFAULT)
+    input_queues, npy = make_queues(Device.DEFAULT)
     np.random.seed(seed)
     Tensor.manual_seed(seed)
 
@@ -252,12 +265,12 @@ def _parse_size(s):
 
 def read_file_chunked_to_shm(path):
   from openpilot.common.file_chunker import read_file_chunked
-  from openpilot.system.hardware.hw import Paths
-  shm_path = os.path.join(Paths.shm_path(), os.path.basename(path))
-  atexit.register(lambda: os.path.exists(shm_path) and os.remove(shm_path))
-  with open(shm_path, 'wb') as f:
+  from openpilot.common.hardware.hw import Paths
+  with tempfile.NamedTemporaryFile(prefix='compile_modeld_', dir=Paths.shm_path(), delete=False) as f:
     f.write(read_file_chunked(path))
-  return shm_path
+    tmp_path = f.name
+  atexit.register(lambda: os.path.exists(tmp_path) and os.remove(tmp_path))
+  return tmp_path
 
 
 if __name__ == "__main__":
@@ -268,31 +281,30 @@ if __name__ == "__main__":
   p.add_argument('--model-size', type=_parse_size, required=True, help='model input WxH')
   p.add_argument('--camera-resolutions', type=_parse_size, nargs='+', required=True,
                  help='camera resolutions WxH (one or more)')
-  p.add_argument('--vision-onnx', required=True)
-  p.add_argument('--on-policy-onnx', required=True)
+  p.add_argument('--onnx', required=True)
   p.add_argument('--output', required=True)
   p.add_argument('--frame-skip', type=int, required=True)
   args = p.parse_args()
 
-  out = defaultdict(dict)
-  vision_path, on_policy_path = read_file_chunked_to_shm(args.vision_onnx), read_file_chunked_to_shm(args.on_policy_onnx)
+  model_path = read_file_chunked_to_shm(args.onnx)
   model_w, model_h = args.model_size
 
-  vision_runner = OnnxRunner(vision_path)
-  on_policy_runner = OnnxRunner(on_policy_path)
-  vision_metadata, on_policy_metadata = make_metadata_dict(vision_path), make_metadata_dict(on_policy_path)
+  model_runner = OnnxRunner(model_path)
+  out = {'metadata': make_metadata_dict(model_path)}
 
-  run_policy_jit = TinyJit(make_run_policy(vision_runner, on_policy_runner, vision_metadata['output_slices']['hidden_state'], args.frame_skip), prune=True)
-  out['metadata']['vision'], out['metadata']['on_policy'] = vision_metadata, on_policy_metadata
+  run_policy_jit = TinyJit(make_run_policy(model_runner, out['metadata'], args.frame_skip), prune=True)
 
-  make_random_model_inputs = partial(make_random_images, keys=['img', 'big_img'], shape=vision_metadata['input_shapes']['img'])
-  out['run_policy'] = compile_jit(run_policy_jit, make_random_model_inputs, POLICY_INPUTS, args.frame_skip, vision_metadata, on_policy_metadata)
+  make_policy_queues = partial(make_input_queues, out['metadata']['input_shapes'], args.frame_skip)
+  make_random_model_inputs = partial(make_random_images, keys=['img', 'big_img'], shape=out['metadata']['input_shapes']['img'])
+  out['run_policy'] = compile_jit(run_policy_jit, make_random_model_inputs, POLICY_INPUTS,
+                                  make_policy_queues)
 
   for cam_w, cam_h in args.camera_resolutions:
     nv12 = NV12Frame(cam_w, cam_h, *get_nv12_info(cam_w, cam_h))
     make_random_warp_inputs = partial(make_random_images, keys=['frame', 'big_frame'], shape=nv12.size, device=WARP_DEV)
     warp_enqueue = TinyJit(make_warp(nv12, model_w, model_h, args.frame_skip), prune=True)
-    out[(cam_w,cam_h)] = compile_jit(warp_enqueue, make_random_warp_inputs, WARP_INPUTS, args.frame_skip, vision_metadata, on_policy_metadata)
+    make_warp_queues = partial(make_warp_input_queues, out['metadata']['input_shapes'], args.frame_skip)
+    out[(cam_w,cam_h)] = compile_jit(warp_enqueue, make_random_warp_inputs, WARP_INPUTS, make_warp_queues)
 
   with open(args.output, "wb") as f:
     pickle.dump(out, f)

selfdrive/modeld/fill_model_msg.py

@@ -56,26 +56,29 @@ def fill_lane_line_meta(builder, lane_lines, lane_line_probs):
   builder.rightY = lane_lines[2].y[0]
   builder.rightProb = lane_line_probs[2]
 
-def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._DynamicStructBuilder,
-                   net_output_data: dict[str, np.ndarray], action: log.ModelDataV2.Action,
+def fill_driving_model_data(msg: capnp._DynamicStructBuilder, modelv2_send: capnp._DynamicStructBuilder) -> None:
+  msg.valid = modelv2_send.valid
+  modelV2 = modelv2_send.modelV2
+  driving_model_data = msg.drivingModelData
+  driving_model_data.frameId = modelV2.frameId
+  driving_model_data.frameIdExtra = modelV2.frameIdExtra
+  driving_model_data.frameDropPerc = modelV2.frameDropPerc
+  driving_model_data.modelExecutionTime = modelV2.modelExecutionTime
+  driving_model_data.action = modelV2.action
+  driving_model_data.meta.laneChangeState = modelV2.meta.laneChangeState
+  driving_model_data.meta.laneChangeDirection = modelV2.meta.laneChangeDirection
+  fill_lane_line_meta(driving_model_data.laneLineMeta, modelV2.laneLines, modelV2.laneLineProbs)
+  fill_xyz_poly(driving_model_data.path, ModelConstants.POLY_PATH_DEGREE, modelV2.position.x, modelV2.position.y, modelV2.position.z)
+
+def fill_model_msg(msg: capnp._DynamicStructBuilder, net_output_data: dict[str, np.ndarray], action: log.ModelDataV2.Action,
                    publish_state: PublishState, vipc_frame_id: int, vipc_frame_id_extra: int,
                    frame_id: int, frame_drop: float, timestamp_eof: int, model_execution_time: float,
                    valid: bool) -> None:
   frame_age = frame_id - vipc_frame_id if frame_id > vipc_frame_id else 0
   frame_drop_perc = frame_drop * 100
-  extended_msg.valid = valid
-  base_msg.valid = valid
+  msg.valid = valid
 
-  driving_model_data = base_msg.drivingModelData
-
-  driving_model_data.frameId = vipc_frame_id
-  driving_model_data.frameIdExtra = vipc_frame_id_extra
-  driving_model_data.frameDropPerc = frame_drop_perc
-  driving_model_data.modelExecutionTime = model_execution_time
-
-  driving_model_data.action = action
-
-  modelV2 = extended_msg.modelV2
+  modelV2 = msg.modelV2
   modelV2.frameId = vipc_frame_id
   modelV2.frameIdExtra = vipc_frame_id_extra
   modelV2.frameAge = frame_age
@@ -90,9 +93,6 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   fill_xyzt(modelV2.orientation, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.T_FROM_CURRENT_EULER].T)
   fill_xyzt(modelV2.orientationRate, ModelConstants.T_IDXS, *net_output_data['plan'][0,:,Plan.ORIENTATION_RATE].T)
 
-  # poly path
-  fill_xyz_poly(driving_model_data.path, ModelConstants.POLY_PATH_DEGREE, *net_output_data['plan'][0,:,Plan.POSITION].T)
-
   # action
   modelV2.action = action
 
@@ -107,8 +107,6 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
   modelV2.laneLineStds = net_output_data['lane_lines_stds'][0,:,0,0].tolist()
   modelV2.laneLineProbs = net_output_data['lane_lines_prob'][0,1::2].tolist()
 
-  fill_lane_line_meta(driving_model_data.laneLineMeta, modelV2.laneLines, modelV2.laneLineProbs)
-
   # road edges
   modelV2.init('roadEdges', 2)
   for i in range(2):

selfdrive/modeld/modeld.py

@@ -21,7 +21,7 @@ from openpilot.selfdrive.controls.lib.desire_helper import DesireHelper
 from openpilot.selfdrive.controls.lib.drive_helpers import get_accel_from_plan, smooth_value, get_curvature_from_plan
 from openpilot.selfdrive.modeld.parse_model_outputs import Parser
 from openpilot.selfdrive.modeld.compile_modeld import make_input_queues, WARP_INPUTS, POLICY_INPUTS
-from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState
+from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_driving_model_data, fill_pose_msg, PublishState
 from openpilot.common.file_chunker import read_file_chunked, get_manifest_path
 from openpilot.selfdrive.modeld.constants import ModelConstants, Plan
 from openpilot.selfdrive.modeld.helpers import usbgpu_present, modeld_pkl_path, get_tg_input_devices
@@ -84,19 +84,15 @@ class ModelState(ModelStateBase):
     input_devices = get_tg_input_devices(PROCESS_NAME, usbgpu)
     self.WARP_DEV, self.QUEUE_DEV = input_devices['WARP_DEV'], input_devices['QUEUE_DEV']
     jits = pickle.loads(read_file_chunked(modeld_pkl_path(usbgpu)))
-    vision_metadata = jits['metadata']['vision']
-    self.vision_input_shapes = vision_metadata['input_shapes']
-    self.vision_input_names = list(self.vision_input_shapes.keys())
-    self.vision_output_slices = vision_metadata['output_slices']
-
-    policy_metadata = jits['metadata']['on_policy']
-    self.policy_input_shapes = policy_metadata['input_shapes']
-    self.policy_output_slices = policy_metadata['output_slices']
+    metadata = jits['metadata']
+    self.input_shapes = metadata['input_shapes']
+    self.vision_input_names = [k for k in self.input_shapes if 'img' in k]
+    self.output_slices = metadata['output_slices']
 
     self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)
 
     self.frame_skip = ModelConstants.MODEL_RUN_FREQ // ModelConstants.MODEL_CONTEXT_FREQ
-    self.input_queues, self.npy = make_input_queues(self.vision_input_shapes, self.policy_input_shapes, self.frame_skip, device=self.QUEUE_DEV)
+    self.input_queues, self.npy = make_input_queues(self.input_shapes, self.frame_skip, device=self.QUEUE_DEV)
     self.full_frames: dict[str, Tensor] = {}
     self._blob_cache: dict[int, Tensor] = {}
     self.parser = Parser()
@@ -133,19 +129,16 @@ class ModelState(ModelStateBase):
     if prepare_only:
       return None
 
-    vision_output, on_policy_output = self.run_policy(
-      **{k: self.input_queues[k] for k in POLICY_INPUTS}, img=img, big_img=big_img
+    outs, = self.run_policy(
+      **{k: self.input_queues[k] for k in POLICY_INPUTS if k in self.input_queues}, img=img, big_img=big_img
     )
-
-    vision_output = vision_output.numpy().flatten()
-    on_policy_output = on_policy_output.numpy().flatten()
-    vision_outputs_dict = self.parser.parse_vision_outputs(self.slice_outputs(vision_output, self.vision_output_slices))
-    policy_outputs_dict = self.parser.parse_policy_outputs(self.slice_outputs(on_policy_output, self.policy_output_slices))
-    combined_outputs_dict = {**vision_outputs_dict, **policy_outputs_dict}
+    model_output = outs.numpy()[0]
+    outputs_dict = self.parser.parse_outputs(self.slice_outputs(model_output, self.output_slices))
+    self.npy['prev_feat'][:] = model_output[self.output_slices['hidden_state']]
 
     if SEND_RAW_PRED:
-      combined_outputs_dict['raw_pred'] = np.concatenate([vision_output.copy(), on_policy_output.copy()])
-    return combined_outputs_dict
+      outputs_dict['raw_pred'] = model_output.copy()
+    return outputs_dict
 
 
 def main(demo=False):
@@ -158,10 +151,7 @@ def main(demo=False):
   params.put_bool("UsbGpuPresent", _present)
   params.put_bool("UsbGpuCompiled", _compiled)
 
-  if not USBGPU:
-    # USB GPU currently saturates a core so can't do this yet,
-    # also need to move the aux USB interrupts for good timings
-    config_realtime_process(7, 54)
+  config_realtime_process(7, 54)
 
   # visionipc clients
   while True:
@@ -317,7 +307,7 @@ def main(demo=False):
 
       action = get_action_from_model(model_output, prev_action, lat_action_t, long_action_t, v_ego)
       prev_action = action
-      fill_model_msg(drivingdata_send, modelv2_send, model_output, action,
+      fill_model_msg(modelv2_send, model_output, action,
                      publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
                      frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen)
 
@@ -329,9 +319,8 @@ def main(demo=False):
       modelv2_send.modelV2.meta.laneChangeState = DH.lane_change_state
       modelv2_send.modelV2.meta.laneChangeDirection = DH.lane_change_direction
       mdv2sp_send.modelDataV2SP.laneTurnDirection = DH.lane_turn_direction
-      drivingdata_send.drivingModelData.meta.laneChangeState = DH.lane_change_state
-      drivingdata_send.drivingModelData.meta.laneChangeDirection = DH.lane_change_direction
 
+      fill_driving_model_data(drivingdata_send, modelv2_send)
       fill_pose_msg(posenet_send, model_output, meta_main.frame_id, vipc_dropped_frames, meta_main.timestamp_eof, live_calib_seen)
       pm.send('modelV2', modelv2_send)
       pm.send('drivingModelData', drivingdata_send)

selfdrive/modeld/models/big_driving_on_policy.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:565e53c38dcd64c50dd3fe4d5ee1530213aeefd66c3f6b67ea6a72a32612a6bf
-size 14061419

selfdrive/modeld/models/big_driving_supercombo.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:471f372751d5931e939320c211e35b7255f8fa8015125b3b3fd48ef43020257e
+size 195490097

selfdrive/modeld/models/big_driving_vision.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1f0cab5033fe9e3bc5e174a2e790fa277f7d9fc44c65822d734064d2f899a9a0
-size 296203378

selfdrive/modeld/models/driving_on_policy.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:78477124cbf3ffe30fa951ebada8410b43c4242c6054584d656f1d329b067e15
-size 14060847

selfdrive/modeld/models/driving_supercombo.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:659727c4d4839adc4992a254409a54259a8756a743f2d567bf5fdc6579f8009b
+size 60881999

selfdrive/modeld/models/driving_vision.onnx

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ee29ee5bce84d1ce23e9ff381280de9b4e4d96d2934cd751740354884e112c66
-size 46877473

selfdrive/pandad/main.cc

@@ -3,7 +3,7 @@
 #include "selfdrive/pandad/pandad.h"
 #include "common/swaglog.h"
 #include "common/util.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 
 int main(int argc, char *argv[]) {
   LOGW("starting pandad");

selfdrive/pandad/pandad.cc

@@ -1,7 +1,6 @@
 #include "selfdrive/pandad/pandad.h"
 
 #include <array>
-#include <atomic>
 #include <bitset>
 #include <cassert>
 #include <cerrno>
@@ -16,7 +15,7 @@
 #include "common/swaglog.h"
 #include "common/timing.h"
 #include "common/util.h"
-#include "system/hardware/hw.h"
+#include "common/hardware/hw.h"
 
 #define MAX_IR_PANDA_VAL 50
 #define CUTOFF_IL 400
@@ -24,14 +23,6 @@
 
 ExitHandler do_exit;
 
-struct HwmonState {
-  std::atomic<uint32_t> voltage{0};
-  std::atomic<uint32_t> current{0};
-  std::atomic<bool> initialized{false};
-};
-
-HwmonState hwmon_state;
-
 bool check_connected(Panda *panda) {
   if (!panda->connected()) {
     do_exit = true;
@@ -126,26 +117,6 @@ void can_recv(Panda *panda, PubMaster *pm) {
   }
 }
 
-void hwmon_thread() {
-  util::set_thread_name("pandad_hwmon");
-
-  while (!do_exit) {
-    double read_time = millis_since_boot();
-    uint32_t voltage = Hardware::get_voltage();
-    uint32_t current = Hardware::get_current();
-    read_time = millis_since_boot() - read_time;
-    if (read_time > 50) {
-      LOGW("reading hwmon took %lfms", read_time);
-    }
-
-    hwmon_state.voltage.store(voltage);
-    hwmon_state.current.store(current);
-    hwmon_state.initialized.store(true);
-
-    util::sleep_for(500);
-  }
-}
-
 void fill_panda_state(cereal::PandaState::Builder &ps, cereal::PandaState::PandaType hw_type, const health_t &health) {
   ps.setVoltage(health.voltage_pkt);
   ps.setCurrent(health.current_pkt);
@@ -278,7 +249,8 @@ std::optional<bool> send_panda_states(PubMaster *pm, Panda *panda, bool is_onroa
 }
 
 void send_peripheral_state(Panda *panda, PubMaster *pm) {
-  if (!hwmon_state.initialized.load()) {
+  auto health_opt = panda->get_state();
+  if (!health_opt) {
     return;
   }
 
@@ -290,18 +262,9 @@ void send_peripheral_state(Panda *panda, PubMaster *pm) {
   auto ps = evt.initPeripheralState();
   ps.setPandaType(panda->hw_type);
 
-  ps.setVoltage(hwmon_state.voltage.load());
-  ps.setCurrent(hwmon_state.current.load());
-
-  // fall back to panda's voltage and current measurement
-  if (ps.getVoltage() == 0 && ps.getCurrent() == 0) {
-    auto health_opt = panda->get_state();
-    if (health_opt) {
-      health_t health = *health_opt;
-      ps.setVoltage(health.voltage_pkt);
-      ps.setCurrent(health.current_pkt);
-    }
-  }
+  health_t health = *health_opt;
+  ps.setVoltage(health.voltage_pkt);
+  ps.setCurrent(health.current_pkt);
 
   uint16_t fan_speed_rpm = panda->get_fan_speed();
   ps.setFanSpeedRpm(fan_speed_rpm);
@@ -413,9 +376,8 @@ void pandad_run(Panda *panda) {
   const bool spoofing_started = getenv("STARTED") != nullptr;
   const bool fake_send = getenv("FAKESEND") != nullptr;
 
-  // Start helper threads for event-driven sendcan and slow non-Panda reads.
+  // Start helper thread for event-driven sendcan.
   std::thread send_thread(can_send_thread, panda, fake_send);
-  std::thread hardware_thread(hwmon_thread);
 
   RateKeeper rk("pandad", 100);
   SubMaster sm({"selfdriveState", "deviceState", "selfdriveStateSP"});
@@ -475,7 +437,6 @@ void pandad_run(Panda *panda) {
   }
 
   send_thread.join();
-  hardware_thread.join();
 }
 
 void pandad_main_thread(std::string serial) {

selfdrive/pandad/pandad.py

@@ -9,7 +9,7 @@ import subprocess
 from panda import Panda, PandaDFU, PandaProtocolMismatch, McuType, FW_PATH
 from openpilot.common.basedir import BASEDIR
 from openpilot.common.params import Params
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 from openpilot.common.swaglog import cloudlog
 
 from openpilot.sunnypilot.selfdrive.pandad.rivian_long_flasher import flash_rivian_long

selfdrive/pandad/tests/test_pandad.py

@@ -7,8 +7,8 @@ from cereal import log
 from openpilot.common.gpio import gpio_set, gpio_init
 from panda import Panda, PandaDFU
 from openpilot.system.manager.process_config import managed_processes
-from openpilot.system.hardware import HARDWARE
-from openpilot.system.hardware.tici.pins import GPIO
+from openpilot.common.hardware import HARDWARE
+from openpilot.common.hardware.tici.pins import GPIO
 
 HERE = os.path.dirname(os.path.realpath(__file__))
 

selfdrive/pandad/tests/test_pandad_spi.py

@@ -80,7 +80,6 @@ class TestBoarddSpi:
             ps = m.peripheralState
             assert ps.pandaType == "tres"
             assert 4000 < ps.voltage < 14000
-            assert 50 < ps.current < 1000
             assert ps.fanSpeedRpm < 10000
 
       time.sleep(0.5)

selfdrive/selfdrived/events.py

@@ -10,7 +10,7 @@ from openpilot.common.realtime import DT_CTRL
 from openpilot.selfdrive.locationd.calibrationd import MIN_SPEED_FILTER
 from openpilot.system.micd import SAMPLE_RATE, SAMPLE_BUFFER
 from openpilot.selfdrive.ui.feedback.feedbackd import FEEDBACK_MAX_DURATION
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 
 from openpilot.sunnypilot.selfdrive.selfdrived.events_base import EventsBase, Priority, ET, Alert, \
   NoEntryAlert, SoftDisableAlert, UserSoftDisableAlert, ImmediateDisableAlert, EngagementAlert, NormalPermanentAlert, \

selfdrive/selfdrived/selfdrived.py

@@ -22,7 +22,7 @@ from openpilot.selfdrive.selfdrived.state import StateMachine
 from openpilot.selfdrive.selfdrived.alertmanager import AlertManager, set_offroad_alert
 
 from openpilot.system.version import get_build_metadata
-from openpilot.system.hardware import HARDWARE
+from openpilot.common.hardware import HARDWARE
 
 from openpilot.sunnypilot.mads.mads import ModularAssistiveDrivingSystem
 from openpilot.sunnypilot import get_sanitize_int_param
@@ -376,12 +376,13 @@ class SelfdriveD(CruiseHelper):
           self.events.add(EventName.cameraFrameRate)
     if not REPLAY and self.rk.lagging:
       self.events.add(EventName.selfdrivedLagging)
-    if self.sm['radarState'].radarErrors.canError:
-      self.events.add(EventName.canError)
-    elif self.sm['radarState'].radarErrors.radarUnavailableTemporary:
-      self.events.add(EventName.radarTempUnavailable)
-    elif any(self.sm['radarState'].radarErrors.to_dict().values()):
-      self.events.add(EventName.radarFault)
+    if self.CP.openpilotLongitudinalControl:
+      if self.sm['radarState'].radarErrors.canError:
+        self.events.add(EventName.canError)
+      elif self.sm['radarState'].radarErrors.radarUnavailableTemporary:
+        self.events.add(EventName.radarTempUnavailable)
+      elif any(self.sm['radarState'].radarErrors.to_dict().values()):
+        self.events.add(EventName.radarFault)
     if not self.sm.valid['pandaStates']:
       self.events.add(EventName.usbError)
     if CS.canTimeout:

selfdrive/selfdrived/state.py

@@ -24,14 +24,14 @@ class StateMachine:
     # ENABLED, SOFT DISABLING, PRE ENABLING, OVERRIDING
     if self.state != State.disabled:
       # user and immediate disable always have priority in a non-disabled state
-      if events.contains(ET.USER_DISABLE):
-        self.state = State.disabled
-        self.current_alert_types.append(ET.USER_DISABLE)
-
-      elif events.contains(ET.IMMEDIATE_DISABLE):
+      if events.contains(ET.IMMEDIATE_DISABLE):
         self.state = State.disabled
         self.current_alert_types.append(ET.IMMEDIATE_DISABLE)
 
+      elif events.contains(ET.USER_DISABLE):
+        self.state = State.disabled
+        self.current_alert_types.append(ET.USER_DISABLE)
+
       else:
         # ENABLED
         if self.state == State.enabled:

selfdrive/test/.gitignore

@@ -1,5 +1,4 @@
 out/
-docker_out/
 
 process_replay/diff.txt
 process_replay/model_diff.txt

selfdrive/test/docker_build.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -e
-
-SCRIPT_DIR=$(dirname "$0")
-OPENPILOT_DIR=$SCRIPT_DIR/../../
-
-DOCKER_IMAGE=openpilot
-DOCKER_FILE=Dockerfile.openpilot
-DOCKER_REGISTRY=ghcr.io/commaai
-COMMIT_SHA=$(git rev-parse HEAD)
-
-if [ -n "$TARGET_ARCHITECTURE" ]; then
-  PLATFORM="linux/$TARGET_ARCHITECTURE"
-  TAG_SUFFIX="-$TARGET_ARCHITECTURE"
-else
-  PLATFORM="linux/$(uname -m)"
-  TAG_SUFFIX=""
-fi
-
-LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX
-REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG
-REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA
-
-DOCKER_BUILDKIT=1 docker buildx build --provenance false --pull --platform $PLATFORM --load -t $DOCKER_IMAGE:latest -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR
-
-if [ -n "$PUSH_IMAGE" ]; then
-  docker push $REMOTE_TAG
-  docker tag $REMOTE_TAG $REMOTE_SHA_TAG
-  docker push $REMOTE_SHA_TAG
-fi

selfdrive/test/process_replay/model_replay.py

@@ -12,7 +12,7 @@ import numpy as np
 from openpilot.common.utils import tabulate
 
 from openpilot.common.git import get_commit
-from openpilot.system.hardware import PC
+from openpilot.common.hardware import PC
 from openpilot.tools.lib.openpilotci import get_url
 from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
 from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process

selfdrive/test/process_replay/process_replay.py

@@ -12,7 +12,7 @@ from typing import Any
 from collections.abc import Callable, Iterable
 from tqdm import tqdm
 import capnp
-from openpilot.system.hardware.hw import Paths
+from openpilot.common.hardware.hw import Paths
 
 import cereal.messaging as messaging
 from cereal import car

selfdrive/test/test_onroad.py

@@ -18,7 +18,7 @@ from openpilot.common.timeout import Timeout
 from openpilot.common.params import Params
 from openpilot.selfdrive.selfdrived.events import EVENTS, ET
 from openpilot.selfdrive.test.helpers import set_params_enabled, release_only
-from openpilot.system.hardware.hw import Paths
+from openpilot.common.hardware.hw import Paths
 from openpilot.tools.lib.logreader import LogReader
 from openpilot.tools.lib.log_time_series import msgs_to_time_series
 
@@ -68,7 +68,7 @@ PROCS = {
   "system.loggerd.deleter": 1.0,
   "./pandad": 19.0,
   "system.qcomgpsd.qcomgpsd": 1.0,
-  "system.hardware.tici.modem": 10.0,
+  "common.hardware.tici.modem": 10.0,
 }
 
 TIMINGS = {
@@ -284,7 +284,7 @@ class TestOnroad:
     print("--------------- Memory Usage -------------------")
     print("------------------------------------------------")
 
-    from openpilot.selfdrive.debug.mem_usage import print_report
+    from openpilot.tools.scripts.mem_usage import print_report
     print_report(self.msgs['procLog'], self.msgs['deviceState'])
 
     offset = int(SERVICE_LIST['deviceState'].frequency * LOG_OFFSET)

selfdrive/test/test_power_draw.py

@@ -10,7 +10,7 @@ from cereal.services import SERVICE_LIST
 from opendbc.car.car_helpers import get_demo_car_params
 from openpilot.common.mock import mock_messages
 from openpilot.common.params import Params
-from openpilot.system.hardware.tici.power_monitor import get_power
+from openpilot.common.hardware.tici.power_monitor import get_power
 from openpilot.system.manager.process_config import managed_processes
 from openpilot.system.manager.manager import manager_cleanup