Merge branch 'master' into ccnc-port-alert-dedup

This reverts commit 15e5d2efb9.

.github/workflows/test_models.yml

@@ -1,82 +0,0 @@
-name: Test Models Compatibility With Tinygrad Changes
-on:
-  push:
-    paths:
-      - 'tinygrad_repo'
-  pull_request:
-    paths:
-      - 'tinygrad_repo'
-  workflow_dispatch:
-
-jobs:
-  generate-matrix:
-    runs-on: ubuntu-latest
-    outputs:
-      models: ${{ steps.set-matrix.outputs.models }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Fetch and Parse json
-        id: set-matrix
-        run: |
-          python3 -c '
-          import json, urllib.request, os, re
-
-          with open("openpilot/sunnypilot/models/fetcher.py", "r") as f:
-            urls = re.findall(r"MODEL_URL(?:_CHESTNUT)?\s*=\s*[\"'"'"']([^\"'"'"']+)[\"'"'"']", f.read())
-
-          artifacts = []
-          for url in urls:
-            data = json.loads(urllib.request.urlopen(url).read())
-            for bundle in data.get("bundles", []):
-              for model in bundle.get("models", []):
-                if "artifact" in model:
-                  artifacts.append(model["artifact"])
-
-          with open(os.environ["GITHUB_OUTPUT"], "a") as f:
-            f.write(f"models={json.dumps(artifacts)}\n")
-          '
-
-  test-model:
-    name: Test ${{ matrix.artifact.file_name }}
-    needs: generate-matrix
-    runs-on: ubuntu-latest
-    container: ghcr.io/commaai/openpilot-base:latest
-    strategy:
-      fail-fast: false
-      matrix:
-        artifact: ${{ fromJson(needs.generate-matrix.outputs.models) }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: true
-
-      - name: Download Model Chunks in Parallel
-        run: |
-          mkdir -p /tmp/model_chunks
-          echo '${{ toJson(matrix.artifact.chunks) }}' > chunks.json
-
-          BASE_URL="${{ matrix.artifact.download_uri.url }}"
-          export BASE_DIR=$(dirname "$BASE_URL")
-
-          python3 -c '
-          import json, os
-          with open("chunks.json") as f:
-            chunks = json.load(f)
-          manifest_path = f"/tmp/model_chunks/${{ matrix.artifact.file_name }}.chunkmanifest"
-          with open(manifest_path, "w") as f:
-            f.write(str(len(chunks)))
-          base_dir = os.environ["BASE_DIR"]
-          with open("/tmp/curl_config.txt", "w") as f:
-            for c in chunks:
-              fn = c["file_name"]
-              f.write(f"url = \"{base_dir}/{fn}\"\noutput = \"/tmp/model_chunks/{fn}\"\n")
-          '
-          curl -Z --parallel-immediate --parallel-max 16 -s -S -f -L -K /tmp/curl_config.txt
-
-      - name: Run Model Compatibility Test
-        env:
-          MODEL_BASE_NAME: ${{ matrix.artifact.file_name }}
-          MODEL_CHUNK_DIR: "/tmp/model_chunks"
-          PYTHONPATH: ".:./tinygrad_repo"
-        run: |
-          python3 -m pytest openpilot/sunnypilot/modeld_v2/tests/test_models.py

openpilot/selfdrive/selfdrived/selfdrived.py

@@ -576,7 +576,7 @@ class SelfdriveD(CruiseHelper):
     clear_event_types = set()
     if ET.WARNING not in self.state_machine.current_alert_types:
       clear_event_types.add(ET.WARNING)
-    if self.enabled:
+    if self.enabled or self.mads.clear_no_entry:
       clear_event_types.add(ET.NO_ENTRY)
 
     pers = LONGITUDINAL_PERSONALITY_MAP[self.personality]

openpilot/sunnypilot/mads/mads.py

@@ -10,6 +10,7 @@ from openpilot.cereal import log, custom
 from opendbc.car import structs
 from opendbc.car.hyundai.values import HyundaiFlags
 from openpilot.common.params import Params
+from openpilot.selfdrive.selfdrived.events import ET
 from openpilot.sunnypilot.mads.helpers import MadsSteeringModeOnBrake, read_steering_mode_param, MADS_NO_ACC_MAIN_BUTTON
 from openpilot.sunnypilot.mads.state import StateMachine, GEARS_ALLOW_PAUSED_SILENT
 
@@ -33,6 +34,7 @@ class ModularAssistiveDrivingSystem:
     self.enabled = False
     self.active = False
     self.available = False
+    self.clear_no_entry = False
     self.lateral_mismatch_counter = 0
     self.allow_always = False
     self.no_main_cruise = False
@@ -218,5 +220,8 @@ class ModularAssistiveDrivingSystem:
     if not self.CP.passive and self.selfdrive.initialized:
       self.enabled, self.active = self.state_machine.update()
 
+    cat = self.selfdrive.state_machine.current_alert_types
+    self.clear_no_entry = (ET.ENABLE in cat or ET.USER_DISABLE in cat) and ET.NO_ENTRY not in cat
+
     # Copy of previous SelfdriveD states for MADS events handling
     self.selfdrive.enabled_prev = self.selfdrive.enabled

openpilot/sunnypilot/modeld_v2/helpers.py

@@ -1,93 +0,0 @@
-import io
-import struct
-import pickle
-import inspect
-import importlib
-import enum
-
-
-def _pad_args(func, args, kwargs):
-  try:
-    sig = inspect.signature(func)
-  except Exception:
-    return args, kwargs
-  params = list(sig.parameters.values())
-  if inspect.isfunction(func) and params and params[0].name in ('cls', 'self'):
-    params = params[1:]
-
-  new_args = list(args)
-  has_varargs = any(p.kind == inspect.Parameter.VAR_POSITIONAL for p in params)
-  if len(new_args) > len(params) and not has_varargs:
-    new_args = new_args[:len(params)]
-
-  for i in range(len(new_args), len(params)):
-    param = params[i]
-    if param.kind in (inspect.Parameter.VAR_POSITIONAL, inspect.Parameter.VAR_KEYWORD):
-      continue
-    val = param.default if param.default is not inspect.Parameter.empty else None
-    new_args.append(val)
-  return new_args, kwargs
-
-
-def _enum_factory(enum_class):
-  def factory(*args, **kwargs):
-    try:
-      return enum_class(*args, **kwargs)
-    # OptOps and UOp objects in the .pkl are left over from the compilation phase,
-    # reassignment does nothing because they aren't tied to the execution graph
-    # It never executes or evaluates the UOp nodes again.
-    except ValueError:
-      return list(enum_class)[0]
-  factory.__name__ = enum_class.__name__
-  factory.__module__ = enum_class.__module__
-  return factory
-
-
-def _dynamic_factory(real_class):
-  if isinstance(real_class, type) and issubclass(real_class, enum.Enum):
-    return _enum_factory(real_class)
-
-  def factory(*args, **kwargs):
-    try:
-      return real_class(*args, **kwargs)
-    except TypeError:
-      new_args, new_kwargs = _pad_args(real_class, args, kwargs)
-      return real_class(*new_args, **new_kwargs)
-
-  class DynamicMeta(type(real_class)):
-    def __call__(cls, *args, **kwargs):
-      return factory(*args, **kwargs)
-
-  class DynamicProxy(real_class, metaclass=DynamicMeta):
-    __slots__ = ()
-
-    def __new__(cls, *args, **kwargs):
-      return factory(*args, **kwargs)
-
-  DynamicProxy.__name__ = real_class.__name__
-  DynamicProxy.__module__ = real_class.__module__
-  return DynamicProxy
-
-
-class DynamicTinygradUnpickler(pickle.Unpickler):
-  def find_class(self, module, name):
-    if module == "tinygrad.ops":
-      try:
-        importlib.import_module("tinygrad.uops")
-        module = "tinygrad.uops"
-      except ImportError:
-        pass
-    real_class = getattr(importlib.import_module(module), name)
-    if module.startswith("tinygrad"):
-      return _dynamic_factory(real_class)
-    return real_class
-
-
-def load_oob(f):
-  opcodes = f.read(struct.unpack('<q', f.read(8))[0])
-  def buffers():
-    while (h := f.read(8)):
-      pb = pickle.PickleBuffer(bytearray(struct.unpack('<q', h)[0]))
-      f.readinto(pb)
-      yield pb
-  return DynamicTinygradUnpickler(io.BytesIO(opcodes), buffers=buffers()).load()

openpilot/sunnypilot/modeld_v2/modeld.py

@@ -17,7 +17,7 @@ from tinygrad.tensor import Tensor
 
 import openpilot.cereal.messaging as messaging
 from openpilot.common.hardware import COMMA_HARDWARE
-from openpilot.selfdrive.modeld.helpers import chestnut_present
+from openpilot.selfdrive.modeld.helpers import chestnut_present, load_oob
 from openpilot.cereal import log
 from opendbc.car.structs import car
 from openpilot.cereal.services import SERVICE_LIST
@@ -52,7 +52,6 @@ from openpilot.sunnypilot.modeld_v2.compile_modeld import (derive_frame_skip, ma
                                                            WARP_INPUTS, POLICY_INPUTS)
 from openpilot.sunnypilot.livedelay.helpers import get_lat_delay
 from openpilot.sunnypilot.modeld_v2.modeld_base import ModelStateBase
-from openpilot.sunnypilot.modeld_v2.helpers import load_oob
 from openpilot.sunnypilot.models.helpers import get_active_bundle
 from openpilot.sunnypilot.selfdrive.controls.lib.relc import RoadEdgeLaneChangeController
 

openpilot/sunnypilot/modeld_v2/tests/test_models.py

@@ -1,36 +0,0 @@
-import os
-import unittest
-from unittest.mock import patch
-from openpilot.common.file_chunker import open_file_chunked
-from openpilot.sunnypilot.modeld_v2.helpers import load_oob
-from tinygrad.device import Device
-
-
-class TestLegacyModels(unittest.TestCase):
-  def test_legacy_model_load(self):
-    base_name = os.environ.get("MODEL_BASE_NAME")
-    if not base_name:
-      raise unittest.SkipTest("MODEL_BASE_NAME env var not set, skipping integration test.")
-    chunk_dir = os.environ.get("MODEL_CHUNK_DIR", "/tmp/model_chunks")
-    base_path = os.path.join(chunk_dir, base_name)
-
-    try:
-      f = open_file_chunked(base_path)
-    except Exception as error:
-      self.fail(f"Failed to open chunked file {base_path}: {error}")
-    self.addCleanup(f.close)
-
-    real_getitem = Device.__class__.__getitem__
-
-    def safe_getitem(device_self, ix):
-      if ix == "QCOM" and not os.path.exists("/dev/kgsl-3d0"):
-        return real_getitem(device_self, "CPU")
-      if ix == "AMD" and not os.path.exists("/dev/kfd"):
-        return real_getitem(device_self, "CPU")
-      return real_getitem(device_self, ix)
-
-    with patch.object(Device.__class__, "__getitem__", safe_getitem):
-      obj = load_oob(f)
-
-    assert isinstance(obj, dict), "Parsed object is not a dictionary"
-    assert "metadata" in obj, "Metadata key is missing"

openpilot/sunnypilot/models/tests/test_tinygrad_ref.py

@@ -0,0 +1,24 @@
+import requests
+
+from openpilot.sunnypilot.models.tinygrad_ref import get_tinygrad_ref
+from openpilot.sunnypilot.models.fetcher import ModelFetcher
+from openpilot.common.test import OpenpilotTestCase
+
+def fetch_tinygrad_ref():
+  response = requests.get(ModelFetcher.MODEL_URL, timeout=10)
+  response.raise_for_status()
+  json_data = response.json()
+  return json_data.get("tinygrad_ref")
+
+
+class TestTinygradRef(OpenpilotTestCase):
+  def test_tinygrad_ref(self):
+    current_ref = get_tinygrad_ref()
+    remote_ref = fetch_tinygrad_ref()
+    assert remote_ref == current_ref, (
+      f"""tinygrad_repo ref does not match remote tinygrad_ref of current compiled driving models json.
+    Current: {current_ref}
+    Remote: {remote_ref}
+    Please run build-all workflow to update models."""
+    )
+    print("tinygrad_repo ref matches current compiled driving models json ref.")