Merge branch 'master' into clippy

* new new

* Simplify model removal

* use a var

.github/workflows/build-all-tinygrad-models.yaml

@@ -74,7 +74,7 @@ jobs:
         env:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           git checkout main
           git sparse-checkout set --no-cone models/
@@ -191,7 +191,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}

.github/workflows/build-single-tinygrad-model.yaml

@@ -109,7 +109,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}

.github/workflows/sunnypilot-build-model.yaml

@@ -156,6 +156,8 @@ jobs:
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
           path: ${{ github.workspace }}/selfdrive/modeld/models
+      - run: |
+          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |

.gitignore

@@ -109,3 +109,7 @@ Pipfile
 !.idea/customTargets.xml
 !.idea/tools/*
 !.run/*
+
+### clippy ###
+clippy_stats.json
+clippy.log

CHANGELOG.md

@@ -1,6 +1,30 @@
-sunnypilot Version 2025.002.000 (2025-xx-xx)
+sunnypilot Version 2025.003.000 (20xx-xx-xx)
 ========================
 
+sunnypilot Version 2025.002.000 (2025-11-06)
+========================
+* What's Changed (sunnypilot/sunnypilot)
+  * models: bump model json to v8 by @Discountchubbs
+  * Bug: Model UI Crash Fix by @nayan8teen
+  * controlsd: add `CP_SP` to `get_pid_accel_limits` by @THERoenPR
+  * sunnylink: update uploader button logic to support novice tier and above by @devtekve
+  * Tesla: Coop Steering by @AmyJeanes
+  * ui: update discord references and add forum widget by @devtekve
+  * ui: Fix spacing in sunnylink panel by @devtekve
+  * docs: Update README installation branches and discord links by @mpurnell1 in
+  * stats: sunnylink integration by @devtekve
+  * bug: Fix initial registration for sunnylink by @devtekve
+* What's Changed (sunnypilot/opendbc)
+  * Honda: add brake hold messages for Clarity by @mvl-boston
+  * interface: add `CP_SP` to `get_pid_accel_limits` method signature by @roenthomas
+  * Honda: use fixed accel min/max constants for Gas Interceptor by @roenthomas
+  * Tesla: Coop Steering by @AmyJeanes
+* New Contributors (sunnypilot/sunnypilot)
+  * @THERoenPR made their first contribution in "controlsd: add `CP_SP` to `get_pid_accel_limits`"
+  * @AmyJeanes made their first contribution in "Tesla: Coop Steering"
+  * @mpurnell1 made their first contribution in "docs: Update README installation branches and discord links"
+* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2025.001.000...v2025.002.000
+
 sunnypilot Version 2025.001.000 (2025-10-25)
 ========================
 * 🛠️ Major rewrite

common/model.h

@@ -1 +1 @@
-#define DEFAULT_MODEL "TCPv3 + gWMv9 (Default)"
+#define DEFAULT_MODEL "Firehose (Default)"

pyproject.toml

@@ -73,6 +73,10 @@ dependencies = [
 
   # ui
   "qrcode",
+
+  # clippy
+  "discord-py",
+  "flask",
 ]
 
 [project.optional-dependencies]

selfdrive/ui/qt/onroad/annotated_camera.cc

@@ -118,7 +118,7 @@ void AnnotatedCameraWidget::paintGL() {
       } else if (v_ego > 15) {
         wide_cam_requested = false;
       }
-      wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode();
+      // wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode();
     }
     CameraWidget::setStreamType(wide_cam_requested ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
     CameraWidget::setFrameId(sm["modelV2"].getModelV2().getFrameId());

sunnypilot/common/version.h

@@ -1 +1 @@
-#define SUNNYPILOT_VERSION "2025.002.000"
+#define SUNNYPILOT_VERSION "2025.003.000"

sunnypilot/models/tests/model_hash

@@ -1 +1 @@
-5584d697233d147e0b6402e485b7cbf8fdddb70bde4b9e3b2f6919ed5f69475f
+70406ab4dd66d0e384734a8a56632ae4a62bc9670c2e630a0f71588c4e212cd8

sunnypilot/selfdrive/car/interfaces.py

@@ -15,6 +15,8 @@ from openpilot.sunnypilot.selfdrive.controls.lib.speed_limit.helpers import set_
 
 import openpilot.system.sentry as sentry
 
+from sunnypilot.sunnylink.statsd import STATSLOGSP
+
 
 def log_fingerprint(CP: structs.CarParams) -> None:
   if CP.carFingerprint == "MOCK":
@@ -100,6 +102,9 @@ def setup_interfaces(CI: CarInterfaceBase, params: Params = None) -> None:
   _initialize_torque_lateral_control(CI, CP, enforce_torque, nnlc_enabled)
   _cleanup_unsupported_params(CP, CP_SP)
 
+  STATSLOGSP.raw('sunnypilot.car_params', CP.to_dict())
+  # STATSLOGSP.raw('sunnypilot_params.car_params_sp', CP_SP.to_dict()) # https://github.com/sunnypilot/opendbc/pull/361
+
 
 def initialize_params(params) -> list[dict[str, Any]]:
   keys: list = []

sunnypilot/sunnylink/athena/sunnylinkd.py

@@ -16,8 +16,9 @@ from functools import partial
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
 from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware.hw import Paths
 from openpilot.system.athena.athenad import ws_send, jsonrpc_handler, \
-  recv_queue, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage, log_handler, start_local_proxy_shim, upload_handler
+  recv_queue, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage, log_handler, start_local_proxy_shim, upload_handler, stat_handler
 from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException,
                        create_connection, WebSocketConnectionClosedException)
 
@@ -33,9 +34,6 @@ SUNNYLINK_RECONNECT_TIMEOUT_S = 70  # FYI changing this will also would require
 DISALLOW_LOG_UPLOAD = threading.Event()
 
 params = Params()
-sunnylink_dongle_id = params.get("SunnylinkDongleId")
-sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
-
 
 def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
   cloudlog.info("sunnylinkd.handle_long_poll started")
@@ -51,7 +49,7 @@ def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
               threading.Thread(target=ws_queue, args=(end_event,), name='ws_queue'),
               threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler'),
               # threading.Thread(target=sunny_log_handler, args=(end_event, comma_prime_cellular_end_event), name='log_handler'),
-              # threading.Thread(target=stat_handler, args=(end_event,), name='stat_handler'),
+              threading.Thread(target=stat_handler, args=(end_event, Paths.stats_sp_root(), True), name='stat_handler'),
             ] + [
               threading.Thread(target=jsonrpc_handler, args=(end_event, partial(startLocalProxy, end_event),), name=f'worker_{x}')
               for x in range(HANDLER_THREADS)
@@ -132,6 +130,8 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None:
 
 
 def ws_queue(end_event: threading.Event) -> None:
+  sunnylink_dongle_id = params.get("SunnylinkDongleId")
+  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
   resume_requested = False
   tries = 0
 
@@ -233,6 +233,9 @@ def saveParams(params_to_update: dict[str, str], compression: bool = False) -> N
 
 
 def startLocalProxy(global_end_event: threading.Event, remote_ws_uri: str, local_port: int) -> dict[str, int]:
+  sunnylink_dongle_id = params.get("SunnylinkDongleId")
+  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
+
   cloudlog.debug("athena.startLocalProxy.starting")
   ws = create_connection(
     remote_ws_uri,
@@ -254,6 +257,8 @@ def main(exit_event: threading.Event = None):
     cloudlog.info("Waiting for sunnylink registration to complete")
     time.sleep(10)
 
+  sunnylink_dongle_id = params.get("SunnylinkDongleId")
+  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
   UploadQueueCache.initialize(upload_queue)
 
   ws_uri = f"{SUNNYLINK_ATHENA_HOST}"

sunnypilot/sunnylink/statsd.py

@@ -0,0 +1,278 @@
+#!/usr/bin/env python3
+import base64
+import json
+import os
+import threading
+import traceback
+
+import zmq
+import time
+import uuid
+from pathlib import Path
+from collections import defaultdict
+from datetime import datetime, UTC
+
+from openpilot.common.params import Params
+from cereal.messaging import SubMaster
+from openpilot.system.hardware.hw import Paths
+from openpilot.common.swaglog import cloudlog
+from openpilot.system.hardware import HARDWARE
+from openpilot.common.file_helpers import atomic_write_in_dir
+from openpilot.system.version import get_build_metadata
+from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
+from openpilot.system.statsd import METRIC_TYPE, StatLogSP
+from openpilot.common.realtime import Ratekeeper
+
+STATSLOGSP = StatLogSP(intercept=False)
+
+def sp_stats(end_event):
+  """Collect sunnypilot-specific statistics and send as raw metrics."""
+  rk = Ratekeeper(.1, print_delay_threshold=None)
+  statlogsp = STATSLOGSP
+  params = Params()
+
+  def flatten_dict(d, parent_key='', sep='.'):
+    items = {}
+    if isinstance(d, dict):
+      for k, v in d.items():
+        new_key = f"{parent_key}{sep}{k}" if parent_key else k
+        items.update(flatten_dict(v, new_key, sep=sep))
+    elif isinstance(d, (list, tuple)):
+      for i, v in enumerate(d):
+        new_key = f"{parent_key}[{i}]"
+        items.update(flatten_dict(v, new_key, sep=sep))
+    else:
+      items[parent_key] = d
+    return items
+
+  # Collect sunnypilot parameters
+  stats_dict = {}
+
+  param_keys = [
+    'SunnylinkEnabled',
+    'AutoLaneChangeBsmDelay',
+    'AutoLaneChangeTimer',
+    'CarPlatformBundle',
+    'CurrentRoute',
+    'DevUIInfo',
+    'EnableCopyparty',
+    'IntelligentCruiseButtonManagement',
+    'QuietMode',
+    'RainbowMode',
+    'ShowAdvancedControls',
+    'Mads',
+    'MadsMainCruiseAllowed',
+    'MadsSteeringMode',
+    'MadsUnifiedEngagementMode',
+    'ModelManager_ActiveBundle',
+    'ModelManager_Favs',
+    'EnableSunnylinkUploader',
+    'SunnylinkEnabled',
+    'InstallDate',
+    'UptimeOffroad',
+    'UptimeOnroad',
+  ]
+
+  while not end_event.is_set():
+    try:
+      for key in param_keys:
+
+        try:
+          value = params.get(key)
+        except Exception as e:
+          stats_dict[key] = e
+          continue
+
+        if value is None:
+          continue
+
+        if isinstance(value, (dict, list, tuple)):
+          stats_dict.update(flatten_dict(value, key))
+        else:
+          stats_dict[key] = value
+
+      if stats_dict:
+        statlogsp.raw('sunnypilot.device_params', stats_dict)
+    except Exception as e:
+      cloudlog.error(f"Exception {e}")
+    finally:
+      rk.keep_time()
+
+
+def stats_main(end_event):
+  comma_dongle_id = Params().get("DongleId")
+  sunnylink_dongle_id = Params().get("SunnylinkDongleId")
+
+  def get_influxdb_line(measurement: str, value: float | dict[str, float], timestamp: datetime, tags: dict) -> str:
+    res = f"{measurement}"
+    for k, v in tags.items():
+      res += f",{k}={str(v)}"
+    res += " "
+
+    if isinstance(value, float):
+      value = {'value': value}
+
+    for k, v in value.items():
+      res += f"{k}={str(v)},"
+
+    res += f"sunnylink_dongle_id=\"{sunnylink_dongle_id}\",comma_dongle_id=\"{comma_dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
+    return res
+
+  def get_influxdb_line_raw(measurement: str, value: dict, timestamp: datetime, tags: dict) -> str:
+    res = f"{measurement}"
+    try:
+      custom_tags = ""
+      for k, v in tags.items():
+        custom_tags += f",{k}={str(v)}"
+      res += custom_tags
+
+      fields = ""
+      for k, v in value.items():
+        # Skip complex types - only keep simple scalar values
+        if isinstance(v, (dict, list, bytes, bytearray)):
+          continue
+
+        fields += f"{k}={json.dumps(v)},"
+
+      res += f" {fields}"
+    except Exception as e:
+      cloudlog.error(f"Unable to get influxdb line for: {value}")
+      res += f",invalid=1 reason={e},"
+
+    res += f"sunnylink_dongle_id=\"{sunnylink_dongle_id}\",comma_dongle_id=\"{comma_dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
+    return res
+
+  # open statistics socket
+  ctx = zmq.Context.instance()
+  sock = ctx.socket(zmq.PULL)
+  sock.bind(f"{STATS_SOCKET}_sp")
+
+  STATS_DIR = Paths.stats_sp_root()
+
+  # initialize stats directory
+  Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
+
+  build_metadata = get_build_metadata()
+
+  # initialize tags
+  tags = {
+    'started': False,
+    'version': build_metadata.openpilot.version,
+    'branch': build_metadata.channel,
+    'dirty': build_metadata.openpilot.is_dirty,
+    'origin': build_metadata.openpilot.git_normalized_origin,
+    'deviceType': HARDWARE.get_device_type(),
+  }
+
+  # subscribe to deviceState for started state
+  sm = SubMaster(['deviceState'])
+
+  idx = 0
+  boot_uid = str(uuid.uuid4())[:8]
+  last_flush_time = time.monotonic()
+  gauges = {}
+  samples: dict[str, list[float]] = defaultdict(list)
+  raws: dict = defaultdict()
+  try:
+    while not end_event.is_set():
+      started_prev = sm['deviceState'].started
+      sm.update()
+
+      # Update metrics
+      while True:
+        try:
+          metric = sock.recv_string(zmq.NOBLOCK)
+          try:
+            metric_type = metric.split('|')[1]
+            metric_name = metric.split(':')[0]
+            metric_value_raw = metric.split('|')[0].split(':')[1]
+
+            if metric_type == METRIC_TYPE.GAUGE:
+              metric_value = float(metric_value_raw)
+              gauges[metric_name] = metric_value
+            elif metric_type == METRIC_TYPE.SAMPLE:
+              metric_value = float(metric_value_raw)
+              samples[metric_name].append(metric_value)
+            elif metric_type == METRIC_TYPE.RAW:
+              raws[metric_name] = metric_value_raw
+            else:
+              cloudlog.event("unknown metric type", metric_type=metric_type)
+          except Exception:
+            print(traceback.format_exc())
+            cloudlog.event("malformed metric", metric=metric)
+        except zmq.error.Again:
+          break
+
+      # flush when started state changes or after FLUSH_TIME_S
+      if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
+        result = ""
+        current_time = datetime.now(UTC)
+        tags['started'] = sm['deviceState'].started
+
+        for key, value in raws.items():
+          decoded_value = json.loads(base64.b64decode(value).decode('utf-8'))
+          result += get_influxdb_line_raw(key, decoded_value, current_time, tags)
+
+        for key, value in gauges.items():
+          result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
+
+        for key, values in samples.items():
+          values.sort()
+          sample_count = len(values)
+          sample_sum = sum(values)
+
+          stats = {
+            'count': sample_count,
+            'min': values[0],
+            'max': values[-1],
+            'mean': sample_sum / sample_count,
+          }
+          for percentile in [0.05, 0.5, 0.95]:
+            value = values[int(round(percentile * (sample_count - 1)))]
+            stats[f"p{int(percentile * 100)}"] = value
+
+          result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
+
+        # clear intermediate data
+        gauges.clear()
+        samples.clear()
+        last_flush_time = time.monotonic()
+
+        # check that we aren't filling up the drive
+        if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
+          if len(result) > 0:
+            stats_path = os.path.join(STATS_DIR, f"{boot_uid}_{idx}")
+            with atomic_write_in_dir(stats_path) as f:
+              f.write(result)
+            idx += 1
+        else:
+          cloudlog.error("stats dir full")
+  finally:
+    sock.close()
+    ctx.term()
+
+
+def main():
+  rk = Ratekeeper(1, print_delay_threshold=None)
+  end_event = threading.Event()
+
+  threads = [
+    threading.Thread(target=stats_main, args=(end_event,)),
+    threading.Thread(target=sp_stats, args=(end_event,)),
+  ]
+
+  for t in threads:
+    t.start()
+
+  try:
+    while all(t.is_alive() for t in threads):
+      rk.keep_time()
+  finally:
+    end_event.set()
+
+  for t in threads:
+    t.join()
+
+
+if __name__ == "__main__":
+  main()

system/athena/athenad.py

@@ -744,26 +744,40 @@ def log_handler(end_event: threading.Event, log_attr_name=LOG_ATTR_NAME) -> None
       cloudlog.exception("athena.log_handler.exception")
 
 
-def stat_handler(end_event: threading.Event) -> None:
-  STATS_DIR = Paths.stats_root()
+def stat_handler(end_event: threading.Event, stats_dir=None, is_sunnylink=False) -> None:
+  stats_dir = stats_dir or Paths.stats_root()
   last_scan = 0.0
 
   while not end_event.is_set():
     curr_scan = time.monotonic()
     try:
       if curr_scan - last_scan > 10:
-        stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
+        stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(stats_dir)))
         if len(stat_filenames) > 0:
-          stat_path = os.path.join(STATS_DIR, stat_filenames[0])
+          stat_path = os.path.join(stats_dir, stat_filenames[0])
           with open(stat_path) as f:
+            payload = f.read()
+            is_compressed = False
+
+            # Log the current size of the file
+            if is_sunnylink:
+              # Compress and encode the data if it exceeds the maximum size
+              compressed_data = gzip.compress(payload.encode())
+              payload = base64.b64encode(compressed_data).decode()
+              is_compressed = True
+
             jsonrpc = {
               "method": "storeStats",
               "params": {
-                "stats": f.read()
+                "stats": payload
               },
               "jsonrpc": "2.0",
               "id": stat_filenames[0]
             }
+
+            if is_sunnylink and is_compressed:
+              jsonrpc["params"]["compressed"] = is_compressed
+
             low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
           os.remove(stat_path)
         last_scan = curr_scan

system/hardware/hw.py

@@ -55,6 +55,13 @@ class Paths:
     else:
       return "/data/stats/"
 
+  @staticmethod
+  def stats_sp_root() -> str:
+    if PC:
+      return str(Path(Paths.comma_home()) / "stats")
+    else:
+      return "/data/stats_sp/"
+
   @staticmethod
   def config_root() -> str:
     if PC:

system/manager/process_config.py

@@ -164,6 +164,7 @@ procs = [
   # sunnylink <3
   DaemonProcess("manage_sunnylinkd", "sunnypilot.sunnylink.athena.manage_sunnylinkd", "SunnylinkdPid"),
   PythonProcess("sunnylink_registration_manager", "sunnypilot.sunnylink.registration_manager", sunnylink_need_register_shim),
+  PythonProcess("statsd_sp", "sunnypilot.sunnylink.statsd", and_(always_run, sunnylink_ready_shim)),
 ]
 
 # sunnypilot

system/statsd.py

@@ -1,11 +1,15 @@
 #!/usr/bin/env python3
+import base64
+import json
 import os
+from decimal import Decimal
+
 import zmq
 import time
 import uuid
 from pathlib import Path
 from collections import defaultdict
-from datetime import datetime, UTC
+from datetime import datetime, UTC, date
 from typing import NoReturn
 
 from openpilot.common.params import Params
@@ -21,18 +25,21 @@ from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET,
 class METRIC_TYPE:
   GAUGE = 'g'
   SAMPLE = 'sa'
+  RAW = 'r'
+
 
 class StatLog:
   def __init__(self):
     self.pid = None
     self.zctx = None
     self.sock = None
+    self.stats_socket = STATS_SOCKET
 
   def connect(self) -> None:
-    self.zctx = zmq.Context()
+    self.zctx = zmq.Context.instance() or zmq.Context()
     self.sock = self.zctx.socket(zmq.PUSH)
     self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect(STATS_SOCKET)
+    self.sock.connect(self.stats_socket)
     self.pid = os.getpid()
 
   def __del__(self):
@@ -60,6 +67,50 @@ class StatLog:
     self._send(f"{name}:{value}|{METRIC_TYPE.SAMPLE}")
 
 
+class StatLogSP(StatLog):
+  def __init__(self, intercept=True):
+    """
+    Initializes the class instance with an optional parameter to determine
+    if statistical logging should be configured or not.
+
+    :param intercept: A boolean flag that indicates whether to initialize
+        the `comma_statlog`. If True, the `comma_statlog` attribute is
+        instantiated as a `StatLog` object. Defaults to True.
+    """
+    super().__init__()
+    self.comma_statlog = StatLog() if intercept else None
+    self.stats_socket = f"{STATS_SOCKET}_sp"
+
+  def connect(self) -> None:
+    super().connect()
+    if self.comma_statlog:
+      self.comma_statlog.connect()
+
+  def __del__(self):
+    super().__del__()
+    if self.comma_statlog:
+      self.comma_statlog.__del__()
+
+  def _send(self, metric: str) -> None:
+    super()._send(metric)
+    if self.comma_statlog:
+      self.comma_statlog._send(metric)
+
+  @staticmethod
+  def default_converter(obj):
+    if isinstance(obj, (datetime, date)):
+      return obj.isoformat()
+    if isinstance(obj, set):
+      return list(obj)
+    if isinstance(obj, Decimal):
+      return float(obj)
+    return str(obj)  # fallback for unknown types
+
+  def raw(self, name: str, value: dict) -> None:
+    encoded_dict = base64.b64encode(json.dumps(value, default=self.default_converter).encode("utf-8")).decode("utf-8")
+    self._send(f"{name}:{encoded_dict}|{METRIC_TYPE.RAW}")
+
+
 def main() -> NoReturn:
   dongle_id = Params().get("DongleId")
   def get_influxdb_line(measurement: str, value: float | dict[str, float],  timestamp: datetime, tags: dict) -> str:
@@ -180,4 +231,4 @@ def main() -> NoReturn:
 if __name__ == "__main__":
   main()
 else:
-  statlog = StatLog()
+  statlog = StatLogSP(intercept=True)

tools/clip/clippy.py

@@ -0,0 +1,676 @@
+import asyncio
+import hashlib
+import json
+import logging
+import os
+import re
+import shutil
+import sys
+import html
+from collections import deque
+from logging.handlers import RotatingFileHandler
+
+import discord
+from discord import app_commands
+from discord.ext import commands
+
+from openpilot.tools.lib.api import CommaApi, UnauthorizedError
+from openpilot.tools.lib.route import Route
+
+import threading
+from flask import Flask, send_file, abort, make_response
+from pathlib import Path
+
+if not (CLIPPY_TOKEN := os.getenv("CLIPPY_TOKEN")):
+  sys.exit("❌ CLIPPY_TOKEN is missing – set it in the environment")
+
+ALLOWED_GUILD_IDS = {880416502577266699, 1368811404689276958}
+
+CLIPPY_BASE_URL = "https://clippy.royjr.com"
+
+WORKING_DIR = os.path.expanduser("~/github/sunnypilot/tools/clip")
+CLIPS_DIR = os.path.join(WORKING_DIR, "clips")
+STATS_PATH = os.path.join(WORKING_DIR, "clippy_stats.json")
+LOG_PATH = os.path.join(WORKING_DIR, "clippy.log")
+os.makedirs(CLIPS_DIR, exist_ok=True)
+
+MAX_TOTAL_JOBS         = 20
+MAX_CONCURRENT_CLIPS       = 3
+MAX_CONCURRENT_CLIPS_PER_USER  = 3
+MAX_CLIP_DURATION        = 60 * 5
+
+CLIPPY_STATS_ALLOWED_ROLES    = ["sunnypilot-dev"]
+CLIPPY_UNLIMITED_ALLOWED_ROLES  = ["sunnypilot-dev"]
+
+TAIL_LINES = 25
+tail_buffer = deque(maxlen=TAIL_LINES)
+
+intents = discord.Intents.default()
+intents.message_content = True
+bot = commands.Bot(command_prefix=lambda bot, msg: [], intents=intents)
+
+clip_queue    = []
+clip_semaphore  = asyncio.Semaphore(MAX_CONCURRENT_CLIPS)
+user_cooldowns  = {}
+
+
+async def queue_monitor():
+  while True:
+    print("\033c", end="")
+    w = shutil.get_terminal_size().columns
+    bar = "-" * w
+    print(f"{bar}\nTotal: {stats['total']} | ✅ {stats['success']} | ❌ {stats['fail']}\n{bar}")
+    print("\n".join(f"{i+1:02d}. {j['status']} {j['user']}: {j['route']}" for i, j in enumerate(clip_queue)) or "No jobs in queue.")
+    print(f"{bar}\n" + "\n".join(line[:w] for line in tail_buffer) + f"\n{bar}")
+    await asyncio.sleep(1)
+
+def start_clip_server():
+  clip_dir_resolved = Path(CLIPS_DIR).resolve()
+  app = Flask("clippy")
+
+  @app.route('/<path:filename>')
+  def get_clip(filename):
+    full_path = (clip_dir_resolved / filename).resolve()
+    try:
+      full_path.relative_to(clip_dir_resolved)
+    except ValueError:
+      abort(404)
+    if not full_path.name.endswith(".mp4"):
+      abort(404)
+    try:
+      if not full_path.is_file() or not full_path.samefile(full_path):
+        abort(404)
+    except Exception:
+      abort(404)
+    response = make_response(send_file(
+      str(full_path),
+      mimetype="video/mp4",
+      as_attachment=False,
+      conditional=True,
+    ))
+    response.headers.update({
+      "Cache-Control": "no-store",
+      "Accept-Ranges": "bytes",
+      "Content-Disposition": f'inline; filename="{filename}"',
+      "X-Content-Type-Options": "nosniff",
+    })
+    return response
+
+  @app.errorhandler(404)
+  def not_found(_):
+    return "clip not found", 404
+
+  app.run(host="127.0.0.1", port=5000)
+
+def has_any_role(user, role_list):
+  if isinstance(user, discord.Member):
+    return any(role.name in role_list for role in user.roles)
+  return False
+
+def user_tag(user: discord.User) -> str:
+  return f"{user.display_name} ({user.name})"
+
+def load_stats():
+  if os.path.exists(STATS_PATH):
+    with open(STATS_PATH, "r") as f:
+      return json.load(f)
+  return {"total": 0, "success": 0, "fail": 0}
+
+def save_stats():
+  with open(STATS_PATH, "w") as f:
+    json.dump(stats, f)
+
+stats = load_stats()
+
+
+class SanitizeFilter(logging.Filter):
+  def filter(self, record: logging.LogRecord) -> bool:
+    if isinstance(record.msg, str):
+      record.msg = re.compile(r'[\x00-\x1f\x7f-\x9f]').sub('', record.msg)
+    return True
+
+
+class DequeHandler(logging.Handler):
+  def __init__(self, buf):
+    super().__init__()
+    self.buf = buf
+    self.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
+
+  def emit(self, record):
+    try:
+      self.buf.append(self.format(record))
+    except Exception:
+      self.handleError(record)
+
+log_handler = RotatingFileHandler(LOG_PATH, maxBytes=5*1024*1024, backupCount=3)
+log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
+
+root = logging.getLogger()
+root.setLevel(logging.INFO)
+root.addHandler(log_handler)
+root.addHandler(DequeHandler(tail_buffer))
+root.addFilter(SanitizeFilter())
+
+
+class DeletePublishedView(discord.ui.View):
+  def __init__(self, message: discord.Message, author_id: int, video_path):
+    super().__init__(timeout=300)
+    self.message = message
+    self.author_id = author_id
+    self.video_path = video_path
+
+  @discord.ui.button(label="Unpublish Clip", style=discord.ButtonStyle.primary)
+  async def unpublish(self, interaction: discord.Interaction, _button: discord.ui.Button):
+    if interaction.user.id != self.author_id:
+      logging.error(f"🚫 {user_tag(interaction.user)} cant unpublish {self.message.id}")
+      await interaction.response.send_message("🚫 You can't unpublish this clip.", ephemeral=True)
+      return
+
+    try:
+      await self.message.delete()
+      logging.info(f"🗑️ {user_tag(interaction.user)} unpublished {self.message.id}")
+      await interaction.response.edit_message(content="🗑️ Unpublished clip.", view=None)
+    except Exception as e:
+      logging.error(f"❌ Failed to unpublish clip {self.message.id}: {e}")
+      await interaction.response.send_message(f"❌ Failed to unpublish clip.", ephemeral=True)
+
+  @discord.ui.button(label="Unpublish + Delete Clip", style=discord.ButtonStyle.danger)
+  async def delete(self, interaction: discord.Interaction, _button: discord.ui.Button):
+    if interaction.user.id != self.author_id:
+      logging.error(f"🚫 {user_tag(interaction.user)} cant unpublish and delete {self.message.id}")
+      await interaction.response.send_message("🚫 You can't unpublish and delete this clip.", ephemeral=True)
+      return
+
+    try:
+      if not os.path.realpath(self.video_path).startswith(os.path.realpath(CLIPS_DIR) + os.sep):
+        logging.error(f"❌ Unsafe delete attempt: {self.video_path}")
+        await interaction.response.send_message("❌ Unsafe delete attempt.", ephemeral=True)
+        return
+      os.remove(self.video_path)
+      await self.message.delete()
+      logging.info(f"🗑️ {user_tag(interaction.user)} unpublished {self.message.id}")
+      await interaction.response.edit_message(content="🗑️ Unpublished and deleted clip.", view=None)
+    except Exception as e:
+      logging.error(f"❌ Failed to unpublish clip {self.message.id}: {e}")
+      await interaction.response.send_message(f"❌ Failed to unpublish and delete clip.", ephemeral=True)
+
+
+class PublishView(discord.ui.View):
+  def __init__(self, route_str, title, video_path, author_id, file_size, safe_name):
+    super().__init__(timeout=300)
+    self.route_str = route_str
+    self.title = title
+    self.video_path = video_path
+    self.author_id = author_id
+    self.file_size = file_size
+    self.safe_name = safe_name
+
+  @discord.ui.button(label="Publish Clip", style=discord.ButtonStyle.success)
+  async def publish(self, interaction: discord.Interaction, _button: discord.ui.Button):
+    if interaction.user.id != self.author_id:
+      await interaction.response.send_message("🚫 You can't publish this clip.", ephemeral=True)
+      return
+
+    if not os.path.exists(self.video_path):
+      logging.error(f"❌ {user_tag(interaction.user)} failed to publish {self.route_str} – file missing")
+      await interaction.response.edit_message(
+        content="❌ Clip could not be published. File missing.",
+        attachments=[], view=None
+      )
+      self.stop()
+      return
+
+    logging.info(f"✅ {user_tag(interaction.user)} published {self.route_str}")
+
+    if not (1 <= self.file_size <= 9):
+      published_msg = await interaction.channel.send(
+        f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{self.safe_name}.mp4) from [{self.route_str}](https://connect.comma.ai/{self.route_str})\n{self.title}"
+      )
+    else:
+      published_msg = await interaction.channel.send(
+        f"{interaction.user.mention} shared a clip from [{self.route_str}](https://connect.comma.ai/{self.route_str})\n{self.title}",
+        file=discord.File(self.video_path)
+      )
+    await interaction.response.edit_message(
+      content="✅ Clip published to channel.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, self.video_path),
+    )
+    self.stop()
+
+  @discord.ui.button(label="Delete Clip", style=discord.ButtonStyle.danger)
+  async def delete(self, interaction: discord.Interaction, _button: discord.ui.Button):
+    if interaction.user.id != self.author_id:
+      logging.error(f"🚫 {user_tag(interaction.user)} cant delete {self.video_path}")
+      await interaction.response.send_message("🚫 You can't delete this clip.", ephemeral=True)
+      return
+
+    try:
+      if not os.path.realpath(self.video_path).startswith(os.path.realpath(CLIPS_DIR) + os.sep):
+        logging.error(f"❌ Unsafe delete attempt: {self.video_path}")
+        await interaction.response.send_message("❌ Unsafe delete attempt.", ephemeral=True)
+        return
+      os.remove(self.video_path)
+      logging.info(f"🗑️ {user_tag(interaction.user)} deleted {self.route_str}")
+      await interaction.response.edit_message(content="🗑️ Clip deleted.", attachments=[], view=None)
+      self.stop()
+    except Exception as e:
+      logging.error(f"❌ Failed to delete {self.route_str}")
+      await interaction.response.edit_message(content=f"❌ Failed to delete clip.", view=None)
+
+
+@bot.tree.command(name="clippy", description="Generate a driving clip - make sure you upload logs first!")
+@app_commands.describe(
+  input="connect link or dongle/route/starttime/endtime or dongle/route/startsegment-endsegment",
+  title="Title (default: none)",
+  quality="Video quality (default: high)",
+  wide="Use wide view if uploaded (default: true)",
+  speed="Playback speed (default: 1)",
+  cache="Set to false to regenerate clip if its already cached (default: true)",
+  private="If true, only you will see the preview (default: true)",
+  bookmarks="Automatically clip bookmarks (default: false)",
+  filesize="Max filesize (MB), set to 0 for unlimited (default: 9)",
+  developer="Show the developer UI (default: Off)"
+)
+@app_commands.choices(
+  quality=[
+    app_commands.Choice(name="high", value="high"),
+    app_commands.Choice(name="low",  value="low"),
+  ],
+  developer=[
+    app_commands.Choice(name="Right",  value="1"),
+    app_commands.Choice(name="Right & Bottom",  value="2"),
+  ]
+)
+async def clippy(
+  interaction: discord.Interaction,
+  input: str,
+  title: str = None,
+  quality: app_commands.Choice[str] | None = None,
+  wide: bool = True,
+  speed: int = 1,
+  cache: bool = True,
+  private: bool = True,
+  bookmarks: bool = False,
+  filesize: int = 9,
+  developer: app_commands.Choice[str] | None = None,
+):
+
+  if interaction.guild_id not in ALLOWED_GUILD_IDS:
+    logging.error(f"❌ This bot is not available in this server {interaction.guild_id}")
+    await interaction.response.send_message("❌ This bot is not available in this server.", ephemeral=True)
+    return
+
+  if len(clip_queue) >= MAX_TOTAL_JOBS:
+    await interaction.response.send_message(
+      "🚫 Server busy – too many jobs in queue. Please try again later.",
+      ephemeral=True
+    )
+    return
+
+  user_id = interaction.user.id
+  if not has_any_role(interaction.user, CLIPPY_UNLIMITED_ALLOWED_ROLES):
+    if user_cooldowns.get(user_id, 0) >= MAX_CONCURRENT_CLIPS_PER_USER:
+      logging.error(f"🚫 {user_tag(interaction.user)} hit the cooldown limit")
+      await interaction.response.send_message(
+        "🚫 You already have a clip running. Wait for it to finish.",
+        ephemeral=True
+      )
+      return
+
+  user_cooldowns[user_id] = user_cooldowns.get(user_id, 0) + 1
+
+  try:
+    await interaction.response.defer(ephemeral=True)
+
+    quality_value = quality.value if quality else "high"
+    title_cmd = title[:80] if title else ""
+    title = f"> ### **{html.unescape(title[:80])}**" if title else ""
+    stats["total"] += 1
+
+    # ── fast‑fail validation ────────────────────────────────────────────────────
+    def fail(msg: str):
+      stats["fail"] += 1
+      save_stats()
+      return interaction.followup.send(f"❌ {msg}", ephemeral=True)
+
+    input = input.removeprefix("https://connect.comma.ai/")
+
+    if bookmarks:
+      match = re.match(r'^([a-z0-9]+)/([a-zA-Z0-9\-]+)$', input)
+      if not match:
+        logging.error(f"❌ {user_tag(interaction.user)} entered bad input {input}")
+        await fail("Use connect link, `dongle/route/starttime/endtime` or `dongle/route/startsegment-endsegment` (endsegment optional).\n```\n--- CONNECT ---\nhttps://connect.comma.ai/a2a0ccea32023010/2023-07-27--13-01-19/5/10\n\n--- EXAMPLES ---\na2a0ccea32023010/2023-07-27--13-01-19/0     segment 0\na2a0ccea32023010/2023-07-27--13-01-19/0-1   segments 0 through 1\na2a0ccea32023010/2023-07-27--13-01-19/5/10  from 5 to 10 seconds\na2a0ccea32023010/2023-07-27--13-01-19       when using bookmark option\n```")
+        return
+      else:
+        dongle, route = match.groups()
+        start = 0
+        end = 0
+    else:
+
+      match = re.match(r'^([a-z0-9]+)/([a-zA-Z0-9\-]+)/(\d+)/(\d+)$', input)
+      if not match:
+        match = re.match(r"^([a-z0-9]+)/([A-Za-z0-9\-]+)/(\d+)(?:-(\d+))?$", input)
+        if not match:
+          logging.error(f"❌ {user_tag(interaction.user)} entered bad input {input}")
+          await fail("Use connect link, `dongle/route/starttime/endtime` or `dongle/route/startsegment-endsegment` (endsegment optional).\n```\n--- CONNECT ---\nhttps://connect.comma.ai/a2a0ccea32023010/2023-07-27--13-01-19/5/10\n\n--- EXAMPLES ---\na2a0ccea32023010/2023-07-27--13-01-19/0     segment 0\na2a0ccea32023010/2023-07-27--13-01-19/0-1   segments 0 through 1\na2a0ccea32023010/2023-07-27--13-01-19/5/10  from 5 to 10 seconds\na2a0ccea32023010/2023-07-27--13-01-19       when using bookmark option\n```")
+          return
+        else:
+          dongle, route, seg_start, seg_end = match.groups()
+
+          if int(seg_start) == 0:
+            # in_start = 2  # fix for 2s
+            in_start = 0
+          else:
+            in_start = int(seg_start) * 60
+
+          if seg_end is None:
+            in_end = 60 if int(seg_start) == 0 else in_start + 60
+          else:
+            in_end = 60 if int(seg_end) == 0 else (int(seg_end) + 1) * 60
+      else:
+        dongle, route, in_start, in_end = match.groups()
+
+      start = int(in_start)
+      end = int(in_end)
+
+      # fix for 2s
+      # if start < 2 or end <= start:
+      #   await fail("Start must be at least 2 and end must be greater than start.")
+      #   return
+
+      if end <= start:
+        logging.error(f"❌ {user_tag(interaction.user)} entered bad times {input}")
+        await fail("End must be greater than start time.")
+        return
+      duration = end - start
+      if duration > MAX_CLIP_DURATION:
+        logging.error(f"❌ {user_tag(interaction.user)} hit the max duration limit {input}")
+        await fail(f"Clips must be {int(MAX_CLIP_DURATION / 60)} minutes or less.")
+        return
+
+    status_msg = await interaction.followup.send(
+      "🕐 Waiting in queue..", ephemeral=private
+    )
+
+    if speed == 0:
+      speed = 1
+    if speed > 1:
+      end = start + int(duration / speed)
+    elif speed < 1:
+      end = start + int(duration / speed)
+    if bookmarks:
+      route_str  = f"{dongle}/{route}"
+      connect_route_str  = f"{dongle}/{route}"
+      base   = f"{dongle}_{route}_bookmarks_{quality_value}"
+    else:
+      route_str  = f"{dongle}/{route}/{start}/{end}"
+      connect_start = 1 if start == 0 else start
+      connect_route_str  = f"{dongle}/{route}/{connect_start}/{end}"
+      base   = f"{dongle}_{route}_{start}_{end}_{quality_value}"
+    if wide:
+      base += "_wide"
+    if speed:
+      base += f"_{speed}"
+    base += f"_s{filesize}"
+    clean_base = re.sub(r'[^A-Za-z0-9_-]+', '_', base)
+    if title_cmd:
+      title_hash = hashlib.sha1(title_cmd.encode()).hexdigest()[:10]
+      safe_name  = f"{clean_base}_{title_hash}"
+    else:
+      safe_name  = clean_base
+
+    safe_name = re.sub(r'[^a-zA-Z0-9_-]', '_', safe_name)
+
+    if any(job["route"] == safe_name for job in clip_queue):
+      await status_msg.edit(content="❌ That clip is already in the queue or processing – wait for it to finish.")
+      return
+
+    try:
+      logs = CommaApi().get(f"/v1/route/{dongle}|{route}/files").get("logs")
+
+      segments = [
+        re.search(r'/(\d+)/rlog\.(?:zst|bz2)', url).group(1)
+        for url in logs
+        if re.search(r'/(\d+)/rlog\.(?:zst|bz2)', url)
+      ]
+
+      startsegment = start // 60
+      endsegment = (end - 1) // 60
+
+      segment_set = set(int(s) for s in segments)
+      if bookmarks:
+        missing = False
+      else:
+        missing = [i for i in range(startsegment, endsegment + 1) if i not in segment_set]
+
+      if missing:
+        logging.error(f"❌ {user_tag(interaction.user)} segments missing {missing}")
+        await status_msg.edit(content=f"❌ You need to upload the missing logs for segments `{missing}` using [connect.comma.ai](https://connect.comma.ai/{connect_route_str})")
+        return
+      else:
+        if bookmarks:
+          logging.info(f"🕐 {user_tag(interaction.user)} getting bookmarks {route_str}")
+          await status_msg.edit(content=f"🕐 Getting bookmarks")
+        else:
+          logging.info(f"☑️ {user_tag(interaction.user)} segments present {route_str}")
+          await status_msg.edit(content=f"☑️ All required segments are present.")
+
+    except UnauthorizedError as e:
+      logging.error(f"❌ {user_tag(interaction.user)} unauthorized: {e}")
+      await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
+      return
+
+    except Exception as e:
+      logging.error(f"❌ {user_tag(interaction.user)} unexpected error: {e}")
+      await status_msg.edit(content=f"❌ Error: unexpected error")
+      return
+
+    if bookmarks:
+      try:
+        route = Route(route_str)
+        user_flags_at_time = []
+
+        for segment in route.segments:
+          for event in segment.events:
+            if event['type'] == 'user_flag':
+              user_flags_at_time.append(round(event['route_offset_millis'] / 1000))
+
+      except Exception as e:
+        logging.error(f"❌ {user_tag(interaction.user)} unauthorized: {e}")
+        await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
+        return
+
+      if len(user_flags_at_time) == 0:
+        logging.error(f"❌ {user_tag(interaction.user)} no bookmarks found")
+        await status_msg.edit(content=f"❌ No bookmarks found")
+        return
+      else:
+        bookmarklinks = ''
+        for user_flag_at_time in user_flags_at_time:
+          bookmarklinks += f"```{connect_route_str}/{user_flag_at_time - 10}/{user_flag_at_time + 5}```"
+        logging.info(f"✅ {user_tag(interaction.user)} {len(user_flags_at_time)} bookmarks found! - {user_flags_at_time}")
+        await status_msg.edit(content=f"✅ {len(user_flags_at_time)} bookmarks found! - {user_flags_at_time}{bookmarklinks}")
+        return
+
+    full_path   = os.path.join(CLIPS_DIR, f"{safe_name}.mp4")
+
+    clip_queue.append({"user": interaction.user.display_name,
+               "route": safe_name,
+               "duration": duration,
+               "status": "🕐"})
+    save_stats()
+    if private:
+      logging.info(f"🕐 {user_tag(interaction.user)} queued (PRIVATE) {route_str}")
+    else:
+      logging.info(f"🕐 {user_tag(interaction.user)} queued {route_str}")
+
+    if os.path.exists(full_path) and cache:
+      stats["success"] += 1
+      save_stats()
+      for j in clip_queue:
+        if j["route"] == safe_name:
+          j["status"] = "✅"
+
+      if private:
+        logging.info(f"📁 {user_tag(interaction.user)} used cache (PRIVATE) {route_str}")
+        await status_msg.edit(content="📁 Used cached clip.")
+        if not (1 <= filesize <= 9):
+          await interaction.followup.send(
+            content=f"Preview for [`{route_str}`]({CLIPPY_BASE_URL}/{safe_name}.mp4)\n{title}",
+            view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
+            ephemeral=True
+          )
+        else:
+          await interaction.followup.send(
+            content=f"Preview for `{route_str}`\n{title}",
+            file=discord.File(full_path),
+            view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
+            ephemeral=True
+          )
+      else:
+        logging.info(f"📁 {user_tag(interaction.user)} used cache {route_str}")
+
+        if not (1 <= filesize <= 9):
+          published_msg = await interaction.channel.send(
+            f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{safe_name}.mp4) from [{route_str}](https://connect.comma.ai/{route_str})\n{title}"
+          )
+        else:
+          published_msg = await interaction.channel.send(
+            f"{interaction.user.mention} shared a clip from [{route_str}](https://connect.comma.ai/{route_str})\n{title}",
+            file=discord.File(full_path)
+          )
+        await status_msg.edit(
+          content="📁 Used cached clip.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, full_path),
+        )
+
+    else:
+      async with clip_semaphore:
+        for j in clip_queue:
+          if j["route"] == safe_name:
+            j["status"] = "🔄"
+        logging.info(f"🔄 {user_tag(interaction.user)} processing {route_str}")
+        await status_msg.edit(content=f"🔄 Processing {j['duration']}s clip..")
+
+        cmd = ["python3", "run.py", route_str, "-q", quality_value, "-x", str(speed), "-o", full_path]
+        if not (in_start and in_end):
+          if in_start != 0: # fix for 2s
+            cmd += ["-s", str(start), "-e", str(end)]
+
+        if title_cmd:
+          cmd += ["-t", str(title_cmd)]
+
+        if wide:
+          cmd += ["-w"]
+
+        if filesize:
+          cmd += ["-f", str(filesize)]
+
+        if developer:
+          dev_mode = int(developer.value)
+        else:
+          dev_mode = 0
+        cmd += ["-z", str(dev_mode)]
+
+        proc = await asyncio.create_subprocess_exec(
+          *cmd, cwd=WORKING_DIR,
+          stdout=asyncio.subprocess.PIPE,
+          stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await proc.communicate()
+        clean_err = "\n".join(stderr.decode().splitlines()[3:]) if stderr else ""
+
+        if proc.returncode != 0 or not os.path.exists(full_path):
+          for j in clip_queue:
+            if j["route"] == safe_name:
+              j["status"] = "❌"
+          stats["fail"] += 1
+          save_stats()
+          logging.error(f"❌ {user_tag(interaction.user)} failed {route_str}\n{clean_err}")
+
+          if clean_err == "clip.py: error: failed to get route: Unauthorized. Authenticate with tools/lib/auth.py":
+            await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
+          elif clean_err == "clip.py: error: failed to get route: 404:The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
+            await status_msg.edit(content="❌ This route does not exist, please try another.")
+          else:
+            await status_msg.edit(content="❌ Clip failed to generate.")
+        else:
+          for j in clip_queue:
+            if j["route"] == safe_name:
+              j["status"] = "✅"
+          stats["success"] += 1
+          save_stats()
+
+          if private:
+            logging.info(f"✅ {user_tag(interaction.user)} success (PRIVATE) {route_str}")
+            await status_msg.edit(content="✅ Clip ready.")
+
+            if not (1 <= filesize <= 9):
+              await interaction.followup.send(
+                content=f"Preview for [`{route_str}`]({CLIPPY_BASE_URL}/{safe_name}.mp4)\n{title}",
+                view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
+                ephemeral=True
+              )
+            else:
+              await interaction.followup.send(
+                content=f"Preview for `{route_str}`\n{title}",
+                file=discord.File(full_path),
+                view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
+                ephemeral=True
+              )
+          else:
+            logging.info(f"✅ {user_tag(interaction.user)} success {route_str}")
+
+            if not (1 <= filesize <= 9):
+              published_msg = await interaction.channel.send(
+                f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{safe_name}.mp4) from [{route_str}](https://connect.comma.ai/{route_str})\n{title}"
+              )
+            else:
+              published_msg = await interaction.channel.send(
+                f"{interaction.user.mention} shared a clip from [{route_str}](https://connect.comma.ai/{route_str})\n{title}",
+                file=discord.File(full_path)
+              )
+            await status_msg.edit(
+              content="✅ Clip ready.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, full_path),
+            )
+
+    await asyncio.sleep(1)
+    clip_queue[:] = [j for j in clip_queue if j["route"] != safe_name]
+
+  finally:
+    if user_id in user_cooldowns:
+      user_cooldowns[user_id] = max(0, user_cooldowns[user_id] - 1)
+    clip_queue[:] = [j for j in clip_queue if j["route"] != safe_name]
+
+
+@bot.tree.command(name="clippy-stats", description="View clippy stats")
+async def clippy_stats(interaction: discord.Interaction):
+
+  if interaction.guild_id not in ALLOWED_GUILD_IDS:
+    logging.error(f"❌ This bot is not available in this server {interaction.guild_id}")
+    await interaction.response.send_message("❌ This bot is not available in this server.", ephemeral=True)
+    return
+
+  if not has_any_role(interaction.user, CLIPPY_STATS_ALLOWED_ROLES):
+    logging.error(f"🚫 {user_tag(interaction.user)} not allowed to use /clippy-stats")
+    await interaction.response.send_message("🚫 You don't have permission.", ephemeral=True)
+    return
+
+  stat  = f"Total: {stats['total']} | ✅ {stats['success']} | ❌ {stats['fail']}"
+  queue = "\n".join(f"{j['status']} {j['user']}: {j['route']}" for j in clip_queue) or "No active jobs."
+  tail  = "\n".join(list(tail_buffer)[-5:][::-1]) or "[no log records yet]"
+
+  content = f"```{stat}``````{queue}``````{tail}"
+  await interaction.response.send_message(content[:1997] + "```", ephemeral=True)
+  logging.info(f"✅ {user_tag(interaction.user)} used /clippy-stats")
+
+
+@bot.event
+async def on_ready():
+  await bot.tree.sync()
+  for guild in bot.guilds:
+    logging.info(f"Connected to guild: {guild.name} ({guild.id})")
+  await bot.change_presence(activity=discord.Game(name="your clips"))
+  asyncio.create_task(queue_monitor())
+  print(f"Logged in as {bot.user}")
+
+threading.Thread(target=start_clip_server, daemon=True).start()
+bot.run(CLIPPY_TOKEN)

tools/clip/run.py

@@ -28,7 +28,7 @@ DEMO_ROUTE = 'a2a0ccea32023010/2023-07-27--13-01-19'
 FRAMERATE = 20
 PIXEL_DEPTH = '24'
 RESOLUTION = '2160x1080'
-SECONDS_TO_WARM = 2
+SECONDS_TO_WARM = 0.5   # fix for 2s
 PROC_WAIT_SECONDS = 30*10
 
 OPENPILOT_FONT = str(Path(BASEDIR, 'selfdrive/assets/fonts/Inter-Regular.ttf').resolve())
@@ -104,8 +104,9 @@ def parse_args(parser: ArgumentParser):
     args.end = int(parts[3])
   if args.end <= args.start:
     parser.error(f'end ({args.end}) must be greater than start ({args.start})')
-  if args.start < SECONDS_TO_WARM:
-    parser.error(f'start must be greater than {SECONDS_TO_WARM}s to allow the UI time to warm up')
+  # fix for 2s
+  # if args.start < SECONDS_TO_WARM:
+  #   parser.error(f'start must be greater than {SECONDS_TO_WARM}s to allow the UI time to warm up')
 
   try:
     args.route = Route(args.route, data_dir=args.data_dir)
@@ -113,16 +114,16 @@ def parse_args(parser: ArgumentParser):
     parser.error(f'failed to get route: {e}')
 
   # FIXME: length isn't exactly max segment seconds, simplify to replay exiting at end of data
-  length = round(args.route.max_seg_number * 60)
-  if args.start >= length:
-    parser.error(f'start ({args.start}s) cannot be after end of route ({length}s)')
-  if args.end > length:
-    parser.error(f'end ({args.end}s) cannot be after end of route ({length}s)')
+  # length = round(args.route.max_seg_number * 60)
+  # if args.start >= length:
+  #   parser.error(f'start ({args.start}s) cannot be after end of route ({length}s)')
+  # if args.end > length:
+  #   parser.error(f'end ({args.end}s) cannot be after end of route ({length}s)')
 
   return args
 
 
-def populate_car_params(lr: LogReader):
+def populate_car_params(lr: LogReader, developer: int):
   init_data = lr.first('initData')
   assert init_data is not None
 
@@ -131,10 +132,14 @@ def populate_car_params(lr: LogReader):
   for cp in entries:
     key, value = cp.key, cp.value
     try:
+      if key == "OSMDownloadProgress":
+        continue
       params.put(key, params.cpp2python(key, value))
     except UnknownKeyName:
       # forks of openpilot may have other Params keys configured. ignore these
-      logger.warning(f"unknown Params key '{key}', skipping")
+      pass
+  if developer is not None:
+    params.put("DevUIInfo", developer)
   logger.debug('persisted CarParams')
 
 
@@ -179,6 +184,7 @@ def wait_for_frames(procs: list[Popen]):
 def clip(
   data_dir: str | None,
   quality: Literal['low', 'high'],
+  wide: bool,
   prefix: str,
   route: Route,
   out: str,
@@ -187,8 +193,9 @@ def clip(
   speed: int,
   target_mb: int,
   title: str | None,
+  developer: int,
 ):
-  logger.info(f'clipping route {route.name.canonical_name}, start={start} end={end} quality={quality} target_filesize={target_mb}MB')
+  logger.info(f'clipping route {route.name.canonical_name}, start={start} end={end} quality={quality} wide={wide} target_filesize={target_mb}MB')
   lr = get_logreader(route)
 
   begin_at = max(start - SECONDS_TO_WARM, 0)
@@ -224,8 +231,6 @@ def clip(
     '-draw_mouse', '0',
     '-i', display,
     '-c:v', 'libx264',
-    '-maxrate', f'{bit_rate_kbps}k',
-    '-bufsize', f'{bit_rate_kbps*2}k',
     '-crf', '23',
     '-filter:v', ','.join(overlays),
     '-preset', 'ultrafast',
@@ -234,12 +239,19 @@ def clip(
     '-movflags', '+faststart',
     '-f', 'mp4',
     '-t', str(duration),
-    out,
   ]
 
-  replay_cmd = [REPLAY, '--ecam', '-c', '1', '-s', str(begin_at), '--prefix', prefix]
+  if target_mb > 0:
+    ffmpeg_cmd += ['-maxrate', f'{bit_rate_kbps}k']
+    ffmpeg_cmd += ['-bufsize', f'{bit_rate_kbps*2}k']
+
+  ffmpeg_cmd.append(out)
+
+  replay_cmd = [REPLAY, '-c', '1', '-s', str(begin_at), '--prefix', prefix]
   if data_dir:
     replay_cmd.extend(['--data_dir', data_dir])
+  if wide:
+    replay_cmd.append('--ecam')
   if quality == 'low':
     replay_cmd.append('--qcam')
   replay_cmd.append(route.name.canonical_name)
@@ -248,7 +260,7 @@ def clip(
   xvfb_cmd = ['Xvfb', display, '-terminate', '-screen', '0', f'{RESOLUTION}x{PIXEL_DEPTH}']
 
   with OpenpilotPrefix(prefix, shared_download_cache=True):
-    populate_car_params(lr)
+    populate_car_params(lr, developer)
     env = os.environ.copy()
     env['DISPLAY'] = display
 
@@ -262,7 +274,7 @@ def clip(
       with managed_proc(ffmpeg_cmd, env) as ffmpeg_proc:
         procs.append(ffmpeg_proc)
         logger.info(f'recording in progress ({duration}s)...')
-        ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
+        ffmpeg_proc.wait((duration * 2) + PROC_WAIT_SECONDS)
         check_for_failure(procs)
         logger.info(f'recording complete: {Path(out).resolve()}')
 
@@ -279,15 +291,18 @@ def main():
   p.add_argument('-o', '--output', help='output clip to (.mp4)', type=validate_output_file, default=DEFAULT_OUTPUT)
   p.add_argument('-p', '--prefix', help='openpilot prefix', default=f'clip_{randint(100, 99999)}')
   p.add_argument('-q', '--quality', help='quality of camera (low = qcam, high = hevc)', choices=['low', 'high'], default='high')
+  p.add_argument('-w', '--wide', help='enable wide view if uploaded', action='store_true',)
   p.add_argument('-x', '--speed', help='record the clip at this speed multiple', type=int, default=1)
   p.add_argument('-s', '--start', help='start clipping at <start> seconds', type=int)
   p.add_argument('-t', '--title', help='overlay this title on the video (e.g. "Chill driving across the Golden Gate Bridge")', type=validate_title)
+  p.add_argument('-z', '--developer', help='developer', type=int, default=0)
   args = parse_args(p)
   exit_code = 1
   try:
     clip(
       data_dir=args.data_dir,
       quality=args.quality,
+      wide=args.wide,
       prefix=args.prefix,
       route=args.route,
       out=args.output,
@@ -296,6 +311,7 @@ def main():
       speed=args.speed,
       target_mb=args.file_size,
       title=args.title,
+      developer=args.developer,
     )
     exit_code = 0
   except KeyboardInterrupt as e:

tools/replay/replay.cc

@@ -191,6 +191,7 @@ void Replay::startStream(const std::shared_ptr<Segment> segment) {
     auto bytes = words.asBytes();
     Params().put("CarParams", (const char *)bytes.begin(), bytes.size());
     Params().put("CarParamsPersistent", (const char *)bytes.begin(), bytes.size());
+    publishMessage(&(*it));
   } else {
     rWarning("failed to read CarParams from current segment");
   }

uv.lock

@@ -195,6 +195,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5b/64/63dbfdd83b31200ac58820a7951ddfdeed1fbee9285b0f3eae12d1357155/azure_storage_blob-12.26.0-py3-none-any.whl", hash = "sha256:8c5631b8b22b4f53ec5fff2f3bededf34cfef111e2af613ad42c9e6de00a77fe", size = 412907, upload-time = "2025-07-16T21:34:09.367Z" },
 ]
 
+[[package]]
+name = "blinker"
+version = "1.9.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" },
+]
+
 [[package]]
 name = "casadi"
 version = "3.7.1"
@@ -475,6 +484,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" },
 ]
 
+[[package]]
+name = "discord-py"
+version = "2.5.2"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "aiohttp" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/7f/dd/5817c7af5e614e45cdf38cbf6c3f4597590c442822a648121a34dee7fa0f/discord_py-2.5.2.tar.gz", hash = "sha256:01cd362023bfea1a4a1d43f5280b5ef00cad2c7eba80098909f98bf28e578524", size = 1054879, upload-time = "2025-03-05T01:15:29.798Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/57/a8/dc908a0fe4cd7e3950c9fa6906f7bf2e5d92d36b432f84897185e1b77138/discord_py-2.5.2-py3-none-any.whl", hash = "sha256:81f23a17c50509ffebe0668441cb80c139e74da5115305f70e27ce821361295a", size = 1155105, upload-time = "2025-03-05T01:15:27.323Z" },
+]
+
 [[package]]
 name = "dnspython"
 version = "2.7.0"
@@ -523,6 +544,23 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
 ]
 
+[[package]]
+name = "flask"
+version = "3.1.1"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "blinker" },
+    { name = "click" },
+    { name = "itsdangerous" },
+    { name = "jinja2" },
+    { name = "markupsafe" },
+    { name = "werkzeug" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/c0/de/e47735752347f4128bcf354e0da07ef311a78244eba9e3dc1d4a5ab21a98/flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e", size = 753440, upload-time = "2025-05-13T15:01:17.447Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/3d/68/9d4508e893976286d2ead7f8f571314af6c2037af34853a30fd769c02e9d/flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c", size = 103305, upload-time = "2025-05-13T15:01:15.591Z" },
+]
+
 [[package]]
 name = "fonttools"
 version = "4.59.2"
@@ -705,6 +743,15 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" },
 ]
 
+[[package]]
+name = "itsdangerous"
+version = "2.2.0"
+source = { registry = "https://pypi.org/simple" }
+sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
+]
+
 [[package]]
 name = "jeepney"
 version = "0.9.0"
@@ -1261,6 +1308,8 @@ dependencies = [
     { name = "cffi" },
     { name = "crcmod" },
     { name = "cython" },
+    { name = "discord-py" },
+    { name = "flask" },
     { name = "dearpygui" },
     { name = "future-fstrings" },
     { name = "inputs" },
@@ -1355,6 +1404,8 @@ requires-dist = [
     { name = "dbus-next", marker = "extra == 'dev'" },
     { name = "dearpygui", specifier = ">=2.1.0" },
     { name = "dictdiffer", marker = "extra == 'dev'" },
+    { name = "discord-py" },
+    { name = "flask" },
     { name = "future-fstrings" },
     { name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
     { name = "inputs" },
@@ -4942,6 +4993,18 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" },
 ]
 
+[[package]]
+name = "werkzeug"
+version = "3.1.3"
+source = { registry = "https://pypi.org/simple" }
+dependencies = [
+    { name = "markupsafe" },
+]
+sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
+wheels = [
+    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
+]
+
 [[package]]
 name = "xattr"
 version = "1.2.0"