TCPv3 plus gWM9 Model

CgWM + ST

.github/workflows/build-all-tinygrad-models.yaml

@@ -74,7 +74,7 @@ jobs:
         env:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
           cd gitlab_docs
           git checkout main
           git sparse-checkout set --no-cone models/
@@ -191,7 +191,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}

.github/workflows/build-single-tinygrad-model.yaml

@@ -109,7 +109,7 @@ jobs:
           GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
         run: |
           echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
+          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/docs.sunnypilot.ai2.git gitlab_docs
           cd gitlab_docs
           echo "checkout models/${RECOMPILED_DIR}"
           git sparse-checkout set --no-cone models/${RECOMPILED_DIR}

.github/workflows/sunnypilot-build-model.yaml

@@ -156,8 +156,6 @@ jobs:
         with:
           name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
           path: ${{ github.workspace }}/selfdrive/modeld/models
-      - run: |
-          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
 
       - name: Build Model
         run: |

.gitignore

@@ -109,7 +109,3 @@ Pipfile
 !.idea/customTargets.xml
 !.idea/tools/*
 !.run/*
-
-### clippy ###
-clippy_stats.json
-clippy.log

CHANGELOG.md

@@ -1,30 +1,6 @@
-sunnypilot Version 2025.003.000 (20xx-xx-xx)
+sunnypilot Version 2025.002.000 (2025-xx-xx)
 ========================
 
-sunnypilot Version 2025.002.000 (2025-11-06)
-========================
-* What's Changed (sunnypilot/sunnypilot)
-  * models: bump model json to v8 by @Discountchubbs
-  * Bug: Model UI Crash Fix by @nayan8teen
-  * controlsd: add `CP_SP` to `get_pid_accel_limits` by @THERoenPR
-  * sunnylink: update uploader button logic to support novice tier and above by @devtekve
-  * Tesla: Coop Steering by @AmyJeanes
-  * ui: update discord references and add forum widget by @devtekve
-  * ui: Fix spacing in sunnylink panel by @devtekve
-  * docs: Update README installation branches and discord links by @mpurnell1 in
-  * stats: sunnylink integration by @devtekve
-  * bug: Fix initial registration for sunnylink by @devtekve
-* What's Changed (sunnypilot/opendbc)
-  * Honda: add brake hold messages for Clarity by @mvl-boston
-  * interface: add `CP_SP` to `get_pid_accel_limits` method signature by @roenthomas
-  * Honda: use fixed accel min/max constants for Gas Interceptor by @roenthomas
-  * Tesla: Coop Steering by @AmyJeanes
-* New Contributors (sunnypilot/sunnypilot)
-  * @THERoenPR made their first contribution in "controlsd: add `CP_SP` to `get_pid_accel_limits`"
-  * @AmyJeanes made their first contribution in "Tesla: Coop Steering"
-  * @mpurnell1 made their first contribution in "docs: Update README installation branches and discord links"
-* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2025.001.000...v2025.002.000
-
 sunnypilot Version 2025.001.000 (2025-10-25)
 ========================
 * 🛠️ Major rewrite

common/model.h

@@ -1 +1 @@
-#define DEFAULT_MODEL "Firehose (Default)"
+#define DEFAULT_MODEL "TCPv3 + gWMv9 (Default)"

pyproject.toml

@@ -73,10 +73,6 @@ dependencies = [
 
   # ui
   "qrcode",
-
-  # clippy
-  "discord-py",
-  "flask",
 ]
 
 [project.optional-dependencies]

selfdrive/ui/qt/onroad/annotated_camera.cc

@@ -118,7 +118,7 @@ void AnnotatedCameraWidget::paintGL() {
       } else if (v_ego > 15) {
         wide_cam_requested = false;
       }
-      // wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode();
+      wide_cam_requested = wide_cam_requested && sm["selfdriveState"].getSelfdriveState().getExperimentalMode();
     }
     CameraWidget::setStreamType(wide_cam_requested ? VISION_STREAM_WIDE_ROAD : VISION_STREAM_ROAD);
     CameraWidget::setFrameId(sm["modelV2"].getModelV2().getFrameId());

sunnypilot/common/version.h

@@ -1 +1 @@
-#define SUNNYPILOT_VERSION "2025.003.000"
+#define SUNNYPILOT_VERSION "2025.002.000"

sunnypilot/models/tests/model_hash

@@ -1 +1 @@
-70406ab4dd66d0e384734a8a56632ae4a62bc9670c2e630a0f71588c4e212cd8
+5584d697233d147e0b6402e485b7cbf8fdddb70bde4b9e3b2f6919ed5f69475f

sunnypilot/selfdrive/car/interfaces.py

@@ -15,8 +15,6 @@ from openpilot.sunnypilot.selfdrive.controls.lib.speed_limit.helpers import set_
 
 import openpilot.system.sentry as sentry
 
-from sunnypilot.sunnylink.statsd import STATSLOGSP
-
 
 def log_fingerprint(CP: structs.CarParams) -> None:
   if CP.carFingerprint == "MOCK":
@@ -102,9 +100,6 @@ def setup_interfaces(CI: CarInterfaceBase, params: Params = None) -> None:
   _initialize_torque_lateral_control(CI, CP, enforce_torque, nnlc_enabled)
   _cleanup_unsupported_params(CP, CP_SP)
 
-  STATSLOGSP.raw('sunnypilot.car_params', CP.to_dict())
-  # STATSLOGSP.raw('sunnypilot_params.car_params_sp', CP_SP.to_dict()) # https://github.com/sunnypilot/opendbc/pull/361
-
 
 def initialize_params(params) -> list[dict[str, Any]]:
   keys: list = []

sunnypilot/sunnylink/athena/sunnylinkd.py

@@ -16,9 +16,8 @@ from functools import partial
 from openpilot.common.params import Params
 from openpilot.common.realtime import set_core_affinity
 from openpilot.common.swaglog import cloudlog
-from openpilot.system.hardware.hw import Paths
 from openpilot.system.athena.athenad import ws_send, jsonrpc_handler, \
-  recv_queue, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage, log_handler, start_local_proxy_shim, upload_handler, stat_handler
+  recv_queue, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage, log_handler, start_local_proxy_shim, upload_handler
 from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException,
                        create_connection, WebSocketConnectionClosedException)
 
@@ -34,6 +33,9 @@ SUNNYLINK_RECONNECT_TIMEOUT_S = 70  # FYI changing this will also would require
 DISALLOW_LOG_UPLOAD = threading.Event()
 
 params = Params()
+sunnylink_dongle_id = params.get("SunnylinkDongleId")
+sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
+
 
 def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
   cloudlog.info("sunnylinkd.handle_long_poll started")
@@ -49,7 +51,7 @@ def handle_long_poll(ws: WebSocket, exit_event: threading.Event | None) -> None:
               threading.Thread(target=ws_queue, args=(end_event,), name='ws_queue'),
               threading.Thread(target=upload_handler, args=(end_event,), name='upload_handler'),
               # threading.Thread(target=sunny_log_handler, args=(end_event, comma_prime_cellular_end_event), name='log_handler'),
-              threading.Thread(target=stat_handler, args=(end_event, Paths.stats_sp_root(), True), name='stat_handler'),
+              # threading.Thread(target=stat_handler, args=(end_event,), name='stat_handler'),
             ] + [
               threading.Thread(target=jsonrpc_handler, args=(end_event, partial(startLocalProxy, end_event),), name=f'worker_{x}')
               for x in range(HANDLER_THREADS)
@@ -130,8 +132,6 @@ def ws_ping(ws: WebSocket, end_event: threading.Event) -> None:
 
 
 def ws_queue(end_event: threading.Event) -> None:
-  sunnylink_dongle_id = params.get("SunnylinkDongleId")
-  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
   resume_requested = False
   tries = 0
 
@@ -233,9 +233,6 @@ def saveParams(params_to_update: dict[str, str], compression: bool = False) -> N
 
 
 def startLocalProxy(global_end_event: threading.Event, remote_ws_uri: str, local_port: int) -> dict[str, int]:
-  sunnylink_dongle_id = params.get("SunnylinkDongleId")
-  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
-
   cloudlog.debug("athena.startLocalProxy.starting")
   ws = create_connection(
     remote_ws_uri,
@@ -257,8 +254,6 @@ def main(exit_event: threading.Event = None):
     cloudlog.info("Waiting for sunnylink registration to complete")
     time.sleep(10)
 
-  sunnylink_dongle_id = params.get("SunnylinkDongleId")
-  sunnylink_api = SunnylinkApi(sunnylink_dongle_id)
   UploadQueueCache.initialize(upload_queue)
 
   ws_uri = f"{SUNNYLINK_ATHENA_HOST}"

sunnypilot/sunnylink/statsd.py

@@ -1,278 +0,0 @@
-#!/usr/bin/env python3
-import base64
-import json
-import os
-import threading
-import traceback
-
-import zmq
-import time
-import uuid
-from pathlib import Path
-from collections import defaultdict
-from datetime import datetime, UTC
-
-from openpilot.common.params import Params
-from cereal.messaging import SubMaster
-from openpilot.system.hardware.hw import Paths
-from openpilot.common.swaglog import cloudlog
-from openpilot.system.hardware import HARDWARE
-from openpilot.common.file_helpers import atomic_write_in_dir
-from openpilot.system.version import get_build_metadata
-from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
-from openpilot.system.statsd import METRIC_TYPE, StatLogSP
-from openpilot.common.realtime import Ratekeeper
-
-STATSLOGSP = StatLogSP(intercept=False)
-
-def sp_stats(end_event):
-  """Collect sunnypilot-specific statistics and send as raw metrics."""
-  rk = Ratekeeper(.1, print_delay_threshold=None)
-  statlogsp = STATSLOGSP
-  params = Params()
-
-  def flatten_dict(d, parent_key='', sep='.'):
-    items = {}
-    if isinstance(d, dict):
-      for k, v in d.items():
-        new_key = f"{parent_key}{sep}{k}" if parent_key else k
-        items.update(flatten_dict(v, new_key, sep=sep))
-    elif isinstance(d, (list, tuple)):
-      for i, v in enumerate(d):
-        new_key = f"{parent_key}[{i}]"
-        items.update(flatten_dict(v, new_key, sep=sep))
-    else:
-      items[parent_key] = d
-    return items
-
-  # Collect sunnypilot parameters
-  stats_dict = {}
-
-  param_keys = [
-    'SunnylinkEnabled',
-    'AutoLaneChangeBsmDelay',
-    'AutoLaneChangeTimer',
-    'CarPlatformBundle',
-    'CurrentRoute',
-    'DevUIInfo',
-    'EnableCopyparty',
-    'IntelligentCruiseButtonManagement',
-    'QuietMode',
-    'RainbowMode',
-    'ShowAdvancedControls',
-    'Mads',
-    'MadsMainCruiseAllowed',
-    'MadsSteeringMode',
-    'MadsUnifiedEngagementMode',
-    'ModelManager_ActiveBundle',
-    'ModelManager_Favs',
-    'EnableSunnylinkUploader',
-    'SunnylinkEnabled',
-    'InstallDate',
-    'UptimeOffroad',
-    'UptimeOnroad',
-  ]
-
-  while not end_event.is_set():
-    try:
-      for key in param_keys:
-
-        try:
-          value = params.get(key)
-        except Exception as e:
-          stats_dict[key] = e
-          continue
-
-        if value is None:
-          continue
-
-        if isinstance(value, (dict, list, tuple)):
-          stats_dict.update(flatten_dict(value, key))
-        else:
-          stats_dict[key] = value
-
-      if stats_dict:
-        statlogsp.raw('sunnypilot.device_params', stats_dict)
-    except Exception as e:
-      cloudlog.error(f"Exception {e}")
-    finally:
-      rk.keep_time()
-
-
-def stats_main(end_event):
-  comma_dongle_id = Params().get("DongleId")
-  sunnylink_dongle_id = Params().get("SunnylinkDongleId")
-
-  def get_influxdb_line(measurement: str, value: float | dict[str, float], timestamp: datetime, tags: dict) -> str:
-    res = f"{measurement}"
-    for k, v in tags.items():
-      res += f",{k}={str(v)}"
-    res += " "
-
-    if isinstance(value, float):
-      value = {'value': value}
-
-    for k, v in value.items():
-      res += f"{k}={str(v)},"
-
-    res += f"sunnylink_dongle_id=\"{sunnylink_dongle_id}\",comma_dongle_id=\"{comma_dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
-    return res
-
-  def get_influxdb_line_raw(measurement: str, value: dict, timestamp: datetime, tags: dict) -> str:
-    res = f"{measurement}"
-    try:
-      custom_tags = ""
-      for k, v in tags.items():
-        custom_tags += f",{k}={str(v)}"
-      res += custom_tags
-
-      fields = ""
-      for k, v in value.items():
-        # Skip complex types - only keep simple scalar values
-        if isinstance(v, (dict, list, bytes, bytearray)):
-          continue
-
-        fields += f"{k}={json.dumps(v)},"
-
-      res += f" {fields}"
-    except Exception as e:
-      cloudlog.error(f"Unable to get influxdb line for: {value}")
-      res += f",invalid=1 reason={e},"
-
-    res += f"sunnylink_dongle_id=\"{sunnylink_dongle_id}\",comma_dongle_id=\"{comma_dongle_id}\" {int(timestamp.timestamp() * 1e9)}\n"
-    return res
-
-  # open statistics socket
-  ctx = zmq.Context.instance()
-  sock = ctx.socket(zmq.PULL)
-  sock.bind(f"{STATS_SOCKET}_sp")
-
-  STATS_DIR = Paths.stats_sp_root()
-
-  # initialize stats directory
-  Path(STATS_DIR).mkdir(parents=True, exist_ok=True)
-
-  build_metadata = get_build_metadata()
-
-  # initialize tags
-  tags = {
-    'started': False,
-    'version': build_metadata.openpilot.version,
-    'branch': build_metadata.channel,
-    'dirty': build_metadata.openpilot.is_dirty,
-    'origin': build_metadata.openpilot.git_normalized_origin,
-    'deviceType': HARDWARE.get_device_type(),
-  }
-
-  # subscribe to deviceState for started state
-  sm = SubMaster(['deviceState'])
-
-  idx = 0
-  boot_uid = str(uuid.uuid4())[:8]
-  last_flush_time = time.monotonic()
-  gauges = {}
-  samples: dict[str, list[float]] = defaultdict(list)
-  raws: dict = defaultdict()
-  try:
-    while not end_event.is_set():
-      started_prev = sm['deviceState'].started
-      sm.update()
-
-      # Update metrics
-      while True:
-        try:
-          metric = sock.recv_string(zmq.NOBLOCK)
-          try:
-            metric_type = metric.split('|')[1]
-            metric_name = metric.split(':')[0]
-            metric_value_raw = metric.split('|')[0].split(':')[1]
-
-            if metric_type == METRIC_TYPE.GAUGE:
-              metric_value = float(metric_value_raw)
-              gauges[metric_name] = metric_value
-            elif metric_type == METRIC_TYPE.SAMPLE:
-              metric_value = float(metric_value_raw)
-              samples[metric_name].append(metric_value)
-            elif metric_type == METRIC_TYPE.RAW:
-              raws[metric_name] = metric_value_raw
-            else:
-              cloudlog.event("unknown metric type", metric_type=metric_type)
-          except Exception:
-            print(traceback.format_exc())
-            cloudlog.event("malformed metric", metric=metric)
-        except zmq.error.Again:
-          break
-
-      # flush when started state changes or after FLUSH_TIME_S
-      if (time.monotonic() > last_flush_time + STATS_FLUSH_TIME_S) or (sm['deviceState'].started != started_prev):
-        result = ""
-        current_time = datetime.now(UTC)
-        tags['started'] = sm['deviceState'].started
-
-        for key, value in raws.items():
-          decoded_value = json.loads(base64.b64decode(value).decode('utf-8'))
-          result += get_influxdb_line_raw(key, decoded_value, current_time, tags)
-
-        for key, value in gauges.items():
-          result += get_influxdb_line(f"gauge.{key}", value, current_time, tags)
-
-        for key, values in samples.items():
-          values.sort()
-          sample_count = len(values)
-          sample_sum = sum(values)
-
-          stats = {
-            'count': sample_count,
-            'min': values[0],
-            'max': values[-1],
-            'mean': sample_sum / sample_count,
-          }
-          for percentile in [0.05, 0.5, 0.95]:
-            value = values[int(round(percentile * (sample_count - 1)))]
-            stats[f"p{int(percentile * 100)}"] = value
-
-          result += get_influxdb_line(f"sample.{key}", stats, current_time, tags)
-
-        # clear intermediate data
-        gauges.clear()
-        samples.clear()
-        last_flush_time = time.monotonic()
-
-        # check that we aren't filling up the drive
-        if len(os.listdir(STATS_DIR)) < STATS_DIR_FILE_LIMIT:
-          if len(result) > 0:
-            stats_path = os.path.join(STATS_DIR, f"{boot_uid}_{idx}")
-            with atomic_write_in_dir(stats_path) as f:
-              f.write(result)
-            idx += 1
-        else:
-          cloudlog.error("stats dir full")
-  finally:
-    sock.close()
-    ctx.term()
-
-
-def main():
-  rk = Ratekeeper(1, print_delay_threshold=None)
-  end_event = threading.Event()
-
-  threads = [
-    threading.Thread(target=stats_main, args=(end_event,)),
-    threading.Thread(target=sp_stats, args=(end_event,)),
-  ]
-
-  for t in threads:
-    t.start()
-
-  try:
-    while all(t.is_alive() for t in threads):
-      rk.keep_time()
-  finally:
-    end_event.set()
-
-  for t in threads:
-    t.join()
-
-
-if __name__ == "__main__":
-  main()

system/athena/athenad.py

@@ -744,40 +744,26 @@ def log_handler(end_event: threading.Event, log_attr_name=LOG_ATTR_NAME) -> None
       cloudlog.exception("athena.log_handler.exception")
 
 
-def stat_handler(end_event: threading.Event, stats_dir=None, is_sunnylink=False) -> None:
-  stats_dir = stats_dir or Paths.stats_root()
+def stat_handler(end_event: threading.Event) -> None:
+  STATS_DIR = Paths.stats_root()
   last_scan = 0.0
 
   while not end_event.is_set():
     curr_scan = time.monotonic()
     try:
       if curr_scan - last_scan > 10:
-        stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(stats_dir)))
+        stat_filenames = list(filter(lambda name: not name.startswith(tempfile.gettempprefix()), os.listdir(STATS_DIR)))
         if len(stat_filenames) > 0:
-          stat_path = os.path.join(stats_dir, stat_filenames[0])
+          stat_path = os.path.join(STATS_DIR, stat_filenames[0])
           with open(stat_path) as f:
-            payload = f.read()
-            is_compressed = False
-
-            # Log the current size of the file
-            if is_sunnylink:
-              # Compress and encode the data if it exceeds the maximum size
-              compressed_data = gzip.compress(payload.encode())
-              payload = base64.b64encode(compressed_data).decode()
-              is_compressed = True
-
             jsonrpc = {
               "method": "storeStats",
               "params": {
-                "stats": payload
+                "stats": f.read()
               },
               "jsonrpc": "2.0",
               "id": stat_filenames[0]
             }
-
-            if is_sunnylink and is_compressed:
-              jsonrpc["params"]["compressed"] = is_compressed
-
             low_priority_send_queue.put_nowait(json.dumps(jsonrpc))
           os.remove(stat_path)
         last_scan = curr_scan

system/hardware/hw.py

@@ -55,13 +55,6 @@ class Paths:
     else:
       return "/data/stats/"
 
-  @staticmethod
-  def stats_sp_root() -> str:
-    if PC:
-      return str(Path(Paths.comma_home()) / "stats")
-    else:
-      return "/data/stats_sp/"
-
   @staticmethod
   def config_root() -> str:
     if PC:

system/manager/process_config.py

@@ -164,7 +164,6 @@ procs = [
   # sunnylink <3
   DaemonProcess("manage_sunnylinkd", "sunnypilot.sunnylink.athena.manage_sunnylinkd", "SunnylinkdPid"),
   PythonProcess("sunnylink_registration_manager", "sunnypilot.sunnylink.registration_manager", sunnylink_need_register_shim),
-  PythonProcess("statsd_sp", "sunnypilot.sunnylink.statsd", and_(always_run, sunnylink_ready_shim)),
 ]
 
 # sunnypilot

system/statsd.py

@@ -1,15 +1,11 @@
 #!/usr/bin/env python3
-import base64
-import json
 import os
-from decimal import Decimal
-
 import zmq
 import time
 import uuid
 from pathlib import Path
 from collections import defaultdict
-from datetime import datetime, UTC, date
+from datetime import datetime, UTC
 from typing import NoReturn
 
 from openpilot.common.params import Params
@@ -25,21 +21,18 @@ from openpilot.system.loggerd.config import STATS_DIR_FILE_LIMIT, STATS_SOCKET,
 class METRIC_TYPE:
   GAUGE = 'g'
   SAMPLE = 'sa'
-  RAW = 'r'
-
 
 class StatLog:
   def __init__(self):
     self.pid = None
     self.zctx = None
     self.sock = None
-    self.stats_socket = STATS_SOCKET
 
   def connect(self) -> None:
-    self.zctx = zmq.Context.instance() or zmq.Context()
+    self.zctx = zmq.Context()
     self.sock = self.zctx.socket(zmq.PUSH)
     self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect(self.stats_socket)
+    self.sock.connect(STATS_SOCKET)
     self.pid = os.getpid()
 
   def __del__(self):
@@ -67,50 +60,6 @@ class StatLog:
     self._send(f"{name}:{value}|{METRIC_TYPE.SAMPLE}")
 
 
-class StatLogSP(StatLog):
-  def __init__(self, intercept=True):
-    """
-    Initializes the class instance with an optional parameter to determine
-    if statistical logging should be configured or not.
-
-    :param intercept: A boolean flag that indicates whether to initialize
-        the `comma_statlog`. If True, the `comma_statlog` attribute is
-        instantiated as a `StatLog` object. Defaults to True.
-    """
-    super().__init__()
-    self.comma_statlog = StatLog() if intercept else None
-    self.stats_socket = f"{STATS_SOCKET}_sp"
-
-  def connect(self) -> None:
-    super().connect()
-    if self.comma_statlog:
-      self.comma_statlog.connect()
-
-  def __del__(self):
-    super().__del__()
-    if self.comma_statlog:
-      self.comma_statlog.__del__()
-
-  def _send(self, metric: str) -> None:
-    super()._send(metric)
-    if self.comma_statlog:
-      self.comma_statlog._send(metric)
-
-  @staticmethod
-  def default_converter(obj):
-    if isinstance(obj, (datetime, date)):
-      return obj.isoformat()
-    if isinstance(obj, set):
-      return list(obj)
-    if isinstance(obj, Decimal):
-      return float(obj)
-    return str(obj)  # fallback for unknown types
-
-  def raw(self, name: str, value: dict) -> None:
-    encoded_dict = base64.b64encode(json.dumps(value, default=self.default_converter).encode("utf-8")).decode("utf-8")
-    self._send(f"{name}:{encoded_dict}|{METRIC_TYPE.RAW}")
-
-
 def main() -> NoReturn:
   dongle_id = Params().get("DongleId")
   def get_influxdb_line(measurement: str, value: float | dict[str, float],  timestamp: datetime, tags: dict) -> str:
@@ -231,4 +180,4 @@ def main() -> NoReturn:
 if __name__ == "__main__":
   main()
 else:
-  statlog = StatLogSP(intercept=True)
+  statlog = StatLog()

tools/clip/clippy.py

@@ -1,676 +0,0 @@
-import asyncio
-import hashlib
-import json
-import logging
-import os
-import re
-import shutil
-import sys
-import html
-from collections import deque
-from logging.handlers import RotatingFileHandler
-
-import discord
-from discord import app_commands
-from discord.ext import commands
-
-from openpilot.tools.lib.api import CommaApi, UnauthorizedError
-from openpilot.tools.lib.route import Route
-
-import threading
-from flask import Flask, send_file, abort, make_response
-from pathlib import Path
-
-if not (CLIPPY_TOKEN := os.getenv("CLIPPY_TOKEN")):
-  sys.exit("❌ CLIPPY_TOKEN is missing – set it in the environment")
-
-ALLOWED_GUILD_IDS = {880416502577266699, 1368811404689276958}
-
-CLIPPY_BASE_URL = "https://clippy.royjr.com"
-
-WORKING_DIR = os.path.expanduser("~/github/sunnypilot/tools/clip")
-CLIPS_DIR = os.path.join(WORKING_DIR, "clips")
-STATS_PATH = os.path.join(WORKING_DIR, "clippy_stats.json")
-LOG_PATH = os.path.join(WORKING_DIR, "clippy.log")
-os.makedirs(CLIPS_DIR, exist_ok=True)
-
-MAX_TOTAL_JOBS         = 20
-MAX_CONCURRENT_CLIPS       = 3
-MAX_CONCURRENT_CLIPS_PER_USER  = 3
-MAX_CLIP_DURATION        = 60 * 5
-
-CLIPPY_STATS_ALLOWED_ROLES    = ["sunnypilot-dev"]
-CLIPPY_UNLIMITED_ALLOWED_ROLES  = ["sunnypilot-dev"]
-
-TAIL_LINES = 25
-tail_buffer = deque(maxlen=TAIL_LINES)
-
-intents = discord.Intents.default()
-intents.message_content = True
-bot = commands.Bot(command_prefix=lambda bot, msg: [], intents=intents)
-
-clip_queue    = []
-clip_semaphore  = asyncio.Semaphore(MAX_CONCURRENT_CLIPS)
-user_cooldowns  = {}
-
-
-async def queue_monitor():
-  while True:
-    print("\033c", end="")
-    w = shutil.get_terminal_size().columns
-    bar = "-" * w
-    print(f"{bar}\nTotal: {stats['total']} | ✅ {stats['success']} | ❌ {stats['fail']}\n{bar}")
-    print("\n".join(f"{i+1:02d}. {j['status']} {j['user']}: {j['route']}" for i, j in enumerate(clip_queue)) or "No jobs in queue.")
-    print(f"{bar}\n" + "\n".join(line[:w] for line in tail_buffer) + f"\n{bar}")
-    await asyncio.sleep(1)
-
-def start_clip_server():
-  clip_dir_resolved = Path(CLIPS_DIR).resolve()
-  app = Flask("clippy")
-
-  @app.route('/<path:filename>')
-  def get_clip(filename):
-    full_path = (clip_dir_resolved / filename).resolve()
-    try:
-      full_path.relative_to(clip_dir_resolved)
-    except ValueError:
-      abort(404)
-    if not full_path.name.endswith(".mp4"):
-      abort(404)
-    try:
-      if not full_path.is_file() or not full_path.samefile(full_path):
-        abort(404)
-    except Exception:
-      abort(404)
-    response = make_response(send_file(
-      str(full_path),
-      mimetype="video/mp4",
-      as_attachment=False,
-      conditional=True,
-    ))
-    response.headers.update({
-      "Cache-Control": "no-store",
-      "Accept-Ranges": "bytes",
-      "Content-Disposition": f'inline; filename="{filename}"',
-      "X-Content-Type-Options": "nosniff",
-    })
-    return response
-
-  @app.errorhandler(404)
-  def not_found(_):
-    return "clip not found", 404
-
-  app.run(host="127.0.0.1", port=5000)
-
-def has_any_role(user, role_list):
-  if isinstance(user, discord.Member):
-    return any(role.name in role_list for role in user.roles)
-  return False
-
-def user_tag(user: discord.User) -> str:
-  return f"{user.display_name} ({user.name})"
-
-def load_stats():
-  if os.path.exists(STATS_PATH):
-    with open(STATS_PATH, "r") as f:
-      return json.load(f)
-  return {"total": 0, "success": 0, "fail": 0}
-
-def save_stats():
-  with open(STATS_PATH, "w") as f:
-    json.dump(stats, f)
-
-stats = load_stats()
-
-
-class SanitizeFilter(logging.Filter):
-  def filter(self, record: logging.LogRecord) -> bool:
-    if isinstance(record.msg, str):
-      record.msg = re.compile(r'[\x00-\x1f\x7f-\x9f]').sub('', record.msg)
-    return True
-
-
-class DequeHandler(logging.Handler):
-  def __init__(self, buf):
-    super().__init__()
-    self.buf = buf
-    self.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
-
-  def emit(self, record):
-    try:
-      self.buf.append(self.format(record))
-    except Exception:
-      self.handleError(record)
-
-log_handler = RotatingFileHandler(LOG_PATH, maxBytes=5*1024*1024, backupCount=3)
-log_handler.setFormatter(logging.Formatter("%(asctime)s [%(levelname)s] %(message)s"))
-
-root = logging.getLogger()
-root.setLevel(logging.INFO)
-root.addHandler(log_handler)
-root.addHandler(DequeHandler(tail_buffer))
-root.addFilter(SanitizeFilter())
-
-
-class DeletePublishedView(discord.ui.View):
-  def __init__(self, message: discord.Message, author_id: int, video_path):
-    super().__init__(timeout=300)
-    self.message = message
-    self.author_id = author_id
-    self.video_path = video_path
-
-  @discord.ui.button(label="Unpublish Clip", style=discord.ButtonStyle.primary)
-  async def unpublish(self, interaction: discord.Interaction, _button: discord.ui.Button):
-    if interaction.user.id != self.author_id:
-      logging.error(f"🚫 {user_tag(interaction.user)} cant unpublish {self.message.id}")
-      await interaction.response.send_message("🚫 You can't unpublish this clip.", ephemeral=True)
-      return
-
-    try:
-      await self.message.delete()
-      logging.info(f"🗑️ {user_tag(interaction.user)} unpublished {self.message.id}")
-      await interaction.response.edit_message(content="🗑️ Unpublished clip.", view=None)
-    except Exception as e:
-      logging.error(f"❌ Failed to unpublish clip {self.message.id}: {e}")
-      await interaction.response.send_message(f"❌ Failed to unpublish clip.", ephemeral=True)
-
-  @discord.ui.button(label="Unpublish + Delete Clip", style=discord.ButtonStyle.danger)
-  async def delete(self, interaction: discord.Interaction, _button: discord.ui.Button):
-    if interaction.user.id != self.author_id:
-      logging.error(f"🚫 {user_tag(interaction.user)} cant unpublish and delete {self.message.id}")
-      await interaction.response.send_message("🚫 You can't unpublish and delete this clip.", ephemeral=True)
-      return
-
-    try:
-      if not os.path.realpath(self.video_path).startswith(os.path.realpath(CLIPS_DIR) + os.sep):
-        logging.error(f"❌ Unsafe delete attempt: {self.video_path}")
-        await interaction.response.send_message("❌ Unsafe delete attempt.", ephemeral=True)
-        return
-      os.remove(self.video_path)
-      await self.message.delete()
-      logging.info(f"🗑️ {user_tag(interaction.user)} unpublished {self.message.id}")
-      await interaction.response.edit_message(content="🗑️ Unpublished and deleted clip.", view=None)
-    except Exception as e:
-      logging.error(f"❌ Failed to unpublish clip {self.message.id}: {e}")
-      await interaction.response.send_message(f"❌ Failed to unpublish and delete clip.", ephemeral=True)
-
-
-class PublishView(discord.ui.View):
-  def __init__(self, route_str, title, video_path, author_id, file_size, safe_name):
-    super().__init__(timeout=300)
-    self.route_str = route_str
-    self.title = title
-    self.video_path = video_path
-    self.author_id = author_id
-    self.file_size = file_size
-    self.safe_name = safe_name
-
-  @discord.ui.button(label="Publish Clip", style=discord.ButtonStyle.success)
-  async def publish(self, interaction: discord.Interaction, _button: discord.ui.Button):
-    if interaction.user.id != self.author_id:
-      await interaction.response.send_message("🚫 You can't publish this clip.", ephemeral=True)
-      return
-
-    if not os.path.exists(self.video_path):
-      logging.error(f"❌ {user_tag(interaction.user)} failed to publish {self.route_str} – file missing")
-      await interaction.response.edit_message(
-        content="❌ Clip could not be published. File missing.",
-        attachments=[], view=None
-      )
-      self.stop()
-      return
-
-    logging.info(f"✅ {user_tag(interaction.user)} published {self.route_str}")
-
-    if not (1 <= self.file_size <= 9):
-      published_msg = await interaction.channel.send(
-        f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{self.safe_name}.mp4) from [{self.route_str}](https://connect.comma.ai/{self.route_str})\n{self.title}"
-      )
-    else:
-      published_msg = await interaction.channel.send(
-        f"{interaction.user.mention} shared a clip from [{self.route_str}](https://connect.comma.ai/{self.route_str})\n{self.title}",
-        file=discord.File(self.video_path)
-      )
-    await interaction.response.edit_message(
-      content="✅ Clip published to channel.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, self.video_path),
-    )
-    self.stop()
-
-  @discord.ui.button(label="Delete Clip", style=discord.ButtonStyle.danger)
-  async def delete(self, interaction: discord.Interaction, _button: discord.ui.Button):
-    if interaction.user.id != self.author_id:
-      logging.error(f"🚫 {user_tag(interaction.user)} cant delete {self.video_path}")
-      await interaction.response.send_message("🚫 You can't delete this clip.", ephemeral=True)
-      return
-
-    try:
-      if not os.path.realpath(self.video_path).startswith(os.path.realpath(CLIPS_DIR) + os.sep):
-        logging.error(f"❌ Unsafe delete attempt: {self.video_path}")
-        await interaction.response.send_message("❌ Unsafe delete attempt.", ephemeral=True)
-        return
-      os.remove(self.video_path)
-      logging.info(f"🗑️ {user_tag(interaction.user)} deleted {self.route_str}")
-      await interaction.response.edit_message(content="🗑️ Clip deleted.", attachments=[], view=None)
-      self.stop()
-    except Exception as e:
-      logging.error(f"❌ Failed to delete {self.route_str}")
-      await interaction.response.edit_message(content=f"❌ Failed to delete clip.", view=None)
-
-
-@bot.tree.command(name="clippy", description="Generate a driving clip - make sure you upload logs first!")
-@app_commands.describe(
-  input="connect link or dongle/route/starttime/endtime or dongle/route/startsegment-endsegment",
-  title="Title (default: none)",
-  quality="Video quality (default: high)",
-  wide="Use wide view if uploaded (default: true)",
-  speed="Playback speed (default: 1)",
-  cache="Set to false to regenerate clip if its already cached (default: true)",
-  private="If true, only you will see the preview (default: true)",
-  bookmarks="Automatically clip bookmarks (default: false)",
-  filesize="Max filesize (MB), set to 0 for unlimited (default: 9)",
-  developer="Show the developer UI (default: Off)"
-)
-@app_commands.choices(
-  quality=[
-    app_commands.Choice(name="high", value="high"),
-    app_commands.Choice(name="low",  value="low"),
-  ],
-  developer=[
-    app_commands.Choice(name="Right",  value="1"),
-    app_commands.Choice(name="Right & Bottom",  value="2"),
-  ]
-)
-async def clippy(
-  interaction: discord.Interaction,
-  input: str,
-  title: str = None,
-  quality: app_commands.Choice[str] | None = None,
-  wide: bool = True,
-  speed: int = 1,
-  cache: bool = True,
-  private: bool = True,
-  bookmarks: bool = False,
-  filesize: int = 9,
-  developer: app_commands.Choice[str] | None = None,
-):
-
-  if interaction.guild_id not in ALLOWED_GUILD_IDS:
-    logging.error(f"❌ This bot is not available in this server {interaction.guild_id}")
-    await interaction.response.send_message("❌ This bot is not available in this server.", ephemeral=True)
-    return
-
-  if len(clip_queue) >= MAX_TOTAL_JOBS:
-    await interaction.response.send_message(
-      "🚫 Server busy – too many jobs in queue. Please try again later.",
-      ephemeral=True
-    )
-    return
-
-  user_id = interaction.user.id
-  if not has_any_role(interaction.user, CLIPPY_UNLIMITED_ALLOWED_ROLES):
-    if user_cooldowns.get(user_id, 0) >= MAX_CONCURRENT_CLIPS_PER_USER:
-      logging.error(f"🚫 {user_tag(interaction.user)} hit the cooldown limit")
-      await interaction.response.send_message(
-        "🚫 You already have a clip running. Wait for it to finish.",
-        ephemeral=True
-      )
-      return
-
-  user_cooldowns[user_id] = user_cooldowns.get(user_id, 0) + 1
-
-  try:
-    await interaction.response.defer(ephemeral=True)
-
-    quality_value = quality.value if quality else "high"
-    title_cmd = title[:80] if title else ""
-    title = f"> ### **{html.unescape(title[:80])}**" if title else ""
-    stats["total"] += 1
-
-    # ── fast‑fail validation ────────────────────────────────────────────────────
-    def fail(msg: str):
-      stats["fail"] += 1
-      save_stats()
-      return interaction.followup.send(f"❌ {msg}", ephemeral=True)
-
-    input = input.removeprefix("https://connect.comma.ai/")
-
-    if bookmarks:
-      match = re.match(r'^([a-z0-9]+)/([a-zA-Z0-9\-]+)$', input)
-      if not match:
-        logging.error(f"❌ {user_tag(interaction.user)} entered bad input {input}")
-        await fail("Use connect link, `dongle/route/starttime/endtime` or `dongle/route/startsegment-endsegment` (endsegment optional).\n```\n--- CONNECT ---\nhttps://connect.comma.ai/a2a0ccea32023010/2023-07-27--13-01-19/5/10\n\n--- EXAMPLES ---\na2a0ccea32023010/2023-07-27--13-01-19/0     segment 0\na2a0ccea32023010/2023-07-27--13-01-19/0-1   segments 0 through 1\na2a0ccea32023010/2023-07-27--13-01-19/5/10  from 5 to 10 seconds\na2a0ccea32023010/2023-07-27--13-01-19       when using bookmark option\n```")
-        return
-      else:
-        dongle, route = match.groups()
-        start = 0
-        end = 0
-    else:
-
-      match = re.match(r'^([a-z0-9]+)/([a-zA-Z0-9\-]+)/(\d+)/(\d+)$', input)
-      if not match:
-        match = re.match(r"^([a-z0-9]+)/([A-Za-z0-9\-]+)/(\d+)(?:-(\d+))?$", input)
-        if not match:
-          logging.error(f"❌ {user_tag(interaction.user)} entered bad input {input}")
-          await fail("Use connect link, `dongle/route/starttime/endtime` or `dongle/route/startsegment-endsegment` (endsegment optional).\n```\n--- CONNECT ---\nhttps://connect.comma.ai/a2a0ccea32023010/2023-07-27--13-01-19/5/10\n\n--- EXAMPLES ---\na2a0ccea32023010/2023-07-27--13-01-19/0     segment 0\na2a0ccea32023010/2023-07-27--13-01-19/0-1   segments 0 through 1\na2a0ccea32023010/2023-07-27--13-01-19/5/10  from 5 to 10 seconds\na2a0ccea32023010/2023-07-27--13-01-19       when using bookmark option\n```")
-          return
-        else:
-          dongle, route, seg_start, seg_end = match.groups()
-
-          if int(seg_start) == 0:
-            # in_start = 2  # fix for 2s
-            in_start = 0
-          else:
-            in_start = int(seg_start) * 60
-
-          if seg_end is None:
-            in_end = 60 if int(seg_start) == 0 else in_start + 60
-          else:
-            in_end = 60 if int(seg_end) == 0 else (int(seg_end) + 1) * 60
-      else:
-        dongle, route, in_start, in_end = match.groups()
-
-      start = int(in_start)
-      end = int(in_end)
-
-      # fix for 2s
-      # if start < 2 or end <= start:
-      #   await fail("Start must be at least 2 and end must be greater than start.")
-      #   return
-
-      if end <= start:
-        logging.error(f"❌ {user_tag(interaction.user)} entered bad times {input}")
-        await fail("End must be greater than start time.")
-        return
-      duration = end - start
-      if duration > MAX_CLIP_DURATION:
-        logging.error(f"❌ {user_tag(interaction.user)} hit the max duration limit {input}")
-        await fail(f"Clips must be {int(MAX_CLIP_DURATION / 60)} minutes or less.")
-        return
-
-    status_msg = await interaction.followup.send(
-      "🕐 Waiting in queue..", ephemeral=private
-    )
-
-    if speed == 0:
-      speed = 1
-    if speed > 1:
-      end = start + int(duration / speed)
-    elif speed < 1:
-      end = start + int(duration / speed)
-    if bookmarks:
-      route_str  = f"{dongle}/{route}"
-      connect_route_str  = f"{dongle}/{route}"
-      base   = f"{dongle}_{route}_bookmarks_{quality_value}"
-    else:
-      route_str  = f"{dongle}/{route}/{start}/{end}"
-      connect_start = 1 if start == 0 else start
-      connect_route_str  = f"{dongle}/{route}/{connect_start}/{end}"
-      base   = f"{dongle}_{route}_{start}_{end}_{quality_value}"
-    if wide:
-      base += "_wide"
-    if speed:
-      base += f"_{speed}"
-    base += f"_s{filesize}"
-    clean_base = re.sub(r'[^A-Za-z0-9_-]+', '_', base)
-    if title_cmd:
-      title_hash = hashlib.sha1(title_cmd.encode()).hexdigest()[:10]
-      safe_name  = f"{clean_base}_{title_hash}"
-    else:
-      safe_name  = clean_base
-
-    safe_name = re.sub(r'[^a-zA-Z0-9_-]', '_', safe_name)
-
-    if any(job["route"] == safe_name for job in clip_queue):
-      await status_msg.edit(content="❌ That clip is already in the queue or processing – wait for it to finish.")
-      return
-
-    try:
-      logs = CommaApi().get(f"/v1/route/{dongle}|{route}/files").get("logs")
-
-      segments = [
-        re.search(r'/(\d+)/rlog\.(?:zst|bz2)', url).group(1)
-        for url in logs
-        if re.search(r'/(\d+)/rlog\.(?:zst|bz2)', url)
-      ]
-
-      startsegment = start // 60
-      endsegment = (end - 1) // 60
-
-      segment_set = set(int(s) for s in segments)
-      if bookmarks:
-        missing = False
-      else:
-        missing = [i for i in range(startsegment, endsegment + 1) if i not in segment_set]
-
-      if missing:
-        logging.error(f"❌ {user_tag(interaction.user)} segments missing {missing}")
-        await status_msg.edit(content=f"❌ You need to upload the missing logs for segments `{missing}` using [connect.comma.ai](https://connect.comma.ai/{connect_route_str})")
-        return
-      else:
-        if bookmarks:
-          logging.info(f"🕐 {user_tag(interaction.user)} getting bookmarks {route_str}")
-          await status_msg.edit(content=f"🕐 Getting bookmarks")
-        else:
-          logging.info(f"☑️ {user_tag(interaction.user)} segments present {route_str}")
-          await status_msg.edit(content=f"☑️ All required segments are present.")
-
-    except UnauthorizedError as e:
-      logging.error(f"❌ {user_tag(interaction.user)} unauthorized: {e}")
-      await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
-      return
-
-    except Exception as e:
-      logging.error(f"❌ {user_tag(interaction.user)} unexpected error: {e}")
-      await status_msg.edit(content=f"❌ Error: unexpected error")
-      return
-
-    if bookmarks:
-      try:
-        route = Route(route_str)
-        user_flags_at_time = []
-
-        for segment in route.segments:
-          for event in segment.events:
-            if event['type'] == 'user_flag':
-              user_flags_at_time.append(round(event['route_offset_millis'] / 1000))
-
-      except Exception as e:
-        logging.error(f"❌ {user_tag(interaction.user)} unauthorized: {e}")
-        await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
-        return
-
-      if len(user_flags_at_time) == 0:
-        logging.error(f"❌ {user_tag(interaction.user)} no bookmarks found")
-        await status_msg.edit(content=f"❌ No bookmarks found")
-        return
-      else:
-        bookmarklinks = ''
-        for user_flag_at_time in user_flags_at_time:
-          bookmarklinks += f"```{connect_route_str}/{user_flag_at_time - 10}/{user_flag_at_time + 5}```"
-        logging.info(f"✅ {user_tag(interaction.user)} {len(user_flags_at_time)} bookmarks found! - {user_flags_at_time}")
-        await status_msg.edit(content=f"✅ {len(user_flags_at_time)} bookmarks found! - {user_flags_at_time}{bookmarklinks}")
-        return
-
-    full_path   = os.path.join(CLIPS_DIR, f"{safe_name}.mp4")
-
-    clip_queue.append({"user": interaction.user.display_name,
-               "route": safe_name,
-               "duration": duration,
-               "status": "🕐"})
-    save_stats()
-    if private:
-      logging.info(f"🕐 {user_tag(interaction.user)} queued (PRIVATE) {route_str}")
-    else:
-      logging.info(f"🕐 {user_tag(interaction.user)} queued {route_str}")
-
-    if os.path.exists(full_path) and cache:
-      stats["success"] += 1
-      save_stats()
-      for j in clip_queue:
-        if j["route"] == safe_name:
-          j["status"] = "✅"
-
-      if private:
-        logging.info(f"📁 {user_tag(interaction.user)} used cache (PRIVATE) {route_str}")
-        await status_msg.edit(content="📁 Used cached clip.")
-        if not (1 <= filesize <= 9):
-          await interaction.followup.send(
-            content=f"Preview for [`{route_str}`]({CLIPPY_BASE_URL}/{safe_name}.mp4)\n{title}",
-            view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
-            ephemeral=True
-          )
-        else:
-          await interaction.followup.send(
-            content=f"Preview for `{route_str}`\n{title}",
-            file=discord.File(full_path),
-            view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
-            ephemeral=True
-          )
-      else:
-        logging.info(f"📁 {user_tag(interaction.user)} used cache {route_str}")
-
-        if not (1 <= filesize <= 9):
-          published_msg = await interaction.channel.send(
-            f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{safe_name}.mp4) from [{route_str}](https://connect.comma.ai/{route_str})\n{title}"
-          )
-        else:
-          published_msg = await interaction.channel.send(
-            f"{interaction.user.mention} shared a clip from [{route_str}](https://connect.comma.ai/{route_str})\n{title}",
-            file=discord.File(full_path)
-          )
-        await status_msg.edit(
-          content="📁 Used cached clip.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, full_path),
-        )
-
-    else:
-      async with clip_semaphore:
-        for j in clip_queue:
-          if j["route"] == safe_name:
-            j["status"] = "🔄"
-        logging.info(f"🔄 {user_tag(interaction.user)} processing {route_str}")
-        await status_msg.edit(content=f"🔄 Processing {j['duration']}s clip..")
-
-        cmd = ["python3", "run.py", route_str, "-q", quality_value, "-x", str(speed), "-o", full_path]
-        if not (in_start and in_end):
-          if in_start != 0: # fix for 2s
-            cmd += ["-s", str(start), "-e", str(end)]
-
-        if title_cmd:
-          cmd += ["-t", str(title_cmd)]
-
-        if wide:
-          cmd += ["-w"]
-
-        if filesize:
-          cmd += ["-f", str(filesize)]
-
-        if developer:
-          dev_mode = int(developer.value)
-        else:
-          dev_mode = 0
-        cmd += ["-z", str(dev_mode)]
-
-        proc = await asyncio.create_subprocess_exec(
-          *cmd, cwd=WORKING_DIR,
-          stdout=asyncio.subprocess.PIPE,
-          stderr=asyncio.subprocess.PIPE
-        )
-        stdout, stderr = await proc.communicate()
-        clean_err = "\n".join(stderr.decode().splitlines()[3:]) if stderr else ""
-
-        if proc.returncode != 0 or not os.path.exists(full_path):
-          for j in clip_queue:
-            if j["route"] == safe_name:
-              j["status"] = "❌"
-          stats["fail"] += 1
-          save_stats()
-          logging.error(f"❌ {user_tag(interaction.user)} failed {route_str}\n{clean_err}")
-
-          if clean_err == "clip.py: error: failed to get route: Unauthorized. Authenticate with tools/lib/auth.py":
-            await status_msg.edit(content=f"❌ You need to make the route public using [connect.comma.ai](https://connect.comma.ai/{route_str}). `/clippy-auth` is no longer supported.")
-          elif clean_err == "clip.py: error: failed to get route: 404:The requested URL was not found on the server. If you entered the URL manually please check your spelling and try again.":
-            await status_msg.edit(content="❌ This route does not exist, please try another.")
-          else:
-            await status_msg.edit(content="❌ Clip failed to generate.")
-        else:
-          for j in clip_queue:
-            if j["route"] == safe_name:
-              j["status"] = "✅"
-          stats["success"] += 1
-          save_stats()
-
-          if private:
-            logging.info(f"✅ {user_tag(interaction.user)} success (PRIVATE) {route_str}")
-            await status_msg.edit(content="✅ Clip ready.")
-
-            if not (1 <= filesize <= 9):
-              await interaction.followup.send(
-                content=f"Preview for [`{route_str}`]({CLIPPY_BASE_URL}/{safe_name}.mp4)\n{title}",
-                view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
-                ephemeral=True
-              )
-            else:
-              await interaction.followup.send(
-                content=f"Preview for `{route_str}`\n{title}",
-                file=discord.File(full_path),
-                view=PublishView(route_str, title, full_path, interaction.user.id, filesize, safe_name),
-                ephemeral=True
-              )
-          else:
-            logging.info(f"✅ {user_tag(interaction.user)} success {route_str}")
-
-            if not (1 <= filesize <= 9):
-              published_msg = await interaction.channel.send(
-                f"{interaction.user.mention} shared a [clip]({CLIPPY_BASE_URL}/{safe_name}.mp4) from [{route_str}](https://connect.comma.ai/{route_str})\n{title}"
-              )
-            else:
-              published_msg = await interaction.channel.send(
-                f"{interaction.user.mention} shared a clip from [{route_str}](https://connect.comma.ai/{route_str})\n{title}",
-                file=discord.File(full_path)
-              )
-            await status_msg.edit(
-              content="✅ Clip ready.", attachments=[], view=DeletePublishedView(published_msg, interaction.user.id, full_path),
-            )
-
-    await asyncio.sleep(1)
-    clip_queue[:] = [j for j in clip_queue if j["route"] != safe_name]
-
-  finally:
-    if user_id in user_cooldowns:
-      user_cooldowns[user_id] = max(0, user_cooldowns[user_id] - 1)
-    clip_queue[:] = [j for j in clip_queue if j["route"] != safe_name]
-
-
-@bot.tree.command(name="clippy-stats", description="View clippy stats")
-async def clippy_stats(interaction: discord.Interaction):
-
-  if interaction.guild_id not in ALLOWED_GUILD_IDS:
-    logging.error(f"❌ This bot is not available in this server {interaction.guild_id}")
-    await interaction.response.send_message("❌ This bot is not available in this server.", ephemeral=True)
-    return
-
-  if not has_any_role(interaction.user, CLIPPY_STATS_ALLOWED_ROLES):
-    logging.error(f"🚫 {user_tag(interaction.user)} not allowed to use /clippy-stats")
-    await interaction.response.send_message("🚫 You don't have permission.", ephemeral=True)
-    return
-
-  stat  = f"Total: {stats['total']} | ✅ {stats['success']} | ❌ {stats['fail']}"
-  queue = "\n".join(f"{j['status']} {j['user']}: {j['route']}" for j in clip_queue) or "No active jobs."
-  tail  = "\n".join(list(tail_buffer)[-5:][::-1]) or "[no log records yet]"
-
-  content = f"```{stat}``````{queue}``````{tail}"
-  await interaction.response.send_message(content[:1997] + "```", ephemeral=True)
-  logging.info(f"✅ {user_tag(interaction.user)} used /clippy-stats")
-
-
-@bot.event
-async def on_ready():
-  await bot.tree.sync()
-  for guild in bot.guilds:
-    logging.info(f"Connected to guild: {guild.name} ({guild.id})")
-  await bot.change_presence(activity=discord.Game(name="your clips"))
-  asyncio.create_task(queue_monitor())
-  print(f"Logged in as {bot.user}")
-
-threading.Thread(target=start_clip_server, daemon=True).start()
-bot.run(CLIPPY_TOKEN)

tools/clip/run.py

@@ -28,7 +28,7 @@ DEMO_ROUTE = 'a2a0ccea32023010/2023-07-27--13-01-19'
 FRAMERATE = 20
 PIXEL_DEPTH = '24'
 RESOLUTION = '2160x1080'
-SECONDS_TO_WARM = 0.5   # fix for 2s
+SECONDS_TO_WARM = 2
 PROC_WAIT_SECONDS = 30*10
 
 OPENPILOT_FONT = str(Path(BASEDIR, 'selfdrive/assets/fonts/Inter-Regular.ttf').resolve())
@@ -104,9 +104,8 @@ def parse_args(parser: ArgumentParser):
     args.end = int(parts[3])
   if args.end <= args.start:
     parser.error(f'end ({args.end}) must be greater than start ({args.start})')
-  # fix for 2s
-  # if args.start < SECONDS_TO_WARM:
-  #   parser.error(f'start must be greater than {SECONDS_TO_WARM}s to allow the UI time to warm up')
+  if args.start < SECONDS_TO_WARM:
+    parser.error(f'start must be greater than {SECONDS_TO_WARM}s to allow the UI time to warm up')
 
   try:
     args.route = Route(args.route, data_dir=args.data_dir)
@@ -114,16 +113,16 @@ def parse_args(parser: ArgumentParser):
     parser.error(f'failed to get route: {e}')
 
   # FIXME: length isn't exactly max segment seconds, simplify to replay exiting at end of data
-  # length = round(args.route.max_seg_number * 60)
-  # if args.start >= length:
-  #   parser.error(f'start ({args.start}s) cannot be after end of route ({length}s)')
-  # if args.end > length:
-  #   parser.error(f'end ({args.end}s) cannot be after end of route ({length}s)')
+  length = round(args.route.max_seg_number * 60)
+  if args.start >= length:
+    parser.error(f'start ({args.start}s) cannot be after end of route ({length}s)')
+  if args.end > length:
+    parser.error(f'end ({args.end}s) cannot be after end of route ({length}s)')
 
   return args
 
 
-def populate_car_params(lr: LogReader, developer: int):
+def populate_car_params(lr: LogReader):
   init_data = lr.first('initData')
   assert init_data is not None
 
@@ -132,14 +131,10 @@ def populate_car_params(lr: LogReader, developer: int):
   for cp in entries:
     key, value = cp.key, cp.value
     try:
-      if key == "OSMDownloadProgress":
-        continue
       params.put(key, params.cpp2python(key, value))
     except UnknownKeyName:
       # forks of openpilot may have other Params keys configured. ignore these
-      pass
-  if developer is not None:
-    params.put("DevUIInfo", developer)
+      logger.warning(f"unknown Params key '{key}', skipping")
   logger.debug('persisted CarParams')
 
 
@@ -184,7 +179,6 @@ def wait_for_frames(procs: list[Popen]):
 def clip(
   data_dir: str | None,
   quality: Literal['low', 'high'],
-  wide: bool,
   prefix: str,
   route: Route,
   out: str,
@@ -193,9 +187,8 @@ def clip(
   speed: int,
   target_mb: int,
   title: str | None,
-  developer: int,
 ):
-  logger.info(f'clipping route {route.name.canonical_name}, start={start} end={end} quality={quality} wide={wide} target_filesize={target_mb}MB')
+  logger.info(f'clipping route {route.name.canonical_name}, start={start} end={end} quality={quality} target_filesize={target_mb}MB')
   lr = get_logreader(route)
 
   begin_at = max(start - SECONDS_TO_WARM, 0)
@@ -231,6 +224,8 @@ def clip(
     '-draw_mouse', '0',
     '-i', display,
     '-c:v', 'libx264',
+    '-maxrate', f'{bit_rate_kbps}k',
+    '-bufsize', f'{bit_rate_kbps*2}k',
     '-crf', '23',
     '-filter:v', ','.join(overlays),
     '-preset', 'ultrafast',
@@ -239,19 +234,12 @@ def clip(
     '-movflags', '+faststart',
     '-f', 'mp4',
     '-t', str(duration),
+    out,
   ]
 
-  if target_mb > 0:
-    ffmpeg_cmd += ['-maxrate', f'{bit_rate_kbps}k']
-    ffmpeg_cmd += ['-bufsize', f'{bit_rate_kbps*2}k']
-
-  ffmpeg_cmd.append(out)
-
-  replay_cmd = [REPLAY, '-c', '1', '-s', str(begin_at), '--prefix', prefix]
+  replay_cmd = [REPLAY, '--ecam', '-c', '1', '-s', str(begin_at), '--prefix', prefix]
   if data_dir:
     replay_cmd.extend(['--data_dir', data_dir])
-  if wide:
-    replay_cmd.append('--ecam')
   if quality == 'low':
     replay_cmd.append('--qcam')
   replay_cmd.append(route.name.canonical_name)
@@ -260,7 +248,7 @@ def clip(
   xvfb_cmd = ['Xvfb', display, '-terminate', '-screen', '0', f'{RESOLUTION}x{PIXEL_DEPTH}']
 
   with OpenpilotPrefix(prefix, shared_download_cache=True):
-    populate_car_params(lr, developer)
+    populate_car_params(lr)
     env = os.environ.copy()
     env['DISPLAY'] = display
 
@@ -274,7 +262,7 @@ def clip(
       with managed_proc(ffmpeg_cmd, env) as ffmpeg_proc:
         procs.append(ffmpeg_proc)
         logger.info(f'recording in progress ({duration}s)...')
-        ffmpeg_proc.wait((duration * 2) + PROC_WAIT_SECONDS)
+        ffmpeg_proc.wait(duration + PROC_WAIT_SECONDS)
         check_for_failure(procs)
         logger.info(f'recording complete: {Path(out).resolve()}')
 
@@ -291,18 +279,15 @@ def main():
   p.add_argument('-o', '--output', help='output clip to (.mp4)', type=validate_output_file, default=DEFAULT_OUTPUT)
   p.add_argument('-p', '--prefix', help='openpilot prefix', default=f'clip_{randint(100, 99999)}')
   p.add_argument('-q', '--quality', help='quality of camera (low = qcam, high = hevc)', choices=['low', 'high'], default='high')
-  p.add_argument('-w', '--wide', help='enable wide view if uploaded', action='store_true',)
   p.add_argument('-x', '--speed', help='record the clip at this speed multiple', type=int, default=1)
   p.add_argument('-s', '--start', help='start clipping at <start> seconds', type=int)
   p.add_argument('-t', '--title', help='overlay this title on the video (e.g. "Chill driving across the Golden Gate Bridge")', type=validate_title)
-  p.add_argument('-z', '--developer', help='developer', type=int, default=0)
   args = parse_args(p)
   exit_code = 1
   try:
     clip(
       data_dir=args.data_dir,
       quality=args.quality,
-      wide=args.wide,
       prefix=args.prefix,
       route=args.route,
       out=args.output,
@@ -311,7 +296,6 @@ def main():
       speed=args.speed,
       target_mb=args.file_size,
       title=args.title,
-      developer=args.developer,
     )
     exit_code = 0
   except KeyboardInterrupt as e:

tools/replay/replay.cc

@@ -191,7 +191,6 @@ void Replay::startStream(const std::shared_ptr<Segment> segment) {
     auto bytes = words.asBytes();
     Params().put("CarParams", (const char *)bytes.begin(), bytes.size());
     Params().put("CarParamsPersistent", (const char *)bytes.begin(), bytes.size());
-    publishMessage(&(*it));
   } else {
     rWarning("failed to read CarParams from current segment");
   }

uv.lock

@@ -195,15 +195,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5b/64/63dbfdd83b31200ac58820a7951ddfdeed1fbee9285b0f3eae12d1357155/azure_storage_blob-12.26.0-py3-none-any.whl", hash = "sha256:8c5631b8b22b4f53ec5fff2f3bededf34cfef111e2af613ad42c9e6de00a77fe", size = 412907, upload-time = "2025-07-16T21:34:09.367Z" },
 ]
 
-[[package]]
-name = "blinker"
-version = "1.9.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" },
-]
-
 [[package]]
 name = "casadi"
 version = "3.7.1"
@@ -484,18 +475,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/47/ef/4cb333825d10317a36a1154341ba37e6e9c087bac99c1990ef07ffdb376f/dictdiffer-0.9.0-py2.py3-none-any.whl", hash = "sha256:442bfc693cfcadaf46674575d2eba1c53b42f5e404218ca2c2ff549f2df56595", size = 16754, upload-time = "2021-07-22T13:24:26.783Z" },
 ]
 
-[[package]]
-name = "discord-py"
-version = "2.5.2"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "aiohttp" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/7f/dd/5817c7af5e614e45cdf38cbf6c3f4597590c442822a648121a34dee7fa0f/discord_py-2.5.2.tar.gz", hash = "sha256:01cd362023bfea1a4a1d43f5280b5ef00cad2c7eba80098909f98bf28e578524", size = 1054879, upload-time = "2025-03-05T01:15:29.798Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/57/a8/dc908a0fe4cd7e3950c9fa6906f7bf2e5d92d36b432f84897185e1b77138/discord_py-2.5.2-py3-none-any.whl", hash = "sha256:81f23a17c50509ffebe0668441cb80c139e74da5115305f70e27ce821361295a", size = 1155105, upload-time = "2025-03-05T01:15:27.323Z" },
-]
-
 [[package]]
 name = "dnspython"
 version = "2.7.0"
@@ -544,23 +523,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/42/14/42b2651a2f46b022ccd948bca9f2d5af0fd8929c4eec235b8d6d844fbe67/filelock-3.19.1-py3-none-any.whl", hash = "sha256:d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d", size = 15988, upload-time = "2025-08-14T16:56:01.633Z" },
 ]
 
-[[package]]
-name = "flask"
-version = "3.1.1"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "blinker" },
-    { name = "click" },
-    { name = "itsdangerous" },
-    { name = "jinja2" },
-    { name = "markupsafe" },
-    { name = "werkzeug" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/c0/de/e47735752347f4128bcf354e0da07ef311a78244eba9e3dc1d4a5ab21a98/flask-3.1.1.tar.gz", hash = "sha256:284c7b8f2f58cb737f0cf1c30fd7eaf0ccfcde196099d24ecede3fc2005aa59e", size = 753440, upload-time = "2025-05-13T15:01:17.447Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/3d/68/9d4508e893976286d2ead7f8f571314af6c2037af34853a30fd769c02e9d/flask-3.1.1-py3-none-any.whl", hash = "sha256:07aae2bb5eaf77993ef57e357491839f5fd9f4dc281593a81a9e4d79a24f295c", size = 103305, upload-time = "2025-05-13T15:01:15.591Z" },
-]
-
 [[package]]
 name = "fonttools"
 version = "4.59.2"
@@ -743,15 +705,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" },
 ]
 
-[[package]]
-name = "itsdangerous"
-version = "2.2.0"
-source = { registry = "https://pypi.org/simple" }
-sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" },
-]
-
 [[package]]
 name = "jeepney"
 version = "0.9.0"
@@ -1308,8 +1261,6 @@ dependencies = [
     { name = "cffi" },
     { name = "crcmod" },
     { name = "cython" },
-    { name = "discord-py" },
-    { name = "flask" },
     { name = "dearpygui" },
     { name = "future-fstrings" },
     { name = "inputs" },
@@ -1404,8 +1355,6 @@ requires-dist = [
     { name = "dbus-next", marker = "extra == 'dev'" },
     { name = "dearpygui", specifier = ">=2.1.0" },
     { name = "dictdiffer", marker = "extra == 'dev'" },
-    { name = "discord-py" },
-    { name = "flask" },
     { name = "future-fstrings" },
     { name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
     { name = "inputs" },
@@ -4993,18 +4942,6 @@ wheels = [
     { url = "https://files.pythonhosted.org/packages/5a/84/44687a29792a70e111c5c477230a72c4b957d88d16141199bf9acb7537a3/websocket_client-1.8.0-py3-none-any.whl", hash = "sha256:17b44cc997f5c498e809b22cdf2d9c7a9e71c02c8cc2b6c56e7c2d1239bfa526", size = 58826, upload-time = "2024-04-23T22:16:14.422Z" },
 ]
 
-[[package]]
-name = "werkzeug"
-version = "3.1.3"
-source = { registry = "https://pypi.org/simple" }
-dependencies = [
-    { name = "markupsafe" },
-]
-sdist = { url = "https://files.pythonhosted.org/packages/9f/69/83029f1f6300c5fb2471d621ab06f6ec6b3324685a2ce0f9777fd4a8b71e/werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746", size = 806925, upload-time = "2024-11-08T15:52:18.093Z" }
-wheels = [
-    { url = "https://files.pythonhosted.org/packages/52/24/ab44c871b0f07f491e5d2ad12c9bd7358e527510618cb1b803a88e986db1/werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e", size = 224498, upload-time = "2024-11-08T15:52:16.132Z" },
-]
-
 [[package]]
 name = "xattr"
 version = "1.2.0"