Merge remote-tracking branch 'origin/master' into master3

2026-07-13 17:02:06 +08:00 · 2024-07-10 17:20:41 +08:00
parent c28bbcdccc fbc53a24a3
commit f2f9ae709c
94 changed files with 1446 additions and 2516 deletions
@@ -44,7 +44,7 @@ subaru:

 tesla:
  - changed-files:
-    - any-glob-to-all-files: 'selfdrive/car/telsa/*'
+    - any-glob-to-all-files: 'selfdrive/car/tesla/*'

 toyota:
  - changed-files:
@@ -0,0 +1,101 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
+jobs:
+  setup:
+    if: github.repository == 'commaai/openpilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
+    steps:
+      - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
+        run: |
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.run_number }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
+              report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
+                case "failure":
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
+                case "cancelled":
+                  report[jobName].cancelled.push({ "run_number": run, "link": job.html_url }); break;
+              }
+            });
+            return JSON.stringify({"jobs": report});
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
+
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY
@@ -0,0 +1,21 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  selfdrive_tests:
+    uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
+  tools_tests:
+    uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
@@ -5,35 +5,35 @@ on:
    branches:
      - master
  pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

-env:
-  BASE_IMAGE: openpilot-base
-
-  BUILD: selfdrive/test/docker_build.sh base
-
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
-
 jobs:
  docs:
    name: build docs
    runs-on: ubuntu-latest
-    timeout-minutes: 45
+    timeout-minutes: 1
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
-    - uses: ./.github/workflows/setup-with-retry
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
+
+    # Build
    - name: Build docs
      run: |
-        ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html"
+        # TODO: can we install just the "docs" dependency group without the normal deps?
+        pip install mkdocs mkdocs-terminal
+        cd docs
+        mkdocs build

+    # Push to docs.comma.ai
    - uses: actions/checkout@v4
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      with:
@@ -48,16 +48,17 @@ jobs:
        source release/identity.sh

        cd openpilot-docs
-
        git checkout --orphan tmp
        git rm -rf .

-        cp -r ../build/docs/html/ docs/
-        cp -r ../docs/README.md .
+        # copy over docs
+        cp -r ../docs/site/ docs/
+
+        # GitHub pages config
        touch docs/.nojekyll
        echo -n docs.comma.ai > docs/CNAME
-        git add -f .

+        git add -f .
        git commit -m "build docs"

        # docs live in different repo to not bloat openpilot's full clone size
@@ -43,8 +43,9 @@ jobs:
    - uses: actions/checkout@v4
    - name: uv lock
      run: |
-        pip install uv
-        uv lock
+        python3 -m ensurepip --upgrade
+        pip3 install uv
+        uv lock --upgrade
    - name: pre-commit autoupdate
      run: |
        git config --global --add safe.directory '*'
@@ -6,9 +6,15 @@ on:
      - master
  pull_request:
  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string

 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

 env:
@@ -100,8 +106,17 @@ jobs:
        SKIP_PROMPT: 1
        # package install has DeprecationWarnings
        PYTHONWARNINGS: default
-    - name: Test openpilot environment
-      run: . .venv/bin/activate && scons -h
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - name: Getting scons cache
+      uses: 'actions/cache@v4'
+      with:
+        path: /tmp/scons_cache
+        key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}-macos
+    - name: Building openpilot
+      run: . .venv/bin/activate && scons -j$(nproc)

  static_analysis:
    name: static analysis
@@ -320,5 +335,55 @@ jobs:
      - name: Upload Test Report
        uses: actions/upload-artifact@v4
        with:
-          name: report
-          path: selfdrive/ui/tests/test_ui/report
+          name: report-${{ inputs.run_number }}
+          path: selfdrive/ui/tests/test_ui/report_${{ inputs.run_number }}
+      - name: Get changes to selfdrive/ui
+        if: ${{ github.event_name == 'pull_request' }}
+        id: changed-files
+        uses: tj-actions/changed-files@v44
+        with:
+         files: |
+            selfdrive/ui/**
+      - name: Checkout ci-artifacts
+        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+        uses: actions/checkout@v4
+        with:
+          repository: commaai/ci-artifacts
+          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
+          path: ${{ github.workspace }}/ci-artifacts
+          ref: master
+      - name: Push Screenshots
+        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+        working-directory: ${{ github.workspace }}/ci-artifacts
+        run: |
+          git checkout -b openpilot/pr-${{ github.event.pull_request.number }}
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          sudo mv ${{ github.workspace }}/selfdrive/ui/tests/test_ui/report/screenshots/* .
+          git add .
+          git commit -m "screenshots for PR #${{ github.event.pull_request.number }}"
+          git push origin openpilot/pr-${{ github.event.pull_request.number }} --force
+      - name: Comment Screenshots on PR
+        if: ${{ github.event_name == 'pull_request' && steps.changed-files.outputs.any_changed == 'true' }}
+        uses: thollander/actions-comment-pull-request@v2
+        with:
+          message: |
+            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
+            ## UI Screenshots
+            <table>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/homescreen.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad.png"></td>
+              </tr>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_map.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/onroad_sidebar.png"></td>
+              </tr>
+              <tr>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_network.png"></td>
+                <td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.pull_request.number }}/settings_device.png"></td>
+              </tr>
+            </table>
+          comment_tag: run_id_screenshots
+          pr_number: ${{ github.event.pull_request.number }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -5,9 +5,14 @@ on:
    branches:
      - master
  pull_request:
-
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
 concurrency:
-  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

 env:
@@ -33,7 +33,7 @@ repos:
        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints,whit,indexIn
        - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.5.0
+    rev: v0.5.1
    hooks:
    -   id: ruff
        exclude: '^(third_party/)|(msgq/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
@@ -93,6 +93,6 @@ repos:
        pass_filenames: false
        files: '^selfdrive/ui/translations/'
 -   repo: https://github.com/python-jsonschema/check-jsonschema
-    rev: 0.28.5
+    rev: 0.28.6
    hooks:
    -   id: check-github-workflows
@@ -17,8 +17,8 @@ from cereal.services import SERVICE_LIST
 NO_TRAVERSAL_LIMIT = 2**64-1


-def log_from_bytes(dat: bytes) -> capnp.lib.capnp._DynamicStructReader:
-  with log.Event.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
+def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event) -> capnp.lib.capnp._DynamicStructReader:
+  with struct.from_bytes(dat, traversal_limit_in_words=NO_TRAVERSAL_LIMIT) as msg:
    return msg


@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 import os
 import capnp
 import multiprocessing
@@ -6,8 +5,8 @@ import numbers
 import random
 import threading
 import time
-import unittest
 from parameterized import parameterized
+import pytest

 from cereal import log, car
 import cereal.messaging as messaging
@@ -28,12 +27,6 @@ def zmq_sleep(t=1):
  if "ZMQ" in os.environ:
    time.sleep(t)

-def zmq_expected_failure(func):
-  if "ZMQ" in os.environ:
-    return unittest.expectedFailure(func)
-  else:
-    return func
-

 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
@@ -58,12 +51,12 @@ def delayed_send(delay, sock, dat):
  threading.Timer(delay, send_func).start()


-class TestMessaging(unittest.TestCase):
+class TestMessaging:
  def setUp(self):
    # TODO: ZMQ tests are too slow; all sleeps will need to be
    # replaced with logic to block on the necessary condition
    if "ZMQ" in os.environ:
-      raise unittest.SkipTest
+      pytest.skip()

    # ZMQ pub socket takes too long to die
    # sleep to prevent multiple publishers error between tests
@@ -75,9 +68,9 @@ class TestMessaging(unittest.TestCase):
      msg = messaging.new_message(evt)
    except capnp.lib.capnp.KjException:
      msg = messaging.new_message(evt, random.randrange(200))
-    self.assertLess(time.monotonic() - msg.logMonoTime, 0.1)
-    self.assertFalse(msg.valid)
-    self.assertEqual(evt, msg.which())
+    assert (time.monotonic() - msg.logMonoTime) < 0.1
+    assert not msg.valid
+    assert evt == msg.which()

  @parameterized.expand(events)
  def test_pub_sock(self, evt):
@@ -99,8 +92,8 @@ class TestMessaging(unittest.TestCase):

    # no wait and no msgs in queue
    msgs = func(sub_sock)
-    self.assertIsInstance(msgs, list)
-    self.assertEqual(len(msgs), 0)
+    assert isinstance(msgs, list)
+    assert len(msgs) == 0

    # no wait but msgs are queued up
    num_msgs = random.randrange(3, 10)
@@ -108,9 +101,9 @@ class TestMessaging(unittest.TestCase):
      pub_sock.send(messaging.new_message(sock).to_bytes())
    time.sleep(0.1)
    msgs = func(sub_sock)
-    self.assertIsInstance(msgs, list)
-    self.assertTrue(all(isinstance(msg, expected_type) for msg in msgs))
-    self.assertEqual(len(msgs), num_msgs)
+    assert isinstance(msgs, list)
+    assert all(isinstance(msg, expected_type) for msg in msgs)
+    assert len(msgs) == num_msgs

  def test_recv_sock(self):
    sock = "carState"
@@ -120,14 +113,14 @@ class TestMessaging(unittest.TestCase):

    # no wait and no msg in queue, socket should timeout
    recvd = messaging.recv_sock(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None

    # no wait and one msg in queue
    msg = random_carstate()
    pub_sock.send(msg.to_bytes())
    time.sleep(0.01)
    recvd = messaging.recv_sock(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
    # https://github.com/python/mypy/issues/13038
    assert_carstate(msg.carState, recvd.carState)

@@ -139,16 +132,16 @@ class TestMessaging(unittest.TestCase):

    # no msg in queue, socket should timeout
    recvd = messaging.recv_one(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None

    # one msg in queue
    msg = random_carstate()
    pub_sock.send(msg.to_bytes())
    recvd = messaging.recv_one(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
    assert_carstate(msg.carState, recvd.carState)

-  @zmq_expected_failure
+  @pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
  def test_recv_one_or_none(self):
    sock = "carState"
    pub_sock = messaging.pub_sock(sock)
@@ -157,13 +150,13 @@ class TestMessaging(unittest.TestCase):

    # no msg in queue, socket shouldn't block
    recvd = messaging.recv_one_or_none(sub_sock)
-    self.assertTrue(recvd is None)
+    assert recvd is None

    # one msg in queue
    msg = random_carstate()
    pub_sock.send(msg.to_bytes())
    recvd = messaging.recv_one_or_none(sub_sock)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert isinstance(recvd, capnp._DynamicStructReader)
    assert_carstate(msg.carState, recvd.carState)

  def test_recv_one_retry(self):
@@ -179,7 +172,7 @@ class TestMessaging(unittest.TestCase):
      p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
      p.start()
      time.sleep(sock_timeout*15)
-      self.assertTrue(p.is_alive())
+      assert p.is_alive()
      p.terminate()

    # wait 15 socket timeouts before sending
@@ -187,9 +180,6 @@ class TestMessaging(unittest.TestCase):
    delayed_send(sock_timeout*15, pub_sock, msg.to_bytes())
    start_time = time.monotonic()
    recvd = messaging.recv_one_retry(sub_sock)
-    self.assertGreaterEqual(time.monotonic() - start_time, sock_timeout*15)
-    self.assertIsInstance(recvd, capnp._DynamicStructReader)
+    assert (time.monotonic() - start_time) >= sock_timeout*15
+    assert isinstance(recvd, capnp._DynamicStructReader)
    assert_carstate(msg.carState, recvd.carState)
-
-if __name__ == "__main__":
-  unittest.main()
@@ -1,8 +1,6 @@
-#!/usr/bin/env python3
 import random
 import time
 from typing import Sized, cast
-import unittest

 import cereal.messaging as messaging
 from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
@@ -10,9 +8,9 @@ from cereal.messaging.tests.test_messaging import events, random_sock, random_so
                                                  zmq_sleep


-class TestSubMaster(unittest.TestCase):
+class TestSubMaster:

-  def setUp(self):
+  def setup_method(self):
    # ZMQ pub socket takes too long to die
    # sleep to prevent multiple publishers error between tests
    zmq_sleep(3)
@@ -21,21 +19,21 @@ class TestSubMaster(unittest.TestCase):
    sm = messaging.SubMaster(events)
    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(cast(Sized, p)), len(events))
+      assert len(cast(Sized, p)) == len(events)

  def test_init_state(self):
    socks = random_socks()
    sm = messaging.SubMaster(socks)
-    self.assertEqual(sm.frame, -1)
-    self.assertFalse(any(sm.updated.values()))
-    self.assertFalse(any(sm.alive.values()))
-    self.assertTrue(all(t == 0. for t in sm.recv_time.values()))
-    self.assertTrue(all(f == 0 for f in sm.recv_frame.values()))
-    self.assertTrue(all(t == 0 for t in sm.logMonoTime.values()))
+    assert sm.frame == -1
+    assert not any(sm.updated.values())
+    assert not any(sm.alive.values())
+    assert all(t == 0. for t in sm.recv_time.values())
+    assert all(f == 0 for f in sm.recv_frame.values())
+    assert all(t == 0 for t in sm.logMonoTime.values())

    for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
              sm.sock, sm.data, sm.logMonoTime, sm.valid]:
-      self.assertEqual(len(cast(Sized, p)), len(socks))
+      assert len(cast(Sized, p)) == len(socks)

  def test_getitem(self):
    sock = "carState"
@@ -59,8 +57,8 @@ class TestSubMaster(unittest.TestCase):
      msg = messaging.new_message(sock)
      pub_sock.send(msg.to_bytes())
      sm.update(1000)
-      self.assertEqual(sm.frame, i)
-      self.assertTrue(all(sm.updated.values()))
+      assert sm.frame == i
+      assert all(sm.updated.values())

  def test_update_timeout(self):
    sock = random_sock()
@@ -70,9 +68,9 @@ class TestSubMaster(unittest.TestCase):
      start_time = time.monotonic()
      sm.update(timeout)
      t = time.monotonic() - start_time
-      self.assertGreaterEqual(t, timeout/1000.)
-      self.assertLess(t, 5)
-      self.assertFalse(any(sm.updated.values()))
+      assert t >= timeout/1000.
+      assert t < 5
+      assert not any(sm.updated.values())

  def test_avg_frequency_checks(self):
    for poll in (True, False):
@@ -118,12 +116,12 @@ class TestSubMaster(unittest.TestCase):
      pub_sock.send(msg.to_bytes())
      time.sleep(0.01)
    sm.update(1000)
-    self.assertEqual(sm[sock].vEgo, n)
+    assert sm[sock].vEgo == n


-class TestPubMaster(unittest.TestCase):
+class TestPubMaster:

-  def setUp(self):
+  def setup_method(self):
    # ZMQ pub socket takes too long to die
    # sleep to prevent multiple publishers error between tests
    zmq_sleep(3)
@@ -156,8 +154,4 @@ class TestPubMaster(unittest.TestCase):
        if capnp:
          msg.clear_write_flag()
          msg = msg.to_bytes()
-        self.assertEqual(msg, recvd, i)
-
-
-if __name__ == "__main__":
-  unittest.main()
+        assert msg == recvd, i
@@ -1,25 +1,21 @@
-#!/usr/bin/env python3
 import os
 import tempfile
 from typing import Dict
-import unittest
 from parameterized import parameterized

 import cereal.services as services
 from cereal.services import SERVICE_LIST


-class TestServices(unittest.TestCase):
+class TestServices:

  @parameterized.expand(SERVICE_LIST.keys())
  def test_services(self, s):
    service = SERVICE_LIST[s]
-    self.assertTrue(service.frequency <= 104)
+    assert service.frequency <= 104
+    assert service.decimation != 0

  def test_generated_header(self):
    with tempfile.NamedTemporaryFile(suffix=".h") as f:
      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
-      self.assertEqual(ret, 0, "generated services header is not valid C")
-
-if __name__ == "__main__":
-  unittest.main()
+      assert ret == 0, "generated services header is not valid C"
@@ -61,7 +61,7 @@ _services: dict[str, tuple] = {
  "wideRoadEncodeIdx": (False, 20., 1),
  "wideRoadCameraState": (True, 20., 20),
  "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., 0),
+  "modelV2": (True, 20.),
  "managerState": (True, 2., 1),
  "uploaderState": (True, 0., 1),
  "navInstruction": (True, 1., 10),
@@ -0,0 +1 @@
+/site/
@@ -4,7 +4,7 @@

 A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified.

-# 288 Supported Cars
+# 287 Supported Cars

 |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|<a href="##"><img width=2000></a>Hardware Needed<br>&nbsp;|Video|
 |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|
@@ -25,8 +25,7 @@ A supported vehicle is one that just works when you install a comma device. All
 |Chrysler|Pacifica 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica 2019-20|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2019-20">Buy Here</a></sub></details>||
 |Chrysler|Pacifica 2021-23|All|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica 2021-23">Buy Here</a></sub></details>||
-|Chrysler|Pacifica Hybrid 2017|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2017">Buy Here</a></sub></details>||
-|Chrysler|Pacifica Hybrid 2018|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2018">Buy Here</a></sub></details>||
+|Chrysler|Pacifica Hybrid 2017-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2017-18">Buy Here</a></sub></details>||
 |Chrysler|Pacifica Hybrid 2019-24|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 FCA connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Chrysler&model=Pacifica Hybrid 2019-24">Buy Here</a></sub></details>||
 |comma|body|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|None||
 |CUPRA|Ateca 2018-23|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=CUPRA&model=Ateca 2018-23">Buy Here</a></sub></details>||
@@ -1,53 +0,0 @@
-# Minimal makefile for Sphinx documentation
-#
-
-OPENPILOT_ROOT = `git rev-parse --show-toplevel`
-
-# You can set these variables from the command line, and also
-# from the environment for the first two.
-SPHINXOPTS    ?=
-SPHINXBUILD   ?= sphinx-build
-DOCSDIR       = "$(OPENPILOT_ROOT)/docs"
-SOURCEDIR     = "$(OPENPILOT_ROOT)/build/docs"
-DOCSBUILDDIR  = "$(OPENPILOT_ROOT)/build/docs"
-BUILDDIR      = "$(OPENPILOT_ROOT)/build"
-
-# Put it first so that "make" without argument is like "make help".
-help:
-	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(DOCSBUILDDIR)" $(SPHINXOPTS) $(O)
-
-clean:
-	@echo "Cleaning build folder..."
-	rm -rf "$(BUILDDIR)"
-
-.PHONY: help Makefile
-
-# Catch-all target: route all unknown targets to Sphinx using the new
-# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
-%: Makefile
-	@echo "Cleaning build folder..."
-	rm -rf "$(BUILDDIR)"
-	mkdir -p "$(DOCSBUILDDIR)"
-
-	@echo "Copying docs & config to build folder..."
-	cp -a "$(DOCSDIR)" "$(BUILDDIR)"
-	cd "$(OPENPILOT_ROOT)" && \
-		find . -type f \( -name "*.md" -o -name "*.rst" -o -name "*.png" -o -name "*.jpg" -o -name "*.svg" \) \
-			-not -path "*/.*" \
-			-not -path "./build/*" \
-			-not -path "./docs/*" \
-			-not -path "./xx/*" \
-			-exec cp --parents "{}" ./build/docs/ \;
-
-	@echo "Building rst files..."
-	sphinx-apidoc -o "$(DOCSBUILDDIR)" ../ \
-		../xx ../rednose_repo ../notebooks ../panda_jungle \
-		../third_party \
-		../panda/examples \
-		../scripts \
-		../selfdrive/modeld \
-		../selfdrive/debug \
-		$(shell find .. -type d -name "*test* -not -path "**.venv**" \")
-
-	@echo "Building html files..."
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(DOCSBUILDDIR)" $(SPHINXOPTS) $(O)
@@ -1,3 +1,22 @@
-# openpilot-docs
+# openpilot docs

-These docs are autogenerated from [this folder](https://github.com/commaai/openpilot/tree/master/docs) in the main openpilot repository.
+This is the source for [docs.comma.ai](https://docs.comma.ai).
+The site is updated on pushes to master by this [workflow](../.github/workflows/docs.yaml).
+
+## development
+```
+# install the docs dependencies
+pip install .[docs]
+
+cd docs/
+
+# for a development server
+mkdocs serve
+
+# build the site
+mkdocs build
+```
+
+References:
+* https://www.mkdocs.org/getting-started/
+* https://github.com/ntno/mkdocs-terminal
@@ -9,6 +9,7 @@ Most development happens on normal Ubuntu workstations, and not in cars or direc
 ```bash
 # get the latest stuff
 git pull
+git lfs pull
 git submodule update --init --recursive

 # update dependencies
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:42bc589954cad7f16cef694887dee2c819378f7dacfdddea2ff833ff65a109fd
-size 1122
@@ -1,2 +0,0 @@
-User-agent: *
-Sitemap: https://docs.comma.ai/sitemap.xml
@@ -1,87 +0,0 @@
-openpilot
-==========
-
-
-opendbc
------
-.. autodoxygenindex::
-   :project: opendbc_can
-
-
-cereal
------
-
-messaging
-^^^^^^^^^
-.. autodoxygenindex::
-   :project: msgq_repo_msgq
-
-visionipc
-^^^^^^^^^
-.. autodoxygenindex::
-   :project: msgq_repo_msgq_visionipc
-
-
-selfdrive
---------
-
-camerad
-^^^^^^^
-.. autodoxygenindex::
-   :project: system_camerad_cameras
-
-locationd
-^^^^^^^^^
-.. autodoxygenindex::
-   :project: selfdrive_locationd
-
-ui
-^^
-
-.. autodoxygenindex::
-   :project: selfdrive_ui
-
-replay
-""""""
-.. autodoxygenindex::
-   :project: tools_replay
-
-qt
-""
-.. autodoxygenindex::
-   :project: selfdrive_ui_qt_offroad
-
-proclogd
-^^^^^^^^
-.. autodoxygenindex::
-   :project: system_proclogd
-
-modeld
-^^^^^^
-.. autodoxygenindex::
-   :project: selfdrive_modeld_transforms
-.. autodoxygenindex::
-   :project: selfdrive_modeld_models
-.. autodoxygenindex::
-   :project: selfdrive_modeld_runners
-
-common
-^^^^^^
-.. autodoxygenindex::
-   :project: common
-
-sensorsd
-^^^^^^^^
-.. autodoxygenindex::
-   :project: system_sensord_sensors
-
-pandad
-^^^^^^
-.. autodoxygenindex::
-   :project: selfdrive_pandad
-
-
-rednose
-------
-.. autodoxygenindex::
-   :project: rednose_repo_rednose_helpers
@@ -1,147 +0,0 @@
-# type: ignore
-
-# Configuration file for the Sphinx documentation builder.
-#
-# This file only contains a selection of the most common options. For a full
-# list see the documentation:
-# https://www.sphinx-doc.org/en/master/usage/configuration.html
-
-# -- Path setup --------------------------------------------------------------
-
-# If extensions (or modules to document with autodoc) are in another directory,
-# add these directories to sys.path here. If the directory is relative to the
-# documentation root, use os.path.abspath to make it absolute, like shown here.
-#
-import os
-import sys
-from os.path import exists
-
-from openpilot.common.basedir import BASEDIR
-from openpilot.system.version import get_version
-
-sys.path.insert(0, os.path.abspath('.'))
-sys.path.insert(0, os.path.abspath('..'))
-
-VERSION = get_version()
-
-
-# -- Project information -----------------------------------------------------
-
-project = 'openpilot docs'
-copyright = '2021, comma.ai' # noqa: A001
-author = 'comma.ai'
-version = VERSION
-release = VERSION
-language = 'en'
-
-
-# -- General configuration ---------------------------------------------------
-
-# Add any Sphinx extension module names here, as strings. They can be
-# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
-# ones.
-extensions = [
-  'sphinx.ext.autodoc',   # Auto-generate docs
-  'sphinx.ext.viewcode',  # Add view code link to modules
-  'sphinx_rtd_theme',     # Read The Docs theme
-  'myst_parser',          # Markdown parsing
-  'breathe',              # Doxygen C/C++ integration
-  'sphinx_sitemap',       # sitemap generation for SEO
-]
-
-myst_html_meta = {
-  "description": "openpilot docs",
-  "keywords": "op, openpilot, docs, documentation",
-  "robots": "all,follow",
-  "googlebot": "index,follow,snippet,archive",
-  "property=og:locale": "en_US",
-  "property=og:site_name": "docs.comma.ai",
-  "property=og:url": "https://docs.comma.ai",
-  "property=og:title": "openpilot Documentation",
-  "property=og:type": "website",
-  "property=og:image:type": "image/jpeg",
-  "property=og:image:width": "400",
-  "property=og:image": "https://docs.comma.ai/_static/logo.png",
-  "property=og:image:url": "https://docs.comma.ai/_static/logo.png",
-  "property=og:image:secure_url": "https://docs.comma.ai/_static/logo.png",
-  "property=og:description": "openpilot Documentation",
-  "property=twitter:card": "summary_large_image",
-  "property=twitter:logo": "https://docs.comma.ai/_static/logo.png",
-  "property=twitter:title": "openpilot Documentation",
-  "property=twitter:description": "openpilot Documentation"
-}
-
-html_baseurl = 'https://docs.comma.ai/'
-sitemap_filename = "sitemap.xml"
-
-# Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
-
-# List of patterns, relative to source directory, that match files and
-# directories to ignore when looking for source files.
-# This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = []
-
-
-# -- c docs configuration ---------------------------------------------------
-
-# Breathe Configuration
-# breathe_default_project = "c_docs"
-breathe_build_directory = f"{BASEDIR}/build/docs/html/xml"
-breathe_separate_member_pages = True
-breathe_default_members = ('members', 'private-members', 'undoc-members')
-breathe_domain_by_extension = {
-  "h": "cc",
-}
-breathe_implementation_filename_extensions = ['.c', '.cc']
-breathe_doxygen_config_options = {}
-breathe_projects_source = {}
-
-# only document files that have accompanying .cc files next to them
-print("searching for c_docs...")
-for root, _, files in os.walk(BASEDIR):
-  found = False
-  breath_src = {}
-  breathe_srcs_list = []
-
-  for file in files:
-    ccFile = os.path.join(root, file)[:-2] + ".cc"
-
-    if file.endswith(".h") and exists(ccFile):
-      f = os.path.join(root, file)
-
-      parent_dir_abs = os.path.dirname(f)
-      parent_dir = parent_dir_abs[len(BASEDIR) + 1:]
-      parent_project = parent_dir.replace('/', '_')
-      print(f"\tFOUND: {f} in {parent_project}")
-
-      breathe_srcs_list.append(file)
-      found = True
-
-    if found:
-      breath_src[parent_project] = (parent_dir_abs, breathe_srcs_list)
-      breathe_projects_source.update(breath_src)
-
-print(f"breathe_projects_source: {breathe_projects_source.keys()}")
-
-# -- Options for HTML output -------------------------------------------------
-
-# The theme to use for HTML and HTML Help pages.  See the documentation for
-# a list of builtin themes.
-#
-html_theme = 'sphinx_rtd_theme'
-html_show_copyright = True
-
-# Add any paths that contain custom static files (such as style sheets) here,
-# relative to this directory. They are copied after the builtin static files,
-# so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
-html_logo = '_static/logo.png'
-html_favicon = '_static/favicon.ico'
-html_theme_options = {
-  'logo_only': False,
-  'display_version': True,
-  'vcs_pageview_mode': 'blob',
-  'style_nav_header_background': '#000000',
-}
-html_extra_path = ['_static']
@@ -0,0 +1,22 @@
+# What is a car port?
+
+A car port enables openpilot support on a particular car. Each car model openpilot supports needs to be individually ported. All car ports live in `openpilot/selfdrive/car/`.
+
+The complexity of a car port varies depending on many factors including:
+* existing openpilot support for similar cars
+* architecture and APIs available in the car
+
+
+# Structure of a car port
+* `interface.py`: Interface for the car, defines the CarInterface class
+* `carstate.py`: Reads CAN from car and builds openpilot CarState message
+* `carcontroller.py`: Builds CAN messages to send to car
+* `values.py`: Limits for actuation, general constants for cars, and supported car documentation
+* `radar_interface.py`: Interface for parsing radar points from the car
+
+
+# Overiew
+
+[Jason Young](https://github.com/jyoung8607) gave a talk at COMMA_CON with an overview of the car porting process. The talk is available on YouTube:
+
+https://youtu.be/KcfzEHB6ms4?si=5szh1PX6TksOCKmM
@@ -0,0 +1 @@
+# Architecture
@@ -0,0 +1,5 @@
+# Roadmap
+
+Coming soon...
+
+For now, check out our GitHub [milestones](https://github.com/commaai/openpilot/milestones) and [bounties](https://comma.ai/bounties).
@@ -1,6 +1,6 @@
 # What is openpilot?

-[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md).
+[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](https://github.com/commaai/openpilot/blob/master/docs/INTEGRATION.md) and [limitations](https://github.com/commaai/openpilot/blob/master/docs/LIMITATIONS.md).


 ## How do I use it?
@@ -1,39 +1,42 @@
-# SSH
+# connect to a comma 3/3X

-## Quick Start
+A comma 3/3X is a normal [Linux](https://github.com/commaai/agnos-builder) computer that exposes [SSH](https://wiki.archlinux.org/title/Secure_Shell) and a [serial console](https://wiki.archlinux.org/title/Working_with_the_serial_console).
+
+## Serial Console
+
+On both the comma three and 3X, the serial console is accessible from the main OBD-C port.
+Connect the comma 3/3X to your computer with a normal USB C cable, or use a [comma serial](https://comma.ai/shop/comma-serial) for steady 12V power.
+
+On the comma three, the serial console is exposed through a UART-to-USB chip, and `tools/serial/connect.sh` can be used to connect.
+
+On the comma 3X, the serial console is accessible through the [panda](https://github.com/commaai/panda) using the `panda/tests/som_debug.sh` script.
+
+## SSH

 In order to SSH into your device, you'll need a GitHub account with SSH keys. See this [GitHub article](https://docs.github.com/en/github/authenticating-to-github/connecting-to-github-with-ssh) for getting your account setup with SSH keys.

 * Enable SSH in your device's settings
 * Enter your GitHub username in the device's settings
 * Connect to your device
-  * Username: `comma`
-  * Port: `22` or `8022`
+    * Username: `comma`
+    * Port: `22`

 Here's an example command for connecting to your device using its tethered connection:<br />
 `ssh comma@192.168.43.1`

 For doing development work on device, it's recommended to use [SSH agent forwarding](https://docs.github.com/en/developers/overview/using-ssh-agent-forwarding).

-## Notes
+### Notes

 The public keys are only fetched from your GitHub account once. In order to update your device's authorized keys, you'll need to re-enter your GitHub username.

 The `id_rsa` key in this directory only works while your device is in the setup state with no software installed. After installation, that default key will be removed.

-See the [community wiki](https://github.com/commaai/openpilot/wiki/SSH) for more detailed instructions and information.
+#### ssh.comma.ai proxy

-# Connecting to ssh.comma.ai
-SSH into your comma device from anywhere with `ssh.comma.ai`. Requires a [comma prime subscription](https://comma.ai/connect).
+With a [comma prime subscription](https://comma.ai/connect), you can SSH into your comma device from anywhere.

-## Setup
-
-With software version 0.6.1 or newer, enter your GitHub username on your device under Developer Settings. Your GitHub authorized public keys will become your authorized SSH keys for `ssh.comma.ai`. You can add any additional keys in `/system/comma/home/.ssh/authorized_keys.persist`.
-
-## Recommended .ssh/config
-
-With the below SSH configuration, you can type `ssh comma-{dongleid}` to connect to your device through `ssh.comma.ai`.<br />
-For example: `ssh comma-ffffffffffffffff`
+With the below SSH configuration, you can type `ssh comma-{dongleid}` to connect to your device through `ssh.comma.ai`.

 ```
 Host comma-*
@@ -41,6 +44,7 @@ Host comma-*
  User comma
  IdentityFile ~/.ssh/my_github_key
  ProxyCommand ssh %h@ssh.comma.ai -W %h:%p
+
 Host ssh.comma.ai
  Hostname ssh.comma.ai
  Port 22
@@ -0,0 +1 @@
+getting-started/what-is-openpilot.md
@@ -1,42 +0,0 @@
-# openpilot Documentation
-
-```{include} README.md
-```
-
-```{toctree}
-:caption: 'General'
-:maxdepth: 4
-
-CARS.md
-CONTRIBUTING.md
-INTEGRATION.md
-LIMITATIONS.md
-SAFETY.md
-```
-
-```{toctree}
-:caption: 'Overview'
-:maxdepth: 2
-
-overview.rst
-```
-
-## API Documentation
-
- {ref}`genindex`
- {ref}`modindex`
- {ref}`search`
-
-```{toctree}
-:caption: 'Python API'
-:maxdepth: 2
-
-modules.rst
-```
-
-```{toctree}
-:caption: 'C/C++ API'
-:maxdepth: 4
-
-c_docs.rst
-```
@@ -0,0 +1,32 @@
+site_name: openpilot docs
+docs_dir: docs
+repo_url: https://github.com/commaai/openpilot/
+site_url: https://docs.comma.ai
+
+strict: true
+
+theme:
+  name: terminal
+  features:
+    - navigation.side.toc.hide
+
+nav:
+  - Getting Started:
+    - What is openpilot?: getting-started/what-is-openpilot.md
+  - How-to:
+    #- Make my first pull request: how-to/first-pr.md
+    - Connect to a comma 3/3X: how-to/connect-to-comma.md
+  - Car Porting:
+    - What is a car port?: car-porting/what-is-a-car-port.md
+    - Porting a car brand: car-porting/brand-port.md
+    - Porting a car model: car-porting/model-port.md
+  - Contributing:
+    - Roadmap: contributing/roadmap.md
+    #- Architecture: contributing/architecture.md
+    - Contributing: https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md
+  - Links:
+    - Blog: https://blog.comma.ai
+    - Bounties: https://comma.ai/bounties
+    - GitHub: https://github.com/commaai
+    - Discord: https://discord.comma.ai
+    - X: https://x.com/comma_ai
@@ -1,12 +0,0 @@
-This is the source for a new https://docs.comma.ai. It's not hosted anywhere yet, but it's easy to run locally.
-
-https://www.mkdocs.org/getting-started/
-
-```
-pip install mkdocs mkdocs-terminal
-mkdocs serve
-```
-
-inspiration:
-* https://rerun.io/docs/
-* https://docs.expo.dev/
@@ -1,9 +0,0 @@
-# What is a car port?
-
-All car ports live in `openpilot/selfdrive/car/`.
-
-* interface.py: Interface for the car, defines the CarInterface class
-* carstate.py: Reads CAN from car and builds openpilot CarState message
-* carcontroller.py: Builds CAN messages to send to car
-* values.py: Limits for actuation, general constants for cars, and supported car documentation
-* radar_interface.py: Interface for parsing radar points from the car
@@ -1,18 +0,0 @@
-site_name: openpilot docs
-docs_dir: docs
-repo_url: https://github.com/commaai/openpilot/
-
-theme:
-  name: terminal
-  features:
-    - navigation.side.toc.hide
-
-nav:
-  - Getting Started:
-    - What is openpilot?: getting-started/what-is-openpilot.md
-  - How-to:
-    - Turn the speed blue: how-to/turning-the-speed-blue.md
-  - Car Porting:
-    - What is a car port?: car-porting/what-is-a-car-port.md
-    - Porting a car brand: car-porting/brand-port.md
-    - Porting a car model: car-porting/model-port.md
@@ -1,72 +0,0 @@
-openpilot
-=========
-
-.. toctree::
-   :maxdepth: 4
-
-   Debugging <selfdrive/debug/README.md>
-   system/loggerd/README.md
-   Driver Monitoring <selfdrive/monitoring/README.md>
-   Process Replay <selfdrive/test/process_replay/README.md>
-
-cereal
-=========
-
-.. toctree::
-   :maxdepth: 4
-
-   cereal/README.md
-   cereal/messaging/msgq.md
-
-models
-=========
-
-.. toctree::
-   :maxdepth: 4
-
-   models/README.md
-
-opendbc
-=========
-
-.. toctree::
-   :maxdepth: 4
-
-   opendbc/README.md
-
-panda
-=========
-
-.. toctree::
-   :maxdepth: 4
-
-   panda/README.md
-   panda/UPDATING.md
-   panda/board/README.md
-   panda/drivers/linux/README.md
-   panda/drivers/windows/README.md
-
-
-rednose
-=========
-.. toctree::
-   :maxdepth: 4
-
-   rednose_repo/README.md
-
-
-tools
-=========
-.. toctree::
-   :maxdepth: 4
-
-   tools/CTF.md
-   tools/joystick/README.md
-   tools/lib/README.md
-   tools/plotjuggler/README.md
-   tools/replay/README.md
-   tools/serial/README.md
-   Simulator <tools/sim/README.md>
-   tools/ssh/README.md
-   Webcam <tools/webcam/README.md>
-   tools/cabana/README.md
@@ -66,9 +66,8 @@ dependencies = [
 [project.optional-dependencies]
 docs = [
  "Jinja2",
-  "sphinx",
-  "sphinx-rtd-theme",
-  "sphinx-sitemap"
+  "mkdocs",
+  "mkdocs-terminal",
 ]

 testing = [
@@ -137,7 +136,7 @@ packages = [ "." ]

 [tool.pytest.ini_options]
 minversion = "6.0"
-addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/  -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
+addopts = "--ignore=openpilot/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=teleoprtc_repo/ --ignore=msgq/  -Werror --strict-config --strict-markers --durations=10 -n auto --dist=loadgroup"
 cpp_files = "test_*"
 cpp_harness = "selfdrive/test/cpp_harness.py"
 python_files = "test_*.py"
@@ -0,0 +1,357 @@
+#!/bin/bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+SRC=/tmp/openpilot/
+SRC_CLONE=/tmp/openpilot-clone/
+OUT=/tmp/openpilot-tiny/
+
+LOG_FILE=$DIR/git-rewrite-log-$(date +"%Y-%m-%dT%H:%M:%S%z").txt
+exec > >(tee -a "$LOG_FILE") 2>&1
+
+# INSTALL git-filter-repo
+if [ ! -f /tmp/git-filter-repo ]; then
+  echo "Installing git-filter-repo..."
+  curl -sSo /tmp/git-filter-repo https://raw.githubusercontent.com/newren/git-filter-repo/main/git-filter-repo
+  chmod +x /tmp/git-filter-repo
+fi
+
+# MIRROR openpilot
+if [ ! -d $SRC ]; then
+  echo "Mirroring openpilot..."
+  git clone --mirror https://github.com/commaai/openpilot.git $SRC # 4.18 GiB (488034 objects)
+
+  cd $SRC
+
+  echo "Starting size $(du -sh .)"
+
+  git remote update
+
+  # the git-filter-repo analysis is bliss - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "Pushing to openpilot-archive..."
+  # push to archive repo - in smaller parts because the 2 GB push limit - https://docs.github.com/en/get-started/using-git/troubleshooting-the-2-gb-push-limit
+  ARCHIVE_REPO=git@github.com:commaai/openpilot-archive.git
+  git push --prune $ARCHIVE_REPO +refs/heads/master:refs/heads/master # push master first so it's the default branch (when openpilot-archive is an empty repo)
+  git push --prune $ARCHIVE_REPO +refs/heads/*:refs/heads/* # 956.39 MiB (110725 objects)
+  git push --prune $ARCHIVE_REPO +refs/tags/*:refs/tags/* # 1.75 GiB (21694 objects)
+  # git push --mirror $ARCHIVE_REPO || true # fails to push refs/pull/* (deny updating a hidden ref) for pull requests
+  # we fail and continue - more reading: https://stackoverflow.com/a/34266401/639708
+fi
+
+# REWRITE master and tags
+if [ ! -d $SRC_CLONE ]; then
+  echo "Cloning $SRC..."
+  GIT_LFS_SKIP_SMUDGE=1 git clone $SRC $SRC_CLONE
+
+  cd $SRC_CLONE
+
+  echo "Checking out old history..."
+
+  git checkout tags/v0.7.1
+  # checkout as main, since we need master ref later
+  git checkout -b main
+
+  echo "Creating setup commits..."
+
+  # rm these so we don't get conflicts later
+  git rm -r cereal opendbc panda selfdrive/ui/ui > /dev/null
+  git commit -m "removed conflicting files"
+
+  # skip-smudge to get rid of some lfs errors that it can't find the reference of some lfs files
+  # we don't care about fetching/pushing lfs right now
+  git lfs install --skip-smudge --local
+
+  # squash initial setup commits
+  git cherry-pick -n -X theirs 6c33a5c..59b3d06
+  git commit -m "switching to master" -m "$(git log --reverse --format=%B 6c33a5c..59b3d06)"
+
+  # get commits we want to cherry-pick
+  # will start with the next commit after #59b3d06 tools is local now
+  COMMITS=$(git rev-list --reverse 59b3d06..master)
+
+  # we need this for logging
+  TOTAL_COMMITS=$(echo $COMMITS | wc -w)
+  CURRENT_COMMIT_NUMBER=0
+
+  # empty this file
+  > commit-map.txt
+
+  echo "Rewriting master commits..."
+
+  for COMMIT in $COMMITS; do
+      CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+      echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Cherry-picking commit: $COMMIT"\\r
+
+      # set environment variables to preserve author/committer and dates
+      export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+      export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+      export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+      export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+      export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+      export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+      # cherry-pick the commit
+      if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+        # check if the failure is because of an empty commit
+        if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+          echo "Empty commit detected. Skipping commit $COMMIT"
+          git cherry-pick --skip
+          # log it was empty to the mapping file
+          echo "$COMMIT EMPTY" >> commit-map.txt
+        else
+          # handle other errors or conflicts
+          echo "Cherry-pick failed. Handling error..."
+          echo "$GIT_OUTPUT"
+          exit 1
+        fi
+      else
+        # capture the new commit hash
+        NEW_COMMIT=$(git rev-parse HEAD)
+
+        # save the old and new commit hashes to the mapping file
+        echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+        # append the old commit ID to the commit message
+        git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+      fi
+  done
+
+  echo "Rewriting tags..."
+
+  # remove all old tags
+  git tag -l | xargs git tag -d
+
+  # read each line from the tag-commit-map.txt
+  while IFS=' ' read -r TAG OLD_COMMIT; do
+    # search for the new commit in commit-map.txt corresponding to the old commit
+    NEW_COMMIT=$(grep "^$OLD_COMMIT " "commit-map.txt" | awk '{print $2}')
+
+    # check if this is a rebased commit
+    if [ -z "$NEW_COMMIT" ]; then
+      # if not, then just use old commit hash
+      NEW_COMMIT=$OLD_COMMIT
+    fi
+
+    printf "Rewriting tag %s from commit %s\n" "$TAG" "$NEW_COMMIT"
+    git tag -f "$TAG" "$NEW_COMMIT"
+  done < "$DIR/tag-commit-map.txt"
+
+  # uninstall lfs since we don't want to touch (push to) lfs right now
+  # git push will also push lfs, if we don't uninstall (--local so just for this repo)
+  git lfs uninstall --local
+
+  # force push new master
+  git push --force origin main:master
+
+  # force push new tags
+  git push --force --tags
+fi
+
+# REWRITE branches based on master
+if [ ! -f "$SRC_CLONE/branch-diff.txt" ]; then
+  cd $SRC_CLONE
+
+  # empty file
+  > branch-diff.txt
+
+  echo "Rewriting branches based on master..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # get a list of all branches except master
+  BRANCHES=$(git branch -r | grep -v ' -> ' | sed 's/origin\///' | grep -v '^master$')
+
+  for BRANCH in $BRANCHES; do
+    # check if the branch is based on master history
+    MERGE_BASE=$(git merge-base master origin/$BRANCH) || true
+    if [ -n "$MERGE_BASE" ]; then
+      echo "Rewriting branch: $BRANCH"
+      
+      # create a new branch based on the new master
+      NEW_MERGE_BASE=$(grep "^$MERGE_BASE " "commit-map.txt" | awk '{print $2}')
+      if [ -z "$NEW_MERGE_BASE" ]; then
+        echo "Error: could not find new merge base for branch $BRANCH" >> branch-diff.txt
+        continue
+      fi
+      git checkout -b ${BRANCH}_new $NEW_MERGE_BASE
+      
+      # get the range of commits unique to this branch
+      COMMITS=$(git rev-list --reverse $MERGE_BASE..origin/${BRANCH})
+      
+      HAS_ERROR=0
+
+      # simple delimiter
+      echo "BRANCH ${BRANCH}" >> commit-map.txt
+
+      for COMMIT in $COMMITS; do
+        # set environment variables to preserve author/committer and dates
+        export GIT_AUTHOR_NAME=$(git show -s --format='%an' $COMMIT)
+        export GIT_AUTHOR_EMAIL=$(git show -s --format='%ae' $COMMIT)
+        export GIT_COMMITTER_NAME=$(git show -s --format='%cn' $COMMIT)
+        export GIT_COMMITTER_EMAIL=$(git show -s --format='%ce' $COMMIT)
+        export GIT_AUTHOR_DATE=$(git show -s --format='%ad' $COMMIT)
+        export GIT_COMMITTER_DATE=$(git show -s --format='%cd' $COMMIT)
+
+        # cherry-pick the commit
+        if ! GIT_OUTPUT=$(git cherry-pick -m 1 -X theirs $COMMIT 2>&1); then
+          # check if the failure is because of an empty commit
+          if [[ "$GIT_OUTPUT" == *"The previous cherry-pick is now empty"* ]]; then
+            echo "Empty commit detected. Skipping commit $COMMIT"
+            git cherry-pick --skip
+            # log it was empty to the mapping file
+            echo "$COMMIT EMPTY" >> commit-map.txt
+          else
+            # handle other errors or conflicts
+            echo "Cherry-pick of ${BRANCH} branch failed. Removing branch upstream..." >> branch-diff.txt
+            echo "$GIT_OUTPUT" > "differences/branch-${BRANCH}"
+            git cherry-pick --abort
+            git push --delete origin ${BRANCH}
+            HAS_ERROR=1
+            break
+          fi
+        else
+          # capture the new commit hash
+          NEW_COMMIT=$(git rev-parse HEAD)
+
+          # save the old and new commit hashes to the mapping file
+          echo "$COMMIT $NEW_COMMIT" >> commit-map.txt
+
+          # append the old commit ID to the commit message
+          git commit --amend -m "$(git log -1 --pretty=%B)" -m "Former-commit-id: $COMMIT" > /dev/null
+        fi
+      done
+      
+      # force push the new branch
+      if [ $HAS_ERROR -eq 0 ]; then
+        # git lfs goes haywire here, so we need to install and uninstall
+        # git lfs install --skip-smudge --local
+        git lfs uninstall --local > /dev/null
+        git push -f origin ${BRANCH}_new:${BRANCH}
+      fi
+      
+      # clean up local branch
+      git checkout master > /dev/null
+      git branch -D ${BRANCH}_new > /dev/null
+    else
+      # echo "Skipping branch $BRANCH as it's not based on master history" >> branch-diff.txt
+       echo "Deleting branch $BRANCH as it's not based on master history" >> branch-diff.txt
+       git push --delete origin ${BRANCH}
+    fi
+  done
+fi
+
+# VALIDATE cherry-pick
+if [ ! -f "$SRC_CLONE/commit-diff.txt" ]; then
+  cd $SRC_CLONE
+
+  TOTAL_COMMITS=$(grep -cve '^\s*$' commit-map.txt)
+  CURRENT_COMMIT_NUMBER=0
+  COUNT_SAME=0
+  COUNT_DIFF=0
+  VALIDATE_IGNORE_FILES=(
+    ".github/ISSUE_TEMPLATE/bug_report.md"
+    ".github/pull_request_template.md"
+  )
+
+  # empty file
+  > commit-diff.txt
+
+  echo "Validating commits..."
+
+  # will store raw diffs here, if exist
+  mkdir -p differences
+
+  # read each line from commit-map.txt
+  while IFS=' ' read -r OLD_COMMIT NEW_COMMIT; do
+    if [ "$NEW_COMMIT" == "EMPTY" ]; then
+      continue
+    fi
+    if [ "$OLD_COMMIT" == "BRANCH" ]; then
+      echo "Branch ${NEW_COMMIT} below:" >> commit-diff.txt
+      continue
+    fi
+    CURRENT_COMMIT_NUMBER=$((CURRENT_COMMIT_NUMBER + 1))
+    # retrieve short hashes and dates for the old and new commits
+    OLD_COMMIT_SHORT=$(git rev-parse --short $OLD_COMMIT)
+    NEW_COMMIT_SHORT=$(git rev-parse --short $NEW_COMMIT)
+    OLD_DATE=$(git show -s --format='%cd' $OLD_COMMIT)
+    NEW_DATE=$(git show -s --format='%cd' $NEW_COMMIT)
+
+    echo -ne "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Comparing old commit $OLD_COMMIT_SHORT ($OLD_DATE) with new commit $NEW_COMMIT_SHORT ($NEW_DATE)"\\r
+    
+    # generate lists of files and their hashes for the old and new commits, excluding ignored files
+    OLD_FILES=$(git ls-tree -r $OLD_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+    NEW_FILES=$(git ls-tree -r $NEW_COMMIT | grep -vE "$(IFS='|'; echo "${VALIDATE_IGNORE_FILES[*]}")")
+
+    # Compare the diffs
+    if diff <(echo "$OLD_FILES") <(echo "$NEW_FILES") > /dev/null; then
+      # echo "Old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT are equivalent."
+      COUNT_SAME=$((COUNT_SAME + 1))
+    else
+      echo "[$CURRENT_COMMIT_NUMBER/$TOTAL_COMMITS] Difference found between old commit $OLD_COMMIT_SHORT and new commit $NEW_COMMIT_SHORT" >> commit-diff.txt
+      COUNT_DIFF=$((COUNT_DIFF + 1))
+      set +e
+      diff -u <(echo "$OLD_FILES") <(echo "$NEW_FILES") > "differences/$CURRENT_COMMIT_NUMBER-$OLD_COMMIT_SHORT-$NEW_COMMIT_SHORT"
+      set -e
+    fi
+  done < "commit-map.txt"
+
+  echo "Summary:" >> commit-diff.txt
+  echo "Equivalent commits: $COUNT_SAME" >> commit-diff.txt
+  echo "Different commits: $COUNT_DIFF" >> commit-diff.txt
+fi
+
+if [ ! -d $OUT ]; then
+  cp -r $SRC $OUT
+
+  cd $OUT
+
+  # remove all non-master branches
+  # TODO: need to see if we "redo" the other branches (except master, master-ci, devel, devel-staging, release3, release3-staging, dashcam3, dashcam3-staging, testing-closet*, hotfix-*)
+  # git branch | grep -v "^  master$" | grep -v "\*" | xargs git branch -D
+
+  # echo "cleaning up refs"
+  # delete pull request refs since we can't alter them anyway (https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/reviewing-changes-in-pull-requests/checking-out-pull-requests-locally#error-failed-to-push-some-refs)
+  # git for-each-ref --format='%(refname)' | grep '^refs/pull/' | xargs -I {} git update-ref -d {}
+
+  echo "importing new lfs files"
+  # import "almost" everything to lfs
+  git lfs migrate import --everything --include="*.dlc,*.onnx,*.svg,*.png,*.gif,*.ttf,*.wav,selfdrive/car/tests/test_models_segs.txt,system/hardware/tici/updater,selfdrive/ui/qt/spinner_larch64,selfdrive/ui/qt/text_larch64,third_party/**/*.a,third_party/**/*.so,third_party/**/*.so.*,third_party/**/*.dylib,third_party/acados/*/t_renderer,third_party/qt5/larch64/bin/lrelease,third_party/qt5/larch64/bin/lupdate,third_party/catch2/include/catch2/catch.hpp,*.apk,*.apkpatch,*.jar,*.pdf,*.jpg,*.mp3,*.thneed,*.tar.gz,*.npy,*.csv,*.a,*.so*,*.dylib,*.o,*.b64,selfdrive/hardware/tici/updater,selfdrive/boardd/tests/test_boardd,selfdrive/ui/qt/spinner_aarch64,installer/updater/updater,selfdrive/debug/profiling/simpleperf/**/*,selfdrive/hardware/eon/updater,selfdrive/ui/qt/text_aarch64,selfdrive/debug/profiling/pyflame/**/*,installer/installers/installer_openpilot,installer/installers/installer_dashcam,selfdrive/ui/text/text,selfdrive/ui/android/text/text,selfdrive/ui/spinner/spinner,selfdrive/visiond/visiond,selfdrive/loggerd/loggerd,selfdrive/sensord/sensord,selfdrive/sensord/gpsd,selfdrive/ui/android/spinner/spinner,selfdrive/ui/qt/spinner,selfdrive/ui/qt/text,_stringdefs.py,dfu-util-aarch64-linux,dfu-util-aarch64,dfu-util-x86_64-linux,dfu-util-x86_64,stb_image.h,clpeak3,clwaste,apk/**/*,external/**/*,phonelibs/**/*,third_party/boringssl/**/*,flask/**/*,panda/**/*,board/**/*,messaging/**/*,opendbc/**/*,tools/cabana/chartswidget.cc,third_party/nanovg/**/*,selfdrive/controls/lib/lateral_mpc/lib_mpc_export/**/*,selfdrive/ui/paint.cc,werkzeug/**/*,pyextra/**/*,third_party/android_hardware_libhardware/**/*,selfdrive/controls/lib/lead_mpc_lib/lib_mpc_export/**/*,selfdrive/locationd/laikad.py,selfdrive/locationd/test/test_laikad.py,tools/gpstest/test_laikad.py,selfdrive/locationd/laikad_helpers.py,tools/nui/**/*,jsonrpc/**/*,selfdrive/controls/lib/longitudinal_mpc/lib_mpc_export/**/*,selfdrive/controls/lib/lateral_mpc/mpc_export/**/*,selfdrive/camerad/cameras/camera_qcom.cc,selfdrive/manager.py,selfdrive/modeld/models/driving.cc,third_party/curl/**/*,selfdrive/modeld/thneed/debug/**/*,selfdrive/modeld/thneed/include/**/*,third_party/openmax/**/*,selfdrive/controls/lib/longitudinal_mpc/mpc_export/**/*,selfdrive/controls/lib/longitudinal_mpc_model/lib_mpc_export/**/*,Pipfile,Pipfile.lock,gunicorn/**/*,*.qm,jinja2/**/*,click/**/*,dbcs/**/*,websocket/**/*"
+
+  echo "reflog and gc"
+  # this is needed after lfs import
+  git reflog expire --expire=now --all
+  git gc --prune=now --aggressive
+
+  # check the git-filter-repo analysis again - can be found in the repo root/filter-repo/analysis
+  echo "Analyzing with git-filter-repo..."
+  /tmp/git-filter-repo --force --analyze
+
+  echo "New size is $(du -sh .)"
+fi
+
+cd $OUT
+
+# fetch all lfs files from https://github.com/commaai/openpilot.git
+# some lfs files are missing on gitlab, but they can be found on github
+git config lfs.url https://github.com/commaai/openpilot.git/info/lfs
+git config lfs.pushurl ssh://git@github.com/commaai/openpilot.git
+git lfs fetch --all || true
+
+# also fetch all lfs files from https://gitlab.com/commaai/openpilot-lfs.git
+git config lfs.url https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+git config lfs.pushurl ssh://git@gitlab.com/commaai/openpilot-lfs.git
+git lfs fetch --all || true
+
+# final push - will also push lfs
+# TODO: switch to git@github.com:commaai/openpilot.git when ready
+# if using mirror, it will also delete non-master branches (master-ci, nightly, devel, release3, release3-staging, dashcam3, release2)
+# git push --mirror git@github.com:commaai/openpilot-tiny.git
+# using this instead - since this is also what --mirror does - https://blog.plataformatec.com.br/2013/05/how-to-properly-mirror-a-git-repository/
+git push git@github.com:commaai/openpilot-tiny.git +refs/heads/*:refs/heads/* +refs/tags/*:refs/tags/*
@@ -0,0 +1,82 @@
+v0.1 e94a30bec07e719c5a7b037ca1f4db8312702cce
+v0.2 449b482cc3236ccf31829830b4f6a44b2dcc06c2
+v0.2.1 17d9becd3c673091b22f09aa02559a9ed9230f50
+v0.2.2 a64b9aa9b8cb5863c917b6926516291a63c02fe5
+v0.2.3 adaa4ed350acda4067fc0b455ad15b54cdf4c768
+v0.2.4 ecc565aa3fdc4c7e719aadc000e1fdc4d80d4fe0
+v0.2.5 29c58b45882ac79595356caf98580c1d2a626011
+v0.2.6 6c3afeec0fb439070b2912978b8dbb659033b1d9
+v0.2.7 c6ba5dc5391d3ca6cda479bf1923b88ce45509a0
+v0.2.8 95a349abcc050712c50d4d85a1c8a804eee7f6c2
+v0.2.9 693bcb0f83478f2651db6bac9be5ca5ad60d03f3
+v0.3.0 c5d8aec28b5230d34ae4b677c2091cc3dec7e3e8
+v0.3.1 41e3a0f699f5c39cb61a15c0eb7a4aa816d47c24
+v0.3.2 7fe46f1e1df5dec08a940451ba0feefd5c039165
+v0.3.3 5cf91d0496688fed4f2a6c7021349b1fc0e057a2
+v0.3.4 1b8c44b5067525a5d266b6e99799d8097da76a29
+v0.3.5 b111277f464cf66fa34b67819a83ea683e0f64df
+v0.4.0.2 da52d065a4c4f52d6017a537f3a80326f5af8bdc
+v0.4.1 4474b9b3718653aeb0aee26422caefb90460cc0e
+v0.4.2 28c0797d30175043bbfa31307b63aab4197cf996
+v0.4.4 9a9ff839a9b70cb2601d7696af743f5652395389
+v0.4.5 37285038d3f91fa1b49159c4a35a8383168e644f
+v0.4.6 c6df34f55ba8c5a911b60d3f9eb20e3fa45f68c1
+v0.4.7 ae5cb7a0dab8b1bed9d52292f9b4e8e66a0f8ec9
+v0.5 de33bc46452b1046387ee2b3a03191b2c71135fb
+v0.5.1 8f22f52235c48eada586795ac57edb22688e4d08
+v0.5.2 0129a8a4ff8da5314e8e4d4d3336e89667ff6d54
+v0.5.3 285c52eb693265a0a530543e9ca0aeb593a2a55e
+v0.5.4 a422246dc30bce11e970514f13f7c110f4470cc3
+v0.5.5 8f3539a27b28851153454eb737da9624cccaed2d
+v0.5.6 860a48765d1016ba226fb2c64aea35a45fe40e4a
+v0.5.7 9ce3045f139ee29bf0eea5ec59dfe7df9c3d2c51
+v0.5.8 2cee2e05ba0f3824fdbb8b957958800fa99071a1
+v0.5.9 ad145da3bcded0fe75306df02061d07a633963c3
+v0.5.10 ff4c1557d8358f158f4358788ff18ef93d2470ef
+v0.5.11 d1866845df423c6855e2b365ff230cf7d89a420b
+v0.5.12 f6e8ef27546e9a406724841e75f8df71cc4c2c97
+v0.5.13 dd34ccfe288ebda8e2568cf550994ae890379f45
+v0.6 60a20537c5f3fcc7f11946d81aebc8f90c08c117
+v0.6.1 cf5c4aeacb1703d0ffd35bdb5297d3494fee9a22
+v0.6.2 095ef5f9f60fca1b269aabcc3cfd322b17b9e674
+v0.6.3 d5f9caa82d80cdcc7f1b7748f2cf3ccbf94f82a3
+v0.6.4 58f376002e0c654fbc2de127765fa297cf694a33
+v0.6.5 70d17cd69b80e7627dcad8fd5b6438f2309ac307
+v0.6.6 d4eb5a6eafdd4803d09e6f3963918216cca5a81f
+v0.7 a2ae18d1dbd1e59c38ce22fa25ddffbd1d3084e3
+v0.7.1 1e1de64a1e59476b7b3d3558b92149246d5c3292
+v0.7.2 59bd58c940673b4c4a6a86f299022614bcf42b22
+v0.7.3 d7acd8b68f8131e0e714400cf124a3e228638643
+v0.7.4 e93649882c5e914eec4a8b8b593dc0587e497033
+v0.7.5 8abc0afe464626a461d2c7e192c912eeebeccc65
+v0.7.6 69aacd9d179fe6dd3110253a099c38b34cff7899
+v0.7.7 f1caed7299cdba5e45635d8377da6cc1e5fd7072
+v0.7.8 2189fe8741b635d8394d55dee28959425cfd5ad0
+v0.7.9 86dc54b836a973f132ed26db9f5a60b29f9b25b2
+v0.7.10 47a42ff432db8a2494e922ca5e767e58020f0446
+v0.7.11 f46ed718ba8d6bb4d42cd7b0f0150c406017c373
+v0.8 d56e04c0d960c8d3d4ab88b578dc508a2b4e07dc
+v0.8.1 cd6f26664cb8d32a13847d6648567c47c580e248
+v0.8.2 7cc0999aebfe63b6bb6dd83c1dff62c3915c4820
+v0.8.3 986500fe2f10870018f1fba1e5465476b8915977
+v0.8.4 f0d0b82b8d6f5f450952113e234d0a5a49e80c48
+v0.8.5 f5d9ddc6c2a2802a61e5ce590c6b6688bf736a69
+v0.8.6 75904ed7452c6cbfb2a70cd379a899d8a75b97c2
+v0.8.7 4f9e568019492126e236da85b5ca0a059f292900
+v0.8.8 a949a49d5efaaf2d881143d23e9fb5ff9e28e88c
+v0.8.9 a034926264cd1025c69d6ceb3fe444965f960b75
+v0.8.10 59accdd814398b884167c0f41dbf46dcccf0c29c
+v0.8.11 d630ec9092f039cb5e51c5dd6d92fc47b91407e4
+v0.8.12 57871c99031cf597ffa0d819057ac1401e129f32
+v0.8.13 e43e6e876513450d235124fcb711f1724ed9814c
+v0.8.14 71901c94dbbaa2f9f156a80c14cc7ea65219fc7c
+v0.8.15 5a7c2f90361e72e9c35e88abd2e11acdc4aba354
+v0.8.16 f41dc62a12cc0f3cb8c5453c0caa0ba21e1bd01e
+v0.9.0 58b84fb401a804967aa0dd5ee66fafa90194fd30
+v0.9.1 89f68bf0cbf53a81b0553d3816fdbe522f941fa1
+v0.9.2 c7d3b28b93faa6c955fb24bc64031512ee985ee9
+v0.9.3 8704c1ff952b5c85a44f50143bbd1a4f7b4887e2
+v0.9.4 fa310d9e2542cf497d92f007baec8fd751ffa99c
+v0.9.5 3b1e9017c560499786d8a0e46aaaeea65037acac
+v0.9.6 0b4d08fab8e35a264bc7383e878538f8083c33e5
+v0.9.7 f8cb04e4a8b032b72a909f68b808a50936184bee
@@ -4,35 +4,6 @@ from openpilot.selfdrive.car.chrysler.values import CAR
 Ecu = car.CarParams.Ecu

 FW_VERSIONS = {
-  CAR.CHRYSLER_PACIFICA_2017_HYBRID: {
-    (Ecu.combinationMeter, 0x742, None): [
-      b'68239262AH',
-      b'68239262AI',
-      b'68239262AJ',
-      b'68239263AH',
-      b'68239263AJ',
-    ],
-    (Ecu.srs, 0x744, None): [
-      b'68238840AH',
-    ],
-    (Ecu.fwdRadar, 0x753, None): [
-      b'68226356AI',
-    ],
-    (Ecu.eps, 0x75a, None): [
-      b'68288309AC',
-      b'68288309AD',
-    ],
-    (Ecu.engine, 0x7e0, None): [
-      b'68277480AV ',
-      b'68277480AX ',
-      b'68277480AZ ',
-    ],
-    (Ecu.hybrid, 0x7e2, None): [
-      b'05190175BF',
-      b'05190175BH',
-      b'05190226AK',
-    ],
-  },
  CAR.CHRYSLER_PACIFICA_2018: {
    (Ecu.combinationMeter, 0x742, None): [
      b'68227902AF',
@@ -181,26 +152,39 @@ FW_VERSIONS = {
  },
  CAR.CHRYSLER_PACIFICA_2018_HYBRID: {
    (Ecu.combinationMeter, 0x742, None): [
+      b'68239262AH',
+      b'68239262AI',
+      b'68239262AJ',
+      b'68239263AH',
+      b'68239263AJ',
      b'68358439AE',
      b'68358439AG',
    ],
    (Ecu.srs, 0x744, None): [
+      b'68238840AH',
      b'68358990AC',
      b'68405939AA',
    ],
    (Ecu.fwdRadar, 0x753, None): [
      b'04672758AA',
+      b'68226356AI',
    ],
    (Ecu.eps, 0x75a, None): [
+      b'68288309AC',
      b'68288309AD',
      b'68525339AA',
    ],
    (Ecu.engine, 0x7e0, None): [
+      b'68277480AV ',
+      b'68277480AX ',
+      b'68277480AZ ',
      b'68366580AI ',
      b'68366580AK ',
      b'68366580AM ',
    ],
    (Ecu.hybrid, 0x7e2, None): [
+      b'05190175BF',
+      b'05190175BH',
      b'05190226AI',
      b'05190226AK',
      b'05190226AM',
@@ -245,6 +229,7 @@ FW_VERSIONS = {
      b'68416680AE ',
      b'68416680AF ',
      b'68416680AG ',
+      b'68444228AC ',
      b'68444228AD ',
      b'68444228AE ',
      b'68444228AF ',
@@ -35,8 +35,8 @@ class CarInterface(CarInterfaceBase):
        ret.flags |= ChryslerFlags.HIGHER_MIN_STEERING_SPEED.value

    # Chrysler
-    if candidate in (CAR.CHRYSLER_PACIFICA_2017_HYBRID, CAR.CHRYSLER_PACIFICA_2018, CAR.CHRYSLER_PACIFICA_2018_HYBRID, \
-                     CAR.CHRYSLER_PACIFICA_2019_HYBRID, CAR.CHRYSLER_PACIFICA_2020, CAR.DODGE_DURANGO):
+    if candidate in (CAR.CHRYSLER_PACIFICA_2018, CAR.CHRYSLER_PACIFICA_2018_HYBRID, CAR.CHRYSLER_PACIFICA_2019_HYBRID,
+                     CAR.CHRYSLER_PACIFICA_2020, CAR.DODGE_DURANGO):
      ret.lateralTuning.init('pid')
      ret.lateralTuning.pid.kpBP, ret.lateralTuning.pid.kiBP = [[9., 20.], [9., 20.]]
      ret.lateralTuning.pid.kpV, ret.lateralTuning.pid.kiV = [[0.15, 0.30], [0.03, 0.05]]
@@ -32,34 +32,30 @@ class ChryslerCarSpecs(CarSpecs):

 class CAR(Platforms):
  # Chrysler
-  CHRYSLER_PACIFICA_2017_HYBRID = ChryslerPlatformConfig(
-    [ChryslerCarDocs("Chrysler Pacifica Hybrid 2017")],
-    ChryslerCarSpecs(mass=2242., wheelbase=3.089, steerRatio=16.2),
-  )
  CHRYSLER_PACIFICA_2018_HYBRID = ChryslerPlatformConfig(
-    [ChryslerCarDocs("Chrysler Pacifica Hybrid 2018")],
-    CHRYSLER_PACIFICA_2017_HYBRID.specs,
+    [ChryslerCarDocs("Chrysler Pacifica Hybrid 2017-18")],
+    ChryslerCarSpecs(mass=2242., wheelbase=3.089, steerRatio=16.2),
  )
  CHRYSLER_PACIFICA_2019_HYBRID = ChryslerPlatformConfig(
    [ChryslerCarDocs("Chrysler Pacifica Hybrid 2019-24")],
-    CHRYSLER_PACIFICA_2017_HYBRID.specs,
+    CHRYSLER_PACIFICA_2018_HYBRID.specs,
  )
  CHRYSLER_PACIFICA_2018 = ChryslerPlatformConfig(
    [ChryslerCarDocs("Chrysler Pacifica 2017-18")],
-    CHRYSLER_PACIFICA_2017_HYBRID.specs,
+    CHRYSLER_PACIFICA_2018_HYBRID.specs,
  )
  CHRYSLER_PACIFICA_2020 = ChryslerPlatformConfig(
    [
      ChryslerCarDocs("Chrysler Pacifica 2019-20"),
      ChryslerCarDocs("Chrysler Pacifica 2021-23", package="All"),
    ],
-    CHRYSLER_PACIFICA_2017_HYBRID.specs,
+    CHRYSLER_PACIFICA_2018_HYBRID.specs,
  )

  # Dodge
  DODGE_DURANGO = ChryslerPlatformConfig(
    [ChryslerCarDocs("Dodge Durango 2020-21")],
-    CHRYSLER_PACIFICA_2017_HYBRID.specs,
+    CHRYSLER_PACIFICA_2018_HYBRID.specs,
  )

  # Jeep
@@ -130,7 +130,8 @@ MIGRATION = {

  # Removal of platform_str, see https://github.com/commaai/openpilot/pull/31868/
  "COMMA BODY": BODY.COMMA_BODY,
-  "CHRYSLER PACIFICA HYBRID 2017": CHRYSLER.CHRYSLER_PACIFICA_2017_HYBRID,
+  "CHRYSLER PACIFICA HYBRID 2017": CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID,
+  "CHRYSLER_PACIFICA_2017_HYBRID": CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID,
  "CHRYSLER PACIFICA HYBRID 2018": CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID,
  "CHRYSLER PACIFICA HYBRID 2019": CHRYSLER.CHRYSLER_PACIFICA_2019_HYBRID,
  "CHRYSLER PACIFICA 2018": CHRYSLER.CHRYSLER_PACIFICA_2018,
@@ -15,7 +15,7 @@ class CarState(CarStateBase):
    super().__init__(CP)
    can_define = CANDefine(DBC[CP.carFingerprint]["pt"])
    if CP.transmissionType == TransmissionType.automatic:
-      self.shifter_values = can_define.dv["Gear_Shift_by_Wire_FD1"]["TrnRng_D_RqGsm"]
+      self.shifter_values = can_define.dv["PowertrainData_10"]["TrnRng_D_Rq"]

    self.vehicle_sensors_valid = False

@@ -69,7 +69,7 @@ class CarState(CarStateBase):

    # gear
    if self.CP.transmissionType == TransmissionType.automatic:
-      gear = self.shifter_values.get(cp.vl["Gear_Shift_by_Wire_FD1"]["TrnRng_D_RqGsm"])
+      gear = self.shifter_values.get(cp.vl["PowertrainData_10"]["TrnRng_D_Rq"])
      ret.gearShifter = self.parse_gear_shifter(gear)
    elif self.CP.transmissionType == TransmissionType.manual:
      ret.clutchPressed = cp.vl["Engine_Clutch_Data"]["CluPdlPos_Pc_Meas"] > 0
@@ -139,7 +139,7 @@ class CarState(CarStateBase):

    if CP.transmissionType == TransmissionType.automatic:
      messages += [
-        ("Gear_Shift_by_Wire_FD1", 10),
+        ("PowertrainData_10", 10),
      ]
    elif CP.transmissionType == TransmissionType.manual:
      messages += [
@@ -529,6 +529,7 @@ FW_VERSIONS = {
      b'28102-5MX-A900\x00\x00',
      b'28102-5MX-A910\x00\x00',
      b'28102-5MX-C001\x00\x00',
+      b'28102-5MX-C610\x00\x00',
      b'28102-5MX-C910\x00\x00',
      b'28102-5MX-D001\x00\x00',
      b'28102-5MX-D710\x00\x00',
@@ -585,6 +585,7 @@ FW_VERSIONS = {
      b'\xf1\x00DL3 MDPS C 1.00 1.02 56310-L7220 4DLHC102',
    ],
    (Ecu.fwdCamera, 0x7c4, None): [
+      b'\xf1\x00DL3HMFC  AT KOR LHD 1.00 1.01 99210-L2000 191022',
      b'\xf1\x00DL3HMFC  AT KOR LHD 1.00 1.02 99210-L2000 200309',
      b'\xf1\x00DL3HMFC  AT KOR LHD 1.00 1.04 99210-L2000 210527',
    ],
@@ -595,6 +596,7 @@ FW_VERSIONS = {
      b'\xf1\x00OS IEB \x02 210 \x02\x14 58520-K4000',
      b'\xf1\x00OS IEB \x02 212 \x11\x13 58520-K4000',
      b'\xf1\x00OS IEB \x03 210 \x02\x14 58520-K4000',
+      b'\xf1\x00OS IEB \x03 211 \x04\x02 58520-K4000',
      b'\xf1\x00OS IEB \x03 212 \x11\x13 58520-K4000',
      b'\xf1\x00OS IEB \r 105\x18\t\x18 58520-K4000',
    ],
@@ -39,7 +39,7 @@ routes = [

  CarTestRoute("0c94aa1e1296d7c6|2021-05-05--19-48-37", CHRYSLER.JEEP_GRAND_CHEROKEE),
  CarTestRoute("91dfedae61d7bd75|2021-05-22--20-07-52", CHRYSLER.JEEP_GRAND_CHEROKEE_2019),
-  CarTestRoute("420a8e183f1aed48|2020-03-05--07-15-29", CHRYSLER.CHRYSLER_PACIFICA_2017_HYBRID),
+  CarTestRoute("420a8e183f1aed48|2020-03-05--07-15-29", CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID),  # 2017
  CarTestRoute("43a685a66291579b|2021-05-27--19-47-29", CHRYSLER.CHRYSLER_PACIFICA_2018),
  CarTestRoute("378472f830ee7395|2021-05-28--07-38-43", CHRYSLER.CHRYSLER_PACIFICA_2018_HYBRID),
  CarTestRoute("8190c7275a24557b|2020-01-29--08-33-58", CHRYSLER.CHRYSLER_PACIFICA_2019_HYBRID),
@@ -62,6 +62,7 @@ class TestCarInterfaces:

    car_params = CarInterface.get_params(car_name, args['fingerprints'], args['car_fw'],
                                         experimental_long=args['experimental_long'], docs=False)
+    car_params = car_params.as_reader()
    car_interface = CarInterface(car_params, CarController, CarState)
    assert car_params
    assert car_interface
@@ -7,7 +7,6 @@ legend = ["LAT_ACCEL_FACTOR", "MAX_LAT_ACCEL_MEASURED", "FRICTION"]
 "CHEVROLET_VOLT" = [1.5961527626411784, 1.8422651988094612, 0.1572393918005158]
 "CHRYSLER_PACIFICA_2018" = [2.07140, 1.3366521181047952, 0.13776367250652022]
 "CHRYSLER_PACIFICA_2020" = [1.86206, 1.509076559398423, 0.14328246159386085]
-"CHRYSLER_PACIFICA_2017_HYBRID" = [1.79422, 1.06831764583744, 0.116237]
 "CHRYSLER_PACIFICA_2018_HYBRID" = [2.08887, 1.2943025830995154, 0.114818]
 "CHRYSLER_PACIFICA_2019_HYBRID" = [1.90120, 1.1958788168371808, 0.131520]
 "GENESIS_G70" = [3.8520195946707947, 2.354697063349854, 0.06830285485626221]
@@ -441,6 +441,7 @@ FW_VERSIONS = {
      b'\x01896630ZU8000\x00\x00\x00\x00',
      b'\x01896630ZU9000\x00\x00\x00\x00',
      b'\x01896630ZX4000\x00\x00\x00\x00',
+      b'\x01896630ZX7100\x00\x00\x00\x00',
      b'\x018966312L8000\x00\x00\x00\x00',
      b'\x018966312M0000\x00\x00\x00\x00',
      b'\x018966312M9000\x00\x00\x00\x00',
@@ -516,6 +517,7 @@ FW_VERSIONS = {
      b'\x018965B1254000\x00\x00\x00\x00',
      b'\x018965B1255000\x00\x00\x00\x00',
      b'\x018965B1256000\x00\x00\x00\x00',
+      b'\x018965B1270000\x00\x00\x00\x00',
      b'8965B12361\x00\x00\x00\x00\x00\x00',
      b'8965B12451\x00\x00\x00\x00\x00\x00',
      b'8965B16011\x00\x00\x00\x00\x00\x00',
@@ -72,9 +72,7 @@ class Controls:

    if CI is None:
      cloudlog.info("controlsd is waiting for CarParams")
-      with car.CarParams.from_bytes(self.params.get("CarParams", block=True)) as msg:
-        # TODO: this shouldn't need to be a builder
-        self.CP = msg.as_builder()
+      self.CP = messaging.log_from_bytes(self.params.get("CarParams", block=True), car.CarParams)
      cloudlog.info("controlsd got CarParams")

      # Uses car interface helper functions, altering state won't be considered by card for actuation
@@ -25,7 +25,7 @@ LOW_SPEED_Y = [15, 13, 10, 5]
 class LatControlTorque(LatControl):
  def __init__(self, CP, CI):
    super().__init__(CP, CI)
-    self.torque_params = CP.lateralTuning.torque
+    self.torque_params = CP.lateralTuning.torque.as_builder()
    self.pid = PIDController(self.torque_params.kp, self.torque_params.ki,
                             k_f=self.torque_params.kf, pos_limit=self.steer_max, neg_limit=-self.steer_max)
    self.torque_from_lateral_accel = CI.torque_from_lateral_accel()
@@ -21,7 +21,7 @@ class TestLatControl:
    CI = CarInterface(CP, CarController, CarState)
    VM = VehicleModel(CP)

-    controller = controller(CP, CI)
+    controller = controller(CP.as_reader(), CI)

    CS = car.CarState.new_message()
    CS.vEgo = 30
@@ -12,8 +12,7 @@ def plannerd_thread():

  cloudlog.info("plannerd is waiting for CarParams")
  params = Params()
-  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg:
-    CP = msg
+  CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
  cloudlog.info("plannerd got CarParams: %s", CP.carName)

  longitudinal_planner = LongitudinalPlanner(CP)
@@ -288,8 +288,7 @@ def main():

  # wait for stats about the car to come in from controls
  cloudlog.info("radard is waiting for CarParams")
-  with car.CarParams.from_bytes(Params().get("CarParams", block=True)) as msg:
-    CP = msg
+  CP = messaging.log_from_bytes(Params().get("CarParams", block=True), car.CarParams)
  cloudlog.info("radard got CarParams")

  # import the radar from the fingerprint
@@ -38,7 +38,7 @@ class PointBuckets:
  def is_calculable(self) -> bool:
    return all(len(v) > 0 for v in self.buckets.values())

-  def add_point(self, x: float, y: float, bucket_val: float) -> None:
+  def add_point(self, x: float, y: float) -> None:
    raise NotImplementedError

  def get_points(self, num_points: int = None) -> Any:
@@ -5,8 +5,7 @@ import json
 import numpy as np

 import cereal.messaging as messaging
-from cereal import car
-from cereal import log
+from cereal import car, log
 from openpilot.common.params import Params
 from openpilot.common.realtime import config_realtime_process, DT_MDL
 from openpilot.common.numpy_fast import clip
@@ -129,8 +128,7 @@ def main():
  params_reader = Params()
  # wait for stats about the car to come in from controls
  cloudlog.info("paramsd is waiting for CarParams")
-  with car.CarParams.from_bytes(params_reader.get("CarParams", block=True)) as msg:
-    CP = msg
+  CP = messaging.log_from_bytes(params_reader.get("CarParams", block=True), car.CarParams)
  cloudlog.info("paramsd got CarParams")

  min_sr, max_sr = 0.5 * CP.steerRatio, 2.0 * CP.steerRatio
@@ -36,8 +36,8 @@ ALLOWED_CARS = ['toyota', 'hyundai']


 def slope2rot(slope):
-  sin = np.sqrt(slope**2 / (slope**2 + 1))
-  cos = np.sqrt(1 / (slope**2 + 1))
+  sin = np.sqrt(slope ** 2 / (slope ** 2 + 1))
+  cos = np.sqrt(1 / (slope ** 2 + 1))
  return np.array([[cos, -sin], [sin, cos]])


@@ -50,9 +50,10 @@ class TorqueBuckets(PointBuckets):


 class TorqueEstimator(ParameterEstimator):
-  def __init__(self, CP, decimated=False):
+  def __init__(self, CP, decimated=False, track_all_points=False):
    self.hist_len = int(HISTORY / DT_MDL)
-    self.lag = CP.steerActuatorDelay + .2   # from controlsd
+    self.lag = CP.steerActuatorDelay + .2  # from controlsd
+    self.track_all_points = track_all_points  # for offline analysis, without max lateral accel or max steer torque filters
    if decimated:
      self.min_bucket_points = MIN_BUCKET_POINTS / 10
      self.min_points_total = MIN_POINTS_TOTAL_QLOG
@@ -119,7 +120,8 @@ class TorqueEstimator(ParameterEstimator):
    for param in initial_params:
      self.filtered_params[param] = FirstOrderFilter(initial_params[param], self.decay, DT_MDL)

-  def get_restore_key(self, CP, version):
+  @staticmethod
+  def get_restore_key(CP, version):
    a, b = None, None
    if CP.lateralTuning.which() == 'torque':
      a = CP.lateralTuning.torque.friction
@@ -135,6 +137,7 @@ class TorqueEstimator(ParameterEstimator):
                                         min_points_total=self.min_points_total,
                                         points_per_bucket=POINTS_PER_BUCKET,
                                         rowsize=3)
+    self.all_torque_points = []

  def estimate_params(self):
    points = self.filtered_points.get_points(self.fit_points)
@@ -159,25 +162,32 @@ class TorqueEstimator(ParameterEstimator):
  def handle_log(self, t, which, msg):
    if which == "carControl":
      self.raw_points["carControl_t"].append(t + self.lag)
-      self.raw_points["active"].append(msg.latActive)
+      self.raw_points["lat_active"].append(msg.latActive)
    elif which == "carOutput":
      self.raw_points["carOutput_t"].append(t + self.lag)
      self.raw_points["steer_torque"].append(-msg.actuatorsOutput.steer)
    elif which == "carState":
      self.raw_points["carState_t"].append(t + self.lag)
+      # TODO: check if high aEgo affects resulting lateral accel
      self.raw_points["vego"].append(msg.vEgo)
      self.raw_points["steer_override"].append(msg.steeringPressed)
+
+    # calculate lateral accel from past steering torque
    elif which == "liveLocationKalman":
      if len(self.raw_points['steer_torque']) == self.hist_len:
        yaw_rate = msg.angularVelocityCalibrated.value[2]
        roll = msg.orientationNED.value[0]
-        active = np.interp(np.arange(t - MIN_ENGAGE_BUFFER, t, DT_MDL), self.raw_points['carControl_t'], self.raw_points['active']).astype(bool)
+        lat_active = np.interp(np.arange(t - MIN_ENGAGE_BUFFER, t, DT_MDL), self.raw_points['carControl_t'], self.raw_points['lat_active']).astype(bool)
        steer_override = np.interp(np.arange(t - MIN_ENGAGE_BUFFER, t, DT_MDL), self.raw_points['carState_t'], self.raw_points['steer_override']).astype(bool)
        vego = np.interp(t, self.raw_points['carState_t'], self.raw_points['vego'])
-        steer = np.interp(t, self.raw_points['carOutput_t'], self.raw_points['steer_torque'])
-        lateral_acc = (vego * yaw_rate) - (np.sin(roll) * ACCELERATION_DUE_TO_GRAVITY)
-        if all(active) and (not any(steer_override)) and (vego > MIN_VEL) and (abs(steer) > STEER_MIN_THRESHOLD) and (abs(lateral_acc) <= LAT_ACC_THRESHOLD):
-          self.filtered_points.add_point(float(steer), float(lateral_acc))
+        steer = np.interp(t, self.raw_points['carOutput_t'], self.raw_points['steer_torque']).item()
+        lateral_acc = (vego * yaw_rate) - (np.sin(roll) * ACCELERATION_DUE_TO_GRAVITY).item()
+        if all(lat_active) and not any(steer_override) and (vego > MIN_VEL) and (abs(steer) > STEER_MIN_THRESHOLD):
+          if abs(lateral_acc) <= LAT_ACC_THRESHOLD:
+            self.filtered_points.add_point(steer, lateral_acc)
+
+          if self.track_all_points:
+            self.all_torque_points.append([steer, lateral_acc])

  def get_msg(self, valid=True, with_points=False):
    msg = messaging.new_message('liveTorqueParameters')
@@ -223,8 +233,7 @@ def main(demo=False):
  sm = messaging.SubMaster(['carControl', 'carOutput', 'carState', 'liveLocationKalman'], poll='liveLocationKalman')

  params = Params()
-  with car.CarParams.from_bytes(params.get("CarParams", block=True)) as CP:
-    estimator = TorqueEstimator(CP)
+  estimator = TorqueEstimator(messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams))

  while True:
    sm.update()
@@ -243,8 +252,10 @@ def main(demo=False):
      msg = estimator.get_msg(valid=sm.all_checks(), with_points=True)
      params.put_nonblocking("LiveTorqueParameters", msg.to_bytes())

+
 if __name__ == "__main__":
  import argparse
+
  parser = argparse.ArgumentParser(description='Process the --demo argument.')
  parser.add_argument('--demo', action='store_true', help='A boolean for demo mode.')
  args = parser.parse_args()
@@ -172,8 +172,8 @@ def main(demo=False):
  if demo:
    CP = get_demo_car_params()
  else:
-    with car.CarParams.from_bytes(params.get("CarParams", block=True)) as msg:
-      CP = msg
+    CP = messaging.log_from_bytes(params.get("CarParams", block=True), car.CarParams)
+
  cloudlog.info("modeld got CarParams: %s", CP.carName)

  # TODO this needs more thought, use .2s extra for now to estimate other delays
@@ -1 +1 @@
-8737e368e17f859291164286feb4246e00c0b4a5
+f6ff3601bd0496e78d8bc3b019d58bb7739f096b
@@ -116,7 +116,7 @@ CASES = {

 TEST_DIR = pathlib.Path(__file__).parent

-TEST_OUTPUT_DIR = TEST_DIR / "report"
+TEST_OUTPUT_DIR = TEST_DIR / "report_1"
 SCREENSHOTS_DIR = TEST_OUTPUT_DIR / "screenshots"


@@ -289,10 +289,9 @@ void ChartView::appendCanEvents(const cabana::Signal *sig, const std::vector<con
  step_vals.reserve(step_vals.size() + events.capacity() * 2);

  double value = 0;
-  const uint64_t begin_mono_time = can->routeStartTime() * 1e9;
  for (const CanEvent *e : events) {
    if (sig->getValue(e->dat, e->size, &value)) {
-      const double ts = (e->mono_time - std::min(e->mono_time, begin_mono_time)) / 1e9;
+      const double ts = can->toSeconds(e->mono_time);
      vals.emplace_back(ts, value);
      if (!step_vals.empty())
        step_vals.emplace_back(ts, step_vals.back().y());
@@ -312,7 +311,7 @@ void ChartView::updateSeries(const cabana::Signal *sig, const MessageEventsMap *
      auto it = events->find(s.msg_id);
      if (it == events->end() || it->second.empty()) continue;

-      if (s.vals.empty() || (it->second.back()->mono_time / 1e9 - can->routeStartTime()) > s.vals.back().x()) {
+      if (s.vals.empty() || can->toSeconds(it->second.back()->mono_time) > s.vals.back().x()) {
        appendCanEvents(s.sig, it->second, s.vals, s.step_vals);
      } else {
        std::vector<QPointF> vals, step_vals;
@@ -500,8 +499,8 @@ void ChartView::mouseReleaseEvent(QMouseEvent *event) {
    rubber->hide();
    auto rect = rubber->geometry().normalized();
    // Prevent zooming/seeking past the end of the route
-    double min = std::clamp(chart()->mapToValue(rect.topLeft()).x(), 0., can->totalSeconds());
-    double max = std::clamp(chart()->mapToValue(rect.bottomRight()).x(), 0., can->totalSeconds());
+    double min = std::clamp(chart()->mapToValue(rect.topLeft()).x(), can->minSeconds(), can->maxSeconds());
+    double max = std::clamp(chart()->mapToValue(rect.bottomRight()).x(), can->minSeconds(), can->maxSeconds());
    if (rubber->width() <= 0) {
      // no rubber dragged, seek to mouse position
      can->seekTo(min);
@@ -531,7 +530,7 @@ void ChartView::mouseMoveEvent(QMouseEvent *ev) {
  // Scrubbing
  if (is_scrubbing && QApplication::keyboardModifiers().testFlag(Qt::ShiftModifier)) {
    if (plot_area.contains(ev->pos())) {
-      can->seekTo(std::clamp(chart()->mapToValue(ev->pos()).x(), 0., can->totalSeconds()));
+      can->seekTo(std::clamp(chart()->mapToValue(ev->pos()).x(), can->minSeconds(), can->maxSeconds()));
    }
  }

@@ -540,8 +539,7 @@ void ChartView::mouseMoveEvent(QMouseEvent *ev) {
  clearTrackPoints();

  if (!is_zooming && plot_area.contains(ev->pos()) && isActiveWindow()) {
-    const double sec = chart()->mapToValue(ev->pos()).x();
-    charts_widget->showValueTip(sec);
+    charts_widget->showValueTip(secondsAtPoint(ev->pos()));
  } else if (tip_label->isVisible()) {
    charts_widget->showValueTip(-1);
  }
@@ -39,6 +39,7 @@ public:
  void showTip(double sec);
  void hideTip();
  void startAnimation();
+  double secondsAtPoint(const QPointF &pt) const { return chart()->mapToValue(pt).x(); }

  struct SigItem {
    MessageId msg_id;
@@ -93,7 +93,7 @@ ChartsWidget::ChartsWidget(QWidget *parent) : QFrame(parent) {
  current_theme = settings.theme;
  column_count = std::clamp(settings.chart_column_count, 1, MAX_COLUMN_COUNT);
  max_chart_range = std::clamp(settings.chart_range, 1, settings.max_cached_minutes * 60);
-  display_range = {0, max_chart_range};
+  display_range = std::make_pair(can->minSeconds(), can->minSeconds() + max_chart_range);
  range_slider->setValue(max_chart_range);
  updateToolBar();

@@ -192,10 +192,10 @@ void ChartsWidget::updateState() {
  if (!time_range.has_value()) {
    double pos = (cur_sec - display_range.first) / std::max<float>(1.0, max_chart_range);
    if (pos < 0 || pos > 0.8) {
-      display_range.first = std::max(0.0, cur_sec - max_chart_range * 0.1);
+      display_range.first = std::max(can->minSeconds(), cur_sec - max_chart_range * 0.1);
    }
-    double max_sec = std::min(display_range.first + max_chart_range, can->totalSeconds());
-    display_range.first = std::max(0.0, max_sec - max_chart_range);
+    double max_sec = std::min(display_range.first + max_chart_range, can->maxSeconds());
+    display_range.first = std::max(can->minSeconds(), max_sec - max_chart_range);
    display_range.second = display_range.first + max_chart_range;
  }

@@ -435,14 +435,20 @@ void ChartsWidget::alignCharts() {

 bool ChartsWidget::eventFilter(QObject *o, QEvent *e) {
  if (value_tip_visible_ && e->type() == QEvent::MouseMove) {
-    auto pos = static_cast<QMouseEvent *>(e)->globalPos();
-    bool outside_plot_area =std::none_of(charts.begin(), charts.end(), [&pos](auto c) {
-      return c->chart()->plotArea().contains(c->mapFromGlobal(pos));
-    });
+    bool on_tip = qobject_cast<TipLabel *>(o) != nullptr;
+    auto global_pos = static_cast<QMouseEvent *>(e)->globalPos();

-    if (outside_plot_area) {
-      showValueTip(-1);
+    for (const auto &c : charts) {
+      auto local_pos = c->mapFromGlobal(global_pos);
+      if (c->chart()->plotArea().contains(local_pos)) {
+        if (on_tip) {
+          showValueTip(c->secondsAtPoint(local_pos));
+        }
+        return false;
+      }
    }
+
+    showValueTip(-1);
  }
  return false;
 }
@@ -8,10 +8,11 @@

 void Sparkline::update(const MessageId &msg_id, const cabana::Signal *sig, double last_msg_ts, int range, QSize size) {
  const auto &msgs = can->events(msg_id);
-  uint64_t ts = (last_msg_ts + can->routeStartTime()) * 1e9;
-  uint64_t first_ts = (ts > range * 1e9) ? ts - range * 1e9 : 0;
-  auto first = std::lower_bound(msgs.cbegin(), msgs.cend(), first_ts, CompareCanEvent());
-  auto last = std::upper_bound(first, msgs.cend(), ts, CompareCanEvent());
+
+  auto range_start = can->toMonoTime(last_msg_ts - range);
+  auto range_end = can->toMonoTime(last_msg_ts);
+  auto first = std::lower_bound(msgs.cbegin(), msgs.cend(), range_start, CompareCanEvent());
+  auto last = std::upper_bound(first, msgs.cend(), range_end, CompareCanEvent());

  if (first == last || size.isEmpty()) {
    pixmap = QPixmap();
@@ -3,6 +3,8 @@
 #include <QLabel>

 class TipLabel : public QLabel {
+  Q_OBJECT
+
 public:
  TipLabel(QWidget *parent = nullptr);
  void showText(const QPoint &pt, const QString &sec, QWidget *w, const QRect &rect);
@@ -1,5 +1,6 @@
 #include "tools/cabana/dbc/dbcmanager.h"

+#include <QSet>
 #include <algorithm>
 #include <numeric>

@@ -124,16 +125,16 @@ cabana::Msg *DBCManager::msg(uint8_t source, const QString &name) {

 QStringList DBCManager::signalNames() {
  // Used for autocompletion
-  QStringList ret;
+  QSet<QString> names;
  for (auto &f : allDBCFiles()) {
    for (auto &[_, m] : f->getMessages()) {
      for (auto sig : m.getSignals()) {
-        ret << sig->name;
+        names.insert(sig->name);
      }
    }
  }
+  QStringList ret = names.values();
  ret.sort();
-  ret.removeDuplicates();
  return ret;
 }

@@ -13,7 +13,7 @@ QVariant HistoryLogModel::data(const QModelIndex &index, int role) const {
  const auto &m = messages[index.row()];
  const int col = index.column();
  if (role == Qt::DisplayRole) {
-    if (col == 0) return QString::number((m.mono_time / (double)1e9) - can->routeStartTime(), 'f', 3);
+    if (col == 0) return QString::number(can->toSeconds(m.mono_time), 'f', 3);
    if (!isHexMode()) return sigs[col - 1]->formatValue(m.sig_values[col - 1], false);
  } else if (role == Qt::TextAlignmentRole) {
    return (uint32_t)(Qt::AlignRight | Qt::AlignVCenter);
@@ -80,7 +80,7 @@ void HistoryLogModel::updateState(bool clear) {
    messages.clear();
    endRemoveRows();
  }
-  uint64_t current_time = (can->lastMessage(msg_id).ts + can->routeStartTime()) * 1e9 + 1;
+  uint64_t current_time = can->toMonoTime(can->lastMessage(msg_id).ts) + 1;
  fetchData(messages.begin(), current_time, messages.empty() ? 0 : messages.front().mono_time);
 }

@@ -33,13 +33,17 @@ SignalModel::SignalModel(QObject *parent) : root(new Item), QAbstractItemModel(p
  QObject::connect(dbc(), &DBCManager::signalRemoved, this, &SignalModel::handleSignalRemoved);
 }

-void SignalModel::insertItem(SignalModel::Item *parent_item, int pos, const cabana::Signal *sig) {
-  Item *item = new Item{.sig = sig, .parent = parent_item, .title = sig->name, .type = Item::Sig};
-  parent_item->children.insert(pos, item);
+void SignalModel::insertItem(SignalModel::Item *root_item, int pos, const cabana::Signal *sig) {
+  Item *parent_item = new Item{.sig = sig, .parent = root_item, .title = sig->name, .type = Item::Sig};
+  root_item->children.insert(pos, parent_item);
  QString titles[]{"Name", "Size", "Receiver Nodes", "Little Endian", "Signed", "Offset", "Factor", "Type",
                   "Multiplex Value", "Extra Info", "Unit", "Comment", "Minimum Value", "Maximum Value", "Value Table"};
  for (int i = 0; i < std::size(titles); ++i) {
-    item->children.push_back(new Item{.sig = sig, .parent = item, .title = titles[i], .type = (Item::Type)(i + Item::Name)});
+    auto item = new Item{.sig = sig, .parent = parent_item, .title = titles[i], .type = (Item::Type)(i + Item::Name)};
+    parent_item->children.push_back(item);
+    if (item->type == Item::ExtraInfo) {
+      parent_item = item;
+    }
  }
 }

@@ -75,12 +79,7 @@ SignalModel::Item *SignalModel::getItem(const QModelIndex &index) const {
 int SignalModel::rowCount(const QModelIndex &parent) const {
  if (parent.isValid() && parent.column() > 0) return 0;

-  auto parent_item = getItem(parent);
-  int row_count = parent_item->children.size();
-  if (parent_item->type == Item::Sig && !parent_item->extra_expanded) {
-    row_count -= (Item::Desc - Item::ExtraInfo);
-  }
-  return row_count;
+  return getItem(parent)->children.size();
 }

 Qt::ItemFlags SignalModel::flags(const QModelIndex &index) const {
@@ -88,7 +87,7 @@ Qt::ItemFlags SignalModel::flags(const QModelIndex &index) const {

  auto item = getItem(index);
  Qt::ItemFlags flags = Qt::ItemIsSelectable | Qt::ItemIsEnabled;
-  if (index.column() == 1  && item->type != Item::Sig && item->type != Item::ExtraInfo) {
+  if (index.column() == 1  && item->children.empty()) {
    flags |= (item->type == Item::Endian || item->type == Item::Signed) ? Qt::ItemIsUserCheckable : Qt::ItemIsEditable;
  }
  if (item->type == Item::MultiplexValue && item->sig->type != cabana::Signal::Type::Multiplexed) {
@@ -153,8 +152,6 @@ QVariant SignalModel::data(const QModelIndex &index, int role) const {
    } else if (role == Qt::CheckStateRole && index.column() == 1) {
      if (item->type == Item::Endian) return item->sig->is_little_endian ? Qt::Checked : Qt::Unchecked;
      if (item->type == Item::Signed) return item->sig->is_signed ? Qt::Checked : Qt::Unchecked;
-    } else if (role == Qt::DecorationRole && index.column() == 0 && item->type == Item::ExtraInfo) {
-      return utils::icon(item->parent->extra_expanded ? "chevron-compact-down" : "chevron-compact-up");
    } else if (role == Qt::ToolTipRole && item->type == Item::Sig) {
      return (index.column() == 0) ? signalToolTip(item->sig) : QString();
    }
@@ -189,21 +186,6 @@ bool SignalModel::setData(const QModelIndex &index, const QVariant &value, int r
  return ret;
 }

-void SignalModel::showExtraInfo(const QModelIndex &index) {
-  auto item = getItem(index);
-  if (item->type == Item::ExtraInfo) {
-    if (!item->parent->extra_expanded) {
-      item->parent->extra_expanded = true;
-      beginInsertRows(index.parent(), Item::ExtraInfo - 2, Item::Desc - 2);
-      endInsertRows();
-    } else {
-      item->parent->extra_expanded = false;
-      beginRemoveRows(index.parent(), Item::ExtraInfo - 2, Item::Desc - 2);
-      endRemoveRows();
-    }
-  }
-}
-
 bool SignalModel::saveSignal(const cabana::Signal *origin_s, cabana::Signal &s) {
  auto msg = dbc()->msg(msg_id);
  if (s.name != origin_s->name && msg->sig(s.name) != nullptr) {
@@ -283,13 +265,9 @@ QSize SignalItemDelegate::sizeHint(const QStyleOptionViewItem &option, const QMo
      text += item->sig->type == cabana::Signal::Type::Multiplexor ? QString(" M ") : QString(" m%1 ").arg(item->sig->multiplex_value);
      spacing += (option.widget->style()->pixelMetric(QStyle::PM_FocusFrameHMargin) + 1) * 2;
    }
-    auto it = width_cache.find(text);
-    if (it == width_cache.end()) {
-      it = width_cache.insert(text, option.fontMetrics.width(text));
-    }
-    width = std::min<int>(option.widget->size().width() / 3.0, it.value() + spacing);
+    width = std::min<int>(option.widget->size().width() / 3.0, option.fontMetrics.width(text) + spacing);
  }
-  return {width, option.fontMetrics.height()};
+  return {width, option.fontMetrics.height() + option.widget->style()->pixelMetric(QStyle::PM_FocusFrameVMargin) * 2};
 }

 void SignalItemDelegate::updateEditorGeometry(QWidget *editor, const QStyleOptionViewItem &option, const QModelIndex &index) const {
@@ -305,51 +283,55 @@ void SignalItemDelegate::updateEditorGeometry(QWidget *editor, const QStyleOptio
 }

 void SignalItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &option, const QModelIndex &index) const {
-  auto item = (SignalModel::Item *)index.internalPointer();
-  if (item && item->type == SignalModel::Item::Sig) {
-    painter->setRenderHint(QPainter::Antialiasing);
-    if (option.state & QStyle::State_Selected) {
-      painter->fillRect(option.rect, option.palette.brush(QPalette::Normal, QPalette::Highlight));
-    }
+  const int h_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameHMargin) + 1;
+  const int v_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameVMargin);
+  auto item = static_cast<SignalModel::Item*>(index.internalPointer());

-    int h_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameHMargin) + 1;
-    int v_margin = option.widget->style()->pixelMetric(QStyle::PM_FocusFrameVMargin);
-    QRect r = option.rect.adjusted(h_margin, v_margin, -h_margin, -v_margin);
-    if (index.column() == 0) {
+  QRect rect = option.rect.adjusted(h_margin, v_margin, -h_margin, -v_margin);
+  painter->setRenderHint(QPainter::Antialiasing);
+  if (option.state & QStyle::State_Selected) {
+    painter->fillRect(option.rect, option.palette.brush(QPalette::Normal, QPalette::Highlight));
+  }
+
+  if (index.column() == 0) {
+    if (item->type == SignalModel::Item::Sig) {
      // color label
      QPainterPath path;
-      QRect icon_rect{r.x(), r.y(), color_label_width, r.height()};
+      QRect icon_rect{rect.x(), rect.y(), color_label_width, rect.height()};
      path.addRoundedRect(icon_rect, 3, 3);
      painter->setPen(item->highlight ? Qt::white : Qt::black);
      painter->setFont(label_font);
      painter->fillPath(path, item->sig->color.darker(item->highlight ? 125 : 0));
      painter->drawText(icon_rect, Qt::AlignCenter, QString::number(item->row() + 1));

-      r.setLeft(icon_rect.right() + h_margin * 2);
+      rect.setLeft(icon_rect.right() + h_margin * 2);
      // multiplexer indicator
      if (item->sig->type != cabana::Signal::Type::Normal) {
        QString indicator = item->sig->type == cabana::Signal::Type::Multiplexor ? QString(" M ") : QString(" m%1 ").arg(item->sig->multiplex_value);
-        QRect indicator_rect{r.x(), r.y(), option.fontMetrics.width(indicator), r.height()};
+        QRect indicator_rect{rect.x(), rect.y(), option.fontMetrics.width(indicator), rect.height()};
        painter->setBrush(Qt::gray);
        painter->setPen(Qt::NoPen);
        painter->drawRoundedRect(indicator_rect, 3, 3);
        painter->setPen(Qt::white);
        painter->drawText(indicator_rect, Qt::AlignCenter, indicator);
-        r.setLeft(indicator_rect.right() + h_margin * 2);
+        rect.setLeft(indicator_rect.right() + h_margin * 2);
      }
+    } else {
+      rect.setLeft(option.widget->style()->pixelMetric(QStyle::PM_TreeViewIndentation) + color_label_width + h_margin * 3);
+    }

-      // name
-      auto text = option.fontMetrics.elidedText(index.data(Qt::DisplayRole).toString(), Qt::ElideRight, r.width());
-      painter->setPen(option.palette.color(option.state & QStyle::State_Selected ? QPalette::HighlightedText : QPalette::Text));
-      painter->setFont(option.font);
-      painter->drawText(r, option.displayAlignment, text);
-    } else if (index.column() == 1 && !item->sparkline.pixmap.isNull()) {
-      // sparkline
+    // name
+    auto text = option.fontMetrics.elidedText(index.data(Qt::DisplayRole).toString(), Qt::ElideRight, rect.width());
+    painter->setPen(option.palette.color(option.state & QStyle::State_Selected ? QPalette::HighlightedText : QPalette::Text));
+    painter->setFont(option.font);
+    painter->drawText(rect, option.displayAlignment, text);
+  } else if (index.column() == 1) {
+    if (!item->sparkline.pixmap.isNull()) {
      QSize sparkline_size = item->sparkline.pixmap.size() / item->sparkline.pixmap.devicePixelRatio();
-      painter->drawPixmap(QRect(r.topLeft(), sparkline_size), item->sparkline.pixmap);
+      painter->drawPixmap(QRect(rect.topLeft(), sparkline_size), item->sparkline.pixmap);
      // min-max value
      painter->setPen(option.palette.color(option.state & QStyle::State_Selected ? QPalette::HighlightedText : QPalette::Text));
-      QRect rect = r.adjusted(sparkline_size.width() + 1, 0, 0, 0);
+      rect.adjust(sparkline_size.width() + 1, 0, 0, 0);
      int value_adjust = 10;
      if (!item->sparkline.isEmpty() && (item->highlight || option.state & QStyle::State_Selected)) {
        painter->drawLine(rect.topLeft(), rect.bottomLeft());
@@ -361,7 +343,7 @@ void SignalItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op
        painter->drawText(rect, Qt::AlignLeft | Qt::AlignBottom, min);
        QFontMetrics fm(minmax_font);
        value_adjust = std::max(fm.width(min), fm.width(max)) + 5;
-      } else if (!item->sparkline.isEmpty()  && item->sig->type == cabana::Signal::Type::Multiplexed) {
+      } else if (!item->sparkline.isEmpty() && item->sig->type == cabana::Signal::Type::Multiplexed) {
        // display freq of multiplexed signal
        painter->setFont(label_font);
        QString freq = QString("%1 hz").arg(item->sparkline.freq(), 0, 'g', 2);
@@ -373,9 +355,9 @@ void SignalItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op
      rect.adjust(value_adjust, 0, -button_size.width(), 0);
      auto text = option.fontMetrics.elidedText(index.data(Qt::DisplayRole).toString(), Qt::ElideRight, rect.width());
      painter->drawText(rect, Qt::AlignRight | Qt::AlignVCenter, text);
+    } else {
+      QStyledItemDelegate::paint(painter, option, index);
    }
-  } else {
-    QStyledItemDelegate::paint(painter, option, index);
  }
 }

@@ -512,7 +494,6 @@ SignalView::SignalView(ChartsWidget *charts, QWidget *parent) : charts(charts),
 }

 void SignalView::setMessage(const MessageId &id) {
-  max_value_width = 0;
  filter_edit->clear();
  model->setMessage(id);
 }
@@ -549,11 +530,9 @@ void SignalView::rowsChanged() {

 void SignalView::rowClicked(const QModelIndex &index) {
  auto item = model->getItem(index);
-  if (item->type == SignalModel::Item::Sig) {
-    auto sig_index = model->index(index.row(), 0, index.parent());
-    tree->setExpanded(sig_index, !tree->isExpanded(sig_index));
-  } else if (item->type == SignalModel::Item::ExtraInfo) {
-    model->showExtraInfo(index);
+  if (item->type == SignalModel::Item::Sig || item->type == SignalModel::Item::ExtraInfo) {
+    auto expand_index = model->index(index.row(), 0, index.parent());
+    tree->setExpanded(expand_index, !tree->isExpanded(expand_index));
  }
 }

@@ -614,39 +593,54 @@ void SignalView::handleSignalUpdated(const cabana::Signal *sig) {
    updateState();
 }

+std::pair<QModelIndex, QModelIndex> SignalView::visibleSignalRange() {
+  auto topLevelIndex = [](QModelIndex index) {
+    while (index.isValid() && index.parent().isValid()) index = index.parent();
+    return index;
+  };
+
+  const auto viewport_rect = tree->viewport()->rect();
+  QModelIndex first_visible = tree->indexAt(viewport_rect.topLeft());
+  if (first_visible.parent().isValid()) {
+    first_visible = topLevelIndex(first_visible);
+    first_visible = first_visible.siblingAtRow(first_visible.row() + 1);
+  }
+
+  QModelIndex last_visible = topLevelIndex(tree->indexAt(viewport_rect.bottomRight()));
+  if (!last_visible.isValid()) {
+    last_visible = model->index(model->rowCount() - 1, 0);
+  }
+  return {first_visible, last_visible};
+}
+
 void SignalView::updateState(const std::set<MessageId> *msgs) {
  const auto &last_msg = can->lastMessage(model->msg_id);
  if (model->rowCount() == 0 || (msgs && !msgs->count(model->msg_id)) || last_msg.dat.size() == 0) return;

+  int max_value_width = 0;
  for (auto item : model->root->children) {
    double value = 0;
    if (item->sig->getValue(last_msg.dat.data(), last_msg.dat.size(), &value)) {
      item->sig_val = item->sig->formatValue(value);
+      max_value_width = std::max(max_value_width, fontMetrics().width(item->sig_val));
    }
-    max_value_width = std::max(max_value_width, fontMetrics().width(item->sig_val));
  }

-  QModelIndex top = tree->indexAt(QPoint(0, 0));
-  if (top.isValid()) {
-    // update visible sparkline
-    int first_visible_row = top.parent().isValid() ? top.parent().row() + 1 : top.row();
-    int last_visible_row = model->rowCount() - 1;
-    QModelIndex bottom = tree->indexAt(tree->viewport()->rect().bottomLeft());
-    if (bottom.isValid()) {
-      last_visible_row = bottom.parent().isValid() ? bottom.parent().row() : bottom.row();
-    }
-
+  auto [first_visible, last_visible] = visibleSignalRange();
+  if (first_visible.isValid() && last_visible.isValid()) {
    const static int min_max_width = QFontMetrics(delegate->minmax_font).width("-000.00") + 5;
    int available_width = value_column_width - delegate->button_size.width();
    int value_width = std::min<int>(max_value_width + min_max_width, available_width / 2);
    QSize size(available_width - value_width,
               delegate->button_size.height() - style()->pixelMetric(QStyle::PM_FocusFrameVMargin) * 2);
+
    QFutureSynchronizer<void> synchronizer;
-    for (int i = first_visible_row; i <= last_visible_row; ++i) {
+    for (int i = first_visible.row(); i <= last_visible.row(); ++i) {
      auto item = model->getItem(model->index(i, 1));
      synchronizer.addFuture(QtConcurrent::run(
          &item->sparkline, &Sparkline::update, model->msg_id, item->sig, last_msg.ts, settings.sparkline_range, size));
    }
+    synchronizer.waitForFinished();
  }

  for (int i = 0; i < model->rowCount(); ++i) {
@@ -2,6 +2,7 @@

 #include <memory>
 #include <set>
+#include <utility>

 #include <QAbstractItemModel>
 #include <QLabel>
@@ -29,7 +30,6 @@ public:
    const cabana::Signal *sig = nullptr;
    QString title;
    bool highlight = false;
-    bool extra_expanded = false;
    QString sig_val = "-";
    Sparkline sparkline;
  };
@@ -47,10 +47,9 @@ public:
  bool saveSignal(const cabana::Signal *origin_s, cabana::Signal &s);
  Item *getItem(const QModelIndex &index) const;
  int signalRow(const cabana::Signal *sig) const;
-  void showExtraInfo(const QModelIndex &index);

 private:
-  void insertItem(SignalModel::Item *parent_item, int pos, const cabana::Signal *sig);
+  void insertItem(SignalModel::Item *root_item, int pos, const cabana::Signal *sig);
  void handleSignalAdded(MessageId id, const cabana::Signal *sig);
  void handleSignalUpdated(const cabana::Signal *sig);
  void handleSignalRemoved(const cabana::Signal *sig);
@@ -92,7 +91,6 @@ public:
  QFont label_font, minmax_font;
  const int color_label_width = 18;
  mutable QSize button_size;
-  mutable QHash<QString, int> width_cache;
 };

 class SignalView : public QFrame {
@@ -119,6 +117,7 @@ private:
  void handleSignalAdded(MessageId id, const cabana::Signal *sig);
  void handleSignalUpdated(const cabana::Signal *sig);
  void updateState(const std::set<MessageId> *msgs = nullptr);
+  std::pair<QModelIndex, QModelIndex> visibleSignalRange();

  struct TreeView : public QTreeView {
    TreeView(QWidget *parent) : QTreeView(parent) {}
@@ -136,7 +135,6 @@ private:
      QTreeView::leaveEvent(event);
    }
  };
-  int max_value_width = 0;
  int value_column_width = 0;
  TreeView *tree;
  QLabel *sparkline_label;
@@ -145,5 +143,4 @@ private:
  ChartsWidget *charts;
  QLabel *signal_count_lb;
  SignalItemDelegate *delegate;
-  friend SignalItemDelegate;
 };
@@ -1,6 +1,5 @@
 #include "tools/cabana/streams/abstractstream.h"

-#include <algorithm>
 #include <utility>

 #include <QApplication>
@@ -20,9 +19,9 @@ AbstractStream::AbstractStream(QObject *parent) : QObject(parent) {
  assert(parent != nullptr);
  event_buffer_ = std::make_unique<MonotonicBuffer>(EVENT_NEXT_BUFFER_SIZE);

-  QObject::connect(QApplication::instance(), &QCoreApplication::aboutToQuit, this, &AbstractStream::stop);
  QObject::connect(this, &AbstractStream::privateUpdateLastMsgsSignal, this, &AbstractStream::updateLastMessages, Qt::QueuedConnection);
  QObject::connect(this, &AbstractStream::seekedTo, this, &AbstractStream::updateLastMsgsTo);
+  QObject::connect(this, &AbstractStream::seeking, this, [this](double sec) { current_sec_ = sec; });
  QObject::connect(dbc(), &DBCManager::DBCFileChanged, this, &AbstractStream::updateMasks);
  QObject::connect(dbc(), &DBCManager::maskUpdated, this, &AbstractStream::updateMasks);
  QObject::connect(this, &AbstractStream::streamStarted, [this]() {
@@ -64,14 +63,12 @@ void AbstractStream::suppressDefinedSignals(bool suppress) {
 size_t AbstractStream::suppressHighlighted() {
  std::lock_guard lk(mutex_);
  size_t cnt = 0;
-  const double cur_ts = currentSec();
  for (auto &[_, m] : messages_) {
    for (auto &last_change : m.last_changes) {
-      const double dt = cur_ts - last_change.ts;
+      const double dt = current_sec_ - last_change.ts;
      if (dt < 2.0) {
        last_change.suppressed = true;
      }
-      // clear bit change counts
      last_change.bit_change_counts.fill(0);
      cnt += last_change.suppressed;
    }
@@ -90,20 +87,16 @@ void AbstractStream::updateLastMessages() {
  auto prev_src_size = sources.size();
  auto prev_msg_size = last_msgs.size();
  std::set<MessageId> msgs;
+
  {
    std::lock_guard lk(mutex_);
-    double max_sec = 0;
    for (const auto &id : new_msgs_) {
      const auto &can_data = messages_[id];
-      max_sec = std::max(max_sec, can_data.ts);
+      current_sec_ = std::max(current_sec_, can_data.ts);
      last_msgs[id] = can_data;
      sources.insert(id.source);
    }
-
-    if (!new_msgs_.empty()) {
-      msgs = std::move(new_msgs_);
-      current_sec_ = max_sec;
-    }
+    msgs = std::move(new_msgs_);
  }

  if (time_range_ && (current_sec_ < time_range_->first || current_sec_ >= time_range_->second)) {
@@ -138,7 +131,7 @@ const std::vector<const CanEvent *> &AbstractStream::events(const MessageId &id)
  return it != events_.end() ? it->second : empty_events;
 }

-const CanData &AbstractStream::lastMessage(const MessageId &id) {
+const CanData &AbstractStream::lastMessage(const MessageId &id) const {
  static CanData empty_data = {};
  auto it = last_msgs.find(id);
  return it != last_msgs.end() ? it->second : empty_data;
@@ -148,15 +141,13 @@ const CanData &AbstractStream::lastMessage(const MessageId &id) {
 // updateLastMsgsTo is always called in UI thread.
 void AbstractStream::updateLastMsgsTo(double sec) {
  current_sec_ = sec;
-  uint64_t last_ts = (sec + routeStartTime()) * 1e9;
+  uint64_t last_ts = toMonoTime(sec);
  std::unordered_map<MessageId, CanData> msgs;
  msgs.reserve(events_.size());

  for (const auto &[id, ev] : events_) {
    auto it = std::upper_bound(ev.begin(), ev.end(), last_ts, CompareCanEvent());
    if (it != ev.begin()) {
-      auto prev = std::prev(it);
-      double ts = (*prev)->mono_time / 1e9 - routeStartTime();
      auto &m = msgs[id];
      double freq = 0;
      // Keep suppressed bits.
@@ -167,7 +158,9 @@ void AbstractStream::updateLastMsgsTo(double sec) {
                       std::back_inserter(m.last_changes),
                       [](const auto &change) { return CanData::ByteLastChange{.suppressed = change.suppressed}; });
      }
-      m.compute(id, (*prev)->dat, (*prev)->size, ts, getSpeed(), {}, freq);
+
+      auto prev = std::prev(it);
+      m.compute(id, (*prev)->dat, (*prev)->size, toSeconds((*prev)->mono_time), getSpeed(), {}, freq);
      m.count = std::distance(ev.begin(), prev) + 1;
    }
  }
@@ -213,7 +206,6 @@ void AbstractStream::mergeEvents(const std::vector<const CanEvent *> &events) {
    all_events_.insert(pos, events.cbegin(), events.cend());
    emit eventsMerged(msg_events);
  }
-  lastest_event_ts = all_events_.empty() ? 0 : all_events_.back()->mono_time;
 }

 namespace {
@@ -236,14 +228,18 @@ inline QColor blend(const QColor &a, const QColor &b) {
 // Calculate the frequency from the past one minute data
 double calc_freq(const MessageId &msg_id, double current_sec) {
  const auto &events = can->events(msg_id);
-  uint64_t cur_mono_time = (can->routeStartTime() + current_sec) * 1e9;
-  uint64_t first_mono_time = std::max<int64_t>(0, cur_mono_time - 59 * 1e9);
-  auto first = std::lower_bound(events.begin(), events.end(), first_mono_time, CompareCanEvent());
-  auto second = std::lower_bound(first, events.end(), cur_mono_time, CompareCanEvent());
-  if (first != events.end() && second != events.end()) {
-    double duration = ((*second)->mono_time - (*first)->mono_time) / 1e9;
-    uint32_t count = std::distance(first, second);
-    return count / std::max(1.0, duration);
+  if (events.empty()) return 0.0;
+
+  auto current_mono_time = can->toMonoTime(current_sec);
+  auto start_mono_time = can->toMonoTime(current_sec - 59);
+
+  auto first = std::lower_bound(events.begin(), events.end(), start_mono_time, CompareCanEvent());
+  auto last = std::upper_bound(first, events.end(), current_mono_time, CompareCanEvent());
+
+  int count = std::distance(first, last);
+  if (count > 1) {
+    double duration = ((*std::prev(last))->mono_time - (*first)->mono_time) / 1e9;
+    return count / duration;
  }
  return 0;
 }
@@ -1,5 +1,6 @@
 #pragma once

+#include <algorithm>
 #include <array>
 #include <memory>
 #include <mutex>
@@ -28,10 +29,10 @@ struct CanData {
  std::vector<QColor> colors;

  struct ByteLastChange {
-    double ts;
-    int delta;
-    int same_delta_counter;
-    bool suppressed;
+    double ts = 0;
+    int delta = 0;
+    int same_delta_counter = 0;
+    bool suppressed = false;
    std::array<uint32_t, 8> bit_change_counts;
  };
  std::vector<ByteLastChange> last_changes;
@@ -51,12 +52,6 @@ struct CompareCanEvent {
  constexpr bool operator()(uint64_t ts, const CanEvent *const e) const { return ts < e->mono_time; }
 };

-struct BusConfig {
-  int can_speed_kbps = 500;
-  int data_speed_kbps = 2000;
-  bool can_fd = false;
-};
-
 typedef std::unordered_map<MessageId, std::vector<const CanEvent *>> MessageEventsMap;

 class AbstractStream : public QObject {
@@ -66,15 +61,14 @@ public:
  AbstractStream(QObject *parent);
  virtual ~AbstractStream() {}
  virtual void start() = 0;
-  virtual void stop() {}
  virtual bool liveStreaming() const { return true; }
  virtual void seekTo(double ts) {}
  virtual QString routeName() const = 0;
  virtual QString carFingerprint() const { return ""; }
  virtual QDateTime beginDateTime() const { return {}; }
-  virtual double routeStartTime() const { return 0; }
-  inline double currentSec() const { return current_sec_; }
-  virtual double totalSeconds() const { return lastEventMonoTime() / 1e9 - routeStartTime(); }
+  virtual uint64_t beginMonoTime() const { return 0; }
+  virtual double minSeconds() const { return 0; }
+  virtual double maxSeconds() const { return 0; }
  virtual void setSpeed(float speed) {}
  virtual double getSpeed() { return 1; }
  virtual bool isPaused() const { return false; }
@@ -82,10 +76,14 @@ public:
  void setTimeRange(const std::optional<std::pair<double, double>> &range);
  const std::optional<std::pair<double, double>> &timeRange() const { return time_range_; }

+  inline double currentSec() const { return current_sec_; }
+  inline uint64_t toMonoTime(double sec) const { return beginMonoTime() + std::max(sec, 0.0) * 1e9; }
+  inline double toSeconds(uint64_t mono_time) const { return std::max(0.0, (mono_time - beginMonoTime()) / 1e9); }
+
  inline const std::unordered_map<MessageId, CanData> &lastMessages() const { return last_msgs; }
  inline const MessageEventsMap &eventsMap() const { return events_; }
  inline const std::vector<const CanEvent *> &allEvents() const { return all_events_; }
-  const CanData &lastMessage(const MessageId &id);
+  const CanData &lastMessage(const MessageId &id) const;
  const std::vector<const CanEvent *> &events(const MessageId &id) const;

  size_t suppressHighlighted();
@@ -111,12 +109,10 @@ protected:
  void mergeEvents(const std::vector<const CanEvent *> &events);
  const CanEvent *newEvent(uint64_t mono_time, const cereal::CanData::Reader &c);
  void updateEvent(const MessageId &id, double sec, const uint8_t *data, uint8_t size);
-  uint64_t lastEventMonoTime() const { return lastest_event_ts; }

  std::vector<const CanEvent *> all_events_;
  double current_sec_ = 0;
  std::optional<std::pair<double, double>> time_range_;
-  uint64_t lastest_event_ts = 0;

 private:
  void updateLastMessages();
@@ -92,6 +92,8 @@ void LiveStream::timerEvent(QTimerEvent *event) {
      // merge events received from live stream thread.
      std::lock_guard lk(lock);
      mergeEvents(received_events_);
+      uint64_t last_received_ts = !received_events_.empty() ? received_events_.back()->mono_time : 0;
+      lastest_event_ts = std::max(lastest_event_ts, last_received_ts);
      received_events_.clear();
    }
    if (!all_events_.empty()) {
@@ -136,8 +138,8 @@ void LiveStream::updateEvents() {
 void LiveStream::seekTo(double sec) {
  sec = std::max(0.0, sec);
  first_update_ts = nanos_since_boot();
-  current_event_ts = first_event_ts = std::min<uint64_t>(sec * 1e9 + begin_event_ts, lastEventMonoTime());
-  post_last_event = (first_event_ts == lastEventMonoTime());
+  current_event_ts = first_event_ts = std::min<uint64_t>(sec * 1e9 + begin_event_ts, lastest_event_ts);
+  post_last_event = (first_event_ts == lastest_event_ts);
  emit seekedTo((current_event_ts - begin_event_ts) / 1e9);
 }

@@ -1,5 +1,6 @@
 #pragma once

+#include <algorithm>
 #include <memory>
 #include <vector>

@@ -14,9 +15,10 @@ public:
  LiveStream(QObject *parent);
  virtual ~LiveStream();
  void start() override;
-  void stop() override;
+  void stop();
  inline QDateTime beginDateTime() const { return begin_date_time; }
-  inline double routeStartTime() const override { return begin_event_ts / 1e9; }
+  inline uint64_t beginMonoTime() const override { return begin_event_ts; }
+  double maxSeconds() const override { return std::max(1.0, (lastest_event_ts - begin_event_ts) / 1e9); }
  void setSpeed(float speed) override { speed_ = speed; }
  double getSpeed() override { return speed_; }
  bool isPaused() const override { return paused_; }
@@ -41,6 +43,7 @@ private:

  QDateTime begin_date_time;
  uint64_t begin_event_ts = 0;
+  uint64_t lastest_event_ts = 0;
  uint64_t current_event_ts = 0;
  uint64_t first_event_ts = 0;
  uint64_t first_update_ts = 0;
@@ -12,6 +12,12 @@
 const uint32_t speeds[] = {10U, 20U, 50U, 100U, 125U, 250U, 500U, 1000U};
 const uint32_t data_speeds[] = {10U, 20U, 50U, 100U, 125U, 250U, 500U, 1000U, 2000U, 5000U};

+struct BusConfig {
+  int can_speed_kbps = 500;
+  int data_speed_kbps = 2000;
+  bool can_fd = false;
+};
+
 struct PandaStreamConfig {
  QString serial = "";
  std::vector<BusConfig> bus_config;
@@ -73,6 +73,9 @@ bool ReplayStream::loadRoute(const QString &route, const QString &data_dir, uint
    } else if (replay->lastRouteError() == RouteLoadError::NetworkError) {
      QMessageBox::warning(nullptr, tr("Network Error"),
                          tr("Unable to load the route:\n\n %1.\n\nPlease check your network connection and try again.").arg(route));
+    } else if (replay->lastRouteError() == RouteLoadError::FileNotFound) {
+      QMessageBox::warning(nullptr, tr("Route Not Found"),
+                           tr("The specified route could not be found:\n\n %1.\n\nPlease check the route name and try again.").arg(route));
    } else {
      QMessageBox::warning(nullptr, tr("Route Load Failed"), tr("Failed to load route: '%1'").arg(route));
    }
@@ -85,16 +88,10 @@ void ReplayStream::start() {
  replay->start();
 }

-void ReplayStream::stop() {
-  if (replay) {
-    replay->stop();
-  }
-}
-
 bool ReplayStream::eventFilter(const Event *event) {
  static double prev_update_ts = 0;
  if (event->which == cereal::Event::Which::CAN) {
-    double current_sec = event->mono_time / 1e9 - routeStartTime();
+    double current_sec = toSeconds(event->mono_time);
    capnp::FlatArrayMessageReader reader(event->data);
    auto e = reader.getRoot<cereal::Event>();
    for (const auto &c : e.getCan()) {
@@ -112,11 +109,6 @@ bool ReplayStream::eventFilter(const Event *event) {
  return true;
 }

-void ReplayStream::seekTo(double ts) {
-  current_sec_ = ts;
-  replay->seekTo(std::max(double(0), ts), false);
-}
-
 void ReplayStream::pause(bool pause) {
  replay->pause(pause);
  emit(pause ? paused() : resume());
@@ -16,17 +16,16 @@ class ReplayStream : public AbstractStream {
 public:
  ReplayStream(QObject *parent);
  void start() override;
-  void stop() override;
  bool loadRoute(const QString &route, const QString &data_dir, uint32_t replay_flags = REPLAY_FLAG_NONE);
  bool eventFilter(const Event *event);
-  void seekTo(double ts) override;
+  void seekTo(double ts) override { replay->seekTo(std::max(double(0), ts), false); }
  bool liveStreaming() const override { return false; }
  inline QString routeName() const override { return replay->route()->name(); }
  inline QString carFingerprint() const override { return replay->carFingerprint().c_str(); }
-  double totalSeconds() const override { return replay->totalSeconds(); }
+  double minSeconds() const override { return replay->minSeconds(); }
+  double maxSeconds() const { return replay->maxSeconds(); }
  inline QDateTime beginDateTime() const { return replay->routeDateTime(); }
-  inline double routeStartTime() const override { return replay->routeStartTime() / (double)1e9; }
-  inline const Route *route() const { return replay->route(); }
+  inline uint64_t beginMonoTime() const override { return replay->routeStartNanos(); }
  inline void setSpeed(float speed) override { replay->setSpeed(speed); }
  inline float getSpeed() const { return replay->getSpeed(); }
  inline Replay *getReplay() const { return replay.get(); }
@@ -46,7 +46,7 @@ void FindSignalModel::search(std::function<bool(double)> cmp) {
    auto it = std::find_if(first, last, [&](const CanEvent *e) { return cmp(get_raw_value(e->dat, e->size, s.sig)); });
    if (it != last) {
      auto values = s.values;
-      values += QString("(%1, %2)").arg((*it)->mono_time / 1e9 - can->routeStartTime(), 0, 'f', 2).arg(get_raw_value((*it)->dat, (*it)->size, s.sig));
+      values += QString("(%1, %2)").arg(can->toSeconds((*it)->mono_time), 0, 'f', 3).arg(get_raw_value((*it)->dat, (*it)->size, s.sig));
      std::lock_guard lk(lock);
      filtered_signals.push_back({.id = s.id, .mono_time = (*it)->mono_time, .sig = s.sig, .values = values});
    }
@@ -217,10 +217,10 @@ void FindSignalDlg::setInitialSignals() {
  double first_time_val = first_time_edit->text().toDouble();
  double last_time_val = last_time_edit->text().toDouble();
  auto [first_sec, last_sec] = std::minmax(first_time_val, last_time_val);
-  uint64_t first_time = (can->routeStartTime() + first_sec) * 1e9;
+  uint64_t first_time = can->toMonoTime(first_sec);
  model->last_time = std::numeric_limits<uint64_t>::max();
  if (last_sec > 0) {
-    model->last_time = (can->routeStartTime() + last_sec) * 1e9;
+    model->last_time = can->toMonoTime(last_sec);
  }
  model->initial_signals.clear();

@@ -10,11 +10,10 @@ namespace utils {
 void exportToCSV(const QString &file_name, std::optional<MessageId> msg_id) {
  QFile file(file_name);
  if (file.open(QIODevice::ReadWrite | QIODevice::Truncate)) {
-    const uint64_t start_time = can->routeStartTime();
    QTextStream stream(&file);
    stream << "time,addr,bus,data\n";
    for (auto e : msg_id ? can->events(*msg_id) : can->allEvents()) {
-      stream << QString::number((e->mono_time / 1e9) - start_time, 'f', 2) << ","
+      stream << QString::number(can->toSeconds(e->mono_time), 'f', 3) << ","
             << "0x" << QString::number(e->address, 16) << "," << e->src << ","
             << "0x" << QByteArray::fromRawData((const char *)e->dat, e->size).toHex().toUpper() << "\n";
    }
@@ -30,9 +29,8 @@ void exportSignalsToCSV(const QString &file_name, const MessageId &msg_id) {
      stream << "," << s->name;
    stream << "\n";

-    const uint64_t start_time = can->routeStartTime();
    for (auto e : can->events(msg_id)) {
-      stream << QString::number((e->mono_time / 1e9) - start_time, 'f', 2) << ","
+      stream << QString::number(can->toSeconds(e->mono_time), 'f', 3) << ","
             << "0x" << QString::number(e->address, 16) << "," << e->src;
      for (auto s : msg->sigs) {
        double value = 0;
@@ -27,6 +27,13 @@ static const QColor timeline_colors[] = {
  [(int)TimelineType::AlertCritical] = QColor(199, 0, 57),
 };

+static Replay *getReplay() {
+  auto stream = qobject_cast<ReplayStream *>(can);
+  if (!stream) return nullptr;
+
+  return stream->getReplay();
+}
+
 VideoWidget::VideoWidget(QWidget *parent) : QFrame(parent) {
  setFrameStyle(QFrame::StyledPanel | QFrame::Plain);
  auto main_layout = new QVBoxLayout(this);
@@ -76,7 +83,7 @@ QHBoxLayout *VideoWidget::createPlaybackController() {
      // set speed to 1.0
      speed_btn->menu()->actions()[7]->setChecked(true);
      can->pause(false);
-      can->seekTo(can->totalSeconds() + 1);
+      can->seekTo(can->maxSeconds() + 1);
    });
  }

@@ -141,7 +148,7 @@ QWidget *VideoWidget::createCameraWidget() {

  QStackedLayout *stacked = new QStackedLayout();
  stacked->setStackingMode(QStackedLayout::StackAll);
-  stacked->addWidget(cam_widget = new CameraWidget("camerad", VISION_STREAM_ROAD, false));
+  stacked->addWidget(cam_widget = new StreamCameraView("camerad", VISION_STREAM_ROAD, false));
  cam_widget->setMinimumHeight(MIN_VIDEO_HEIGHT);
  cam_widget->setSizePolicy(QSizePolicy::Preferred, QSizePolicy::MinimumExpanding);
  stacked->addWidget(alert_label = new InfoLabel(this));
@@ -149,9 +156,11 @@ QWidget *VideoWidget::createCameraWidget() {

  l->addWidget(slider = new Slider(w));
  slider->setSingleStep(0);
+  slider->setTimeRange(can->minSeconds(), can->maxSeconds());

-  setMaximumTime(can->totalSeconds());
  QObject::connect(slider, &QSlider::sliderReleased, [this]() { can->seekTo(slider->currentSecond()); });
+  QObject::connect(can, &AbstractStream::paused, cam_widget, [c = cam_widget]() { c->showPausedOverlay(); });
+  QObject::connect(can, &AbstractStream::resume, cam_widget, [c = cam_widget]() { c->update(); });
  QObject::connect(can, &AbstractStream::eventsMerged, this, [this]() { slider->update(); });
  QObject::connect(cam_widget, &CameraWidget::clicked, []() { can->pause(!can->isPaused()); });
  QObject::connect(cam_widget, &CameraWidget::vipcAvailableStreamsUpdated, this, &VideoWidget::vipcAvailableStreamsUpdated);
@@ -161,7 +170,7 @@ QWidget *VideoWidget::createCameraWidget() {

  auto replay = static_cast<ReplayStream*>(can)->getReplay();
  QObject::connect(replay, &Replay::qLogLoaded, slider, &Slider::parseQLog, Qt::QueuedConnection);
-  QObject::connect(replay, &Replay::totalSecondsUpdated, this, &VideoWidget::setMaximumTime, Qt::QueuedConnection);
+  QObject::connect(replay, &Replay::minMaxTimeChanged, this, &VideoWidget::timeRangeChanged, Qt::QueuedConnection);
  return w;
 }

@@ -185,7 +194,7 @@ void VideoWidget::vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams
 }

 void VideoWidget::loopPlaybackClicked() {
-  auto replay = qobject_cast<ReplayStream *>(can)->getReplay();
+  auto replay = getReplay();
  if (!replay) return;

  if (replay->hasFlag(REPLAY_FLAG_NO_LOOP)) {
@@ -197,18 +206,15 @@ void VideoWidget::loopPlaybackClicked() {
  }
 }

-void VideoWidget::setMaximumTime(double sec) {
-  maximum_time = sec;
-  slider->setTimeRange(0, sec);
-}
-
-void VideoWidget::timeRangeChanged(const std::optional<std::pair<double, double>> &time_range) {
+void VideoWidget::timeRangeChanged() {
+  const auto time_range = can->timeRange();
  if (can->liveStreaming()) {
    skip_to_end_btn->setEnabled(!time_range.has_value());
    return;
  }
  time_range ? slider->setTimeRange(time_range->first, time_range->second)
-             : slider->setTimeRange(0, maximum_time);
+             : slider->setTimeRange(can->minSeconds(), can->maxSeconds());
+  updateState();
 }

 QString VideoWidget::formatTime(double sec, bool include_milliseconds) {
@@ -219,8 +225,9 @@ QString VideoWidget::formatTime(double sec, bool include_milliseconds) {

 void VideoWidget::updateState() {
  if (slider) {
-    if (!slider->isSliderDown())
+    if (!slider->isSliderDown()) {
      slider->setCurrentSecond(can->currentSec());
+    }
    alert_label->showAlert(slider->alertInfo(can->currentSec()));
    time_btn->setText(QString("%1 / %2").arg(formatTime(can->currentSec(), true),
                                             formatTime(slider->maximum() / slider->factor)));
@@ -242,15 +249,14 @@ Slider::Slider(QWidget *parent) : QSlider(Qt::Horizontal, parent) {
 }

 AlertInfo Slider::alertInfo(double seconds) {
-  uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
+  uint64_t mono_time = can->toMonoTime(seconds);
  auto alert_it = alerts.lower_bound(mono_time);
  bool has_alert = (alert_it != alerts.end()) && ((alert_it->first - mono_time) <= 1e8);
  return has_alert ? alert_it->second : AlertInfo{};
 }

 QPixmap Slider::thumbnail(double seconds)  {
-  uint64_t mono_time = (seconds + can->routeStartTime()) * 1e9;
-  auto it = thumbnails.lowerBound(mono_time);
+  auto it = thumbnails.lowerBound(can->toMonoTime(seconds));
  return it != thumbnails.end() ? it.value() : QPixmap();
 }

@@ -298,16 +304,18 @@ void Slider::paintEvent(QPaintEvent *ev) {
    p.fillRect(r, color);
  };

-  const auto replay = qobject_cast<ReplayStream *>(can)->getReplay();
-  for (auto [begin, end, type] : replay->getTimeline()) {
-    fillRange(begin, end, timeline_colors[(int)type]);
-  }
+  auto replay = getReplay();
+  if (replay) {
+    for (auto [begin, end, type] : replay->getTimeline()) {
+      fillRange(begin, end, timeline_colors[(int)type]);
+    }

-  QColor empty_color = palette().color(QPalette::Window);
-  empty_color.setAlpha(160);
-  for (const auto &[n, seg] : replay->segments()) {
-    if (!(seg && seg->isLoaded()))
-      fillRange(n * 60.0, (n + 1) * 60.0, empty_color);
+    QColor empty_color = palette().color(QPalette::Window);
+    empty_color.setAlpha(160);
+    for (const auto &[n, seg] : replay->segments()) {
+      if (!(seg && seg->isLoaded()))
+        fillRange(n * 60.0, (n + 1) * 60.0, empty_color);
+    }
  }

  QStyleOptionSlider opt;
@@ -411,3 +419,22 @@ void InfoLabel::paintEvent(QPaintEvent *event) {
    p.drawText(text_rect, Qt::AlignTop | Qt::AlignHCenter | Qt::TextWordWrap, text);
  }
 }
+
+StreamCameraView::StreamCameraView(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget *parent)
+    : CameraWidget(stream_name, stream_type, zoom, parent) {
+  fade_animation = new QPropertyAnimation(this, "overlayOpacity");
+  fade_animation->setDuration(500);
+  fade_animation->setStartValue(0.2f);
+  fade_animation->setEndValue(0.7f);
+}
+
+void StreamCameraView::paintGL() {
+  CameraWidget::paintGL();
+
+  if (can->isPaused()) {
+    QPainter p(this);
+    p.setPen(QColor(200, 200, 200, static_cast<int>(255 * overlay_opacity)));
+    p.setFont(QFont(font().family(), 16, QFont::Bold));
+    p.drawText(rect(), Qt::AlignCenter, tr("PAUSED"));
+  }
+}
@@ -4,10 +4,12 @@
 #include <memory>
 #include <optional>
 #include <set>
+#include <string>
 #include <utility>

 #include <QHBoxLayout>
 #include <QFrame>
+#include <QPropertyAnimation>
 #include <QSlider>
 #include <QTabBar>

@@ -57,16 +59,34 @@ private:
  InfoLabel *thumbnail_label;
 };

+class StreamCameraView : public CameraWidget {
+  Q_OBJECT
+  Q_PROPERTY(float overlayOpacity READ overlayOpacity WRITE setOverlayOpacity)
+
+public:
+  StreamCameraView(std::string stream_name, VisionStreamType stream_type, bool zoom, QWidget *parent = nullptr);
+  void paintGL() override;
+  void showPausedOverlay() { fade_animation->start(); }
+  float overlayOpacity() const { return overlay_opacity; }
+  void setOverlayOpacity(float opacity) {
+    overlay_opacity = opacity;
+    update();
+  }
+
+private:
+  float overlay_opacity;
+  QPropertyAnimation *fade_animation;
+};
+
 class VideoWidget : public QFrame {
  Q_OBJECT

 public:
  VideoWidget(QWidget *parnet = nullptr);
-  void setMaximumTime(double sec);

 protected:
  QString formatTime(double sec, bool include_milliseconds = false);
-  void timeRangeChanged(const std::optional<std::pair<double, double>> &time_range);
+  void timeRangeChanged();
  void updateState();
  void updatePlayBtnState();
  QWidget *createCameraWidget();
@@ -74,8 +94,7 @@ protected:
  void loopPlaybackClicked();
  void vipcAvailableStreamsUpdated(std::set<VisionStreamType> streams);

-  CameraWidget *cam_widget;
-  double maximum_time = 0;
+  StreamCameraView *cam_widget;
  QToolButton *time_btn = nullptr;
  ToolButton *seek_backward_btn = nullptr;
  ToolButton *play_btn = nullptr;
@@ -98,10 +98,10 @@ def auto_strategy(rlog_paths: LogPaths, qlog_paths: LogPaths, interactive: bool,
  missing_rlogs = [rlog is None or not valid_file(rlog) for rlog in rlog_paths].count(True)
  if missing_rlogs != 0:
    if interactive:
-      if input(f"{missing_rlogs} rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y":
+      if input(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, would you like to fallback to qlogs for those segments? (y/n) ").lower() != "y":
        return rlog_paths
    else:
-      cloudlog.warning(f"{missing_rlogs} rlogs were not found, falling back to qlogs for those segments...")
+      cloudlog.warning(f"{missing_rlogs}/{len(rlog_paths)} rlogs were not found, falling back to qlogs for those segments...")

    return [rlog if valid_file(rlog) else (qlog if valid_file(qlog) else None)
            for (rlog, qlog) in zip(rlog_paths, qlog_paths, strict=True)]
@@ -63,6 +63,7 @@ brew "qt@5"
 brew "zeromq"
 cask "gcc-arm-embedded"
 brew "portaudio"
+brew "gcc@13"
 EOS

 echo "[ ] finished brew install t=$SECONDS"
@@ -171,7 +171,7 @@ void ConsoleUI::updateStatus() {

  if (status != Status::Paused) {
    auto events = replay->events();
-    uint64_t current_mono_time = replay->routeStartTime() + replay->currentSeconds() * 1e9;
+    uint64_t current_mono_time = replay->routeStartNanos() + replay->currentSeconds() * 1e9;
    bool playing = !events->empty() && events->back().mono_time > current_mono_time;
    status = playing ? Status::Playing : Status::Waiting;
  }
@@ -262,10 +262,10 @@ void ConsoleUI::updateTimeline() {
  mvwhline(win, 2, 0, ' ', width);
  wattroff(win, COLOR_PAIR(Color::Disengaged));

-  const int total_sec = replay->totalSeconds();
+  const int total_sec = replay->maxSeconds() - replay->minSeconds();
  for (auto [begin, end, type] : replay->getTimeline()) {
-    int start_pos = (begin / total_sec) * width;
-    int end_pos = (end / total_sec) * width;
+    int start_pos = ((begin - replay->minSeconds()) / total_sec) * width;
+    int end_pos = ((end - replay->minSeconds()) / total_sec) * width;
    if (type == TimelineType::Engaged) {
      mvwchgat(win, 1, start_pos, end_pos - start_pos + 1, A_COLOR, Color::Engaged, NULL);
      mvwchgat(win, 2, start_pos, end_pos - start_pos + 1, A_COLOR, Color::Engaged, NULL);
@@ -280,7 +280,7 @@ void ConsoleUI::updateTimeline() {
    }
  }

-  int cur_pos = ((double)replay->currentSeconds() / total_sec) * width;
+  int cur_pos = ((replay->currentSeconds() - replay->minSeconds()) / total_sec) * width;
  wattron(win, COLOR_PAIR(Color::BrightWhite));
  mvwaddch(win, 0, cur_pos, ACS_VLINE);
  mvwaddch(win, 3, cur_pos, ACS_VLINE);
@@ -85,6 +85,7 @@ bool Replay::load() {
    return false;
  }
  rInfo("load route %s with %zu valid segments", qPrintable(route_->name()), segments_.size());
+  max_seconds_ = (segments_.rbegin()->first + 1) * 60;
  return true;
 }

@@ -108,7 +109,11 @@ void Replay::seekTo(double seconds, bool relative) {
    target_time = std::max(double(0.0), target_time);
    int target_segment = (int)target_time / 60;
    if (segments_.count(target_segment) == 0) {
-      rWarning("Can't seek to %d s segment %d is invalid", (int)target_time, target_segment);
+      rWarning("Can't seek to %.2f s segment %d is invalid", target_time, target_segment);
+      return true;
+    }
+    if (target_time > max_seconds_) {
+      rWarning("Can't seek to %.2f s, time is invalid", target_time);
      return true;
    }

@@ -193,15 +198,22 @@ void Replay::buildTimeline() {
      }
    }

+    if (it->first == route_segments.rbegin()->first) {
+      if (engaged) {
+        timeline.push_back({toSeconds(engaged_begin), toSeconds(log->events.back().mono_time), TimelineType::Engaged});
+      }
+      if (!alert_type.empty() && alert_size != cereal::ControlsState::AlertSize::NONE) {
+        timeline.push_back({toSeconds(alert_begin), toSeconds(log->events.back().mono_time), timeline_types[(int)alert_status]});
+      }
+
+      max_seconds_ = std::ceil(toSeconds(log->events.back().mono_time));
+      emit minMaxTimeChanged(route_segments.cbegin()->first * 60.0, max_seconds_);
+    }
    {
      std::lock_guard lk(timeline_lock);
      timeline_.insert(timeline_.end(), timeline.begin(), timeline.end());
      std::sort(timeline_.begin(), timeline_.end(), [](auto &l, auto &r) { return std::get<2>(l) < std::get<2>(r); });
    }
-
-    if (it->first == route_segments.rbegin()->first) {
-      emit totalSecondsUpdated(toSeconds(log->events.back().mono_time));
-    }
    emit qLogLoaded(log);
  }
 }
@@ -463,7 +475,7 @@ void Replay::streamThread() {
      int last_segment = segments_.rbegin()->first;
      if (current_segment_ >= last_segment && isSegmentMerged(last_segment)) {
        rInfo("reaches the end of route, restart from beginning");
-        QMetaObject::invokeMethod(this, std::bind(&Replay::seekTo, this, 0, false), Qt::QueuedConnection);
+        QMetaObject::invokeMethod(this, std::bind(&Replay::seekTo, this, minSeconds(), false), Qt::QueuedConnection);
      }
    }
  }
@@ -76,9 +76,10 @@ public:
  inline double currentSeconds() const { return double(cur_mono_time_ - route_start_ts_) / 1e9; }
  inline QDateTime routeDateTime() const { return route_date_time_; }
  inline QDateTime currentDateTime() const { return route_date_time_.addSecs(currentSeconds()); }
-  inline uint64_t routeStartTime() const { return route_start_ts_; }
+  inline uint64_t routeStartNanos() const { return route_start_ts_; }
  inline double toSeconds(uint64_t mono_time) const { return (mono_time - route_start_ts_) / 1e9; }
-  inline int totalSeconds() const { return (!segments_.empty()) ? (segments_.rbegin()->first + 1) * 60 : 0; }
+  inline double minSeconds() const { return !segments_.empty() ? segments_.begin()->first * 60 : 0; }
+  inline double maxSeconds() const { return max_seconds_; }
  inline void setSpeed(float speed) { speed_ = speed; }
  inline float getSpeed() const { return speed_; }
  inline const std::vector<Event> *events() const { return &events_; }
@@ -95,7 +96,7 @@ signals:
  void seeking(double sec);
  void seekedTo(double sec);
  void qLogLoaded(std::shared_ptr<LogReader> qlog);
-  void totalSecondsUpdated(double sec);
+  void minMaxTimeChanged(double min_sec, double max_sec);

 protected slots:
  void segmentLoadFinished(bool success);
@@ -133,6 +134,7 @@ protected:
  QDateTime route_date_time_;
  uint64_t route_start_ts_ = 0;
  std::atomic<uint64_t> cur_mono_time_ = 0;
+  std::atomic<double> max_seconds_ = 0;
  std::vector<Event> events_;
  std::set<int> merged_segments_;

@@ -39,6 +39,7 @@ RouteIdentifier Route::parseRoute(const QString &str) {
 }

 bool Route::load() {
+  err_ = RouteLoadError::None;
  if (route_.str.isEmpty() || (data_dir_.isEmpty() && route_.dongle_id.isEmpty())) {
    rInfo("invalid route format");
    return false;
@@ -76,6 +77,10 @@ bool Route::loadFromServer(int retries) {
      rWarning(">>  Unauthorized. Authenticate with tools/lib/auth.py  <<");
      err_ = RouteLoadError::AccessDenied;
      return false;
+    } else if (err == QNetworkReply::ContentNotFoundError) {
+      rWarning("The specified route could not be found on the server.");
+      err_ = RouteLoadError::FileNotFound;
+      return false;
    } else {
      err_ = RouteLoadError::NetworkError;
    }
@@ -323,7 +323,7 @@ void precise_nano_sleep(int64_t nanoseconds, std::atomic<bool> &should_exit) {
  req.tv_sec = nanoseconds / 1000000000;
  req.tv_nsec = nanoseconds % 1000000000;
  while (!should_exit) {
-#ifdef __APPLE_
+#ifdef __APPLE__
    int ret = nanosleep(&req, &rem);
    if (ret == 0 || errno != EINTR)
      break;