more

.clang-tidy

@@ -0,0 +1,19 @@
+---
+Checks: '
+bugprone-*,
+-bugprone-integer-division,
+-bugprone-narrowing-conversions,
+performance-*,
+clang-analyzer-*,
+misc-*,
+-misc-unused-parameters,
+modernize-*,
+-modernize-avoid-c-arrays,
+-modernize-deprecated-headers,
+-modernize-use-auto,
+-modernize-use-using,
+-modernize-use-nullptr,
+-modernize-use-trailing-return-type,
+'
+CheckOptions:
+...

.codespellignore

@@ -1,6 +0,0 @@
-Wen
-REGIST
-PullRequest
-cancelled
-FOF
-NoO

.devcontainer/.gitignore

@@ -0,0 +1,3 @@
+.Xauthority
+.env
+.host/

.devcontainer/Dockerfile

@@ -0,0 +1,18 @@
+FROM ghcr.io/commaai/openpilot-base:latest
+
+RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
+RUN python3 -m ensurepip --upgrade
+RUN pip3 install ipython jupyter jupyterlab
+
+RUN cd /tmp && \
+    ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
+    curl -L -o virtualgl.deb "https://github.com/VirtualGL/virtualgl/releases/download/3.1.1/virtualgl_3.1.1_$ARCH.deb" && \
+    dpkg -i virtualgl.deb
+
+RUN usermod -aG video batman
+
+USER batman
+
+RUN cd $HOME && \
+    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
+    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc

.devcontainer/container_post_create.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+TARGET_USER=batman
+source .devcontainer/.host/.env
+
+# override display flag for mac hosts
+if [[ $HOST_OS == darwin ]]; then
+  echo "Setting up DISPLAY override for macOS..."
+  cat <<EOF >> /home/$TARGET_USER/.bashrc
+source .devcontainer/.host/.env
+if [ -n "\$HOST_DISPLAY" ]; then
+  DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}')
+  export DISPLAY=host.docker.internal:\$DISPLAY_NUM
+fi
+EOF
+fi
+
+# setup virtualgl for mac hosts
+if [[ $HOST_OS == darwin ]]; then
+  echo "Setting up virtualgl for macOS..."
+  cat <<EOF >> /home/$TARGET_USER/.bashrc
+if [ -n "\$HOST_DISPLAY" ]; then
+  export VGL_PORT=10000
+  export VGL_CLIENT=host.docker.internal
+  export VGL_COMPRESS=rgb
+  export VGL_DISPLAY=:99
+  export VGL_FPS=60
+  # prevent vglrun from running exec
+  alias exec=:; source vglrun :; unalias exec
+fi
+EOF
+fi
+
+# These lines are temporary, to remain backwards compatible with old devcontainers
+# that were running as root and therefore had their caches written as root
+sudo chown -R $TARGET_USER: /tmp/scons_cache
+sudo chown -R $TARGET_USER: /tmp/comma_download_cache
+sudo chown -R $TARGET_USER: /home/batman/.comma

.devcontainer/container_post_start.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source .devcontainer/.host/.env
+
+# setup safe directories for submodules
+SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
+for DIR in $SUBMODULE_DIRS; do 
+  git config --global --add safe.directory "$PWD/$DIR"
+done
+
+# virtual display for virtualgl
+if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then
+  echo "Starting virtual display at :99 ..."
+  tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24
+fi

.devcontainer/devcontainer.json

@@ -0,0 +1,53 @@
+{
+  "name": "openpilot devcontainer",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "postCreateCommand": ".devcontainer/container_post_create.sh",
+  "postStartCommand": ".devcontainer/container_post_start.sh",
+  "initializeCommand": [".devcontainer/host_setup"],
+  "privileged": true,
+  "containerEnv": {
+    "DISPLAY": "${localEnv:DISPLAY}",
+    "PYTHONPATH": "${containerWorkspaceFolder}",
+    "TERM": "xterm-256color",
+    "force_color_prompt": "1"
+  },
+  "runArgs": [
+    "--volume=/dev:/dev",
+    "--volume=/tmp/.X11-unix:/tmp/.X11-unix",
+    "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority",
+    "--volume=${localEnv:HOME}/.comma:/home/batman/.comma",
+    "--volume=${localEnv:HOME}/.azure:/home/batman/.azure",
+    "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
+    "--shm-size=1G",
+    "--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux
+    "--publish=0.0.0.0:8070-8079:8070-8079"  // body ZMQ services
+  ],
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "installZsh": false,
+      "installOhMyZsh": false,
+      "upgradePackages": false,
+      "username": "batman"
+    },
+    "ghcr.io/devcontainers-contrib/features/gh-cli:1": {},
+    "ghcr.io/devcontainers/features/azure-cli:1": {}
+  },
+  "containerUser": "batman",
+  "remoteUser": "batman",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-vscode.cpptools",
+        "ms-toolsai.jupyter",
+        "guyskk.language-cython",
+        "lharri73.dbc"
+      ]
+    }
+  },
+  "mounts": [
+    "type=volume,source=scons_cache,target=/tmp/scons_cache"
+  ]
+}

.devcontainer/host_setup

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# pull base image
+if [[ -z $USE_LOCAL_IMAGE ]]; then
+  echo "Updating openpilot_base image if needed..."
+  docker pull ghcr.io/commaai/openpilot-base:latest
+fi
+
+# setup .host dir
+mkdir -p .devcontainer/.host
+
+# setup links to Xauthority
+XAUTHORITY_LINK=".devcontainer/.host/.Xauthority"
+rm -f $XAUTHORITY_LINK
+if [[ -z $XAUTHORITY ]]; then
+  echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
+  if ! [[ -f $HOME/.Xauthority ]]; then
+    echo "~/.XAuthority file does not exist. GUI tools may not work properly."
+    touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
+  else
+    ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
+  fi
+else
+    ln -sf $XAUTHORITY $XAUTHORITY_LINK
+fi
+
+# setup host env file
+HOST_INFO_FILE=".devcontainer/.host/.env"
+SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
+echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE
+echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE
+
+# run virtualgl if macos
+if [[ $SYSTEM == "darwin" ]]; then
+  echo
+  if [[ -f /opt/VirtualGL/bin/vglclient ]]; then
+    echo "Starting VirtualGL client at port 10000..."
+    VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log"
+    mkdir -p "$(dirname $VGL_LOG_FILE)"
+    /opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach
+  else
+    echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:"
+    echo
+    echo "  brew install --cask virtualgl"
+    echo
+  fi
+fi

.devcontainer/host_setup.cmd

@@ -0,0 +1,10 @@
+:: pull base image
+IF NOT DEFINED USE_LOCAL_IMAGE ^
+echo "Updating openpilot_base image if needed..." && ^
+docker pull ghcr.io/commaai/openpilot-base:latest
+
+:: setup .host dir
+mkdir .devcontainer\.host
+
+:: setup host env file
+echo "" > .devcontainer\.host\.env

.dockerignore

@@ -13,6 +13,27 @@
 *.o-*
 *.os
 *.os-*
+*.so
+*.a
 
 venv/
 .venv/
+
+notebooks
+phone
+massivemap
+neos
+installer
+chffr/app2
+chffr/backend/env
+selfdrive/nav
+selfdrive/baseui
+selfdrive/test/simulator2
+**/cache_data
+xx/plus
+xx/community
+xx/projects
+!xx/projects/eon_testing_master
+!xx/projects/map3d
+xx/ops
+xx/junk

.editorconfig

@@ -5,7 +5,7 @@ end_of_line = lf
 insert_final_newline = true
 trim_trailing_whitespace = true
 
-[*.{py,pyx,pxd}]
+[{*.py, *.pyx, *.pxd}]
 charset = utf-8
 indent_style = space
 indent_size = 2

.gitattributes

@@ -7,12 +7,12 @@
 *.png filter=lfs diff=lfs merge=lfs -text
 *.gif filter=lfs diff=lfs merge=lfs -text
 *.ttf filter=lfs diff=lfs merge=lfs -text
-*.otf filter=lfs diff=lfs merge=lfs -text
 *.wav filter=lfs diff=lfs merge=lfs -text
 
 selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
-system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
 third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text

.github/CODEOWNERS

@@ -1,11 +0,0 @@
-*       @sunnypilot/dev-internal
-/.github/ @devtekve @sunnyhaibin
-/release/ci/ @devtekve @sunnyhaibin
-/tinygrad_repo @devtekve @Discountchubbs
-/tinygrad/ @devtekve @Discountchubbs
-/selfdrive/controls/lib/longitudinal_planner.py @devtekve @Discountchubbs
-/selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py @devtekve @Discountchubbs
-/selfdrive/modeld/ @devtekve @Discountchubbs
-/sunnypilot/model* @devtekve @Discountchubbs
-/sunnypilot/sunnylink/ @devtekve
-/system/athena/ @devtekve

.github/ISSUE_TEMPLATE/car_bug_report.yml

@@ -0,0 +1,53 @@
+name: Car bug report
+description: For issues with a particular car make or model
+labels: ["car", "bug"]
+body:
+
+  - type: markdown
+    attributes:
+      value: >
+        Before creating a **bug report**, please check the following:
+          * Ensure you're running the latest openpilot release.
+          * Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
+          * Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
+          * Ensure you're running stock openpilot. We cannot look into bug reports from forks.
+
+        If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
+
+  - type: textarea
+    attributes:
+      label: Describe the bug
+      description: Also include a description of how to reproduce the bug
+    validations:
+      required: true
+
+  - type: input
+    id: car
+    attributes:
+      label: Which car does this affect?
+      placeholder: Toyota Prius 2017
+    validations:
+      required: true
+
+  - type: input
+    id: route
+    attributes:
+      label: Provide a route where the issue occurs
+      description: Ensure the route is fully uploaded at https://useradmin.comma.ai
+      placeholder: 77611a1fac303767|2020-05-11--16-37-07
+    validations:
+      required: true
+
+  - type: input
+    id: version
+    attributes:
+      label: openpilot version
+      description: If you're not on release, provide the commit hash
+      placeholder: 0.8.10
+    validations:
+      required: true
+
+  - type: textarea
+    attributes:
+      label: Additional info
+

.github/ISSUE_TEMPLATE/config.yml

@@ -1,12 +1,9 @@
 blank_issues_enabled: false
 contact_links:
-  - name: Car bug report
-    url: https://github.com/commaai/opendbc/issues/new
-    about: For issues with a particular car make or model
   - name: Join the Discord
     url: https://discord.comma.ai
     about: The community Discord is for both openpilot development and experience discussion
-  - name: Report driving behavior feedback
+  - name: Report model bugs
     url: https://discord.com/channels/469524606043160576/1254834193066623017
     about: Feedback for the driving and driver monitoring models goes in the #driving-feedback in Discord
   - name: Community Wiki

.github/ISSUE_TEMPLATE/enhancement.md

@@ -0,0 +1,8 @@
+---
+name: Enhancement
+about: For openpilot enhancement suggestions
+title: ''
+labels: 'enhancement'
+assignees: ''
+---
+

.github/PULL_REQUEST_TEMPLATE/bugfix.md

@@ -0,0 +1,15 @@
+---
+name: Bug fix
+about: For openpilot bug fixes
+title: ''
+labels: 'bugfix'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the bug and the fix. Also link the issue if it exists. -->
+
+**Verification**
+
+<!-- Explain how you tested this bug fix. -->

.github/PULL_REQUEST_TEMPLATE/car_bugfix.md

@@ -0,0 +1,19 @@
+---
+name: Car Bug fix
+about: For vehicle/brand specific bug fixes
+title: ''
+labels: 'car bug fix'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the bug and the fix. Also link the issue if it exists. -->
+
+**Verification**
+
+<!-- Explain how you tested this bug fix. -->
+
+**Route**
+
+Route: [a route with the bug fix]

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -0,0 +1,15 @@
+---
+name: Car port
+about: For new car ports
+title: ''
+labels: 'car port'
+assignees: ''
+---
+
+**Checklist**
+
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
+- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
+- [ ] route with openpilot:
+- [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -0,0 +1,13 @@
+---
+name: Fingerprint
+about: For adding fingerprints to existing cars
+title: ''
+labels: 'fingerprint'
+assignees: ''
+---
+
+**Car**
+Which car (make, model, year) this fingerprint is for
+
+**Route**
+A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/refactor.md

@@ -0,0 +1,15 @@
+---
+name: Refactor
+about: For code refactors
+title: ''
+labels: 'refactor'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of the refactor, including the goals it accomplishes. -->
+
+**Verification**
+
+<!-- Explain how you tested the refactor for regressions. -->

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -0,0 +1,31 @@
+---
+name: Tuning
+about: For openpilot tuning changes
+title: ''
+labels: 'tuning'
+assignees: ''
+---
+
+**Description**
+
+<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
+
+**Verification**
+
+<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
+Use the PlotJuggler tuning layout to compare planned versus actual behavior.
+
+Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
+
+Longitudinal:
+* Maintaining speed at 25, 40, 65mph
+* Driving up and down hills
+* Accelerating from a stop
+* Decelerating to a stop
+* Following large changes in set speed
+* Coming to a stop behind a lead car
+
+Lateral:
+* Straight driving at ~25, ~45 and ~65mph
+* Turns driving at ~25, ~45 and ~65mph
+-->

.github/build.py

@@ -0,0 +1,30 @@
+import pathlib
+
+GITHUB_FOLDER = pathlib.Path(__file__).parent
+
+PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
+
+order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
+
+def create_pull_request_template():
+  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
+    f.write("<!-- Please copy and paste the relevant template -->\n\n")
+
+    for t in order:
+      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
+      text = template.read_text()
+
+      # Remove metadata for GitHub
+      start = text.find("---")
+      end = text.find("---", start+1)
+      text = text[end + 4:]
+
+      # Remove comments
+      text = text.replace("<!-- ", "").replace("-->", "")
+
+      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
+      f.write(text)
+      f.write("\n\n")
+      f.write("-->\n\n")
+
+create_pull_request_template()

.github/labeler.yaml

@@ -1,10 +1,6 @@
-ci:
+CI / testing:
   - changed-files:
-    - any-glob-to-all-files: "{.github/**,**/test_*,**/test/**,Jenkinsfile}"
-
-chore:
-  - changed-files:
-      - any-glob-to-all-files: "{.github/**}"
+    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
 
 car:
   - changed-files:
@@ -16,7 +12,7 @@ simulation:
 
 ui:
   - changed-files:
-    - any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
+    - any-glob-to-all-files: 'selfdrive/ui/**'
 
 tools:
   - changed-files:
@@ -28,4 +24,4 @@ multilanguage:
 
 autonomy:
   - changed-files:
-    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit,sunnypilot/modeld*/models/**}"
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/pull_request_template.md

@@ -44,7 +44,7 @@ Explain how you tested this bug fix.
 
 **Checklist**
 
-- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
+- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
 - [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
 - [ ] route with openpilot:
 - [ ] route with stock system:

.github/release-drafter.yml

@@ -1,43 +0,0 @@
-exclude-labels:
-  - 'no-changelog'
-categories:
-  - title: '🚀 Features'
-    labels:
-      - 'feature'
-      - 'enhancement'
-  - title: '🐛 Bug Fixes'
-    collapse-after: 5
-    labels:
-      - 'fix'
-      - 'bugfix'
-      - 'bug'
-  - title: '🧰 Maintenance'
-    collapse-after: 5
-    label: 'chore'
-change-template: '- $TITLE @$AUTHOR (#$NUMBER)'
-change-title-escapes: '\<*_&'
-replacers:
-  - search: '/[Ss][Uu][Nn][Nn][Yy][Pp][Ii][Ll][Oo][Tt]/g'
-    replace: 'sunnypilot'
-  - search: '/\b[Ss][Pp]\b/g'
-    replace: 'SP'
-version-resolver:
-  major:
-    labels:
-      - 'major'
-  minor:
-    labels:
-      - 'minor'
-  patch:
-    labels:
-      - 'patch'
-  default: patch
-name-template: 'v$RESOLVED_VERSION 🚀'
-tag-template: 'v$RESOLVED_VERSION'
-version-template: "0.$MAJOR.$MINOR.$PATCH" # The day OP becomes v1, we need to bump this
-tag-prefix: "v0." # The day OP becomes v1, we need to bump this
-prerelease-identifier: "staging"
-template: |
-  ## Changes
-
-  $CHANGES

.github/workflows/auto-cache/action.yaml

@@ -0,0 +1,49 @@
+name: 'automatically cache based on current runner'
+
+inputs:
+  path:
+    description: 'path to cache'
+    required: true
+  key:
+    description: 'key'
+    required: true
+  restore-keys:
+    description: 'restore-keys'
+    required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'false'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup namespace cache
+      if: ${{ contains(runner.name, 'nsc') }}
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: ${{ inputs.path }}
+
+    - name: setup github cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v4'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -1,7 +1,7 @@
 name: "PR review"
 on:
   pull_request_target:
-    types: [ opened, reopened, synchronize, edited ]
+    types: [opened, reopened, synchronize, edited, edited]
 
 jobs:
   labeler:
@@ -9,15 +9,14 @@ jobs:
     permissions:
       contents: read
       pull-requests: write
-      issues: write
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: false
 
     # Label PRs
-    - uses: actions/labeler@v6
+    - uses: actions/labeler@v5.0.0
       with:
         dot: true
         configuration-path: .github/labeler.yaml
@@ -30,62 +29,25 @@ jobs:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       with:
         target: /^(?!master$).*/
-        exclude: /sunnypilot:.*/
+        exclude: /commaai:.*/
         change-to: ${{ github.base_ref }}
         already-exists-action: close_this
         already-exists-comment: "Your PR should be made against the `master` branch"
 
-  update-pr-labels:
-    name: Update fork's PR Labels
-    runs-on: ubuntu-latest
-    if: (github.event.pull_request.head.repo.fork && (contains(github.event_name, 'pull_request') && github.event.action == 'synchronize'))
-    env:
-      PR_LABEL: 'dev'
-      TRUST_FORK_PR_LABEL: 'trust-fork-pr'
-    steps:
-      - name: Check if PR has dev label
-        id: check-labels
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const prNumber = context.payload.pull_request.number;
-            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber
-            });
+    # Welcome comment
+    - name: "First timers PR"
+      uses: actions/first-interaction@v1
+      if: github.event.pull_request.head.repo.full_name != 'sunnypilot/sunnypilot'
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        pr-message: |
+            <!-- _(run_id **${{ github.run_id }}**)_ -->
+            Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
+            * Convert your PR to a draft unless it's ready to review
+            * Read the [contributing docs](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md)
+            * Before marking as "ready for review", ensure:
+              * the goal is clearly stated in the description
+              * all the tests are passing
+              * the change is [something we merge](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
+              * include a route or your device' dongle ID if relevant
 
-            const hasDevC3Label = labels.some(label => label.name === process.env.PR_LABEL);
-            const hasTrustLabel = labels.some(label => label.name === process.env.TRUST_FORK_PR_LABEL);
-
-            console.log(`PR #${prNumber} has ${process.env.PR_LABEL} label: ${hasDevC3Label}`);
-            console.log(`PR #${prNumber} has ${process.env.TRUST_FORK_PR_LABEL} label: ${hasTrustLabel}`);
-
-            core.setOutput('has-dev', hasDevC3Label ? 'true' : 'false');
-            core.setOutput('has-trust', hasTrustLabel ? 'true' : 'false');
-
-      - name: Remove trust-fork-pr label if present
-        if: steps.check-labels.outputs.has-dev == 'true' && steps.check-labels.outputs.has-trust == 'true'
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const prNumber = context.payload.pull_request.number;
-
-            await github.rest.issues.removeLabel({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              name: process.env.TRUST_FORK_PR_LABEL
-            });
-
-            console.log(`Removed '${process.env.TRUST_FORK_PR_LABEL}' label from PR #${prNumber} as it received new commits`);
-
-            // Add a comment to the PR
-            await github.rest.issues.createComment({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              body: `The \`${process.env.TRUST_FORK_PR_LABEL}\` label has been automatically removed because new commits were pushed to this PR. This PR will need to be re-reviewed before the label can be applied again.`
-            });

.github/workflows/badges.yaml

@@ -5,7 +5,9 @@ on:
   workflow_dispatch:
 
 env:
-  PYTHONPATH: ${{ github.workspace }}
+  BASE_IMAGE: openpilot-base
+  DOCKER_REGISTRY: ghcr.io/commaai
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:
@@ -15,13 +17,13 @@ jobs:
     permissions:
       contents: write
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
-        python3 selfdrive/ui/translations/create_badges.py
+        ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/ui/translations/create_badges.py"
 
         rm .gitattributes
 

.github/workflows/build-all-tinygrad-models.yaml

@@ -1,304 +0,0 @@
-name: Build and push all tinygrad models
-
-on:
-  workflow_dispatch:
-    inputs:
-      set_min_version:
-        description: 'Minimum selector version required for the models (see helpers.py or readme.md)'
-        required: true
-        type: string
-
-jobs:
-  setup:
-    runs-on: ubuntu-latest
-    outputs:
-      json_version: ${{ steps.get-json.outputs.json_version }}
-      recompiled_dir: ${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-      json_file: ${{ steps.get-json.outputs.json_file }}
-      model_matrix: ${{ steps.set-matrix.outputs.model_matrix }}
-      tinygrad_ref: ${{ steps.get-tinygrad-ref.outputs.tinygrad_ref }}
-    steps:
-      - name: Checkout sunnypilot repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot
-          path: sunnypilot
-          submodules: recursive
-
-      - name: Get tinygrad_repo ref
-        id: get-tinygrad-ref
-        run: |
-          cd sunnypilot
-          export PYTHONPATH=$(pwd)
-          ref=$(python3 sunnypilot/models/tinygrad_ref.py)
-          echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
-          echo "tinygrad_ref is $ref"
-
-      - name: Checkout docs repo (sunnypilot-models, gh-pages)
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Get next JSON version to use (from GitHub docs repo)
-        id: get-json
-        run: |
-          cd docs/docs
-          latest=$(ls driving_models_v*.json | sed -E 's/.*_v([0-9]+)\.json/\1/' | sort -n | tail -1)
-          next=$((latest+1))
-          json_file="driving_models_v${next}.json"
-          cp "driving_models_v${latest}.json" "$json_file"
-          echo "json_file=docs/docs/$json_file" >> $GITHUB_OUTPUT
-          echo "json_version=$((next+0))" >> $GITHUB_OUTPUT
-          echo "SRC_JSON_FILE=docs/docs/driving_models_v${latest}.json" >> $GITHUB_ENV
-
-      - name: Extract tinygrad models
-        id: set-matrix
-        working-directory: docs/docs
-        run: |
-          jq -c '[.bundles[] | select(.runner=="tinygrad") | {ref, display_name: (.display_name | gsub(" \\([^)]*\\)"; "")), is_20hz}]' "$(basename "${SRC_JSON_FILE}")" > matrix.json
-          echo "model_matrix=$(cat matrix.json)" >> $GITHUB_OUTPUT
-
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-      - run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo and create new recompiled dir
-        id: create-recompiled-dir
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          git checkout main
-          git sparse-checkout set --no-cone models/
-          cd models
-          latest_dir=$(ls -d recompiled* 2>/dev/null | sed -E 's/recompiled([0-9]+)/\1/' | sort -n | tail -1)
-          if [[ -z "$latest_dir" ]]; then
-            next_dir=1
-          else
-            next_dir=$((latest_dir+1))
-          fi
-          recompiled_dir="${next_dir}"
-          mkdir -p "recompiled${recompiled_dir}"
-          touch "recompiled${recompiled_dir}/.gitkeep"
-          cd ../..
-          echo "recompiled_dir=$recompiled_dir" >> $GITHUB_OUTPUT
-
-      - name: Push empty recompiled dir to GitLab
-        run: |
-          cd gitlab_docs
-          git add models/recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }}
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add recompiled${{ steps.create-recompiled-dir.outputs.recompiled_dir }} for build-all" || echo "No changes to commit"
-          git push origin main
-
-      - name: Push new JSON to GitHub docs repo
-        run: |
-          cd docs
-          git pull origin gh-pages
-          git add docs/"$(basename ${{ steps.get-json.outputs.json_file }})"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Add new ${{ steps.get-json.outputs.json_file }} for build-all" || echo "No changes to commit"
-          git push origin gh-pages
-
-  get_and_build:
-    needs: [setup]
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-    secrets: inherit
-
-  retry_failed_models:
-    needs: [setup, get_and_build]
-    runs-on: ubuntu-latest
-    if: ${{ needs.setup.result != 'failure' && !cancelled() }}
-    outputs:
-      retry_matrix: ${{ steps.set-retry-matrix.outputs.retry_matrix }}
-    steps:
-      - uses: actions/download-artifact@v4
-        with:
-          pattern: model-*
-          path: output
-
-      - id: set-retry-matrix
-        run: |
-          echo '${{ needs.setup.outputs.model_matrix }}' > matrix.json
-          built=(); while IFS= read -r line; do built+=("$line"); done < <(
-            find output -maxdepth 1 -name 'model-*' -printf "%f\n" | sed -E 's/^model-//' | sed -E 's/-[0-9]+$//' | sed -E 's/ \([^)]*\)//' | awk '{gsub(/^ +| +$/, ""); print}'
-          )
-          jq -c --argjson built "$(printf '%s\n' "${built[@]}" | jq -R . | jq -s .)" \
-            'map(select(.display_name as $n | ($built | index($n | gsub("^ +| +$"; "")) | not)))' matrix.json > retry_matrix.json
-          echo "retry_matrix=$(cat retry_matrix.json)" >> $GITHUB_OUTPUT
-
-  retry_get_and_build:
-    needs: [setup, get_and_build, retry_failed_models]
-    if: ${{ needs.get_and_build.result == 'failure' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '') }}
-    strategy:
-      matrix:
-        model: ${{ fromJson(needs.retry_failed_models.outputs.retry_matrix) }}
-      fail-fast: false
-    uses: ./.github/workflows/build-single-tinygrad-model.yaml
-    with:
-      upstream_branch: ${{ matrix.model.ref }}
-      custom_name: ${{ matrix.model.display_name }}
-      recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
-      json_version: ${{ needs.setup.outputs.json_version }}
-      artifact_suffix: -retry
-    secrets: inherit
-
-  publish_models:
-    name: Publish models sequentially
-    needs: [setup, get_and_build, retry_failed_models, retry_get_and_build]
-    if: ${{ !cancelled() && (needs.get_and_build.result != 'failure' || needs.retry_get_and_build.result == 'success' || (needs.retry_failed_models.outputs.retry_matrix != '[]' && needs.retry_failed_models.outputs.retry_matrix != '')) }}
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      max-parallel: 1
-      matrix:
-        model: ${{ fromJson(needs.setup.outputs.model_matrix) }}
-    env:
-      RECOMPILED_DIR: recompiled${{ needs.setup.outputs.recompiled_dir }}
-      JSON_FILE: ${{ needs.setup.outputs.json_file }}
-      ARTIFACT_NAME_INPUT: ${{ matrix.model.display_name }}
-    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
-        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
-          fi
-
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ env.ARTIFACT_NAME_INPUT }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove onnx files bc not needed for recompiled dir since they already exist from single build
-        run: |
-          find output -type f -name '*.onnx' -delete
-          find output -type f -name 'big_*.pkl' -delete
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-
-      - name: Copy model artifacts to gitlab
-        env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
-        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
-
-      - name: Push recompiled dir to GitLab
-        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Update $RECOMPILED_DIR with model from build-all-tinygrad-models" || echo "No changes to commit"
-          git push origin main
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
-      - name: update json
-        run: |
-          ARGS=""
-          [ -n "${{ inputs.set_min_version }}" ] && ARGS="$ARGS --set-min-version \"${{ inputs.set_min_version }}\""
-          ARGS="$ARGS --sort-by-date"
-          ARGS="$ARGS --tinygrad-ref \"${{ needs.setup.outputs.tinygrad_ref }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            $ARGS
-
-      - name: Push updated json to GitHub
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages

.github/workflows/build-single-tinygrad-model.yaml

@@ -1,228 +0,0 @@
-name: Build Single Tinygrad Model and Push
-
-on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream commit to build from'
-        required: true
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
-        required: true
-        type: string
-      json_version:
-        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
-        required: true
-        type: string
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
-      bypass_push:
-        description: 'Bypass pushing to GitLab for build-all'
-        required: false
-        default: true
-        type: boolean
-  workflow_dispatch:
-    inputs:
-      upstream_branch:
-        description: 'Upstream commit to build from'
-        required: true
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      recompiled_dir:
-        description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
-        required: true
-        type: string
-      json_version:
-        description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
-        required: true
-        type: string
-      model_folder:
-        description: 'Model folder'
-        type: choice
-        default: 'None'
-        options:
-          - None
-          - Simple Plan Models
-          - Space Lab Models
-          - TR Models
-          - DTR Models
-          - Custom Merge Models
-          - FOF series models
-          - Other
-      custom_model_folder:
-        description: 'Custom model folder name (if "Other" selected)'
-        required: false
-        type: string
-      generation:
-        description: 'Model generation'
-        required: false
-        type: string
-      version:
-        description: 'Minimum selector version'
-        required: false
-        type: string
-env:
-  RECOMPILED_DIR: recompiled${{ inputs.recompiled_dir }}
-  JSON_FILE: docs/docs/driving_models_v${{ inputs.json_version }}.json
-
-jobs:
-  build_model:
-    uses: ./.github/workflows/sunnypilot-build-model.yaml
-    with:
-      upstream_branch: ${{ inputs.upstream_branch }}
-      custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
-      is_20hz: true
-      artifact_suffix: ${{ inputs.artifact_suffix }}
-    secrets: inherit
-
-  publish_model:
-    if: ${{ !inputs.bypass_push && !cancelled() }}
-    concurrency:
-      group: gitlab-push-${{ inputs.recompiled_dir }}
-      cancel-in-progress: false
-    needs: build_model
-    runs-on: ubuntu-latest
-    steps:
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab.com SSH key to known_hosts
-        run: |
-          mkdir -p ~/.ssh
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Clone GitLab docs repo
-        env:
-          GIT_SSH_COMMAND: 'ssh -o UserKnownHostsFile=~/.ssh/known_hosts'
-        run: |
-          echo "Cloning GitLab"
-          git clone --depth 1 --filter=tree:0 --sparse git@gitlab.com:sunnypilot/public/${{ vars.MODELS_GITLAB }} gitlab_docs
-          cd gitlab_docs
-          echo "checkout models/${RECOMPILED_DIR}"
-          git sparse-checkout set --no-cone models/${RECOMPILED_DIR}
-          git checkout main
-          cd ..
-
-      - name: Checkout docs repo
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot-models
-          ref: gh-pages
-          path: docs
-          ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
-
-      - name: Validate recompiled dir and JSON version
-        run: |
-          if [ ! -d "gitlab_docs/models/$RECOMPILED_DIR" ]; then
-            echo "Recompiled dir $RECOMPILED_DIR does not exist in GitLab repo"
-            exit 1
-          fi
-          if [ ! -f "$JSON_FILE" ]; then
-            echo "JSON file $JSON_FILE does not exist!"
-            exit 1
-          fi
-
-      - name: Download artifact name file
-        uses: actions/download-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: artifact_name
-
-      - name: Read artifact name
-        id: read-artifact-name
-        run: |
-          ARTIFACT_NAME=$(cat artifact_name/artifact_name.txt)
-          echo "artifact_name=$ARTIFACT_NAME" >> $GITHUB_OUTPUT
-
-      - name: Download and extract model artifact
-        uses: actions/download-artifact@v4
-        with:
-          name: ${{ steps.read-artifact-name.outputs.artifact_name }}
-          path: output
-
-      - name: Remove unwanted files
-        run: |
-          find output -type f -name 'dmonitoring_model_tinygrad.pkl' -delete
-          find output -type f -name 'dmonitoring_model.onnx' -delete
-
-      - name: Copy model artifact(s) to GitLab recompiled dir
-        env:
-          ARTIFACT_NAME: ${{ steps.read-artifact-name.outputs.artifact_name }}
-        run: |
-          ARTIFACT_DIR="gitlab_docs/models/${RECOMPILED_DIR}/${ARTIFACT_NAME}"
-          mkdir -p "$ARTIFACT_DIR"
-          for path in output/*; do
-            if [ "$(basename "$path")" = "artifact_name.txt" ]; then
-              continue
-            fi
-            name="$(basename "$path")"
-            if [ -d "$path" ]; then
-              mkdir -p "$ARTIFACT_DIR/$name"
-              cp -r "$path"/* "$ARTIFACT_DIR/$name/"
-              echo "Copied dir $name -> $ARTIFACT_DIR/$name"
-            else
-              cp "$path" "$ARTIFACT_DIR/"
-              echo "Copied file $name -> $ARTIFACT_DIR/"
-            fi
-          done
-
-      - name: Push recompiled dir to GitLab
-        env:
-          GITLAB_SSH_PRIVATE_KEY: ${{ secrets.GITLAB_SSH_PRIVATE_KEY }}
-        run: |
-          cd gitlab_docs
-          git checkout main
-          git pull origin main
-          for d in models/"$RECOMPILED_DIR"/*/; do
-            git sparse-checkout add "$d"
-          done
-          git add models/"$RECOMPILED_DIR"
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git commit -m "Create/Update $RECOMPILED_DIR with new/updated model from build-single-tinygrad-model" || echo "No changes to commit"
-          git push origin main
-
-      - run: |
-          cd docs
-          git pull origin gh-pages
-
-      - name: Run json_parser.py to update JSON
-        run: |
-          FOLDER="${{ inputs.model_folder }}"
-          if [ "$FOLDER" = "Other" ]; then
-            FOLDER="${{ inputs.custom_model_folder }}"
-          fi
-          ARGS=""
-          if [ "$FOLDER" != "None" ] && [ -n "$FOLDER" ]; then
-            ARGS="$ARGS --model-folder \"$FOLDER\""
-          fi
-          [ -n "${{ inputs.generation }}" ] && ARGS="$ARGS --generation \"${{ inputs.generation }}\""
-          [ -n "${{ inputs.version }}" ] && ARGS="$ARGS --version \"${{ inputs.version }}\""
-          eval python3 docs/json_parser.py \
-            --json-path "$JSON_FILE" \
-            --recompiled-dir "gitlab_docs/models/$RECOMPILED_DIR" \
-            --sort-by-date \
-            $ARGS
-
-      - name: Push updated JSON to GitHub docs repo
-        run: |
-          cd docs
-          git config --global user.name "GitHub Action"
-          git config --global user.email "action@github.com"
-          git checkout gh-pages
-          git add docs/"$(basename $JSON_FILE)"
-          git commit -m "Update $(basename $JSON_FILE) after recompiling model" || echo "No changes to commit"
-          git push origin gh-pages

.github/workflows/cereal_validation.yaml

@@ -1,78 +0,0 @@
-name: cereal validation
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    paths:
-      - 'cereal/**'
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: cereal-validation-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  CI: 1
-
-jobs:
-  generate_cereal_artifact:
-    name: Generate cereal validation artifacts
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - run: ./tools/op.sh setup
-      - name: Build openpilot
-        run: scons -j$(nproc) cereal
-      - name: Dump sunnypilot schema
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          python3 cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema.json
-      - name: 'Prepare artifact'
-        run: |
-          mkdir -p "cereal/messaging/tests/cereal_validations"
-          cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
-          cp schema.json "cereal/messaging/tests/cereal_validations/schema.json"
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v4
-        with:
-          name: cereal_validations
-          path: cereal/messaging/tests/cereal_validations
-
-  validate_cereal_with_upstream:
-    name: Validate cereal with Upstream
-    runs-on: ubuntu-24.04
-    needs: generate_cereal_artifact
-    steps:
-      - name: Checkout sunnypilot
-        uses: actions/checkout@v6
-      - name: Checkout upstream openpilot
-        uses: actions/checkout@v6
-        with:
-          repository: 'commaai/openpilot'
-          path: openpilot
-          submodules: true
-          ref: "refs/heads/master"
-      - run: ./tools/op.sh setup
-      - name: Build openpilot
-        working-directory: openpilot
-        run: scons -j$(nproc) cereal
-      - name: Download build artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: cereal_validations
-          path: openpilot/cereal/messaging/tests/cereal_validations
-      - name: 'Validate sunnypilot schema against upstream'
-        run: |
-          export PYTHONPATH=${{ github.workspace }}/openpilot
-          chmod +x openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
-          python3 openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f openpilot/cereal/messaging/tests/cereal_validations/schema.json

.github/workflows/ci_weekly_report.yaml

@@ -0,0 +1,101 @@
+name: weekly CI test report
+on:
+  schedule:
+    - cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
+  workflow_dispatch:
+    inputs:
+      ci_runs:
+        description: 'The amount of runs to trigger in CI test report'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+env:
+  CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
+
+jobs:
+  setup:
+    if: github.repository == 'sunnypilot/sunnypilot'
+    runs-on: ubuntu-latest
+    outputs:
+      ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
+    steps:
+      - id: ci_runs_setup
+        name: CI_RUNS=${{ env.CI_RUNS }}
+        run: |
+          matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
+          echo "matrix=$matrix" >> $GITHUB_OUTPUT
+
+  ci_matrix_run:
+    needs: [ setup ]
+    strategy:
+      fail-fast: false
+      matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
+    uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
+    with:
+      run_number: ${{ matrix.run_number }}
+
+  report:
+    needs: [ci_matrix_run]
+    runs-on: ubuntu-latest
+    if: always()
+    steps:
+      - name: Get job results
+        uses: actions/github-script@v7
+        id: get-job-results
+        with:
+          script: |
+            const jobs = await github
+              .paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
+                owner: "commaai",
+                repo: "${{ github.event.repository.name }}",
+                run_id: "${{ github.run_id }}",
+                attempt: "${{ github.run_attempt }}",
+              })
+            var report = {}
+            jobs.slice(1, jobs.length-1).forEach(job => {
+              if (job.conclusion === "skipped") return;
+              const jobName = job.name.split(" / ")[2];
+              const runRegex = /\((.*?)\)/;
+              const run = job.name.match(runRegex)[1];
+              report[jobName] = report[jobName] || { successes: [], failures: [], canceled: [] };
+              switch (job.conclusion) {
+                case "success":
+                  report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
+                case "failure":
+                  report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
+                case "canceled":
+                  report[jobName].canceled.push({ "run_number": run, "link": job.html_url }); break;
+              }
+            });
+            return JSON.stringify({"jobs": report});
+
+      - name: Add job results to summary
+        env:
+          JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
+        run: |
+          cat <<EOF >> template.html
+          <table>
+            <thead>
+              <tr>
+                <th></th>
+                <th>Job</th>
+                <th>✅ Passing</th>
+                <th>❌ Failure Details</th>
+              </tr>
+            </thead>
+            <tbody>
+              {% for key in jobs.keys() %}<tr>
+                <td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
+                <td>{{ key }}</td>
+                <td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
+                <td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
+                </td>
+              </tr>{% endfor %}
+          </table>
+          EOF
+
+          pip install jinja2-cli
+          echo $JOB_RESULTS | jinja2 template.html > report.html
+          echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
+          cat report.html >> $GITHUB_STEP_SUMMARY

.github/workflows/ci_weekly_run.yaml

@@ -0,0 +1,21 @@
+name: weekly CI test run
+on:
+  workflow_call:
+    inputs:
+      run_number:
+        required: true
+        type: string
+
+concurrency:
+  group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  selfdrive_tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}
+  tools_tests:
+    uses: sunnypilot/sunnypilot/.github/workflows/tools_tests.yaml@master
+    with:
+      run_number: ${{ inputs.run_number }}

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,21 @@
+name: 'compile openpilot'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v4
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/diff_report.yaml

@@ -1,45 +0,0 @@
-name: diff report
-
-on:
-  pull_request_target:
-    types: [opened, synchronize, reopened]
-
-jobs:
-  comment:
-    name: comment
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    permissions:
-      contents: read
-      pull-requests: write
-      actions: read
-    steps:
-    - name: Wait for process replay
-      id: wait
-      continue-on-error: true
-      uses: lewagon/wait-on-check-action@v1.3.4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-        check-name: process replay
-        repo-token: ${{ secrets.GITHUB_TOKEN }}
-        allowed-conclusions: success,failure
-        wait-interval: 20
-    - name: Download diff
-      if: steps.wait.outcome == 'success'
-      uses: dawidd6/action-download-artifact@v6
-      with:
-        github_token: ${{ secrets.GITHUB_TOKEN }}
-        workflow: tests.yaml
-        workflow_conclusion: ''
-        pr: ${{ github.event.number }}
-        name: diff_report_${{ github.event.number }}
-        path: .
-        allow_forks: true
-    - name: Comment on PR
-      if: steps.wait.outcome == 'success'
-      uses: thollander/actions-comment-pull-request@v2
-      with:
-        filePath: diff_report.txt
-        comment_tag: diff_report
-        pr_number: ${{ github.event.number }}
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/docs.yaml

@@ -18,23 +18,22 @@ concurrency:
 jobs:
   docs:
     name: build docs
-    runs-on: ubuntu-24.04
+    runs-on: ubuntu-latest
+    timeout-minutes: 1
     steps:
-    - uses: commaai/timeout@v1
-
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
 
     # Build
     - name: Build docs
       run: |
-        git lfs pull
-        pip install zensical
-        python scripts/docs.py build
+        # TODO: can we install just the "docs" dependency group without the normal deps?
+        pip install mkdocs
+        mkdocs build
 
     # Push to docs.comma.ai
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
       with:
         path: openpilot-docs

.github/workflows/jenkins-pr-trigger.yaml

@@ -5,54 +5,14 @@ on:
     types: [created, edited]
 
 jobs:
-  cleanup-branches:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Delete stale Jenkins branches
-        uses: actions/github-script@v8
-        with:
-          script: |
-            const cutoff = Date.now() - 24 * 60 * 60 * 1000;
-            const prefixes = ['tmp-jenkins', '__jenkins'];
-
-            for await (const response of github.paginate.iterator(github.rest.repos.listBranches, {
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              per_page: 100,
-            })) {
-              for (const branch of response.data) {
-                if (!prefixes.some(p => branch.name.startsWith(p))) continue;
-
-                const { data: commit } = await github.rest.repos.getCommit({
-                  owner: context.repo.owner,
-                  repo: context.repo.repo,
-                  ref: branch.commit.sha,
-                });
-
-                const commitDate = new Date(commit.commit.committer.date).getTime();
-                if (commitDate < cutoff) {
-                  console.log(`Deleting branch: ${branch.name} (last commit: ${commit.commit.committer.date})`);
-                  await github.rest.git.deleteRef({
-                    owner: context.repo.owner,
-                    repo: context.repo.repo,
-                    ref: `heads/${branch.name}`,
-                  });
-                }
-              }
-            }
-
+  # TODO: gc old branches in a separate job in this workflow
   scan-comments:
     runs-on: ubuntu-latest
     if: ${{ github.event.issue.pull_request }}
-    permissions:
-      contents: write
-      issues: write
     steps:
     - name: Check for trigger phrase
       id: check_comment
-      uses: actions/github-script@v8
+      uses: actions/github-script@v7
       with:
         script: |
           const triggerPhrase = "trigger-jenkins";
@@ -72,7 +32,7 @@ jobs:
 
     - name: Checkout repository
       if: steps.check_comment.outputs.result == 'true'
-      uses: actions/checkout@v6
+      uses: actions/checkout@v4
       with:
         ref: refs/pull/${{ github.event.issue.number }}/head
 
@@ -83,14 +43,3 @@ jobs:
         git config --global user.email "github-actions[bot]@users.noreply.github.com"
         git checkout -b tmp-jenkins-${{ github.event.issue.number }}
         GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
-
-    - name: Delete trigger comment
-      if: steps.check_comment.outputs.result == 'true' && always()
-      uses: actions/github-script@v8
-      with:
-        script: |
-          await github.rest.issues.deleteComment({
-            owner: context.repo.owner,
-            repo: context.repo.repo,
-            comment_id: context.payload.comment.id,
-          });

.github/workflows/lfs-maintenance.yaml

@@ -1,72 +0,0 @@
-name: Sync comma's LFS
-
-env:
-  LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
-  LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
-
-on:
-  schedule:
-    - cron: '0 0 * * *'  # Runs at 00:00 UTC every day
-  push:
-    branches:
-      - 'master'
-  pull_request:
-    branches:
-      - 'master'
-  workflow_dispatch: # enables manual triggering
-    inputs:
-      upstream_branch:
-        default: 'master'
-        type: string
-
-jobs:
-  sync:
-    runs-on: ubuntu-latest
-    # Skip if PR is in draft mode
-    if: (github.event_name != 'pull_request' || (github.event_name == 'pull_request' && github.event.pull_request.draft == false)) && !github.event.pull_request.head.repo.fork
-    steps:
-      - name: Checkout Repository
-        uses: actions/checkout@v4
-        with:
-          repository: 'commaai/openpilot'
-          ref: ${{ inputs.upstream_branch }}
-
-      - name: LFS Fetch
-        run: |
-          git lfs fetch
-
-      - name: Set up Git
-        run: |
-          git config --global user.name 'GitHub Action'
-          git config --global user.email 'action@github.com'
-
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab public keys
-        run: |
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Ensure branch
-        run: |
-          if git symbolic-ref -q HEAD >/dev/null; then
-              echo "Already on a branch, proceeding with push"
-            else
-              echo "Detached HEAD state detected, creating temporary branch"
-            git checkout -b temp_branch
-          fi
-
-      - name: Update LFS Config
-        run: |
-          echo '[lfs]' > .lfsconfig
-          echo '    url = ${{ env.LFS_URL }}' >> .lfsconfig
-          echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
-          echo '    locksverify = false' >> .lfsconfig
-
-      - name: Push LFS
-        id: sync-and-commit
-        run: |
-          git lfs ls-files -l
-          git lfs push --all origin

.github/workflows/model_review.yaml

@@ -1,42 +0,0 @@
-name: "model review"
-
-on:
-  pull_request:
-    types: [opened, reopened, synchronize]
-    paths:
-      - 'selfdrive/modeld/models/*.onnx'
-  workflow_dispatch:
-
-jobs:
-  comment:
-    permissions:
-      contents: read
-      pull-requests: write
-    runs-on: ubuntu-latest
-    if: github.repository == 'commaai/openpilot'
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v6
-        with:
-          submodules: true
-      - name: Checkout master
-        uses: actions/checkout@v6
-        with:
-          ref: master
-          path: base
-      - run: git lfs pull
-      - run: cd base && git lfs pull
-
-      - name: scripts/reporter.py
-        id: report
-        run: |
-          echo "content<<EOF" >> $GITHUB_OUTPUT
-          echo "## Model Review" >> $GITHUB_OUTPUT
-          PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
-          echo "EOF" >> $GITHUB_OUTPUT
-
-      - name: Post model report comment
-        uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
-        with:
-          header: model-review
-          message: ${{ steps.report.outputs.content }}

.github/workflows/post-to-discourse/action.yml

@@ -1,105 +0,0 @@
-name: 'Post to Discourse'
-description: 'Posts a message to a Discourse topic (existing or new)'
-
-inputs:
-  discourse-url:
-    description: 'Discourse instance URL (e.g., https://discourse.example.com)'
-    required: true
-  api-key:
-    description: 'Discourse API key'
-    required: true
-  api-username:
-    description: 'Discourse API username'
-    required: true
-  topic-id:
-    description: 'Discourse topic ID to post to (use this OR category-id + title)'
-    required: false
-  category-id:
-    description: 'Category ID for new topic (required if topic-id not provided)'
-    required: false
-  title:
-    description: 'Title for new topic (required if topic-id not provided)'
-    required: false
-  message:
-    description: 'Message content (markdown supported)'
-    required: true
-
-outputs:
-  post-number:
-    description: 'The post number in the topic'
-    value: ${{ steps.post.outputs.post_number }}
-  post-url:
-    description: 'Direct URL to the post'
-    value: ${{ steps.post.outputs.post_url }}
-  topic-id:
-    description: 'The topic ID (useful when creating a new topic)'
-    value: ${{ steps.post.outputs.topic_id }}
-
-runs:
-  using: "composite"
-  steps:
-    - name: Post to Discourse
-      id: post
-      shell: bash
-      run: |
-        # Validate inputs
-        if [ -z "${{ inputs.topic-id }}" ] && ([ -z "${{ inputs.category-id }}" ] || [ -z "${{ inputs.title }}" ]); then
-          echo "❌ Error: Must provide either topic-id OR both category-id and title"
-          exit 1
-        fi
-        
-        if [ -n "${{ inputs.topic-id }}" ] && ([ -n "${{ inputs.category-id }}" ] || [ -n "${{ inputs.title }}" ]); then
-          echo "⚠️  Warning: Both topic-id and category-id/title provided. Will post to existing topic."
-        fi
-        
-        # Determine if creating new topic or posting to existing
-        if [ -n "${{ inputs.topic-id }}" ]; then
-          echo "📝 Posting to existing topic ID: ${{ inputs.topic-id }}"
-        
-          # Create JSON payload for posting to existing topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg topic_id "${{ inputs.topic-id }}" \
-            '{topic_id: $topic_id, raw: $content}')
-        else
-          echo "✨ Creating new topic: ${{ inputs.title }}"
-        
-          # Create JSON payload for new topic
-          PAYLOAD=$(jq -n \
-            --arg content '${{ inputs.message }}' \
-            --arg title "${{ inputs.title }}" \
-            --arg category "${{ inputs.category-id }}" \
-            '{title: $title, category: ($category | tonumber), raw: $content}')
-        fi
-        
-        # Post to Discourse
-        RESPONSE=$(curl -s -w "\n%{http_code}" \
-          -X POST "${{ inputs.discourse-url }}/posts.json" \
-          -H "Content-Type: application/json" \
-          -H "Api-Key: ${{ inputs.api-key }}" \
-          -H "Api-Username: ${{ inputs.api-username }}" \
-          -d "$PAYLOAD")
-        
-        HTTP_CODE=$(echo "$RESPONSE" | tail -n1)
-        BODY=$(echo "$RESPONSE" | sed '$d')
-        
-        if [ "$HTTP_CODE" -ge 200 ] && [ "$HTTP_CODE" -lt 300 ]; then
-          echo "✅ Successfully posted to Discourse!"
-        
-          POST_NUMBER=$(echo "$BODY" | jq -r '.post_number // "unknown"')
-          TOPIC_ID=$(echo "$BODY" | jq -r '.topic_id // "${{ inputs.topic-id }}"')
-          POST_URL="${{ inputs.discourse-url }}/t/${TOPIC_ID}/${POST_NUMBER}"
-        
-          echo "post_number=${POST_NUMBER}" >> $GITHUB_OUTPUT
-          echo "post_url=${POST_URL}" >> $GITHUB_OUTPUT
-          echo "topic_id=${TOPIC_ID}" >> $GITHUB_OUTPUT
-        
-          echo "Topic ID: ${TOPIC_ID}"
-          echo "Post number: ${POST_NUMBER}"
-          echo "URL: ${POST_URL}"
-        else
-          echo "❌ Failed to post to Discourse"
-          echo "HTTP Code: ${HTTP_CODE}"
-          echo "Response: ${BODY}"
-          exit 1
-        fi

.github/workflows/prebuilt.yaml

@@ -6,7 +6,7 @@ on:
 
 env:
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: release/ci/docker_build_sp.sh
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
@@ -28,8 +28,8 @@ jobs:
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
-    - uses: actions/checkout@v6
+        check-regexp: ^((?!.*(build master-ci).*).)*$
+    - uses: actions/checkout@v4
       with:
         submodules: true
     - run: git lfs pull

.github/workflows/release-drafter.yml

@@ -1,28 +0,0 @@
-name: Release Drafter
-
-on:
-  push:
-    branches:
-      - master
-    tags:
-      - 'v*'
-  pull_request_target:
-    types: [opened, reopened, synchronize]
-  workflow_dispatch:
-
-permissions:
-  contents: read
-
-jobs:
-  update_release_draft:
-    permissions:
-      contents: write
-      pull-requests: write
-    runs-on: ubuntu-latest
-    steps:
-      - uses: release-drafter/release-drafter@v6
-        with:
-          config-name: release-drafter.yml
-          prerelease: ${{ !startsWith(github.ref, 'refs/tags/v') }}
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -5,27 +5,50 @@ on:
   workflow_dispatch:
 
 jobs:
-  build___nightly:
-    name: build __nightly
+  build_masterci:
+    name: build master-ci
+    env:
+      TARGET_DIR: /tmp/openpilot
+      ImageOS: ubuntu20
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-latest
     if: github.repository == 'sunnypilot/sunnypilot'
     permissions:
       checks: read
       contents: write
     steps:
+    - name: Install wait-on-check-action dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
-      if: ${{ github.event_name == 'schedule' }}
+      if: ${{ github.event_name != 'workflow_dispatch' }}
       uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
       with:
         ref: master
         wait-interval: 30
-        running-workflow-name: 'build __nightly'
+        running-workflow-name: 'build master-ci'
         repo-token: ${{ secrets.GITHUB_TOKEN }}
-        check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
+        check-regexp: ^((?!.*(build prebuilt).*).)*$
     - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
-    - run: ./tools/op.sh setup
-    - name: Push __nightly
-      run: BRANCH=__nightly release/build_stripped.sh
+    - name: Pull LFS
+      run: |
+        git config --global --add safe.directory '*'
+        git lfs pull
+    - name: Build master-ci
+      run: |
+        release/build_devel.sh
+    - name: Run tests
+      run: |
+        export PYTHONPATH=$TARGET_DIR
+        cd $TARGET_DIR
+        scons -j$(nproc)
+        pytest -n logical selfdrive/car/tests/test_car_interfaces.py
+    - name: Push master-ci
+      run: |
+        unset TARGET_DIR
+        BRANCH=master-ci release/build_devel.sh

.github/workflows/repo-maintenance.yaml

@@ -5,73 +5,41 @@ on:
     - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
   workflow_dispatch:
 
-env:
-  PYTHONPATH: ${{ github.workspace }}
-
 jobs:
   package_updates:
     name: package_updates
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
     if: github.repository == 'sunnypilot/sunnypilot'
     steps:
-    - uses: actions/checkout@v6
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - run: ./tools/op.sh setup
     - name: uv lock
-      run: uv lock --upgrade
-    - name: uv pip tree
-      id: pip_tree
       run: |
-        echo 'PIP_TREE<<EOF' >> $GITHUB_OUTPUT
-        uv pip tree >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
-    - name: venv size
-      id: venv_size
-      run: |
-        echo 'VENV_SIZE<<EOF' >> $GITHUB_OUTPUT
-        echo "Total: $(du -sh .venv | cut -f1)" >> $GITHUB_OUTPUT
-        echo "" >> $GITHUB_OUTPUT
-        echo "Top 10 by size:" >> $GITHUB_OUTPUT
-        du -sh .venv/lib/python*/site-packages/* 2>/dev/null \
-          | grep -v '\.dist-info' \
-          | grep -v '__pycache__' \
-          | sort -rh \
-          | head -10 \
-          | while IFS=$'\t' read size path; do echo "$size  ${path##*/}"; done >> $GITHUB_OUTPUT
-        echo 'EOF' >> $GITHUB_OUTPUT
+        python3 -m ensurepip --upgrade
+        pip3 install uv
+        uv lock --upgrade
     - name: bump submodules
       run: |
-        git config submodule.msgq.update none
-        git config submodule.rednose_repo.update none
-        git config submodule.teleoprtc_repo.update none
-        git config submodule.tinygrad.update none
-        git submodule update --remote
+        git config --global --add safe.directory '*'
+        git -c submodule."tinygrad".update=none submodule update --remote
         git add .
     - name: update car docs
       run: |
-        python selfdrive/car/docs.py
+        scons -j$(nproc) --minimal opendbc
+        PYTHONPATH=. python selfdrive/car/docs.py
         git add docs/CARS.md
     - name: Create Pull Request
-      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
+      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
       with:
-        author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
-        token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
+        author: Vehicle Researcher <user@comma.ai>
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
         commit-message: Update Python packages
         title: '[bot] Update Python packages'
         branch: auto-package-updates
         base: master
         delete-branch: true
-        body: |
-          Automatic PR from repo-maintenance -> package_updates
-
-          ```
-          $ du -sh .venv && du -sh .venv/lib/python*/site-packages/* | sort -rh | head -10
-          ${{ steps.venv_size.outputs.VENV_SIZE }}
-          ```
-
-          ```
-          $ uv pip tree
-          ${{ steps.pip_tree.outputs.PIP_TREE }}
-          ```
+        body: 'Automatic PR from repo-maintenance -> package_updates'
         labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -0,0 +1,401 @@
+name: selfdrive
+
+on:
+  push:
+    branches:
+      - master
+      - master-new
+  pull_request:
+  workflow_dispatch:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+
+concurrency:
+  group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  PYTHONWARNINGS: error
+  BASE_IMAGE: openpilot-base
+  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
+
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
+
+jobs:
+  build_release:
+    if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
+    name: build release
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    env:
+      STRIPPED_DIR: /tmp/releasepilot
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Getting LFS files
+      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
+      with:
+        timeout_minutes: 2
+        max_attempts: 3
+        command: git lfs pull
+    - name: Build devel
+      timeout-minutes: 1
+      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Check submodules
+      if: github.repository == 'sunnypilot/sunnypilot'
+      timeout-minutes: 3
+      run: release/check-submodules.sh
+    - name: Build openpilot and run checks
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "python3 system/manager/build.py"
+    - name: Run tests
+      timeout-minutes: 1
+      run: |
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "release/check-dirty.sh && \
+                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car system/manager"
+    - name: Static analysis
+      timeout-minutes: 1
+      run: |
+        cd $GITHUB_WORKSPACE
+        cp pyproject.toml $STRIPPED_DIR
+        cd $STRIPPED_DIR
+        ${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
+
+  build:
+    strategy:
+      matrix:
+        arch: ${{ fromJson(
+           ((github.repository == 'commaai/openpilot') &&
+              ((github.event_name != 'pull_request') ||
+               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ ((matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8') ||
+                 ((matrix.arch == 'x86_64') && ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16') ||
+                   'ubuntu-latest'}}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Setup docker push
+      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
+
+  build_mac:
+    if: github.repository == 'commaai/openpilot' # Blocking macos builds as well since they have a 10x miltiplier for GH action minutes, waaaay too much!
+    name: build macOS
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: Homebrew cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: ~/Library/Caches/Homebrew
+        key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
+    - name: Install dependencies
+      run: ./tools/mac_setup.sh
+      env:
+        # package install has DeprecationWarnings
+        PYTHONWARNINGS: default
+    - run: git lfs pull
+    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - name: Getting scons cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: /tmp/scons_cache
+        key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
+    - name: Building openpilot
+      run: . .venv/bin/activate && scons -j$(nproc)
+
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [build]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - name: Setup docker
+      run: |
+        $DOCKER_LOGIN
+    - name: Merge x64 and arm64 tags
+      run: |
+        export PUSH_IMAGE=true
+        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
+
+  static_analysis:
+    name: static analysis
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
+    env:
+      PYTHONWARNINGS: default
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Static analysis
+      timeout-minutes: 1
+      run: ${{ env.RUN }} "scripts/lint/lint.sh"
+
+  unit_tests:
+    name: unit tests
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+    - name: Build openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Setup cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/comma_download_cache
+        key: unit_tests_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
+    - name: Run unit tests
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "$PYTEST --collect-only -m 'not slow' &> /dev/null && \
+                        MAX_EXAMPLES=1 $PYTEST -m 'not slow' && \
+                        ./selfdrive/ui/tests/create_test_translations.sh && \
+                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
+                        chmod -R 777 /tmp/comma_download_cache"
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  process_replay:
+    name: process replay
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v4
+      with:
+        path: .ci_cache/comma_download_cache
+        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run replay
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache && \
+                        coverage combine && \
+                        coverage xml"
+    - name: Print diff
+      id: print-diff
+      if: always()
+      run: cat selfdrive/test/process_replay/diff.txt
+    - uses: actions/upload-artifact@v4
+      if: always()
+      continue-on-error: true
+      with:
+        name: process_replay_diff.txt
+        path: selfdrive/test/process_replay/diff.txt
+    - name: Upload reference logs
+      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+      run: |
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python3 selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+    - name: Run regen
+      if: false
+      timeout-minutes: 4
+      run: |
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  test_cars:
+    name: cars
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        job: [0, 1, 2, 3]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/comma_download_cache
+        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
+    - name: Build openpilot
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Test car models
+      timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
+      run: |
+        ${{ env.RUN }} "FILEREADER_CACHE=1 MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
+                        chmod -R 777 /tmp/comma_download_cache"
+      env:
+        NUM_JOBS: 4
+        JOB_ID: ${{ matrix.job }}
+    - name: "Upload coverage to Codecov"
+      uses: codecov/codecov-action@v4
+      with:
+        name: ${{ github.job }}-${{ matrix.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  car_docs_diff:
+    name: PR comments
+    runs-on: ubuntu-latest
+    #if: github.event_name == 'pull_request'
+    if: false  # TODO: run this in opendbc?
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          ref: ${{ github.event.pull_request.base.ref }}
+      - run: git lfs pull
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Get base car info
+        run: |
+          ${{ env.RUN }} "scons -j$(nproc) && python3 selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
+          sudo chown -R $USER:$USER ${{ github.workspace }}
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+          path: current
+      - run: cd current && git lfs pull
+      - name: Save car docs diff
+        id: save_diff
+        run: |
+          cd current
+          ${{ env.RUN }} "scons -j$(nproc)"
+          output=$(${{ env.RUN }} "python3 selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
+          output="${output//$'\n'/'%0A'}"
+          echo "::set-output name=diff::$output"
+      - name: Find comment
+        if: ${{ env.AZURE_TOKEN != '' }}
+        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
+        id: fc
+        with:
+          issue-number: ${{ github.event.pull_request.number }}
+          body-includes: This PR makes changes to
+      - name: Update comment
+        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
+        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
+        with:
+          comment-id: ${{ steps.fc.outputs.comment-id }}
+          issue-number: ${{ github.event.pull_request.number }}
+          body: "${{ steps.save_diff.outputs.diff }}"
+          edit-mode: replace
+      - name: Delete comment
+        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
+        uses: actions/github-script@v7
+        with:
+          script: |
+            github.rest.issues.deleteComment({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              comment_id: ${{ steps.fc.outputs.comment-id }}
+            })
+
+  simulator_driving:
+    name: simulator driving
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Driving test
+      timeout-minutes: 1
+      run: |
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        source selfdrive/test/setup_vsound.sh && \
+                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
+
+  create_ui_report:
+    # This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
+    name: Create UI Report
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') ||
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: caching frames
+        id: frames-cache
+        uses: actions/cache@v4
+        with:
+          path: .ci_cache/comma_download_cache
+          key: ui_screenshots_test_${{ hashFiles('selfdrive/ui/tests/test_ui/run.py') }}
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create Test Report
+        timeout-minutes: ${{ ((steps.frames-cache.outputs.cache-hit == 'true') && 1 || 3) }}
+        run: >
+            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                            source selfdrive/test/setup_xvfb.sh &&
+                            CACHE_ROOT=/tmp/comma_download_cache python3 selfdrive/ui/tests/test_ui/run.py &&
+                            chmod -R 777 /tmp/comma_download_cache"
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ env.REPORT_NAME }}
+          path: selfdrive/ui/tests/test_ui/report_1/screenshots

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,40 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+runs:
+  using: "composite"
+  steps:
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -0,0 +1,75 @@
+name: 'openpilot env setup'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: true
+    default: ''
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
+    required: false
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
+    - shell: bash
+      name: No retries!
+      run: |
+        if [ "${{ github.run_attempt }}" -gt 1 ]; then
+          echo -e "\033[31m"
+          echo "##################################################"
+          echo "    Retries not allowed! Fix the flaky test!      "
+          echo "##################################################"
+          echo -e "\033[0m"
+          exit 1
+        fi
+
+    # do this after checkout to ensure our custom LFS config is used to pull from GitLab
+    - shell: bash
+      run: git lfs pull
+
+    # on BuildJet runners, must be logged into DockerHub to avoid rate limiting
+    # https://buildjet.com/for-github-actions/docs/guides/docker
+    - shell: bash
+      if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }}
+      run: |
+        echo "Need to set the Docker Hub PAT secret as an input to this action"
+        exit 1
+    - name: Login to Docker Hub
+      if: contains(runner.name, 'buildjet')
+      shell: bash
+      run: |
+        docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }}
+
+    # build cache
+    - id: date
+      shell: bash
+      run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
+    - shell: bash
+      run: echo "$CACHE_COMMIT_DATE"
+    - id: scons-cache
+      uses: ./.github/workflows/auto-cache
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        restore-keys: |
+          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
+    # build our docker image
+    - shell: bash
+      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -5,15 +5,14 @@ on:
   workflow_dispatch:
 
 env:
-  DAYS_BEFORE_PR_CLOSE: 7
-  DAYS_BEFORE_PR_STALE: 24
-  DAYS_BEFORE_PR_STALE_DRAFT: 30
+  DAYS_BEFORE_PR_CLOSE: 2
+  DAYS_BEFORE_PR_STALE: 9
 
 jobs:
   stale:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/stale@v10
+      - uses: actions/stale@v9
         with:
           exempt-all-milestones: true
 
@@ -25,28 +24,6 @@ jobs:
           exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
           days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
           days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: false
-
-          # issue config
-          days-before-issue-stale: -1 # ignore issues for now
-
-  # same as above, but give draft PRs more time
-  stale_drafts:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/stale@v10
-        with:
-          exempt-all-milestones: true
-
-          # pull request config
-          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
-          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
-          stale-pr-label: stale
-          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
-          exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
-          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE_DRAFT }}
-          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
-          exempt-draft-pr: true
 
           # issue config
           days-before-issue-stale: -1 # ignore issues for now

.github/workflows/sunnypilot-build-model.yaml

@@ -1,229 +0,0 @@
-name: Build Model from Upstream
-
-env:
-  BUILD_DIR: "/data/openpilot"
-  OUTPUT_DIR: ${{ github.workspace }}/output
-  SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
-  UPSTREAM_REPO: "commaai/openpilot"
-  TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
-  MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
-
-on:
-  workflow_call:
-    inputs:
-      upstream_branch:
-        description: 'Upstream branch to build from'
-        required: true
-        default: 'master'
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
-      artifact_suffix:
-        description: 'Suffix for artifact name'
-        required: false
-        type: string
-        default: ''
-  workflow_dispatch:
-    inputs:
-      upstream_branch:
-        description: 'Upstream branch to build from'
-        required: true
-        default: 'master'
-        type: string
-      custom_name:
-        description: 'Custom name for the model (no date, only name)'
-        required: false
-        type: string
-      is_20hz:
-        description: 'Is this a 20Hz model'
-        required: false
-        type: boolean
-        default: true
-
-
-run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
-
-jobs:
-  get_model:
-    runs-on: ubuntu-latest
-    env:
-      REF: ${{ inputs.upstream_branch }}
-    outputs:
-      model_date: ${{ steps.commit-date.outputs.model_date }}
-    steps:
-      # Note: To allow dynamic models from both openpilot and sunnypilot (merges/mashups), we try commaai as default,
-      # and fallback to sunnypilot if the ref checkout fails.
-      - name: Checkout commaai/openpilot
-        id: checkout_upstream
-        continue-on-error: true
-        uses: actions/checkout@v4
-        with:
-          repository: commaai/openpilot
-          ref: ${{ inputs.upstream_branch }}
-          submodules: recursive
-          path: openpilot
-
-      - name: Fallback to sunnypilot/sunnypilot
-        if: steps.checkout_upstream.outcome == 'failure'
-        uses: actions/checkout@v4
-        with:
-          repository: sunnypilot/sunnypilot
-          ref: ${{ inputs.upstream_branch }}
-          submodules: recursive
-          path: openpilot
-      - name: Get commit date
-        id: commit-date
-        run: |
-          cd ${{ github.workspace }}/openpilot
-          commit_date=$(git log -1 --format=%cd --date=format:'%B %d, %Y')
-          echo "model_date=${commit_date}" >> $GITHUB_OUTPUT
-          cat $GITHUB_OUTPUT
-      - run: |
-          cd ${{ github.workspace }}/openpilot
-          git lfs pull
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v4
-        with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix  }}
-          path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
-
-  build_model:
-    runs-on: [self-hosted, tici]
-    needs: get_model
-    env:
-      MODEL_NAME: ${{ inputs.custom_name || inputs.upstream_branch }} (${{ needs.get_model.outputs.model_date }})
-      REF: ${{ inputs.upstream_branch }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-
-      - run: git lfs pull
-      - name: Cache SCons
-        uses: actions/cache@v4
-        with:
-          path: ${{env.SCONS_CACHE_DIR}}
-          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
-          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
-          restore-keys: |
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ github.head_ref || github.ref_name }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}-model
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_NEW_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.MASTER_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}
-
-      - name: Set environment variables
-        id: set-env
-        run: |
-          # Set up common environment
-          source /etc/profile;
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          printenv >> $GITHUB_ENV
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            cat $GITHUB_OUTPUT
-          fi
-
-      - name: Setup build environment
-        run: |
-          mkdir -p "${BUILD_DIR}/"
-          sudo find $BUILD_DIR/ -mindepth 1 -delete
-          echo "Starting build stage..."
-          echo "BUILD_DIR: ${BUILD_DIR}"
-          echo "CI_DIR: ${CI_DIR}"
-          echo "VERSION: ${{ steps.set-env.outputs.version }}"
-          echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
-          echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
-          echo "-------"
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            printenv
-          fi
-          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
-          rm -rf ${{ env.MODELS_DIR }}/*.onnx
-
-      - name: Download model artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
-          path: ${{ github.workspace }}/selfdrive/modeld/models
-      - run: |
-          rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
-
-      - name: Build Model
-        run: |
-          source /etc/profile
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}"
-
-          # Loop through all .onnx files
-          find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
-            base_name=$(basename "$onnx_file" .onnx)
-            output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl"
-
-            echo "Compiling: $onnx_file -> $output_file"
-            DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1 IMAGE=2 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
-            DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
-          done
-
-      - name: Validate Model Outputs
-        run: |
-          source /etc/profile
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
-            --validate-only \
-            --model-dir "${{ env.MODELS_DIR }}"
-
-      - name: Prepare Output
-        run: |
-          sudo rm -rf ${{ env.OUTPUT_DIR }}
-          mkdir -p ${{ env.OUTPUT_DIR }}
-
-          # Copy the model files
-          rsync -avm \
-            --include='*.dlc' \
-            --include='*.pkl' \
-            --include='*.onnx' \
-            --exclude='*' \
-            --delete-excluded \
-            --chown=comma:comma \
-            ${{ env.MODELS_DIR }}/ ${{ env.OUTPUT_DIR }}/
-
-          python3 "${{ github.workspace }}/release/ci/model_generator.py" \
-            --model-dir "${{ env.MODELS_DIR }}" \
-            --output-dir "${{ env.OUTPUT_DIR }}" \
-            --custom-name "${{ env.MODEL_NAME }}" \
-            --upstream-branch "${{ inputs.upstream_branch }}" \
-            ${{ inputs.is_20hz && '--is-20hz' || '' }}
-
-      - name: Write artifact name to file
-        run: echo "model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}" > ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
-      - name: Upload Build Artifacts
-        id: upload-artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: model-${{ env.MODEL_NAME }}${{ inputs.artifact_suffix }}-${{ github.run_number }}
-          path: ${{ env.OUTPUT_DIR }}
-
-      - name: Upload artifact name file
-        uses: actions/upload-artifact@v4
-        with:
-          name: artifact-name-${{ inputs.custom_name || inputs.upstream_branch }}
-          path: ${{ env.OUTPUT_DIR }}/artifact_name.txt
-
-      - name: Re-enable powersave
-        if: always()
-        run: |
-          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable

.github/workflows/sunnypilot-build-prebuilt.yaml

@@ -1,369 +0,0 @@
-name: sunnypilot prebuilt action
-
-env:
-  BUILD_DIR: "/data/openpilot"
-  OUTPUT_DIR: ${{ github.workspace }}/output
-  CI_DIR: ${{ github.workspace }}/release/ci
-  SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
-  PUBLIC_REPO_URL: "https://github.com/sunnypilot/sunnypilot"
-
-  # Branch configurations
-  STAGING_SOURCE_BRANCH: 'master'
-
-  # Runtime configuration
-  SOURCE_BRANCH: "${{ github.head_ref || github.ref_name }}"
-
-on:
-  push:
-    branches: [ master, master-dev ]
-    tags: [ 'release/*' ]
-  pull_request_target:
-    types: [ labeled ]
-  workflow_dispatch:
-    inputs:
-      wait_for_tests:
-        description: 'Wait for tests to finish'
-        required: false
-        type: boolean
-        default: false
-
-jobs:
-  prepare_strategy:
-    runs-on: ubuntu-24.04
-    if: (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-    outputs:
-      environment: ${{ steps.strategy.outputs.environment }}
-      new_branch: ${{ steps.strategy.outputs.new_branch }}
-      extra_version_identifier: ${{ steps.strategy.outputs.extra_version_identifier }}
-      version: ${{ steps.strategy.outputs.version }}
-      cancel_publish_in_progress: ${{ steps.strategy.outputs.cancel_publish_in_progress }}
-      publish_concurrency_group: ${{ steps.strategy.outputs.publish_concurrency_group }}
-      is_stable_branch: ${{ steps.strategy.outputs.is_stable_branch }}
-      build: ${{ steps.strategy.outputs.build }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Extract deploy strategy
-        id: strategy
-        run: |
-          echo '::group::Strategy Extraction'
-          BRANCH="${{ github.head_ref || github.ref_name }}"
-          echo "Current branch: $BRANCH"
-
-          STRATEGY_JSON='${{ vars.DEPLOY_STRATEGY }}'
-          CONFIG=$(echo "$STRATEGY_JSON" | jq -r --arg branch "$BRANCH" '
-            .configs[] | select(.branch == $branch)
-          ')
-
-          BUILD="$(date '+%Y.%m.%d')-${{ github.run_number }}"
-          if [[ -z "$CONFIG" || "$CONFIG" == "null" ]]; then
-            echo "No exact strategy match found. Falling back to feature/fork logic."
-            IS_FORK="${{ github.event.pull_request.head.repo.fork && 'true' || 'false' }}"
-            FORK_SUFFIX=$( [[ "$IS_FORK" == "true" ]] && echo "-fork" || echo "" )
-            NEW_BRANCH="${BRANCH}${FORK_SUFFIX}-prebuilt"
-
-            echo "new_branch=$NEW_BRANCH" >> $GITHUB_OUTPUT
-            echo "version=$BUILD" >> $GITHUB_OUTPUT
-            echo "cancel_publish_in_progress=true" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}" >> $GITHUB_OUTPUT
-            echo "environment=feature-branch" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=feature-branch" >> $GITHUB_OUTPUT
-          else
-          echo "Matched config: $CONFIG"
-            environment=$(echo "$CONFIG" | jq -r '.environment')
-            echo "environment=$environment" >> $GITHUB_OUTPUT
-            echo "new_branch=$(echo "$CONFIG" | jq -r '.target_branch')" >> $GITHUB_OUTPUT
-            cancel="$(echo "$CONFIG" | jq -r '.cancel_publish_in_progress')";
-            echo "cancel_publish_in_progress=$( [ "$cancel" = "null" ] && echo "true" || echo $cancel)" >> $GITHUB_OUTPUT
-            echo "publish_concurrency_group=publish-${BRANCH}$( [ "$cancel" = "null" ] || [ "$cancel" = "true" ] || echo "${{ github.sha }}" )" >> $GITHUB_OUTPUT
-
-            is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
-            echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
-
-            stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
-            echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
-            echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
-          fi
-          echo "build=$BUILD" >> $GITHUB_OUTPUT
-          cat $GITHUB_OUTPUT
-
-  validate_tests:
-    runs-on: ubuntu-24.04
-    needs: [ prepare_strategy ]
-    if: ${{
-      ((github.event_name == 'workflow_dispatch' && inputs.wait_for_tests) ||
-      (github.event_name == 'push' && needs.prepare_strategy.outputs.is_stable_branch == 'true') ||
-      contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
-    steps:
-      - uses: actions/checkout@v4
-      - name: Wait for Tests
-        uses: ./.github/workflows/wait-for-action # Path to where you place the action
-        with:
-          workflow: tests.yaml # The workflow file to monitor
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          should-wait-for-start: ${{ github.event_name == 'push' && 'true' || 'false' }}
-
-  build:
-    needs: [ validate_tests, prepare_strategy ]
-    concurrency:
-      group: build-${{ github.head_ref || github.ref_name }}
-      cancel-in-progress: false
-    runs-on: [self-hosted, tici]
-    outputs:
-      new_branch: ${{ needs.prepare_strategy.outputs.new_branch }}
-      version: ${{ needs.prepare_strategy.outputs.version }}
-      extra_version_identifier: ${{ needs.prepare_strategy.outputs.extra_version_identifier }}
-      commit_sha: ${{ github.sha }}
-    if: ${{
-      (always() && !cancelled() && !failure()) &&
-      needs.prepare_strategy.result == 'success' &&
-      (needs.validate_tests.result == 'success' || needs.validate_tests.result == 'skipped') &&
-      (!contains(github.event_name, 'pull_request') ||
-      (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      }}
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          submodules: recursive
-          ref: ${{ env.SOURCE_BRANCH }}
-          repository: ${{ github.event.pull_request.head.repo.fork && github.event.pull_request.head.repo.full_name || github.repository }}
-      - run: git lfs pull
-
-      - name: Cache SCons
-        uses: actions/cache@v4
-        with:
-          path: ${{env.SCONS_CACHE_DIR}}
-          key: scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}-${{ github.sha }}
-          # Note: GitHub Actions enforces cache isolation between different build sources (PR builds, workflow dispatches, etc.)
-          #   for security. Only caches from the default branch are shared across all builds. This is by design and cannot be overridden.
-          restore-keys: |
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.SOURCE_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}-${{ env.STAGING_SOURCE_BRANCH }}
-            scons-${{ runner.os }}-${{ runner.arch }}
-
-      - name: Set environment variables
-        id: set-env
-        run: |
-          echo "new_branch=${{ needs.prepare_strategy.outputs.new_branch }}" >> $GITHUB_OUTPUT
-          echo "version=${{ needs.prepare_strategy.outputs.version }}" >> $GITHUB_OUTPUT
-          echo "extra_version_identifier=${{ needs.prepare_strategy.outputs.extra_version_identifier }}" >> $GITHUB_OUTPUT
-          echo "commit_sha=${{ github.sha }}" >> $GITHUB_OUTPUT
-
-          # Set up common environment
-          source /etc/profile;
-          export UV_PROJECT_ENVIRONMENT=${HOME}/venv
-          export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
-          printenv >> $GITHUB_ENV
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            cat $GITHUB_OUTPUT
-          fi
-
-      - name: Setup build environment
-        run: |
-          mkdir -p "${BUILD_DIR}/"
-          sudo find $BUILD_DIR/ -mindepth 1 -delete
-          echo "Starting build stage..."
-          echo "BUILD_DIR: ${BUILD_DIR}"
-          echo "CI_DIR: ${CI_DIR}"
-          echo "VERSION: ${{ steps.set-env.outputs.version }}"
-          echo "UV_PROJECT_ENVIRONMENT: ${UV_PROJECT_ENVIRONMENT}"
-          echo "VIRTUAL_ENV: ${VIRTUAL_ENV}"
-          echo "-------"
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            printenv
-          fi
-          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
-
-      - name: Build Main Project
-        run: |
-          export PYTHONPATH="$BUILD_DIR"
-          ./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
-          cd $BUILD_DIR
-          sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
-          echo "Building sunnypilot's modeld_v2..."
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
-          echo "Building sunnypilot's locationd..."
-          scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
-          echo "Building openpilot's locationd..."
-          scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
-          echo "Building rest of sunnypilot"
-          scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
-          touch ${BUILD_DIR}/prebuilt
-          if [[ "${{ runner.debug }}" == "1" ]]; then
-            ls -la ${BUILD_DIR}
-          fi
-
-      - name: Prepare Output
-        run: |
-          sudo rm -rf ${OUTPUT_DIR}
-          mkdir -p ${OUTPUT_DIR}
-          rsync -am${RUNNER_DEBUG:+v} \
-            --exclude='.sconsign.dblite' \
-            --exclude='*.a' \
-            --exclude='*.o' \
-            --exclude='*.os' \
-            --exclude='*.pyc' \
-            --exclude='moc_*' \
-            --exclude='__pycache__' \
-            --exclude='Jenkinsfile' \
-            --exclude='**/release/' \
-            --exclude='**/.github/' \
-            --exclude='**/selfdrive/ui/replay/' \
-            --exclude='**/__pycache__/' \
-            --exclude='${{env.SCONS_CACHE_DIR}}' \
-            --exclude='**/.git/' \
-            --exclude='**/SConstruct' \
-            --exclude='**/SConscript' \
-            --exclude='**/.venv/' \
-            --exclude='selfdrive/modeld/models/driving_vision.onnx' \
-            --exclude='selfdrive/modeld/models/driving_policy.onnx' \
-            --exclude='third_party/*x86*' \
-            --exclude='third_party/*Darwin*' \
-            --delete-excluded \
-            --chown=comma:comma \
-            ${BUILD_DIR}/ ${OUTPUT_DIR}/
-
-      - name: 'Tar.gz files'
-        run: |
-          tar czf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }} .
-          ls -la prebuilt.tar.gz
-
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v4
-        with:
-          name: prebuilt
-          path: prebuilt.tar.gz
-
-      - name: Re-enable powersave
-        if: always()
-        run: |
-          PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
-
-
-  publish:
-    concurrency:
-      # We do a bit of a hack here to avoid canceling the publishing job if a new commit comes in while we're publishing by adding the sha to the group name.
-      # This means that if multiple commits come in while we're publishing, they will be queued up and publish one after the other.
-      # Otherwise, if a job is waiting to be published due to environment wait time, it would be canceled by a new commit and restart the wait time.
-      group: ${{ needs.prepare_strategy.outputs.publish_concurrency_group }}
-      cancel-in-progress: ${{ needs.prepare_strategy.outputs.cancel_publish_in_progress == 'true' }}
-    if: ${{ (always() && !cancelled() && !failure()) && needs.build.result == 'success' && needs.prepare_strategy.result == 'success' && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt')) }}
-    needs: [ build, prepare_strategy ]
-    runs-on: ubuntu-24.04
-    environment: ${{ needs.prepare_strategy.outputs.environment }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Download build artifacts
-        uses: actions/download-artifact@v4
-        with:
-          name: prebuilt
-
-      - name: Untar prebuilt
-        run: |
-          mkdir -p ${{ env.OUTPUT_DIR }}
-          tar xzf prebuilt.tar.gz -C ${{ env.OUTPUT_DIR }}
-
-      - name: Configure Git
-        run: |
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git config --global user.name "github-actions[bot]"
-
-      - name: Publish to Public Repository
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          echo '${{ toJSON(needs.build.outputs) }}'
-          ls -la ${{ env.OUTPUT_DIR }}
-
-          ${{ env.CI_DIR }}/publish.sh \
-            "${{ github.workspace }}" \
-            "${{ env.OUTPUT_DIR }}" \
-            "${{ needs.build.outputs.new_branch }}" \
-            "${{ needs.build.outputs.version }}" \
-            "https://x-access-token:${{github.token}}@github.com/sunnypilot/sunnypilot.git" \
-            "${{ needs.build.outputs.extra_version_identifier }}"
-
-          echo ""
-          echo "----   ℹ️ To update the list of branches that auto deploy prebuilts  -----"
-          echo ""
-          echo "1. Go to: ${{ github.server_url }}/${{ github.repository }}/settings/variables/actions/AUTO_DEPLOY_PREBUILT_BRANCHES"
-          echo "2. Current value: ${{ vars.AUTO_DEPLOY_PREBUILT_BRANCHES }}"
-          echo "3. Update as needed (JSON array with no spaces)"
-
-      - name: Tag ${{ needs.prepare_strategy.outputs.environment }}
-        if: ${{ needs.prepare_strategy.outputs.is_stable_branch == 'true' && (github.event_name != 'push' || !startsWith(github.ref, 'refs/tags/')) }}
-        run: |
-            TAG="${{ needs.prepare_strategy.outputs.environment }}/${{ needs.prepare_strategy.outputs.version }}/${{ needs.prepare_strategy.outputs.build }}"
-            git tag -f -a ${TAG} -m "${{ needs.prepare_strategy.outputs.environment }} @ ${{ needs.prepare_strategy.outputs.version }} of build ${{ needs.build.outputs.build }}."
-            git push -f origin ${TAG}
-
-  notify:
-    needs:
-    - prepare_strategy
-    - build
-    - publish
-    runs-on: ubuntu-24.04
-    if: ${{ (always() && !cancelled() && !failure())
-      && needs.publish.result == 'success'
-      && (!contains(github.event_name, 'pull_request') || (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-      && (fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name] != null) }}
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Prepare notification message
-        id: message
-        run: |
-          TEMPLATE='${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}'
-          export VERSION="${{ needs.prepare_strategy.outputs.version }}"
-          export branch_name="${{ env.SOURCE_BRANCH }}"
-          export new_branch="${{ needs.prepare_strategy.outputs.new_branch }}"
-          export commit_sha="${{ github.sha }}"
-          export commit_short_sha="${{ github.sha }}"
-          export commit_short_sha="${commit_short_sha:0:7}"
-          export extra_version_identifier="${{ needs.prepare_strategy.outputs.extra_version_identifier || github.run_number }}"
-          export PUBLIC_REPO_URL="${{ env.PUBLIC_REPO_URL }}"
-
-          MESSAGE=$(cat << 'EOF' | envsubst
-          ${{ vars.DISCOURSE_GENERAL_UPDATE_NOTICE }}
-          EOF
-          )
-
-          {
-            echo 'content<<EOFMARKER'
-            echo "$MESSAGE"
-            echo 'EOFMARKER'
-          } >> $GITHUB_OUTPUT
-        shell: bash
-
-      - name: Post to Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: "system"
-          topic-id: ${{ fromJSON(vars.DEV_FEEDBACK_NOTIFICATION_BRANCHES_V2)[github.head_ref || github.ref_name].topic_id }}
-          message: ${{ steps.message.outputs.content }}
-
-  manage-pr-labels:
-    name: Remove prebuilt label
-    runs-on: ubuntu-latest
-    if: (always() && contains(github.event_name, 'pull_request') && (github.event.action == 'labeled' && github.event.label.name == 'prebuilt'))
-    env:
-      LABEL: prebuilt
-    steps:
-      - name: Remove trust-fork-pr label if present
-        uses: actions/github-script@v7
-        with:
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-          script: |
-            const prNumber = context.payload.pull_request.number;
-
-            await github.rest.issues.removeLabel({
-              owner: context.repo.owner,
-              repo: context.repo.repo,
-              issue_number: prNumber,
-              name: process.env.LABEL
-            });
-
-            console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);

.github/workflows/sunnypilot-master-dev-prep.yaml

@@ -1,243 +0,0 @@
-name: Build dev
-
-env:
-  DEFAULT_SOURCE_BRANCH: "master"
-  DEFAULT_TARGET_BRANCH: "master-dev"
-  LFS_URL: 'https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs'
-  LFS_PUSH_URL: 'ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git'
-
-on:
-  push:
-    branches:
-      - master
-  pull_request_target:
-    types: [ labeled ]
-    branches:
-      - 'master'
-  workflow_dispatch:
-    inputs:
-      source_branch:
-        description: 'Source branch to reset from'
-        required: true
-        default: 'master'
-        type: string
-      target_branch:
-        description: 'Target branch to reset and squash into'
-        required: true
-        default: 'master-dev'
-        type: string
-      cancel_in_progress:
-        description: 'Cancel any in-progress runs of this workflow'
-        required: false
-        default: true
-        type: boolean
-
-concurrency:
-  group: ${{ github.workflow }}
-  cancel-in-progress: ${{ inputs.cancel_in_progress || github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }}
-
-jobs:
-  reset-and-squash:
-    runs-on: ubuntu-latest
-    if: (
-      (github.event_name == 'workflow_dispatch')
-      || (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-      || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-      )
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0  # Fetch all history for all branches
-          token: ${{ secrets.GITHUB_TOKEN }}
-          persist-credentials: false
-
-      - name: Wait for Tests
-        uses: ./.github/workflows/wait-for-action # Path to where you place the action
-        if: (
-          (github.event_name == 'push' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch))
-          || (contains(github.event_name, 'pull_request') && ((github.event.action == 'labeled' && (github.event.label.name == vars.PREBUILT_PR_LABEL || github.event.label.name == 'trust-fork-pr') && contains(github.event.pull_request.labels.*.name, vars.PREBUILT_PR_LABEL))))
-          )
-        with:
-          workflow: tests.yaml # The workflow file to monitor
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Configure Git
-        run: |
-          git config --global user.name 'github-actions[bot]'
-          git config --global user.email 'github-actions[bot]@users.noreply.github.com'
-
-      - name: Set up SSH
-        uses: webfactory/ssh-agent@v0.9.0
-        with:
-          ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
-
-      - name: Add GitLab public keys
-        run: |
-          ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
-
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '3.10'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip
-          pip install PyGithub
-
-      - name: Check branches exist
-        run: |
-          # Check if source branch exists
-          if ! git ls-remote --heads origin ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }} | grep -q "${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"; then
-            echo "Source branch ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }} does not exist!"
-            exit 1
-          fi
-
-          # Make sure we have the latest source branch
-          git fetch origin ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}
-
-          # Check if target branch exists
-          if ! git ls-remote --heads origin ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} | grep -q "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"; then
-            echo "Target branch ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} does not exist, creating it from ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-            git checkout -b ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} origin/${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}
-            git push origin ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}
-          else
-            # Fetch target branch if it exists
-            git fetch origin ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}
-          fi
-
-      - name: Reset target branch
-        run: |
-          echo "Resetting ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} to match ${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-          # Delete if exists and recreate pointing to source
-          git branch -D ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} || true
-          git branch ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} origin/${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}
-
-      - name: Get PRs to squash
-        id: get-prs
-        run: |
-          # Use GitHub API to get PRs with specific label, ordered by creation date
-          PR_LIST=$(gh api graphql -f query='
-            query($search_query:String!) {
-              search(query: $search_query, type:ISSUE, first:40) {
-                nodes {
-                  ... on PullRequest {
-                    number
-                    headRefName
-                    title
-                    createdAt
-                    labels(last:10) {
-                      nodes {
-                        name
-                      }
-                    }
-                    headRepository {
-                      name
-                      nameWithOwner
-                      url
-                      isFork
-                    }
-                    commits(last: 1) {
-                      nodes {
-                        commit {
-                          statusCheckRollup {
-                            state
-                          }
-                        }
-                      }
-                    }
-                  }
-                }
-              }
-            }' -F search_query="repo:${{ github.repository }} is:pr is:open label:${{ vars.PREBUILT_PR_LABEL }},${{ vars.PREBUILT_PR_LABEL }}-c3 draft:false sort:created-asc")
-
-          PR_LIST=${PR_LIST//\'/}
-          echo "PR_LIST=${PR_LIST}" >> $GITHUB_OUTPUT
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Process PRs
-        run: |
-          cp ${{ github.workspace }}/release/ci/squash_and_merge.py /tmp/squash_and_merge.py && \
-          chmod +x /tmp/squash_and_merge.py && \
-          python3 ${{ github.workspace }}/release/ci/squash_and_merge_prs.py \
-            --pr-data '${{ steps.get-prs.outputs.PR_LIST }}' \
-            --target-branch ${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }} \
-            --squash-script-path '/tmp/squash_and_merge.py'
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Update LFS Config
-        run: |
-          echo '[lfs]' > .lfsconfig
-          echo '    url = ${{ env.LFS_URL }}' >> .lfsconfig
-          echo '    pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
-          echo '    locksverify = false' >> .lfsconfig
-
-      - name: Restore workflows from source
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-          SOURCE_BRANCH="${{ inputs.source_branch || env.DEFAULT_SOURCE_BRANCH }}"
-
-          # Ensure we are on the target branch
-          git checkout $TARGET_BRANCH
-
-          echo "Restoring .github/workflows from $SOURCE_BRANCH"
-          git checkout origin/$SOURCE_BRANCH -- .github/workflows
-
-          if ! git diff --cached --quiet; then
-            echo "Workflows differ. Committing restoration."
-            git commit -m "chore: restore .github/workflows from $SOURCE_BRANCH"
-          else
-            echo "Workflows match $SOURCE_BRANCH."
-          fi
-
-      - uses: actions/create-github-app-token@v2
-        id: ci-token
-        with:
-          app-id: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_ID }}
-          private-key: ${{ secrets.CI_GITHUB_ACTIONS_TOKEN_APP_PRIVATE_KEY }}
-
-      - name: Push changes if there are diffs
-        id: push-changes
-        run: |
-          TARGET_BRANCH="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-
-          # Use the App Token to set the remote URL with authentication
-          git remote set-url origin "https://x-access-token:${{ steps.ci-token.outputs.token }}@github.com/${{ github.repository }}.git"
-
-          # Fetch the latest from remote
-          git fetch origin $TARGET_BRANCH
-
-          # Check for diffs between local and remote
-          if git diff $TARGET_BRANCH origin/$TARGET_BRANCH --quiet; then
-            echo "No changes to push - local and remote branches are identical"
-            echo "has_changes=false" >> $GITHUB_OUTPUT
-            exit 0
-          fi
-
-          # Push with the authenticated origin
-          if ! git push origin $TARGET_BRANCH --force; then
-            echo "Failed to push changes to $TARGET_BRANCH"
-            exit 1
-          fi
-
-          echo "Branch $TARGET_BRANCH has been reset and updated with squashed PRs"
-          echo "has_changes=true" >> $GITHUB_OUTPUT
-
-      - name: Trigger and wait for selfdrive tests
-        if: steps.push-changes.outputs.has_changes == 'true'
-        run: |
-          echo "Triggering selfdrive tests..."
-          gh workflow run tests.yaml --ref "${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}"
-
-          echo "Sleeping for 120s to give plenty of time for the action to start and then we wait"
-          sleep 120
-
-          echo "Getting latest run ID..."
-          RUN_ID=$(gh run list --workflow=tests.yaml --branch="${{ inputs.target_branch || env.DEFAULT_TARGET_BRANCH }}" --limit=1 --json databaseId --jq '.[0].databaseId')
-
-          echo "Watching run ID: $RUN_ID"
-          gh run watch "$RUN_ID"
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/test-discourse.yaml.yml

@@ -1,78 +0,0 @@
-name: Debug Discourse Posting
-
-on:
-  push:
-
-jobs:
-  test-discourse-post:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Post test message to Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      
-      
-      - name: Create topic on Discourse
-        uses: ./.github/workflows/post-to-discourse
-        with:
-          discourse-url: ${{ vars.DISCOURSE_URL }}
-          api-key: ${{ secrets.DISCOURSE_API_KEY }}
-          api-username: ${{ secrets.DISCOURSE_API_USERNAME }}
-          #topic-id: ${{ vars.DISCOURSE_UPDATES_TOPIC_ID }}
-          category-id: 4
-          title: "This is a test of a new topic instead of a reply"
-          message: |
-            ## 🧪 Test Post from GitHub Actions
-            
-            **This is a test post to verify Discourse integration**
-            
-            - **Workflow**: ${{ github.workflow }}
-            - **Run Number**: #${{ github.run_number }}
-            - **Branch**: `${{ github.ref_name }}`
-            - **Commit**: ${{ github.sha }}
-            - **Actor**: @${{ github.actor }}
-            - **Timestamp**: ${{ github.event.head_commit.timestamp }}
-            
-            ---
-            
-            ### Fake Build Info (for testing)
-            - **Version**: 0.9.8-test
-            - **Build**: #42
-            - **Branch**: release-test
-            
-            [View workflow run](${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }})
-            
-            *This is an automated test message. Drive safe! 🚗💨*
-      - name: Display results
-        if: always()
-        run: |
-          echo "::notice::Discourse post test completed"
-          echo "Check your Discourse topic to verify the post appeared correctly"

.github/workflows/tests.yaml

@@ -1,248 +0,0 @@
-name: tests
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-  workflow_dispatch:
-  workflow_call:
-    inputs:
-      run_number:
-        default: '1'
-        required: true
-        type: string
-
-concurrency:
-  group: tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
-  cancel-in-progress: true
-
-env:
-  CI: 1
-  PYTHONPATH: ${{ github.workspace }}
-  PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
-
-jobs:
-  build_release:
-    name: build release
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    env:
-      STRIPPED_DIR: /tmp/releasepilot
-      PYTHONPATH: /tmp/releasepilot
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Getting LFS files
-      uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
-      with:
-        timeout_minutes: 2
-        max_attempts: 3
-        command: git lfs pull
-    - name: Build devel
-      timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./tools/op.sh setup
-    - name: Build openpilot and run checks
-      timeout-minutes: 30
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: python3 system/manager/build.py
-    - name: Run tests
-      timeout-minutes: 1
-      working-directory: ${{ env.STRIPPED_DIR }}
-      run: release/check-dirty.sh
-    - name: Check submodules
-      if: github.repository == 'sunnypilot/sunnypilot'
-      timeout-minutes: 3
-      run: |
-        if [ "${{ github.ref }}" != "refs/heads/master" ]; then
-          git fetch  origin master:refs/remotes/origin/master
-
-          SUBMODULE_PATHS=$(git diff origin/master HEAD --name-only | grep -E '^[^/]+$' | while read path; do
-            if git ls-files --stage "$path" | grep -q "^160000"; then
-              echo "$path"
-            fi
-          done | tr '\n' ' ')
-
-          if [ -n "$SUBMODULE_PATHS" ]; then
-            echo "Changed submodule paths: $SUBMODULE_PATHS"
-            export SUBMODULE_PATHS="$SUBMODULE_PATHS"
-            export CHECK_PR_REFS=true
-          fi
-        fi
-        release/check-submodules.sh
-
-  build_mac:
-    name: build macOS
-    runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - name: Remove Homebrew from environment
-      run: |
-        FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
-        echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
-    - run: ./tools/op.sh setup
-    - name: Building openpilot
-      run: scons
-
-  static_analysis:
-    name: static analysis
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Static analysis
-      timeout-minutes: 1
-      run: scripts/lint/lint.sh
-
-  unit_tests:
-    name: unit tests
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run unit tests
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        # Pre-compile Python bytecode so each pytest worker doesn't need to
-        $PYTEST --collect-only -m 'not slow' -qq
-        MAX_EXAMPLES=1 $PYTEST -m 'not slow'
-
-  process_replay:
-    name: process replay
-    if: false  # disable process_replay for forks
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Run replay
-      timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
-      continue-on-error: ${{ github.ref == 'refs/heads/master' }}
-      run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
-    - name: Print diff
-      id: print-diff
-      if: always()
-      run: cat selfdrive/test/process_replay/diff.txt
-    - name: Print diff report
-      if: always()
-      run: cat selfdrive/test/process_replay/diff_report.txt
-    - uses: actions/upload-artifact@v6
-      if: always()
-      continue-on-error: true
-      with:
-        name: process_replay_diff.txt
-        path: selfdrive/test/process_replay/diff.txt
-    - name: Upload diff report
-      uses: actions/upload-artifact@v6
-      if: always() && github.event_name == 'pull_request'
-      continue-on-error: true
-      with:
-        name: diff_report_${{ github.event.number }}
-        path: selfdrive/test/process_replay/diff_report.txt
-    - name: Checkout ci-artifacts
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      uses: actions/checkout@v4
-      with:
-        repository: commaai/ci-artifacts
-        ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-        path: ${{ github.workspace }}/ci-artifacts
-    - name: Push refs
-      if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
-      working-directory: ${{ github.workspace }}/ci-artifacts
-      run: |
-        git config user.name "GitHub Actions Bot"
-        git config user.email "<>"
-        git fetch origin process-replay || true
-        git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
-        cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
-        echo "${{ github.sha }}" > ref_commit
-        git add .
-        git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
-        git push origin process-replay --force
-    - name: Run regen
-      if: false
-      timeout-minutes: 4
-      env:
-        ONNXCPU: 1
-      run: $PYTEST selfdrive/test/process_replay/test_regen.py
-
-  simulator_driving:
-    name: simulator driving
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    if: false  # FIXME: Started to timeout recently
-    steps:
-    - uses: actions/checkout@v6
-      with:
-        submodules: true
-    - run: ./tools/op.sh setup
-    - name: Build openpilot
-      run: scons -j$(nproc)
-    - name: Driving test
-      timeout-minutes: 2
-      run: |
-        source selfdrive/test/setup_xvfb.sh
-        pytest -s tools/sim/tests/test_metadrive_bridge.py
-
-  create_ui_report:
-    name: Create UI Report
-    runs-on: ${{
-      (github.repository == 'commaai/openpilot') &&
-      ((github.event_name != 'pull_request') ||
-      (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
-      && fromJSON('["namespace-profile-amd64-8x16"]')
-      || fromJSON('["ubuntu-24.04"]') }}
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
-      - run: ./tools/op.sh setup
-      - name: Build openpilot
-        run: scons -j$(nproc)
-      - name: Create UI Report
-        run: |
-          source selfdrive/test/setup_xvfb.sh
-          python3 selfdrive/ui/tests/diff/replay.py
-          python3 selfdrive/ui/tests/diff/replay.py --big
-      - name: Upload UI Report
-        uses: actions/upload-artifact@v6
-        with:
-          name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-          path: selfdrive/ui/tests/diff/report

.github/workflows/tools_tests.yaml

@@ -0,0 +1,55 @@
+name: tools
+
+on:
+  push:
+    branches:
+      - master
+      - master-new
+  pull_request:
+  workflow_call:
+    inputs:
+      run_number:
+        default: '1'
+        required: true
+        type: string
+concurrency:
+  group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  BASE_IMAGE: openpilot-base
+  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
+
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+
+jobs:
+  devcontainer:
+    name: devcontainer
+    runs-on: ubuntu-latest
+    if: false  # we can re-enable once this is faster
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Use local image for testing devcontainer with latest base image
+      run: |
+        echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
+    - name: Setup Dev Container CLI
+      run: npm install -g @devcontainers/cli
+    - name: Build dev container image
+      run: ./scripts/retry.sh devcontainer build --workspace-folder .
+    - name: Run dev container
+      run: |
+        mkdir -p /tmp/devcontainer_scons_cache/
+        cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
+        devcontainer up --workspace-folder .
+    - name: Test environment
+      run: |
+        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
+        devcontainer exec --workspace-folder . pip3 install pip-install-test
+        devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
+        devcontainer exec --workspace-folder . sudo touch /root/test.txt

.github/workflows/ui_preview.yaml

@@ -3,25 +3,21 @@ on:
   push:
     branches:
       - master
+      - master-new
   pull_request_target:
     types: [assigned, opened, synchronize, reopened, edited]
     branches:
       - 'master'
+      - 'master-new'
     paths:
-      - 'selfdrive/assets/**'
       - 'selfdrive/ui/**'
-      - 'system/ui/**'
   workflow_dispatch:
 
 env:
   UI_JOB_NAME: "Create UI Report"
-  REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
-  SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
-  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}-ui-preview"
-  REPORT_FILES_BRANCH_NAME: "mici-raylib-ui-reports"
-
-  # variant:video_prefix:master_branch
-  VARIANTS: "mici:mici_ui_replay:openpilot_master_ui_mici_raylib big:tizi_ui_replay:openpilot_master_ui_big_raylib"
+  REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
+  SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
+  BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
 
 jobs:
   preview:
@@ -34,9 +30,8 @@ jobs:
       pull-requests: write
       actions: read
     steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: true
+      - name: Waiting for ui generation to start
+        run: sleep 30
 
       - name: Waiting for ui generation to end
         uses: lewagon/wait-on-check-action@v1.3.4
@@ -53,93 +48,98 @@ jobs:
           echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
 
       - name: Getting proposed ui
+        id: download-artifact
         uses: dawidd6/action-download-artifact@v6
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           run_id: ${{ steps.get_run_id.outputs.run_id }}
           search_artifacts: true
-          name: ui-report-1-${{ env.REPORT_NAME }}
+          name: report-1-${{ env.REPORT_NAME }}
           path: ${{ github.workspace }}/pr_ui
 
-      - name: Getting mici master ui
-        uses: actions/checkout@v6
+      - name: Getting master ui
+        uses: actions/checkout@v4
         with:
           repository: sunnypilot/ci-artifacts
           ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_mici
-          ref: openpilot_master_ui_mici_raylib
-
-      - name: Getting big master ui
-        uses: actions/checkout@v6
-        with:
-          repository: sunnypilot/ci-artifacts
-          ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
-          path: ${{ github.workspace }}/master_big
-          ref: openpilot_master_ui_big_raylib
+          path: ${{ github.workspace }}/master_ui
+          ref: openpilot_master_ui
 
       - name: Saving new master ui
-        if: github.ref == 'refs/heads/master' && github.event_name == 'push'
+        if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video branch <<< "$variant"
-            master_dir="${{ github.workspace }}/master_${name}"
-            cd "$master_dir"
-            git checkout --orphan=new_branch
-            git rm -rf *
-            git branch -D "$branch"
-            git branch -m "$branch"
-            git config user.name "GitHub Actions Bot"
-            git config user.email "<>"
-            cp "${{ github.workspace }}/pr_ui/${video}.mp4" .
-            git add .
-            git commit -m "${name} video for commit ${{ env.SHA }}"
-            git push origin "$branch" --force
-          done
+          git checkout --orphan=new_master_ui
+          git rm -rf *
+          git branch -D openpilot_master_ui
+          git branch -m openpilot_master_ui
+          git config user.name "GitHub Actions Bot"
+          git config user.email "<>"
+          mv ${{ github.workspace }}/pr_ui/*.png .
+          git add .
+          git commit -m "screenshots for commit ${{ env.SHA }}"
+          git push origin openpilot_master_ui --force
 
-      - name: Setup FFmpeg
-        uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
-
-      - name: Finding diffs
+      - name: Finding diff
         if: github.event_name == 'pull_request_target'
         id: find_diff
-        run: |
-          export PYTHONPATH=${{ github.workspace }}
-          baseurl="https://github.com/sunnypilot/ci-artifacts/raw/refs/heads/${{ env.BRANCH_NAME }}"
+        run: >-
+          sudo apt-get install -y imagemagick
 
-          COMMENT=""
-          for variant in $VARIANTS; do
-            IFS=':' read -r name video _ <<< "$variant"
-            diff_name="${name}_diff"
+          scenes="homescreen settings_device settings_software settings_toggles settings_developer offroad_alert update_available prime onroad onroad_disengaged onroad_override onroad_sidebar onroad_wide onroad_wide_sidebar onroad_alert_small onroad_alert_mid onroad_alert_full driver_camera body keyboard"
+          A=($scenes)
 
-            mv "${{ github.workspace }}/pr_ui/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_proposed.mp4"
-            cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
+          DIFF=""
+          TABLE="<details><summary>All Screenshots</summary>"
+          TABLE="${TABLE}<table>"
 
-            diff_exit_code=0
-            python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
-              "${{ github.workspace }}/pr_ui/${video}_master.mp4" \
-              "${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
-              "${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
+          for ((i=0; i<${#A[*]}; i=i+1));
+          do
 
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
+            if ! compare -fuzz 2% -highlight-color DeepSkyBlue1 -lowlight-color Black -compose Src ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png; then
+              convert ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png -transparent black mask.png
+              composite mask.png ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png
+              convert -delay 100 ${{ github.workspace }}/master_ui/${A[$i]}.png composite_diff.png -loop 0 ${{ github.workspace }}/pr_ui/${A[$i]}_diff.gif
 
-            REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
-            if [ $diff_exit_code -eq 0 ]; then
-              COMMENT+="**${name}**: Videos are identical! [View Diff Report]($REPORT_URL)"$'\n'
+              mv ${{ github.workspace }}/master_ui/${A[$i]}.png ${{ github.workspace }}/pr_ui/${A[$i]}_master_ref.png
+
+              DIFF="${DIFF}<details open>"
+              DIFF="${DIFF}<summary>${A[$i]} : \$\${\\color{red}\\text{DIFFERENT}}\$\$</summary>"
+              DIFF="${DIFF}<table>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
+              DIFF="${DIFF}  <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}<tr>"
+              DIFF="${DIFF}  <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
+              DIFF="${DIFF}  <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
+              DIFF="${DIFF}</tr>"
+
+              DIFF="${DIFF}</table>"
+              DIFF="${DIFF}</details>"
             else
-              COMMENT+="**${name}**: ⚠️ <strong>Videos differ!</strong> [View Diff Report]($REPORT_URL)"$'\n'
+              rm -f ${{ github.workspace }}/pr_ui/${A[$i]}_diff.png
+            fi
+
+            INDEX=$(($i % 2))
+            if [[ $INDEX -eq 0 ]]; then
+              TABLE="${TABLE}<tr>"
+            fi
+            TABLE="${TABLE}  <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
+            if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
+              TABLE="${TABLE}</tr>"
             fi
           done
 
-          {
-            echo "COMMENT<<EOF"
-            echo "$COMMENT"
-            echo "EOF"
-          } >> "$GITHUB_OUTPUT"
+          TABLE="${TABLE}</table></details>"
+
+          echo "DIFF=$DIFF$TABLE" >> "$GITHUB_OUTPUT"
 
       - name: Saving proposed ui
         if: github.event_name == 'pull_request_target'
-        working-directory: ${{ github.workspace }}/master_mici
+        working-directory: ${{ github.workspace }}/master_ui
         run: |
           git config user.name "GitHub Actions Bot"
           git config user.email "<>"
@@ -147,29 +147,17 @@ jobs:
           git rm -rf *
           mv ${{ github.workspace }}/pr_ui/* .
           git add .
-          git commit -m "ui videos for PR #${{ github.event.number }}"
+          git commit -m "screenshots for PR #${{ github.event.number }}"
           git push origin ${{ env.BRANCH_NAME }} --force
 
-          # Append diff reports to report files branch
-          git fetch origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-          git checkout ${{ env.REPORT_FILES_BRANCH_NAME }}
-          for variant in $VARIANTS; do
-            IFS=':' read -r name _ _ <<< "$variant"
-            diff_name="${name}_diff"
-            cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
-            git add "${diff_name}_pr_${{ github.event.number }}.html"
-          done
-          git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
-          git push origin ${{ env.REPORT_FILES_BRANCH_NAME }}
-
-      - name: Comment on PR
+      - name: Comment Screenshots on PR
         if: github.event_name == 'pull_request_target'
         uses: thollander/actions-comment-pull-request@v2
         with:
           message: |
-            <!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
+            <!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
             ## UI Preview
-            ${{ steps.find_diff.outputs.COMMENT }}
-          comment_tag: run_id_ui_preview
+            ${{ steps.find_diff.outputs.DIFF }}
+          comment_tag: run_id_screenshots
           pr_number: ${{ github.event.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/wait-for-action/action.yaml

@@ -1,52 +0,0 @@
-name: 'Wait for Tests'
-description: 'Action to wait for workflow tests to start and complete'
-inputs:
-  workflow:
-    description: 'The workflow file name to monitor'
-    required: true
-    default: 'tests.yaml'
-  branch:
-    description: 'The branch to monitor (defaults to current branch)'
-    required: false
-    default: ''
-  github-token:
-    description: 'GitHub token for API access'
-    required: true
-  wait-time:
-    description: 'Initial sleep time in seconds before monitoring starts'
-    required: false
-    default: '30'
-  should-wait-for-start:
-    description: 'Whether to wait for tests to start'
-    required: false
-    default: false
-
-runs:
-  using: 'composite'
-  steps:
-    - name: Wait for tests to start
-      if: inputs.should-wait-for-start == 'true'
-      shell: bash
-      run: |
-        echo "Sleeping for ${{ inputs.wait-time }} seconds to give some time for the action to start and then we'll wait"
-        sleep ${{ inputs.wait-time }}
-
-    - name: Wait for tests to finish
-      shell: bash
-      run: |
-        BRANCH="${{ inputs.branch || github.head_ref || github.ref_name }}"
-        
-        echo "Looking for workflow runs of ${{ inputs.workflow }} on branch $BRANCH"
-        RUN_ID=$(gh run list --workflow=${{ inputs.workflow }} --branch="$BRANCH" --limit=1 --json databaseId --jq '.[0].databaseId')
-        echo "Watching run ID: $RUN_ID"
-        gh run watch "$RUN_ID"
-
-        CONCLUSION=$(gh run view "$RUN_ID" --json conclusion --jq '.conclusion')
-        echo "Run concluded with: $CONCLUSION"
-
-        if [[ "$CONCLUSION" != "success" ]]; then
-          echo "❌ Workflow run failed with conclusion: $CONCLUSION"
-          exit 1
-        fi
-      env:
-        GITHUB_TOKEN: ${{ inputs.github-token }}

.gitignore

@@ -10,16 +10,15 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
+model2.png
 a.out
 .hypothesis
-.cache/
-bin/
 
-*.mp4
+/docs_site/
+
 *.dylib
 *.DSYM
 *.d
-*.pem
 *.pyc
 *.pyo
 .*.swp
@@ -36,62 +35,71 @@ bin/
 *.class
 *.pyxbldc
 *.vcd
-*.mo
-*_pyx.cpp
-*.stats
-*.pkl
-*.pkl*
+*.qm
 config.json
+clcache
 compile_commands.json
 compare_runtime*.html
-selfdrive/modeld/models/tg_compiled_flags.json
 
-# build artifacts
-docs_site/
+persist
 selfdrive/pandad/pandad
 cereal/services.h
 cereal/gen
 cereal/messaging/bridge
+selfdrive/logcatd/logcatd
+selfdrive/mapd/default_speeds_by_region.json
+system/proclogd/proclogd
+selfdrive/ui/translations/alerts_generated.h
 selfdrive/ui/translations/tmp
+selfdrive/test/longitudinal_maneuvers/out
 selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 
+notebooks
+hyperthneed
+provisioning
+
 .coverage*
 coverage.xml
 htmlcov
+pandaextra
+
+.mypy_cache/
+flycheck_*
+
+cppcheck_report.txt
+comma*.sh
+
+selfdrive/modeld/thneed/compile
+selfdrive/modeld/models/*.thneed
+selfdrive/modeld/models/*.pkl
 
-# openpilot log files
 *.bz2
 *.zst
-*.rlog
 
 build/
 
 !**/.gitkeep
 
+poetry.toml
+Pipfile
 
 ### VisualStudioCode ###
-*.vsix
-.history
-.ionide
 .vscode/*
-.history/
 !.vscode/settings.json
 !.vscode/tasks.json
 !.vscode/launch.json
 !.vscode/extensions.json
 !.vscode/*.code-snippets
 
-# agents
-.claude/
-.context/
-PLAN.md
-TASK.md
-CLAUDE.md
-SKILL.md
+# Local History for Visual Studio Code
+.history/
 
-### JetBrains ###
-!.idea/customTargets.xml
-!.idea/tools/*
-!.run/*
+# Built Visual Studio Code Extensions
+*.vsix
+
+### VisualStudioCode Patch ###
+# Ignore all local history of files
+.history
+.ionide

.gitmodules

@@ -6,7 +6,7 @@
   url = https://github.com/sunnypilot/opendbc.git
 [submodule "msgq"]
   path = msgq_repo
-  url = https://github.com/commaai/msgq.git
+  url = https://github.com/sunnypilot/msgq.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = https://github.com/commaai/rednose.git
@@ -15,7 +15,4 @@
   url = https://github.com/commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
-  url = https://github.com/sunnypilot/tinygrad.git
-[submodule "sunnypilot/neural_network_data"]
-  path = sunnypilot/neural_network_data
-  url = https://github.com/sunnypilot/neural-network-data.git
+  url = https://github.com/tinygrad/tinygrad.git

.idea/customTargets.xml

@@ -1,25 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<project version="4">
-  <component name="CLionExternalBuildManager">
-    <target id="a62f99e8-5ec4-434c-8122-49efed5af108" name="uv Scons Build Debug" defaultType="TOOL">
-      <configuration id="b93ec964-16e5-4962-a12e-3ed360ce8f02" name="uv Scons Build Debug">
-        <build type="TOOL">
-          <tool actionId="Tool_External Tools_uv Scons Build Debug" />
-        </build>
-        <clean type="TOOL">
-          <tool actionId="Tool_External Tools_uv Scons Clean" />
-        </clean>
-      </configuration>
-    </target>
-    <target id="edd8ad9d-183b-467c-a355-0d9a0ecab026" name="uv Scons Build Release" defaultType="TOOL">
-      <configuration id="09523339-5ce3-4223-ab9e-904f38ad7752" name="uv Scons Build Release">
-        <build type="TOOL">
-          <tool actionId="Tool_External Tools_uv Scons Build Release" />
-        </build>
-        <clean type="TOOL">
-          <tool actionId="Tool_External Tools_uv Scons Clean" />
-        </clean>
-      </configuration>
-    </target>
-  </component>
-</project>

.idea/tools/External Tools.xml

@@ -1,23 +0,0 @@
-<toolSet name="External Tools">
-  <tool name="uv Scons Build Debug" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
-    <exec>
-      <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --ccflags=\&quot;-fno-inline\&quot;&quot;" />
-      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
-    </exec>
-  </tool>
-  <tool name="uv Scons Clean" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
-    <exec>
-      <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -c&quot; " />
-      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
-    </exec>
-  </tool>
-  <tool name="uv Scons Build Release" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
-    <exec>
-      <option name="COMMAND" value="bash" />
-      <option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc)&quot; " />
-      <option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
-    </exec>
-  </tool>
-</toolSet>

.lfsconfig

@@ -1,4 +1,4 @@
 [lfs]
-	url = https://gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git/info/lfs
-	pushurl = ssh://git@gitlab.com/sunnypilot/public/sunnypilot-new-lfs.git
+	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
+	pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
 	locksverify = false

.lfsconfig-comma

@@ -1,4 +0,0 @@
-[lfs]
-	url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
-	pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
-	locksverify = false

.python-version

@@ -1 +0,0 @@
-3.12.13

.run/Build Debug.run.xml

@@ -1,10 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build Debug" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="sunnypilot" TARGET_NAME="uv Scons Build Debug" CONFIG_NAME="uv Scons Build Debug" RUN_PATH="ui">
-    <envs>
-      <env name="QT_DBL_CLICK_DIST" value="150" />
-    </envs>
-    <method v="2">
-      <option name="CLION.EXTERNAL.BUILD" enabled="true" />
-    </method>
-  </configuration>
-</component>

.run/Build Release.run.xml

@@ -1,10 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build Release" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="sunnypilot" TARGET_NAME="uv Scons Build Release" CONFIG_NAME="uv Scons Build Release" RUN_PATH="ui">
-    <envs>
-      <env name="QT_DBL_CLICK_DIST" value="150" />
-    </envs>
-    <method v="2">
-      <option name="CLION.EXTERNAL.BUILD" enabled="true" />
-    </method>
-  </configuration>
-</component>

.run/Build_BIG_UI.run.xml

@@ -1,26 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_BIG_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <envs>
-      <env name="BIG" value="1" />
-    </envs>
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.run/Build_SMALL_UI.run.xml

@@ -1,23 +0,0 @@
-<component name="ProjectRunConfigurationManager">
-  <configuration default="false" name="Build_SMALL_UI" type="PythonConfigurationType" factoryName="Python">
-    <module name="sunnypilot" />
-    <option name="ENV_FILES" value="" />
-    <option name="INTERPRETER_OPTIONS" value="" />
-    <option name="PARENT_ENVS" value="true" />
-    <option name="SDK_HOME" value="" />
-    <option name="WORKING_DIRECTORY" value="$ProjectFileDir$/" />
-    <option name="IS_MODULE_SDK" value="true" />
-    <option name="ADD_CONTENT_ROOTS" value="true" />
-    <option name="ADD_SOURCE_ROOTS" value="true" />
-    <option name="SCRIPT_NAME" value="$ProjectFileDir$/selfdrive/ui/ui.py" />
-    <option name="PARAMETERS" value="" />
-    <option name="SHOW_COMMAND_LINE" value="false" />
-    <option name="EMULATE_TERMINAL" value="false" />
-    <option name="MODULE_MODE" value="false" />
-    <option name="REDIRECT_INPUT" value="false" />
-    <option name="INPUT_FILE" value="" />
-    <method v="2">
-      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_uv Scons Build Debug" />
-    </method>
-  </configuration>
-</component>

.vscode/launch.json

@@ -23,11 +23,6 @@
       "id": "args",
       "description": "Arguments to pass to the process",
       "type": "promptString"
-    },
-    {
-      "id": "replayArg",
-      "type": "promptString",
-      "description": "Enter route or segment to replay."
     }
   ],
   "configurations": [
@@ -45,44 +40,7 @@
       "type": "cppdbg",
       "request": "launch",
       "program": "${workspaceFolder}/${input:cpp_process}",
-      "cwd": "${workspaceFolder}"
-    },
-    {
-      "name": "Attach LLDB to Replay drive",
-      "type": "lldb",
-      "request": "attach",
-      "pid": "${command:pickMyProcess}",
-      "sourceMap": {
-        ".": "${workspaceFolder}/opendbc/safety"
-      },
-      "initCommands": [
-        "script import time; time.sleep(3)"
-      ]
-    },
-    {
-      "name": "Replay drive",
-      "type": "debugpy",
-      "request": "launch",
-      "program": "${workspaceFolder}/opendbc/safety/tests/safety_replay/replay_drive.py",
-      "args": [
-        "${input:replayArg}"
-      ],
-      "console": "integratedTerminal",
-      "justMyCode": false,
-      "env": {
-        "PYTHONPATH": "${workspaceFolder}"
-      },
-      "subProcess": true,
-      "stopOnEntry": false
-    }
-  ],
-  "compounds": [
-    {
-      "name": "Replay drive + Safety LLDB",
-      "configurations": [
-        "Replay drive",
-        "Attach LLDB to Replay drive"
-      ]
+      "cwd": "${workspaceFolder}",
     }
   ]
 }

Dockerfile.openpilot

@@ -1,38 +1,13 @@
-FROM ubuntu:24.04
+FROM ghcr.io/commaai/openpilot-base:latest
 
-ENV PYTHONUNBUFFERED=1
+ENV PYTHONUNBUFFERED 1
 
-ENV DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales && \
-    rm -rf /var/lib/apt/lists/*
+ENV OPENPILOT_PATH /home/batman/openpilot
+ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
 
-RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
-ENV LANG=en_US.UTF-8
-ENV LANGUAGE=en_US:en
-ENV LC_ALL=en_US.UTF-8
-
-ENV NVIDIA_VISIBLE_DEVICES=all
-ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
-
-ARG USER=batman
-ARG USER_UID=1001
-RUN useradd -m -s /bin/bash -u $USER_UID $USER
-RUN usermod -aG sudo $USER
-RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
-USER $USER
-
-ENV OPENPILOT_PATH=/home/$USER/openpilot
 RUN mkdir -p ${OPENPILOT_PATH}
 WORKDIR ${OPENPILOT_PATH}
 
-COPY --chown=$USER . ${OPENPILOT_PATH}/
+COPY . ${OPENPILOT_PATH}/
 
-ENV UV_BIN="/home/$USER/.local/bin/"
-ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
-ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
-RUN tools/setup_dependencies.sh && \
-    sudo rm -rf /var/lib/apt/lists/*
-
-USER root
-RUN git config --global --add safe.directory '*'
+RUN scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -0,0 +1,84 @@
+FROM ubuntu:24.04
+
+ENV PYTHONUNBUFFERED 1
+
+ENV DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
+RUN /tmp/tools/install_ubuntu_dependencies.sh && \
+    rm -rf /var/lib/apt/lists/* /tmp/* && \
+    cd /usr/lib/gcc/arm-none-eabi/* && \
+    rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
+
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    libx11-6 \
+    wget \
+  && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /tmp/opencl-driver-intel && \
+    cd /tmp/opencl-driver-intel && \
+    wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
+    mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
+    tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
+    cd /opt/intel && \
+    tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
+    mkdir -p /etc/ld.so.conf.d && \
+    echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX 1
+
+RUN dbus-uuidgen > /etc/machine-id
+
+ARG USER=batman
+ARG USER_UID=1001
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+COPY --chown=$USER pyproject.toml uv.lock /tmp/
+COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
+
+ENV VIRTUAL_ENV=/home/$USER/.venv
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+RUN cd /tmp && \
+    tools/install_python_dependencies.sh && \
+    mkdir -p $VIRTUAL_ENV && \
+    cp -r /tmp/.venv/* $VIRTUAL_ENV && \
+    rm -rf /tmp/* && \
+    rm -rf /home/$USER/.cache
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Jenkinsfile

@@ -22,7 +22,7 @@ shopt -s huponexit # kill all child processes when the shell exits
 
 export CI=1
 export PYTHONWARNINGS=error
-#export LOGPRINT=debug # this has gotten too spammy...
+export LOGPRINT=debug
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
@@ -31,7 +31,7 @@ export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
 export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
 # only use 1 thread for tici tests since most require HIL
-export PYTEST_ADDOPTS="-n0 -s"
+export PYTEST_ADDOPTS="-n 0"
 
 
 export GIT_SSH_COMMAND="ssh -i /data/gitkey"
@@ -79,10 +79,6 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
         return
     }
 
-    if (isReplay()) {
-      error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.")
-    }
-
     def extra = extra_env.collect { "export ${it}" }.join('\n');
     def branch = env.BRANCH_NAME ?: 'master';
     def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
@@ -103,7 +99,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
             def diffPaths = args.diffPaths ?: []
             def cmdTimeout = args.timeout ?: 9999
 
-            if (branch != "master" && !branch.contains("__jenkins_loop_") && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
+            if (branch != "master" && diffPaths && !hasPathChanged(gitDiff, diffPaths)) {
               println "Skipping ${name}: no changes in ${diffPaths}."
               return
             } else {
@@ -127,11 +123,6 @@ def hasPathChanged(String gitDiff, List<String> paths) {
   return false
 }
 
-def isReplay() {
-  def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
-  return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) }
-}
-
 def setupCredentials() {
   withCredentials([
     string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
@@ -166,11 +157,11 @@ node {
   env.GIT_BRANCH = checkout(scm).GIT_BRANCH
   env.GIT_COMMIT = checkout(scm).GIT_COMMIT
 
-  def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
-                         'release-tici', 'release-tizi', 'release-tizi-staging', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
+  def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
+                         'testing-closet*', 'hotfix-*']
   def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
 
-  if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
+  if (env.BRANCH_NAME != 'master') {
     properties([
         disableConcurrentBuilds(abortPrevious: true)
     ])
@@ -178,20 +169,20 @@ node {
 
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
-      deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh && git push -f origin release-tizi-staging:release-mici-staging"),
+      deviceStage("build release3-staging", "tici-needs-can", [], [
+        step("build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"),
       ])
     }
 
-    if (env.BRANCH_NAME == '__nightly') {
+    if (env.BRANCH_NAME == 'master-ci') {
       parallel (
         'nightly': {
-          deviceStage("build nightly", "tizi-needs-can", [], [
+          deviceStage("build nightly", "tici-needs-can", [], [
             step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
-          deviceStage("build nightly-dev", "tizi-needs-can", [], [
+          deviceStage("build nightly-dev", "tici-needs-can", [], [
             step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
           ])
         },
@@ -200,54 +191,69 @@ node {
 
     if (!env.BRANCH_NAME.matches(excludeRegex)) {
     parallel (
+      // tici tests
       'onroad tests': {
-        deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
+        deviceStage("onroad", "tici-needs-can", ["UNSAFE=1"], [
+          // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
+          //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
           step("build openpilot", "cd system/manager && ./build.py"),
           step("check dirty", "release/check-dirty.sh"),
           step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
+          //["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
         ])
       },
       'HW + Unit Tests': {
-        deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
+        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
-          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
+          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"),
+          step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"),
           step("test manager", "pytest system/manager/test/test_manager.py"),
         ])
       },
-      'camerad OX03C10': {
-        deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
-          step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+      'loopback': {
+        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
+          step("build openpilot", "cd system/manager && ./build.py"),
+          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
         ])
       },
-      'camerad OS04C10': {
-        deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
+      'camerad': {
+        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
-          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"),
-          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
+        ])
+        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 60]),
+          step("test exposure", "pytest system/camerad/test/test_exposure.py"),
         ])
       },
       'sensord': {
-        deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
+        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
+          step("build", "cd system/manager && ./build.py"),
+          step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
+        ])
+        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py"),
           step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
         ])
       },
       'replay': {
-        deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
+        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
           step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
-          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
+          step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/"]]),
         ])
       },
       'tizi': {
         deviceStage("tizi", "tizi", ["UNSAFE=1"], [
           step("build openpilot", "cd system/manager && ./build.py"),
-          step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
+          step("test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
           step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
+          step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
           step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
-          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
+          step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"),
         ])
       },
 

README.md

@@ -1,74 +1,98 @@
-![](https://user-images.githubusercontent.com/47793918/233812617-beab2e71-57b9-479e-8bff-c3931347ca40.png)
+<div align="center" style="text-align: center;">
 
-## 🌞 What is sunnypilot?
-[sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 300+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
+<h1>openpilot</h1>
 
-## 💭 Join our Community Forum
-Join the official sunnypilot community forum to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
-* https://community.sunnypilot.ai/
+<p>
+  <b>openpilot is an operating system for robotics.</b>
+  <br>
+  Currently, it upgrades the driver assistance system in 275+ supported cars.
+</p>
 
-## Documentation
-https://docs.sunnypilot.ai/ is your one stop shop for everything from features to installation to FAQ about the sunnypilot
+<h3>
+  <a href="https://docs.comma.ai">Docs</a>
+  <span> · </span>
+  <a href="https://docs.comma.ai/contributing/roadmap/">Roadmap</a>
+  <span> · </span>
+  <a href="https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md">Contribute</a>
+  <span> · </span>
+  <a href="https://discord.comma.ai">Community</a>
+  <span> · </span>
+  <a href="https://comma.ai/shop">Try it on a comma 3X</a>
+</h3>
 
-## 🚘 Running on a dedicated device in a car
-First, check out this list of items you'll need to [get started](https://community.sunnypilot.ai/t/getting-started-using-sunnypilot-in-your-supported-car/251).
+Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
 
-## Installation
-Next, refer to the sunnypilot community forum for [installation instructions](https://community.sunnypilot.ai/t/read-before-installing-sunnypilot/254), as well as a complete list of [Recommended Branch Installations](https://community.sunnypilot.ai/t/recommended-branch-installations/235).
+![openpilot tests](https://github.com/commaai/openpilot/actions/workflows/selfdrive_tests.yaml/badge.svg)
+[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+[![X Follow](https://img.shields.io/twitter/follow/comma_ai)](https://x.com/comma_ai)
+[![Discord](https://img.shields.io/discord/469524606043160576)](https://discord.comma.ai)
 
-## 🎆 Pull Requests
-We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.
+</div>
 
-Pull requests should be against the most current `master` branch.
+<table>
+  <tr>
+    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://github.com/commaai/openpilot/assets/8762862/2f7112ae-f748-4f39-b617-fabd689c3772"></a></td>
+    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://github.com/commaai/openpilot/assets/8762862/92351544-2833-40d7-9e0b-7ef7ae37ec4c"></a></td>
+    <td><a href="https://youtu.be/SUIZYzxtMQs" title="A drive to Taco Bell"><img src="https://github.com/commaai/openpilot/assets/8762862/05ceefc5-2628-439c-a9b2-89ce77dc6f63"></a></td>
+  </tr>
+</table>
 
-## 📊 User Data
+To start using openpilot in a car
+------
 
-By default, sunnypilot uploads the driving data to comma servers. You can also access your data through [comma connect](https://connect.comma.ai/).
+To use openpilot in a car, you need four things:
+1. **Supported Device:** a comma 3/3X, available at [comma.ai/shop](https://comma.ai/shop/comma-3x).
+2. **Software:** The setup procedure for the comma 3/3X allows users to enter a URL for custom software. Use the URL `openpilot.comma.ai` to install the release version.
+3. **Supported Car:** Ensure that you have one of [the 275+ supported cars](docs/CARS.md).
+4. **Car Harness:** You will also need a [car harness](https://comma.ai/shop/car-harness) to connect your comma 3/3X to your car.
 
-sunnypilot is open source software. The user is free to disable data collection if they wish to do so.
+We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
 
-sunnypilot logs the road-facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
-The driver-facing camera and microphone are only logged if you explicitly opt-in in settings.
+To start developing openpilot
+------
 
-By using this software, you understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot).
 
-## Licensing
+* Join the [community Discord](https://discord.comma.ai)
+* Check out [the contributing docs](docs/CONTRIBUTING.md)
+* Check out the [openpilot tools](tools/)
+* Read about the [development workflow](docs/WORKFLOW.md)
+* Code documentation lives at https://docs.comma.ai
+* Information about running openpilot lives on the [community wiki](https://github.com/commaai/openpilot/wiki)
 
-sunnypilot is released under the [MIT License](LICENSE). This repository includes original work as well as significant portions of code derived from [openpilot by comma.ai](https://github.com/commaai/openpilot), which is also released under the MIT license with additional disclaimers.
+Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs#open-positions) and offers lots of [bounties](https://comma.ai/bounties) for external contributors.
 
-The original openpilot license notice, including comma.ai’s indemnification and alpha software disclaimer, is reproduced below as required:
+Safety and Testing
+----
 
-> openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
->
-> Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
->
-> **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
-> YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
-> NO WARRANTY EXPRESSED OR IMPLIED.**
+* openpilot observes [ISO26262](https://en.wikipedia.org/wiki/ISO_26262) guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
+* openpilot has software-in-the-loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
+* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
+* panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
+* Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
+* panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
+* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
 
-For full license terms, please see the [`LICENSE`](LICENSE) file.
+Licensing
+------
 
-## 💰 Support sunnypilot
-If you find any of the features useful, consider becoming a [sponsor on GitHub](https://github.com/sponsors/sunnyhaibin) to support future feature development and improvements.
+openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
+Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
 
-By becoming a sponsor, you will gain access to exclusive content, early access to new features, and the opportunity to directly influence the project's development.
+**THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
+YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
+NO WARRANTY EXPRESSED OR IMPLIED.**
 
+User Data and comma Account
+------
 
-<h3>GitHub Sponsor</h3>
+By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
 
-<a href="https://github.com/sponsors/sunnyhaibin">
-  <img src="https://user-images.githubusercontent.com/47793918/244135584-9800acbd-69fd-4b2b-bec9-e5fa2d85c817.png" alt="Become a Sponsor" width="300" style="max-width: 100%; height: auto;">
-</a>
-<br>
+openpilot is open source software: the user is free to disable data collection if they wish to do so.
 
-<h3>PayPal</h3>
+openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
 
-<a href="https://paypal.me/sunnyhaibin0850" target="_blank">
-<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" alt="PayPal this" title="PayPal - The safer, easier way to pay online!" border="0" />
-</a>
-<br></br>
-
-Your continuous love and support are greatly appreciated! Enjoy 🥰
-
-<span>-</span> Jason, Founder of sunnypilot
+By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.

RELEASES.md

@@ -1,88 +1,16 @@
-Version 0.11.1 (2026-04-22)
-========================
-* New driver monitoring model
-* Improved image processing pipeline for driver camera
-* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
-
-Version 0.11.0 (2026-03-17)
-========================
-* New driving model #36798
-  * Fully trained using a learned simulator
-  * Improved longitudinal performance in Experimental mode
-* Reduce comma four standby power usage by 77% to 52 mW
-* Kia K7 2017 support thanks to royjr!
-* Lexus LS 2018 support thanks to Hacheoy!
-
-Version 0.10.3 (2025-12-17)
-========================
-* New driving model #36249
-  * New temporal policy architecture
-  * New on-policy training physics noise model
-* New driver monitoring model #36409
-  * Trained on a new dataset, including comma four data
-* Improved inter-process communication memory efficiency
-
-Version 0.10.2 (2025-11-19)
-========================
-* comma four support
-
-Version 0.10.1 (2025-09-08)
-========================
-* New driving model #36276
-  * World Model: removed global localization inputs
-  * World Model: 2x the number of parameters
-  * World Model: trained on 4x the number of segments
-  * VAE Compression Model: new architecture and training objective
-  * Driving Vision Model: trained on 4x the number of segments
-* New Driver Monitoring model #36198
-* Acura TLX 2021 support thanks to MVL!
-* Honda City 2023 support thanks to vanillagorillaa and drFritz!
-* Honda N-Box 2018 support thanks to miettal!
-* Honda Odyssey 2021-25 support thanks to csouers and MVL!
-* Honda Passport 2026 support thanks to vanillagorillaa and MVL!
-
-Version 0.10.0 (2025-08-05)
+Version 0.9.8 (2024-XX-XX)
 ========================
 * New driving model
-  * New training architecture
-     * Described in our CVPR paper: "Learning to Drive from a World Model"
-     * Longitudinal MPC replaced by E2E planning from World Model in Experimental Mode
-     * Action from lateral MPC as training objective replaced by E2E planning from World Model
-  * Low-speed lead car ground-truth fixes
-* Enable live-learned steering actuation delay
-* Opt-in audio recording for dashcam video
-* Acura MDX 2025 support thanks to vanillagorillaa and MVL!
-* Honda Accord 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda CR-V 2023-25 support thanks to vanillagorillaa and MVL!
-* Honda Pilot 2023-25 support thanks to vanillagorillaa and MVL!
-
-Version 0.9.9 (2025-05-23)
-========================
-* New driving model
-  * New training architecture using parts from MLSIM
-* Steering actuation delay is now learned online
-* Ford Escape 2023-24 support thanks to incognitojam!
-* Ford Kuga 2024 support thanks to incognitojam!
-* Hyundai Nexo 2021 support thanks to sunnyhaibin!
-* Tesla Model 3 and Y support thanks to lukasloetkolben!
-* Lexus RC 2023 support thanks to nelsonjchen!
-
-Version 0.9.8 (2025-02-28)
-========================
-* New driving model
-  * Model now gates applying positive acceleration in Chill mode
-* New driver monitoring model
+  * Trained in brand new ML simulator
+  * Model now gates applying positive accel in Chill mode
+* New driving monitoring model
   * Reduced false positives related to passengers
 * Image processing pipeline moved to the ISP
-  * More GPU time for bigger driving models
-  * Power draw reduced 0.5W, which means your device runs cooler
+  * More GPU time for driving models
+  * Power draw reduced 0.5W, which means your device runs cooler 
 * Added toggle to enable driver monitoring even when openpilot is not engaged
-* Localizer rewritten to remove GPS dependency at runtime
-* Firehose Mode for maximizing your training data uploads
 * Enable openpilot longitudinal control for Ford Q3 vehicles
 * New Toyota TSS2 longitudinal tune
-* Rivian R1S and R1T support thanks to lukasloetkolben!
-* Ford F-150, F-150 Hybrid, Mach-E, and Ranger support
 
 Version 0.9.7 (2024-06-13)
 ========================

SConstruct

@@ -3,228 +3,358 @@ import subprocess
 import sys
 import sysconfig
 import platform
-import shlex
-import importlib
 import numpy as np
 
 import SCons.Errors
-from SCons.Defaults import _stripixes
 
 SCons.Warnings.warningAsException(True)
 
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
+TICI = os.path.isfile('/TICI')
+AGNOS = TICI
+
 Decider('MD5-timestamp')
 
-SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
+SetOption('num_jobs', int(os.cpu_count()/2))
+
+AddOption('--kaitai',
+          action='store_true',
+          help='Regenerate kaitai struct parsers')
+
+AddOption('--asan',
+          action='store_true',
+          help='turn on ASAN')
+
+AddOption('--ubsan',
+          action='store_true',
+          help='turn on UBSan')
+
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
+AddOption('--clazy',
+          action='store_true',
+          help='build with clazy')
+
+AddOption('--compile_db',
+          action='store_true',
+          help='build clang compilation database')
+
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
+AddOption('--snpe',
+          action='store_true',
+          help='use SNPE on PC')
+
+AddOption('--external-sconscript',
+          action='store',
+          metavar='FILE',
+          dest='external_sconscript',
+          help='add an external SConscript to the build')
+
+AddOption('--pc-thneed',
+          action='store_true',
+          dest='pc_thneed',
+          help='use thneed on pc')
+
+AddOption('--mutation',
+          action='store_true',
+          help='generate mutation-ready code')
 
-AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
-AddOption('--verbose', action='store_true', default=False, help='show full build commands')
 AddOption('--minimal',
           action='store_false',
           dest='extras',
-          default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
+          default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
           help='the minimum build to run openpilot. no tests, tools, etc.')
 
-# Detect platform
-arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
+real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-elif arch == "aarch64" and os.path.isfile('/TICI'):
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and AGNOS:
   arch = "larch64"
-assert arch in [
-  "larch64",  # linux tici arm64
-  "aarch64",  # linux pc arm64
-  "x86_64",   # linux pc x64
-  "Darwin",   # macOS arm64 (x86 not supported)
-]
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
 
-pkg_names = ['bzip2', 'capnproto', 'eigen', 'ffmpeg', 'libjpeg', 'libyuv', 'ncurses', 'zeromq', 'zstd']
-pkgs = [importlib.import_module(name) for name in pkg_names]
+lenv = {
+  "PATH": os.environ['PATH'],
+  "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
 
-
-# ***** enforce a whitelist of system libraries *****
-# this prevents silently relying on a 3rd party package,
-# e.g. apt-installed libusb. all libraries should either
-# be distributed with all Linux distros and macOS, or
-# vendored in commaai/dependencies.
-allowed_system_libs = {
-  "EGL", "GLESv2", "GL",
-  "Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
-  "dl", "drm", "gbm", "m", "pthread",
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
+  "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
+  "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
 
-def _resolve_lib(env, name):
-  for d in env.Flatten(env.get('LIBPATH', [])):
-    p = Dir(str(d)).abspath
-    for ext in ('.a', '.so', '.dylib'):
-      f = File(os.path.join(p, f'lib{name}{ext}'))
-      if f.exists() or f.has_builder():
-        return name
-  if name in allowed_system_libs:
-    return name
-  raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
+rpath = lenv["LD_LIBRARY_PATH"].copy()
 
-def _libflags(target, source, env, for_signature):
-  libs = []
-  lp = env.subst('$LIBLITERALPREFIX')
-  for lib in env.Flatten(env.get('LIBS', [])):
-    if isinstance(lib, str):
-      if os.sep in lib or lib.startswith('#'):
-        libs.append(File(lib))
-      elif lib.startswith('-') or (lp and lib.startswith(lp)):
-        libs.append(lib)
-      else:
-        libs.append(_resolve_lib(env, lib))
-    else:
-      libs.append(lib)
-  return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
-                    env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
+if arch == "larch64":
+  cpppath = [
+    "#third_party/opencl/include",
+  ]
+
+  libpath = [
+    "/usr/local/lib",
+    "/system/vendor/lib64",
+    f"#third_party/acados/{arch}/lib",
+  ]
+
+  libpath += [
+    "#third_party/snpe/larch64",
+    "#third_party/libyuv/larch64/lib",
+    "/usr/lib/aarch64-linux-gnu"
+  ]
+  cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
+  rpath += ["/usr/local/lib"]
+else:
+  cflags = []
+  cxxflags = []
+  cpppath = []
+  rpath += []
+
+  # MacOS
+  if arch == "Darwin":
+    libpath = [
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
+      f"{brew_prefix}/lib",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
+      "/System/Library/Frameworks/OpenGL.framework/Libraries",
+    ]
+
+    cflags += ["-DGL_SILENCE_DEPRECATION"]
+    cxxflags += ["-DGL_SILENCE_DEPRECATION"]
+    cpppath += [
+      f"{brew_prefix}/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
+    ]
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
+  else:
+    libpath = [
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      "/usr/lib",
+      "/usr/local/lib",
+    ]
+
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
+
+if GetOption('asan'):
+  ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
+  ldflags = ["-fsanitize=address"]
+elif GetOption('ubsan'):
+  ccflags = ["-fsanitize=undefined"]
+  ldflags = ["-fsanitize=undefined"]
+else:
+  ccflags = []
+  ldflags = []
+
+# no --as-needed on mac linker
+if arch != "Darwin":
+  ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
+
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
 
 env = Environment(
-  ENV={
-    "PATH": os.environ['PATH'],
-    "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
-    "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
-    "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
-    "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
-  },
+  ENV=lenv,
   CCFLAGS=[
     "-g",
     "-fPIC",
     "-O2",
     "-Wunused",
     "-Werror",
-    "-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
+    "-Wshadow",
     "-Wno-unknown-warning-option",
     "-Wno-inconsistent-missing-override",
     "-Wno-c99-designator",
     "-Wno-reorder-init-list",
     "-Wno-vla-cxx-extension",
-  ],
-  CFLAGS=["-std=gnu11"],
-  CXXFLAGS=["-std=c++1z"],
-  CPPPATH=[
+  ] + cflags + ccflags,
+
+  CPPPATH=cpppath + [
     "#",
-    "#msgq",
-    "#third_party",
-    "#third_party/json11",
-    "#third_party/linux/include",
     "#third_party/acados/include",
     "#third_party/acados/include/blasfeo/include",
     "#third_party/acados/include/hpipm/include",
     "#third_party/catch2/include",
-    [x.INCLUDE_DIR for x in pkgs],
+    "#third_party/libyuv/include",
+    "#third_party/json11",
+    "#third_party/linux/include",
+    "#third_party/snpe/include",
+    "#third_party",
+    "#msgq",
   ],
-  LIBPATH=[
-    "#common",
+
+  CC='clang',
+  CXX='clang++',
+  LINKFLAGS=ldflags,
+
+  RPATH=rpath,
+
+  CFLAGS=["-std=gnu11"] + cflags,
+  CXXFLAGS=["-std=c++1z"] + cxxflags,
+  LIBPATH=libpath + [
     "#msgq_repo",
     "#third_party",
     "#selfdrive/pandad",
+    "#common",
     "#rednose/helpers",
-    f"#third_party/acados/{arch}/lib",
-    [x.LIB_DIR for x in pkgs],
   ],
-  RPATH=[],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
   REDNOSE_ROOT="#",
   tools=["default", "cython", "compilation_db", "rednose_filter"],
   toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
 )
-if arch != "larch64":
-  env['_LIBFLAGS'] = _libflags
 
-# Arch-specific flags and paths
-if arch == "larch64":
-  env["CC"] = "clang"
-  env["CXX"] = "clang++"
-  env.Append(LIBPATH=[
-    "/usr/lib/aarch64-linux-gnu",
-  ])
-  arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
-  env.Append(CCFLAGS=arch_flags)
-  env.Append(CXXFLAGS=arch_flags)
-elif arch == "Darwin":
-  env.Append(LIBPATH=[
-    "/System/Library/Frameworks/OpenGL.framework/Libraries",
-  ])
-  env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
-  env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
+if arch == "Darwin":
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
-_extra_cc = shlex.split(GetOption('ccflags') or '')
-if _extra_cc:
-  env.Append(CCFLAGS=_extra_cc)
+if GetOption('compile_db'):
+  env.CompilationDatabase('compile_commands.json')
 
-# no --as-needed on mac linker
-if arch != "Darwin":
-  env.Append(LINKFLAGS=["-Wl,--as-needed", "-Wl,--no-undefined"])
+# Setup cache dir
+cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
+CacheDir(cache_dir)
+Clean(["."], cache_dir)
 
-# Shorter build output: show brief descriptions instead of full commands.
-# Full command lines are still printed on failure by scons.
-if not GetOption('verbose'):
-  for action, short in (
-    ("CC",     "CC"),
-    ("CXX",    "CXX"),
-    ("LINK",   "LINK"),
-    ("SHCC",   "CC"),
-    ("SHCXX",  "CXX"),
-    ("SHLINK", "LINK"),
-    ("AR",     "AR"),
-    ("RANLIB", "RANLIB"),
-    ("AS",     "AS"),
-  ):
-    env[f"{action}COMSTR"] = f"  [{short}] $TARGET"
-
-# progress output
 node_interval = 5
 node_count = 0
 def progress_function(node):
   global node_count
   node_count += node_interval
   sys.stderr.write("progress: %d\n" % node_count)
+
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-# ********** Cython build environment **********
+# Cython build environment
+py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
-envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
-envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CPPPATH"] += [py_include, np.get_include()]
+envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
 envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = env["LINKFLAGS"] + ["-bundle", "-undefined", "dynamic_lookup"]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
 np_version = SCons.Script.Value(np.__version__)
 Export('envCython', 'np_version')
 
-Export('env', 'arch')
+# Qt build environment
+qt_env = env.Clone()
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "DBus", "Xml"]
 
-# Setup cache dir
-default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
-cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
-CacheDir(cache_dir)
-Clean(["."], cache_dir)
+qt_libs = []
+if arch == "Darwin":
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
+  qt_dirs = [
+    os.path.join(qt_env['QTDIR'], "include"),
+  ]
+  qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
+  qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
+  qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
+  qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
+else:
+  qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
+  qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
 
-# ********** start building stuff **********
+  qt_env['QTDIR'] = qt_install_prefix
+  qt_dirs = [
+    f"{qt_install_headers}",
+  ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
+  qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
+
+  qt_libs = [f"Qt5{m}" for m in qt_modules]
+  if arch == "larch64":
+    qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
+  elif arch != "Darwin":
+    qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
+
+# compatibility for older SCons versions
+try:
+  qt_env.Tool('qt3')
+except SCons.Errors.UserError:
+  qt_env.Tool('qt')
+
+qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
+qt_flags = [
+  "-D_REENTRANT",
+  "-DQT_NO_DEBUG",
+  "-DQT_WIDGETS_LIB",
+  "-DQT_GUI_LIB",
+  "-DQT_CORE_LIB",
+  "-DQT_MESSAGELOGCONTEXT",
+]
+qt_env['CXXFLAGS'] += qt_flags
+qt_env['LIBPATH'] += ['#selfdrive/ui', ]
+qt_env['LIBS'] = qt_libs
+
+if GetOption("clazy"):
+  checks = [
+    "level0",
+    "level1",
+    "no-range-loop",
+    "no-non-pod-global-static",
+  ]
+  qt_env['CXX'] = 'clazy'
+  qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
+  qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
+
+Export('env', 'qt_env', 'arch', 'real_arch')
 
 # Build common module
 SConscript(['common/SConscript'])
-Import('_common')
+Import('_common', '_gpucommon')
+
 common = [_common, 'json11', 'zmq']
-Export('common')
+gpucommon = [_gpucommon]
+
+Export('common', 'gpucommon')
 
 # Build messaging (cereal + msgq + socketmaster + their dependencies)
 # Enable swaglog include in submodules
 env_swaglog = env.Clone()
 env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
 SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
+SConscript(['opendbc/can/SConscript'], exports={'env': env_swaglog})
 
 SConscript(['cereal/SConscript'])
 
 Import('socketmaster', 'msgq')
-messaging = [socketmaster, msgq, 'capnp', 'kj',]
+messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
 Export('messaging')
 
 
@@ -236,8 +366,15 @@ SConscript(['rednose/SConscript'])
 
 # Build system services
 SConscript([
+  'system/proclogd/SConscript',
+  'system/ubloxd/SConscript',
   'system/loggerd/SConscript',
 ])
+if arch != "Darwin":
+  SConscript([
+    'system/sensord/SConscript',
+    'system/logcatd/SConscript',
+  ])
 
 if arch == "larch64":
   SConscript(['system/camerad/SConscript'])
@@ -245,25 +382,13 @@ if arch == "larch64":
 # Build openpilot
 SConscript(['third_party/SConscript'])
 
-# Build selfdrive
-SConscript([
-  'selfdrive/pandad/SConscript',
-  'selfdrive/controls/lib/lateral_mpc_lib/SConscript',
-  'selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
-  'selfdrive/locationd/SConscript',
-  'selfdrive/modeld/SConscript',
-  'selfdrive/ui/SConscript',
-])
+SConscript(['selfdrive/SConscript'])
 
-SConscript(['sunnypilot/SConscript'])
+if Dir('#tools/cabana/').exists() and GetOption('extras'):
+  SConscript(['tools/replay/SConscript'])
+  if arch != "larch64":
+    SConscript(['tools/cabana/SConscript'])
 
-# Build tools
-if arch != "larch64":
-  SConscript([
-    'tools/replay/SConscript',
-    'tools/cabana/SConscript',
-    'tools/jotpluggler/SConscript',
-  ])
-
-
-env.CompilationDatabase('compile_commands.json')
+external_sconscript = GetOption('external_sconscript')
+if external_sconscript:
+  SConscript([external_sconscript])

cereal/README.md

@@ -29,50 +29,6 @@ then means breaking backwards-compatibility with all old logs of your fork. So w
 [custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
 fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
 
-An example of compatible changes:
-```diff
-diff --git a/cereal/custom.capnp b/cereal/custom.capnp
-index 3348e859e..3365c7b98 100644
---- a/cereal/custom.capnp
-+++ b/cereal/custom.capnp
-@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
- # DO rename the structs
- # DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
--struct CustomReserved0 @0x81c2f05a394cf4af {
-+struct SteeringInfo @0x81c2f05a394cf4af {
-+  active @0 :Bool;
-+  steeringAngleDeg @1 :Float32;
-+  steeringRateDeg @2 :Float32;
-+  steeringAccelDeg @3 :Float32;
- }
-
- struct CustomReserved1 @0xaedffd8f31e7b55d {
-diff --git a/cereal/log.capnp b/cereal/log.capnp
-index 1209f3fd9..b189f58b6 100644
---- a/cereal/log.capnp
-+++ b/cereal/log.capnp
-@@ -2558,14 +2558,14 @@ struct Event {
-
-     # DO change the name of the field
-     # DON'T change anything after the "@"
--    customReservedRawData0 @124 :Data;
-+    rawCanData @124 :Data;
-     customReservedRawData1 @125 :Data;
-     customReservedRawData2 @126 :Data;
-
-     # DO change the name of the field and struct
-     # DON'T change the ID (e.g. @107)
-     # DON'T change which struct it points to
--    customReserved0 @107 :Custom.CustomReserved0;
-+    steeringInfo @107 :Custom.SteeringInfo;
-     customReserved1 @108 :Custom.CustomReserved1;
-     customReserved2 @109 :Custom.CustomReserved2;
-     customReserved3 @110 :Custom.CustomReserved3;
-```
-
----
-
 Example
 ---
 ```python

cereal/SConscript

@@ -4,7 +4,7 @@ cereal_dir = Dir('.')
 gen_dir = Dir('gen')
 
 # Build cereal
-schema_files = ['log.capnp', 'car.capnp', 'deprecated.capnp', 'custom.capnp']
+schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
 env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
             schema_files,
             f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
@@ -13,7 +13,7 @@ cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
 
 # Build messaging
 services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
-env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
+env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc'], LIBS=[msgq, common, 'pthread'])
 
 socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
 

cereal/__init__.py

@@ -1,11 +1,9 @@
 import os
 import capnp
-from importlib.resources import as_file, files
 
+CEREAL_PATH = os.path.dirname(os.path.abspath(__file__))
 capnp.remove_import_hook()
 
-with as_file(files("cereal")) as fspath:
-  CEREAL_PATH = fspath.as_posix()
-  log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
-  car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
-  custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
+log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
+car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
+custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))

cereal/custom.capnp

@@ -7,481 +7,33 @@ $Cxx.namespace("cereal");
 # These structs are guaranteed to remain reserved and empty in mainline
 # cereal, so use these if you want custom events in your fork.
 
-# DO rename the structs
-# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
-
-struct ModularAssistiveDrivingSystem {
-  state @0 :ModularAssistiveDrivingSystemState;
-  enabled @1 :Bool;
-  active @2 :Bool;
-  available @3 :Bool;
-
-  enum ModularAssistiveDrivingSystemState {
-    disabled @0;
-    paused @1;
-    enabled @2;
-    softDisabling @3;
-    overriding @4;
-  }
+# you can rename the struct, but don't change the identifier
+struct CustomReserved0 @0x81c2f05a394cf4af {
 }
 
-struct IntelligentCruiseButtonManagement {
-  state @0 :IntelligentCruiseButtonManagementState;
-  sendButton @1 :SendButtonState;
-  vTarget @2 :Float32;
-
-  enum IntelligentCruiseButtonManagementState {
-    inactive @0;      # No button press or default state
-    preActive @1;     # Pre-active state before transitioning to increasing or decreasing
-    increasing @2;    # Increasing speed
-    decreasing @3;    # Decreasing speed
-    holding @4;       # Holding steady speed
-  }
-
-  enum SendButtonState {
-    none @0;
-    increase @1;
-    decrease @2;
-  }
+struct CustomReserved1 @0xaedffd8f31e7b55d {
 }
 
-# Same struct as Log.RadarState.LeadData
-struct LeadData {
-  dRel @0 :Float32;
-  yRel @1 :Float32;
-  vRel @2 :Float32;
-  aRel @3 :Float32;
-  vLead @4 :Float32;
-  dPath @6 :Float32;
-  vLat @7 :Float32;
-  vLeadK @8 :Float32;
-  aLeadK @9 :Float32;
-  fcw @10 :Bool;
-  status @11 :Bool;
-  aLeadTau @12 :Float32;
-  modelProb @13 :Float32;
-  radar @14 :Bool;
-  radarTrackId @15 :Int32 = -1;
-
-  aLeadDEPRECATED @5 :Float32;
+struct CustomReserved2 @0xf35cc4560bbf6ec2 {
 }
 
-struct SelfdriveStateSP @0x81c2f05a394cf4af {
-  mads @0 :ModularAssistiveDrivingSystem;
-  intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
-
-  enum AudibleAlert {
-    none @0;
-
-    engage @1;
-    disengage @2;
-    refuse @3;
-
-    warningSoft @4;
-    warningImmediate @5;
-
-    prompt @6;
-    promptRepeat @7;
-    promptDistracted @8;
-
-    # unused, these are reserved for upstream events so we don't collide
-    reserved9 @9;
-    reserved10 @10;
-    reserved11 @11;
-    reserved12 @12;
-    reserved13 @13;
-    reserved14 @14;
-    reserved15 @15;
-    reserved16 @16;
-    reserved17 @17;
-    reserved18 @18;
-    reserved19 @19;
-    reserved20 @20;
-    reserved21 @21;
-    reserved22 @22;
-    reserved23 @23;
-    reserved24 @24;
-    reserved25 @25;
-    reserved26 @26;
-    reserved27 @27;
-    reserved28 @28;
-    reserved29 @29;
-    reserved30 @30;
-
-    promptSingleLow @31;
-    promptSingleHigh @32;
-  }
+struct CustomReserved3 @0xda96579883444c35 {
 }
 
-struct ModelManagerSP @0xaedffd8f31e7b55d {
-  activeBundle @0 :ModelBundle;
-  selectedBundle @1 :ModelBundle;
-  availableBundles @2 :List(ModelBundle);
-
-  struct DownloadUri {
-    uri @0 :Text;
-    sha256 @1 :Text;
-  }
-
-  enum DownloadStatus {
-    notDownloading @0;
-    downloading @1;
-    downloaded @2;
-    cached @3;
-    failed @4;
-  }
-
-  struct DownloadProgress {
-    status @0 :DownloadStatus;
-    progress @1 :Float32;
-    eta @2 :UInt32;
-  }
-
-  struct Artifact {
-    fileName @0 :Text;
-    downloadUri @1 :DownloadUri;
-    downloadProgress @2 :DownloadProgress;
-  }
-
-  struct Model {
-    type @0 :Type;
-    artifact @1 :Artifact;  # Main artifact
-    metadata @2 :Artifact;  # Metadata artifact
-
-    enum Type {
-      supercombo @0;
-      navigation @1;
-      vision @2;
-      policy @3;
-      offPolicy @4;
-      onPolicy @5;
-    }
-  }
-
-  enum Runner {
-    snpe @0;
-    tinygrad @1;
-    stock @2;
-  }
-
-  struct Override {
-    key @0 :Text;
-    value @1 :Text;
-  }
-
-  struct ModelBundle {
-    index @0 :UInt32;
-    internalName @1 :Text;
-    displayName @2 :Text;
-    models @3 :List(Model);
-    status @4 :DownloadStatus;
-    generation @5 :UInt32;
-    environment @6 :Text;
-    runner @7 :Runner;
-    is20hz @8 :Bool;
-    ref @9 :Text;
-    minimumSelectorVersion @10 :UInt32;
-    overrides @11 :List(Override);
-  }
+struct CustomReserved4 @0x80ae746ee2596b11 {
 }
 
-struct LongitudinalPlanSP @0xf35cc4560bbf6ec2 {
-  dec @0 :DynamicExperimentalControl;
-  longitudinalPlanSource @1 :LongitudinalPlanSource;
-  smartCruiseControl @2 :SmartCruiseControl;
-  speedLimit @3 :SpeedLimit;
-  vTarget @4 :Float32;
-  aTarget @5 :Float32;
-  events @6 :List(OnroadEventSP.Event);
-  e2eAlerts @7 :E2eAlerts;
-
-  struct DynamicExperimentalControl {
-    state @0 :DynamicExperimentalControlState;
-    enabled @1 :Bool;
-    active @2 :Bool;
-
-    enum DynamicExperimentalControlState {
-      acc @0;
-      blended @1;
-    }
-  }
-
-  struct SmartCruiseControl {
-    vision @0 :Vision;
-    map @1 :Map;
-
-    struct Vision {
-      state @0 :VisionState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      currentLateralAccel @3 :Float32;
-      maxPredictedLateralAccel @4 :Float32;
-      enabled @5 :Bool;
-      active @6 :Bool;
-    }
-
-    struct Map {
-      state @0 :MapState;
-      vTarget @1 :Float32;
-      aTarget @2 :Float32;
-      enabled @3 :Bool;
-      active @4 :Bool;
-    }
-
-    enum VisionState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on vision range.
-      entering @2; # A substantial turn is predicted ahead, adapting speed to turn comfort levels.
-      turning @3; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      leaving @4; # Road ahead straightens. Start to allow positive acceleration.
-      overriding @5; # System overriding with manual control.
-    }
-
-    enum MapState {
-      disabled @0; # System disabled or inactive.
-      enabled @1; # No predicted substantial turn on map range.
-      turning @2; # Actively turning. Managing acceleration to provide a roll on turn feeling.
-      overriding @3; # System overriding with manual control.
-    }
-  }
-
-  struct SpeedLimit {
-    resolver @0 :Resolver;
-    assist @1 :Assist;
-
-    struct Resolver {
-      speedLimit @0 :Float32;
-      distToSpeedLimit @1 :Float32;
-      source @2 :Source;
-      speedLimitOffset @3 :Float32;
-      speedLimitLast @4 :Float32;
-      speedLimitFinal @5 :Float32;
-      speedLimitFinalLast @6 :Float32;
-      speedLimitValid @7 :Bool;
-      speedLimitLastValid @8 :Bool;
-    }
-
-    struct Assist {
-      state @0 :AssistState;
-      enabled @1 :Bool;
-      active @2 :Bool;
-      vTarget @3 :Float32;
-      aTarget @4 :Float32;
-    }
-
-    enum Source {
-      none @0;
-      car @1;
-      map @2;
-    }
-
-    enum AssistState {
-      disabled @0;
-      inactive @1; # No speed limit set or not enabled by parameter.
-      preActive @2;
-      pending @3; # Awaiting new speed limit.
-      adapting @4; # Reducing speed to match new speed limit.
-      active @5; # Cruising at speed limit.
-    }
-  }
-
-  enum LongitudinalPlanSource {
-    cruise @0;
-    sccVision @1;
-    sccMap @2;
-    speedLimitAssist @3;
-  }
-
-  struct E2eAlerts {
-    greenLightAlert @0 :Bool;
-    leadDepartAlert @1 :Bool;
-  }
+struct CustomReserved5 @0xa5cd762cd951a455 {
 }
 
-struct OnroadEventSP @0xda96579883444c35 {
-  events @0 :List(Event);
-
-  struct Event {
-    name @0 :EventName;
-
-    # event types
-    enable @1 :Bool;
-    noEntry @2 :Bool;
-    warning @3 :Bool;   # alerts presented only when  enabled or soft disabling
-    userDisable @4 :Bool;
-    softDisable @5 :Bool;
-    immediateDisable @6 :Bool;
-    preEnable @7 :Bool;
-    permanent @8 :Bool; # alerts presented regardless of openpilot state
-    overrideLateral @10 :Bool;
-    overrideLongitudinal @9 :Bool;
-  }
-
-  enum EventName {
-    lkasEnable @0;
-    lkasDisable @1;
-    manualSteeringRequired @2;
-    manualLongitudinalRequired @3;
-    silentLkasEnable @4;
-    silentLkasDisable @5;
-    silentBrakeHold @6;
-    silentWrongGear @7;
-    silentReverseGear @8;
-    silentDoorOpen @9;
-    silentSeatbeltNotLatched @10;
-    silentParkBrake @11;
-    controlsMismatchLateral @12;
-    hyundaiRadarTracksConfirmed @13;
-    experimentalModeSwitched @14;
-    wrongCarModeAlertOnly @15;
-    pedalPressedAlertOnly @16;
-    laneTurnLeft @17;
-    laneTurnRight @18;
-    speedLimitPreActive @19;
-    speedLimitActive @20;
-    speedLimitChanged @21;
-    speedLimitPending @22;
-    e2eChime @23;
-  }
+struct CustomReserved6 @0xf98d843bfd7004a3 {
 }
 
-struct CarParamsSP @0x80ae746ee2596b11 {
-  flags @0 :UInt32;        # flags for car specific quirks in sunnypilot
-  safetyParam @1 : Int16;  # flags for sunnypilot's custom safety flags
-  pcmCruiseSpeed @3 :Bool;
-  intelligentCruiseButtonManagementAvailable @4 :Bool;
-  enableGasInterceptor @5 :Bool;
-
-  neuralNetworkLateralControl @2 :NeuralNetworkLateralControl;
-
-  struct NeuralNetworkLateralControl {
-    model @0 :Model;
-    fuzzyFingerprint @1 :Bool;
-
-    struct Model {
-      path @0 :Text;
-      name @1 :Text;
-    }
-  }
+struct CustomReserved7 @0xb86e6369214c01c8 {
 }
 
-struct CarControlSP @0xa5cd762cd951a455 {
-  mads @0 :ModularAssistiveDrivingSystem;
-  params @1 :List(Param);
-  leadOne @2 :LeadData;
-  leadTwo @3 :LeadData;
-  intelligentCruiseButtonManagement @4 :IntelligentCruiseButtonManagement;
-
-  struct Param {
-    key @0 :Text;
-    type @2 :ParamType;
-    value @3 :Data;
-
-    valueDEPRECATED @1 :Text; # The data type change may cause issues with backwards compatibility.
-  }
-
-  enum ParamType {
-    string @0;
-    bool @1;
-    int @2;
-    float @3;
-    time @4;
-    json @5;
-    bytes @6;
-  }
+struct CustomReserved8 @0xf416ec09499d9d19 {
 }
 
-struct BackupManagerSP @0xf98d843bfd7004a3 {
-  backupStatus @0 :Status;
-  restoreStatus @1 :Status;
-  backupProgress @2 :Float32;
-  restoreProgress @3 :Float32;
-  lastError @4 :Text;
-  currentBackup @5 :BackupInfo;
-  backupHistory @6 :List(BackupInfo);
-
-  enum Status {
-    idle @0;
-    inProgress @1;
-    completed @2;
-    failed @3;
-  }
-
-  struct Version {
-    major @0 :UInt16;
-    minor @1 :UInt16;
-    patch @2 :UInt16;
-    build @3 :UInt16;
-    branch @4 :Text;
-  }
-
-  struct MetadataEntry {
-    key @0 :Text;
-    value @1 :Text;
-    tags @2 :List(Text);
-  }
-
-  struct BackupInfo {
-    deviceId @0 :Text;
-    version @1 :UInt32;
-    config @2 :Text;
-    isEncrypted @3 :Bool;
-    createdAt @4 :Text;  # ISO timestamp
-    updatedAt @5 :Text;  # ISO timestamp
-    sunnypilotVersion @6 :Version;
-    backupMetadata @7 :List(MetadataEntry);
-  }
-}
-
-struct CarStateSP @0xb86e6369214c01c8 {
-  speedLimit @0 :Float32;
-}
-
-struct LiveMapDataSP @0xf416ec09499d9d19 {
-  speedLimitValid @0 :Bool;
-  speedLimit @1 :Float32;
-  speedLimitAheadValid @2 :Bool;
-  speedLimitAhead @3 :Float32;
-  speedLimitAheadDistance @4 :Float32;
-  roadName @5 :Text;
-}
-
-struct ModelDataV2SP @0xa1680744031fdb2d {
-  laneTurnDirection @0 :TurnDirection;
-
-  enum TurnDirection {
-    none @0;
-    turnLeft @1;
-    turnRight @2;
-  }
-}
-
-struct CustomReserved10 @0xcb9fd56c7057593a {
-}
-
-struct CustomReserved11 @0xc2243c65e0340384 {
-}
-
-struct CustomReserved12 @0x9ccdc8676701b412 {
-}
-
-struct CustomReserved13 @0xcd96dafb67a082d0 {
-}
-
-struct CustomReserved14 @0xb057204d7deadf3f {
-}
-
-struct CustomReserved15 @0xbd443b539493bc68 {
-}
-
-struct CustomReserved16 @0xfc6241ed8877b611 {
-}
-
-struct CustomReserved17 @0xa30662f84033036c {
-}
-
-struct CustomReserved18 @0xc86a3d38d13eb3ef {
-}
-
-struct CustomReserved19 @0xa4f1eb3323f5f582 {
+struct CustomReserved9 @0xa1680744031fdb2d {
 }

cereal/legacy.capnp

@@ -3,7 +3,7 @@ $Cxx.namespace("cereal");
 
 @0x80ef1ec4889c2a63;
 
-# deprecated.capnp: a home for deprecated structs
+# legacy.capnp: a home for deprecated structs
 
 struct LogRotate @0x9811e1f38f62f2d1 {
   segmentNum @0 :Int32;
@@ -571,219 +571,4 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
   pkt @4 :Data;
 }
 
-struct LiveTracksDEPRECATED @0xb16f60103159415a {
-  trackId @0 :Int32;
-  dRel @1 :Float32;
-  yRel @2 :Float32;
-  vRel @3 :Float32;
-  aRel @4 :Float32;
-  timeStamp @5 :Float32;
-  status @6 :Float32;
-  currentTime @7 :Float32;
-  stationary @8 :Bool;
-  oncoming @9 :Bool;
-}
 
-struct LiveMpcData @0x92a5e332a85f32a0 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  psi @2 :List(Float32);
-  curvature @3 :List(Float32);
-  qpIterations @4 :UInt32;
-  calculationTime @5 :UInt64;
-  cost @6 :Float64;
-}
-
-struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
-  xEgo @0 :List(Float32);
-  vEgo @1 :List(Float32);
-  aEgo @2 :List(Float32);
-  xLead @3 :List(Float32);
-  vLead @4 :List(Float32);
-  aLead @5 :List(Float32);
-  aLeadTau @6 :Float32;    # lead accel time constant
-  qpIterations @7 :UInt32;
-  mpcId @8 :UInt32;
-  calculationTime @9 :UInt64;
-  cost @10 :Float64;
-}
-
-struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
-  frameId @0 :UInt32;
-  modelExecutionTime @14 :Float32;
-  dspExecutionTime @16 :Float32;
-  rawPredictions @15 :Data;
-
-  faceOrientation @3 :List(Float32);
-  facePosition @4 :List(Float32);
-  faceProb @5 :Float32;
-  leftEyeProb @6 :Float32;
-  rightEyeProb @7 :Float32;
-  leftBlinkProb @8 :Float32;
-  rightBlinkProb @9 :Float32;
-  faceOrientationStd @11 :List(Float32);
-  facePositionStd @12 :List(Float32);
-  sunglassesProb @13 :Float32;
-  poorVision @17 :Float32;
-  partialFace @18 :Float32;
-  distractedPose @19 :Float32;
-  distractedEyes @20 :Float32;
-  eyesOnRoad @21 :Float32;
-  phoneUse @22 :Float32;
-  occludedProb @23 :Float32;
-
-  readyProb @24 :List(Float32);
-  notReadyProb @25 :List(Float32);
-
-  irPwrDEPRECATED @10 :Float32;
-  descriptorDEPRECATED @1 :List(Float32);
-  stdDEPRECATED @2 :Float32;
-}
-
-struct NavModelData @0xac3de5c437be057a {
-  frameId @0 :UInt32;
-  locationMonoTime @6 :UInt64;
-  modelExecutionTime @1 :Float32;
-  dspExecutionTime @2 :Float32;
-  features @3 :List(Float32);
-  # predicted future position
-  position @4 :XYData;
-  desirePrediction @5 :List(Float32);
-
-  # All SI units and in device frame
-  struct XYData @0xbe09e615b2507e26 {
-    x @0 :List(Float32);
-    y @1 :List(Float32);
-    xStd @2 :List(Float32);
-    yStd @3 :List(Float32);
-  }
-}
-
-struct AndroidBuildInfo @0xfe2919d5c21f426c {
-  board @0 :Text;
-  bootloader @1 :Text;
-  brand @2 :Text;
-  device @3 :Text;
-  display @4 :Text;
-  fingerprint @5 :Text;
-  hardware @6 :Text;
-  host @7 :Text;
-  id @8 :Text;
-  manufacturer @9 :Text;
-  model @10 :Text;
-  product @11 :Text;
-  radioVersion @12 :Text;
-  serial @13 :Text;
-  supportedAbis @14 :List(Text);
-  tags @15 :Text;
-  time @16 :Int64;
-  type @17 :Text;
-  user @18 :Text;
-
-  versionCodename @19 :Text;
-  versionRelease @20 :Text;
-  versionSdk @21 :Int32;
-  versionSecurityPatch @22 :Text;
-}
-
-struct AndroidSensor @0x9b513b93a887dbcd {
-  id @0 :Int32;
-  name @1 :Text;
-  vendor @2 :Text;
-  version @3 :Int32;
-  handle @4 :Int32;
-  type @5 :Int32;
-  maxRange @6 :Float32;
-  resolution @7 :Float32;
-  power @8 :Float32;
-  minDelay @9 :Int32;
-  fifoReservedEventCount @10 :UInt32;
-  fifoMaxEventCount @11 :UInt32;
-  stringType @12 :Text;
-  maxDelay @13 :Int32;
-}
-
-struct IosBuildInfo @0xd97e3b28239f5580 {
-  appVersion @0 :Text;
-  appBuild @1 :UInt32;
-  osVersion @2 :Text;
-  deviceModel @3 :Text;
-}
-
-enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
-  unknown @0;
-  neo @1;
-  chffrAndroid @2;
-  front @3;
-}
-
-struct AndroidCaptureResult @0xbcc3efbac41d2048 {
-  sensitivity @0 :Int32;
-  frameDuration @1 :Int64;
-  exposureTime @2 :Int64;
-  rollingShutterSkew @3 :UInt64;
-  colorCorrectionTransform @4 :List(Int32);
-  colorCorrectionGains @5 :List(Float32);
-  displayRotation @6 :Int8;
-}
-
-enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
-  none @0;
-  client @1;
-  cdp @2;
-  dcp @3;
-}
-
-struct LateralINDIState @0x939463348632375e {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  steeringRateDeg @2 :Float32;
-  steeringAccelDeg @3 :Float32;
-  rateSetPoint @4 :Float32;
-  accelSetPoint @5 :Float32;
-  accelError @6 :Float32;
-  delayedOutput @7 :Float32;
-  delta @8 :Float32;
-  output @9 :Float32;
-  saturated @10 :Bool;
-  steeringAngleDesiredDeg @11 :Float32;
-  steeringRateDesiredDeg @12 :Float32;
-}
-
-struct LateralLQRState @0x9024e2d790c82ade {
-  active @0 :Bool;
-  steeringAngleDeg @1 :Float32;
-  i @2 :Float32;
-  output @3 :Float32;
-  lqrOutput @4 :Float32;
-  saturated @5 :Bool;
-  steeringAngleDesiredDeg @6 :Float32;
-}
-
-struct LateralCurvatureState @0xad9d8095c06f7c61 {
-  active @0 :Bool;
-  actualCurvature @1 :Float32;
-  desiredCurvature @2 :Float32;
-  error @3 :Float32;
-  p @4 :Float32;
-  i @5 :Float32;
-  f @6 :Float32;
-  output @7 :Float32;
-  saturated @8 :Bool;
-}
-
-struct LateralPlannerSolution @0x84caeca5a6b4acfe {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-  yaw @2 :List(Float32);
-  yawRate @3 :List(Float32);
-  xStd @4 :List(Float32);
-  yStd @5 :List(Float32);
-  yawStd @6 :List(Float32);
-  yawRateStd @7 :List(Float32);
-}
-
-struct GpsTrajectory @0x8cfeb072f5301000 {
-  x @0 :List(Float32);
-  y @1 :List(Float32);
-}

cereal/maptile.capnp

@@ -0,0 +1,49 @@
+using Cxx = import "./include/c++.capnp";
+$Cxx.namespace("cereal");
+
+@0xa086df597ef5d7a0;
+
+# Geometry
+struct Point {
+  x @0: Float64;
+  y @1: Float64;
+  z @2: Float64;
+}
+
+struct PolyLine {
+  points @0: List(Point);
+}
+
+# Map features
+struct Lane {
+  id @0 :Text;
+
+  leftBoundary @1 :LaneBoundary;
+  rightBoundary @2 :LaneBoundary;
+
+  leftAdjacentId @3 :Text;
+  rightAdjacentId @4 :Text;
+
+  inboundIds @5 :List(Text);
+  outboundIds @6 :List(Text);
+
+  struct LaneBoundary {
+    polyLine @0 :PolyLine;
+    startHeading @1 :Float32; # WRT north
+  }
+}
+
+# Map tiles
+struct TileSummary {
+  version @0 :Text;
+  updatedAt @1 :UInt64; # Millis since epoch
+
+  level @2 :UInt8;
+  x @3 :UInt16;
+  y @4 :UInt16;
+}
+
+struct MapTile {
+  summary @0 :TileSummary;
+  lanes @1 :List(Lane);
+}

cereal/messaging/__init__.py

@@ -1,35 +1,23 @@
 # must be built with scons
-from msgq import fake_event_handle, drain_sock_raw, MultiplePublishersError, IpcError, \
-                 Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
-                 set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import Context, Poller, SubSocket, PubSocket, SocketEventHandle, toggle_fake_events, \
+                                set_fake_prefix, get_fake_prefix, delete_fake_prefix, wait_for_one_event
+from msgq.ipc_pyx import MultiplePublishersError, IpcError
+from msgq import fake_event_handle, pub_sock, sub_sock, drain_sock_raw
 import msgq
+
 import os
 import capnp
 import time
 
-from typing import Optional, List, Union, Dict
+from typing import Optional, List, Union, Dict, Deque
+from collections import deque
 
 from cereal import log
 from cereal.services import SERVICE_LIST
-from openpilot.common.utils import MovingAverage
 
 NO_TRAVERSAL_LIMIT = 2**64-1
 
 
-def pub_sock(endpoint: str) -> PubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.pub_sock(endpoint, segment_size)
-
-
-def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
-             conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
-  service = SERVICE_LIST.get(endpoint)
-  segment_size = service.queue_size if service else 0
-  return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
-                       timeout=timeout, segment_size=segment_size)
-
-
 def reset_context():
   msgq.context = Context()
 
@@ -120,8 +108,10 @@ class FrequencyTracker:
 
     self.min_freq = min_freq * 0.8
     self.max_freq = max_freq * 1.2
-    self.avg_dt = MovingAverage(int(10 * freq))
-    self.recent_avg_dt = MovingAverage(int(freq))
+    self.recv_dts: Deque[float] = deque(maxlen=int(10 * freq))
+    self.recv_dts_sum = 0.0
+    self.recent_recv_dts: Deque[float] = deque(maxlen=int(freq))
+    self.recent_recv_dts_sum = 0.0
     self.prev_time = 0.0
 
   def record_recv_time(self, cur_time: float) -> None:
@@ -129,21 +119,28 @@ class FrequencyTracker:
     if self.prev_time > 1e-5:
       dt = cur_time - self.prev_time
 
-      self.avg_dt.add_value(dt)
-      self.recent_avg_dt.add_value(dt)
+      if len(self.recv_dts) == self.recv_dts.maxlen:
+        self.recv_dts_sum -= self.recv_dts[0]
+      self.recv_dts.append(dt)
+      self.recv_dts_sum += dt
+
+      if len(self.recent_recv_dts) == self.recent_recv_dts.maxlen:
+        self.recent_recv_dts_sum -= self.recent_recv_dts[0]
+      self.recent_recv_dts.append(dt)
+      self.recent_recv_dts_sum += dt
 
     self.prev_time = cur_time
 
   @property
   def valid(self) -> bool:
-    if self.avg_dt.count == 0:
+    if not self.recv_dts:
       return False
 
-    avg_freq = 1.0 / self.avg_dt.get_average()
+    avg_freq = len(self.recv_dts) / self.recv_dts_sum
     if self.min_freq <= avg_freq <= self.max_freq:
       return True
 
-    avg_freq_recent = 1.0 / self.recent_avg_dt.get_average()
+    avg_freq_recent = len(self.recent_recv_dts) / self.recent_recv_dts_sum
     return self.min_freq <= avg_freq_recent <= self.max_freq
 
 
@@ -157,16 +154,12 @@ class SubMaster:
     self.updated = {s: False for s in services}
     self.recv_time = {s: 0. for s in services}
     self.recv_frame = {s: 0 for s in services}
+    self.alive = {s: False for s in services}
+    self.freq_ok = {s: False for s in services}
     self.sock = {}
     self.data = {}
-    self.logMonoTime = {s: 0 for s in services}
-
-    # zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
-    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
-    self.static_freq_services = set(s for s in services if not on_demand[s])
-    self.alive = {s: on_demand[s] for s in services}
-    self.freq_ok = {s: on_demand[s] for s in services}
-    self.valid = {s: on_demand[s] for s in services}
+    self.valid = {}
+    self.logMonoTime = {}
 
     self.freq_tracker: Dict[str, FrequencyTracker] = {}
     self.poller = Poller()
@@ -193,6 +186,8 @@ class SubMaster:
         data = new_message(s, 0) # lists
 
       self.data[s] = getattr(data.as_reader(), s)
+      self.logMonoTime[s] = 0
+      self.valid[s] = False
       self.freq_tracker[s] = FrequencyTracker(SERVICE_LIST[s].frequency, self.update_freq, s == poll)
 
   def __getitem__(self, s: str) -> capnp.lib.capnp._DynamicStructReader:
@@ -229,10 +224,14 @@ class SubMaster:
       self.logMonoTime[s] = msg.logMonoTime
       self.valid[s] = msg.valid
 
-    for s in self.static_freq_services:
-      # alive if delay is within 10x the expected frequency; checks relaxed in simulator
-      self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
-      self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
+    for s in self.services:
+      if SERVICE_LIST[s].frequency > 1e-5 and not self.simulation:
+        # alive if delay is within 10x the expected frequency
+        self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency)
+        self.freq_ok[s] = self.freq_tracker[s].valid
+      else:
+        self.freq_ok[s] = True
+        self.alive[s] = self.seen[s] if self.simulation else True
 
   def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
     return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)

cereal/messaging/bridge.cc

@@ -6,12 +6,12 @@
 
 ExitHandler do_exit;
 
-static std::vector<std::string> get_services(const std::string &whitelist_str, bool zmq_to_msgq) {
+static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
   std::vector<std::string> service_list;
   for (const auto& it : services) {
     std::string name = it.second.name;
     bool in_whitelist = whitelist_str.find(name) != std::string::npos;
-    if (zmq_to_msgq && !in_whitelist) {
+    if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
       continue;
     }
     service_list.push_back(name);
@@ -25,16 +25,15 @@ void msgq_to_zmq(const std::vector<std::string> &endpoints, const std::string &i
 }
 
 void zmq_to_msgq(const std::vector<std::string> &endpoints, const std::string &ip) {
-  auto poller = std::make_unique<BridgeZmqPoller>();
-  auto pub_context = std::make_unique<Context>();
-  auto sub_context = std::make_unique<BridgeZmqContext>();
-  std::map<BridgeZmqSubSocket *, PubSocket *> sub2pub;
+  auto poller = std::make_unique<ZMQPoller>();
+  auto pub_context = std::make_unique<MSGQContext>();
+  auto sub_context = std::make_unique<ZMQContext>();
+  std::map<SubSocket *, PubSocket *> sub2pub;
 
   for (auto endpoint : endpoints) {
-    auto pub_sock = new PubSocket();
-    auto sub_sock = new BridgeZmqSubSocket();
-    size_t queue_size = services.at(endpoint).queue_size;
-    pub_sock->connect(pub_context.get(), endpoint, true, queue_size);
+    auto pub_sock = new MSGQPubSocket();
+    auto sub_sock = new ZMQSubSocket();
+    pub_sock->connect(pub_context.get(), endpoint);
     sub_sock->connect(sub_context.get(), endpoint, ip, false);
 
     poller->registerSocket(sub_sock);

cereal/messaging/bridge_zmq.cc

@@ -1,170 +0,0 @@
-#include "cereal/messaging/bridge_zmq.h"
-
-#include <cassert>
-#include <cstring>
-#include <unistd.h>
-
-static size_t fnv1a_hash(const std::string &str) {
-  const size_t fnv_prime = 0x100000001b3;
-  size_t hash_value = 0xcbf29ce484222325;
-  for (char c : str) {
-    hash_value ^= (unsigned char)c;
-    hash_value *= fnv_prime;
-  }
-  return hash_value;
-}
-
-// FIXME: This is a hack to get the port number from the socket name, might have collisions.
-static int get_port(std::string endpoint) {
-  size_t hash_value = fnv1a_hash(endpoint);
-  int start_port = 8023;
-  int max_port = 65535;
-  return start_port + (hash_value % (max_port - start_port));
-}
-
-BridgeZmqContext::BridgeZmqContext() {
-  context = zmq_ctx_new();
-}
-
-BridgeZmqContext::~BridgeZmqContext() {
-  if (context != nullptr) {
-    zmq_ctx_term(context);
-  }
-}
-
-void BridgeZmqMessage::init(size_t sz) {
-  size = sz;
-  data = new char[size];
-}
-
-void BridgeZmqMessage::init(char *d, size_t sz) {
-  size = sz;
-  data = new char[size];
-  memcpy(data, d, size);
-}
-
-void BridgeZmqMessage::close() {
-  if (size > 0) {
-    delete[] data;
-  }
-  data = nullptr;
-  size = 0;
-}
-
-BridgeZmqMessage::~BridgeZmqMessage() {
-  close();
-}
-
-int BridgeZmqSubSocket::connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_SUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  zmq_setsockopt(sock, ZMQ_SUBSCRIBE, "", 0);
-
-  if (conflate) {
-    int arg = 1;
-    zmq_setsockopt(sock, ZMQ_CONFLATE, &arg, sizeof(int));
-  }
-
-  int reconnect_ivl = 500;
-  zmq_setsockopt(sock, ZMQ_RECONNECT_IVL_MAX, &reconnect_ivl, sizeof(reconnect_ivl));
-
-  full_endpoint = "tcp://" + address + ":";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  return zmq_connect(sock, full_endpoint.c_str());
-}
-
-void BridgeZmqSubSocket::setTimeout(int timeout) {
-  zmq_setsockopt(sock, ZMQ_RCVTIMEO, &timeout, sizeof(int));
-}
-
-Message *BridgeZmqSubSocket::receive(bool non_blocking) {
-  zmq_msg_t msg;
-  assert(zmq_msg_init(&msg) == 0);
-
-  int flags = non_blocking ? ZMQ_DONTWAIT : 0;
-  int rc = zmq_msg_recv(&msg, sock, flags);
-
-  Message *ret = nullptr;
-  if (rc >= 0) {
-    ret = new BridgeZmqMessage;
-    ret->init((char *)zmq_msg_data(&msg), zmq_msg_size(&msg));
-  }
-
-  zmq_msg_close(&msg);
-  return ret;
-}
-
-BridgeZmqSubSocket::~BridgeZmqSubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-int BridgeZmqPubSocket::connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint) {
-  sock = zmq_socket(context->getRawContext(), ZMQ_PUB);
-  if (sock == nullptr) {
-    return -1;
-  }
-
-  full_endpoint = "tcp://*:";
-  if (check_endpoint) {
-    full_endpoint += std::to_string(get_port(endpoint));
-  } else {
-    full_endpoint += endpoint;
-  }
-
-  // ZMQ pub sockets cannot be shared between processes, so we need to ensure pid stays the same.
-  pid = getpid();
-
-  return zmq_bind(sock, full_endpoint.c_str());
-}
-
-int BridgeZmqPubSocket::sendMessage(Message *message) {
-  assert(pid == getpid());
-  return zmq_send(sock, message->getData(), message->getSize(), ZMQ_DONTWAIT);
-}
-
-int BridgeZmqPubSocket::send(char *data, size_t size) {
-  assert(pid == getpid());
-  return zmq_send(sock, data, size, ZMQ_DONTWAIT);
-}
-
-BridgeZmqPubSocket::~BridgeZmqPubSocket() {
-  if (sock != nullptr) {
-    zmq_close(sock);
-  }
-}
-
-void BridgeZmqPoller::registerSocket(BridgeZmqSubSocket *socket) {
-  assert(num_polls + 1 < (sizeof(polls) / sizeof(polls[0])));
-  polls[num_polls].socket = socket->getRawSocket();
-  polls[num_polls].events = ZMQ_POLLIN;
-
-  sockets.push_back(socket);
-  num_polls++;
-}
-
-std::vector<BridgeZmqSubSocket *> BridgeZmqPoller::poll(int timeout) {
-  std::vector<BridgeZmqSubSocket *> ret;
-
-  int rc = zmq_poll(polls, num_polls, timeout);
-  if (rc < 0) {
-    return ret;
-  }
-
-  for (size_t i = 0; i < num_polls; i++) {
-    if (polls[i].revents) {
-      ret.push_back(sockets[i]);
-    }
-  }
-
-  return ret;
-}

cereal/messaging/bridge_zmq.h

@@ -1,72 +0,0 @@
-#pragma once
-
-#include <cstddef>
-#include <string>
-#include <vector>
-
-#include <zmq.h>
-
-#include "msgq/ipc.h"
-
-class BridgeZmqContext {
-public:
-  BridgeZmqContext();
-  void *getRawContext() { return context; }
-  ~BridgeZmqContext();
-
-private:
-  void *context = nullptr;
-};
-
-class BridgeZmqMessage : public Message {
-public:
-  void init(size_t size);
-  void init(char *data, size_t size);
-  void close();
-  size_t getSize() { return size; }
-  char *getData() { return data; }
-  ~BridgeZmqMessage();
-
-private:
-  char *data = nullptr;
-  size_t size = 0;
-};
-
-class BridgeZmqSubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, std::string address, bool conflate = false, bool check_endpoint = true);
-  void setTimeout(int timeout);
-  Message *receive(bool non_blocking = false);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqSubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-};
-
-class BridgeZmqPubSocket {
-public:
-  int connect(BridgeZmqContext *context, std::string endpoint, bool check_endpoint = true);
-  int sendMessage(Message *message);
-  int send(char *data, size_t size);
-  void *getRawSocket() { return sock; }
-  ~BridgeZmqPubSocket();
-
-private:
-  void *sock = nullptr;
-  std::string full_endpoint;
-  int pid = -1;
-};
-
-class BridgeZmqPoller {
-public:
-  void registerSocket(BridgeZmqSubSocket *socket);
-  std::vector<BridgeZmqSubSocket *> poll(int timeout);
-
-private:
-  static constexpr size_t MAX_BRIDGE_ZMQ_POLLERS = 128;
-  std::vector<BridgeZmqSubSocket *> sockets;
-  zmq_pollitem_t polls[MAX_BRIDGE_ZMQ_POLLERS] = {};
-  size_t num_polls = 0;
-};

cereal/messaging/msgq_to_zmq.cc

@@ -2,7 +2,6 @@
 
 #include <cassert>
 
-#include "cereal/services.h"
 #include "common/util.h"
 
 extern ExitHandler do_exit;
@@ -22,14 +21,14 @@ static std::string recv_zmq_msg(void *sock) {
 }
 
 void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string &ip) {
-  zmq_context = std::make_unique<BridgeZmqContext>();
-  msgq_context = std::make_unique<Context>();
+  zmq_context = std::make_unique<ZMQContext>();
+  msgq_context = std::make_unique<MSGQContext>();
 
   // Create ZMQPubSockets for each endpoint
   for (const auto &endpoint : endpoints) {
     auto &socket_pair = socket_pairs.emplace_back();
     socket_pair.endpoint = endpoint;
-    socket_pair.pub_sock = std::make_unique<BridgeZmqPubSocket>();
+    socket_pair.pub_sock = std::make_unique<ZMQPubSocket>();
     int ret = socket_pair.pub_sock->connect(zmq_context.get(), endpoint);
     if (ret != 0) {
       printf("Failed to create ZMQ publisher for [%s]: %s\n", endpoint.c_str(), zmq_strerror(zmq_errno()));
@@ -49,7 +48,7 @@ void MsgqToZmq::run(const std::vector<std::string> &endpoints, const std::string
 
       for (auto sub_sock : msgq_poller->poll(100)) {
         // Process messages for each socket
-        BridgeZmqPubSocket *pub_sock = sub2pub.at(sub_sock);
+        ZMQPubSocket *pub_sock = sub2pub.at(sub_sock);
         for (int i = 0; i < MAX_MESSAGES_PER_SOCKET; ++i) {
           auto msg = std::unique_ptr<Message>(sub_sock->receive(true));
           if (!msg) break;
@@ -72,7 +71,7 @@ void MsgqToZmq::zmqMonitorThread() {
   // Set up ZMQ monitor for each pub socket
   for (int i = 0; i < socket_pairs.size(); ++i) {
     std::string addr = "inproc://op-bridge-monitor-" + std::to_string(i);
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, addr.c_str(), ZMQ_EVENT_ACCEPTED | ZMQ_EVENT_DISCONNECTED);
 
     void *monitor_socket = zmq_socket(zmq_context->getRawContext(), ZMQ_PAIR);
     zmq_connect(monitor_socket, addr.c_str());
@@ -109,8 +108,7 @@ void MsgqToZmq::zmqMonitorThread() {
           if (++pair.connected_clients == 1) {
             // Create new MSGQ subscriber socket and map to ZMQ publisher
             pair.sub_sock = std::make_unique<MSGQSubSocket>();
-            size_t queue_size = services.at(pair.endpoint).queue_size;
-            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1", false, true, queue_size);
+            pair.sub_sock->connect(msgq_context.get(), pair.endpoint, "127.0.0.1");
             sub2pub[pair.sub_sock.get()] = pair.pub_sock.get();
             registerSockets();
           }
@@ -130,7 +128,7 @@ void MsgqToZmq::zmqMonitorThread() {
 
   // Clean up monitor sockets
   for (int i = 0; i < pollitems.size(); ++i) {
-    zmq_socket_monitor(socket_pairs[i].pub_sock->getRawSocket(), nullptr, 0);
+    zmq_socket_monitor(socket_pairs[i].pub_sock->sock, nullptr, 0);
     zmq_close(pollitems[i].socket);
   }
   cv.notify_one();

cereal/messaging/msgq_to_zmq.h

@@ -7,8 +7,9 @@
 #include <string>
 #include <vector>
 
+#define private public
 #include "msgq/impl_msgq.h"
-#include "cereal/messaging/bridge_zmq.h"
+#include "msgq/impl_zmq.h"
 
 class MsgqToZmq {
 public:
@@ -21,16 +22,16 @@ protected:
 
   struct SocketPair {
     std::string endpoint;
-    std::unique_ptr<BridgeZmqPubSocket> pub_sock;
+    std::unique_ptr<ZMQPubSocket> pub_sock;
     std::unique_ptr<MSGQSubSocket> sub_sock;
     int connected_clients = 0;
   };
 
-  std::unique_ptr<Context> msgq_context;
-  std::unique_ptr<BridgeZmqContext> zmq_context;
+  std::unique_ptr<MSGQContext> msgq_context;
+  std::unique_ptr<ZMQContext> zmq_context;
   std::mutex mutex;
   std::condition_variable cv;
   std::unique_ptr<MSGQPoller> msgq_poller;
-  std::map<SubSocket *, BridgeZmqPubSocket *> sub2pub;
+  std::map<SubSocket *, ZMQPubSocket *> sub2pub;
   std::vector<SocketPair> socket_pairs;
 };

cereal/messaging/socketmaster.cc

@@ -33,7 +33,7 @@ MessageContext message_context;
 struct SubMaster::SubMessage {
   std::string name;
   SubSocket *socket = nullptr;
-  float freq = 0.0f;
+  int freq = 0;
   bool updated = false, alive = false, valid = false, ignore_alive;
   uint64_t rcv_time = 0, rcv_frame = 0;
   void *allocated_msg_reader = nullptr;
@@ -50,7 +50,7 @@ SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::v
     assert(services.count(std::string(name)) > 0);
 
     service serv = services.at(std::string(name));
-    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true, true, serv.queue_size);
+    SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
     assert(socket != 0);
     bool is_polled = inList(poll, name) || poll.empty();
     if (is_polled) poller_->registerSocket(socket);
@@ -187,8 +187,7 @@ SubMaster::~SubMaster() {
 PubMaster::PubMaster(const std::vector<const char *> &service_list) {
   for (auto name : service_list) {
     assert(services.count(name) > 0);
-    service serv = services.at(std::string(name));
-    PubSocket *socket = PubSocket::create(message_context.context(), name, true, serv.queue_size);
+    PubSocket *socket = PubSocket::create(message_context.context(), name);
     assert(socket);
     sockets_[name] = socket;
   }

cereal/messaging/tests/test_messaging.py

@@ -5,7 +5,7 @@ import numbers
 import random
 import threading
 import time
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 import pytest
 
 from cereal import log, car
@@ -30,7 +30,7 @@ def zmq_sleep(t=1):
 
 # TODO: this should take any capnp struct and returrn a msg with random populated data
 def random_carstate():
-  fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
+  fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
   msg = messaging.new_message("carState")
   cs = msg.carState
   for f in fields:
@@ -177,8 +177,8 @@ class TestMessaging:
 
     # wait 5 socket timeouts before sending
     msg = random_carstate()
-    start_time = time.monotonic()
     delayed_send(sock_timeout*5, pub_sock, msg.to_bytes())
+    start_time = time.monotonic()
     recvd = messaging.recv_one_retry(sub_sock)
     assert (time.monotonic() - start_time) >= sock_timeout*5
     assert isinstance(recvd, capnp._DynamicStructReader)

cereal/messaging/tests/test_pub_sub_master.py

@@ -6,7 +6,6 @@ import cereal.messaging as messaging
 from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
                                                   random_bytes, random_carstate, assert_carstate, \
                                                   zmq_sleep
-from cereal.services import SERVICE_LIST
 
 
 class TestSubMaster:
@@ -27,9 +26,7 @@ class TestSubMaster:
     sm = messaging.SubMaster(socks)
     assert sm.frame == -1
     assert not any(sm.updated.values())
-    assert not any(sm.seen.values())
-    on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in sm.services}
-    assert all(sm.alive[s] == sm.valid[s] == sm.freq_ok[s] == on_demand[s] for s in sm.services)
+    assert not any(sm.alive.values())
     assert all(t == 0. for t in sm.recv_time.values())
     assert all(f == 0 for f in sm.recv_frame.values())
     assert all(t == 0 for t in sm.logMonoTime.values())
@@ -86,7 +83,6 @@ class TestSubMaster:
         "cameraOdometry": (20, 10),
         "liveCalibration": (4, 4),
         "carParams": (None, None),
-        "userBookmark": (None, None),
       }
 
       for service, (max_freq, min_freq) in checks.items():

cereal/messaging/tests/test_services.py

@@ -1,7 +1,7 @@
 import os
 import tempfile
 from typing import Dict
-from openpilot.common.parameterized import parameterized
+from parameterized import parameterized
 
 import cereal.services as services
 from cereal.services import SERVICE_LIST
@@ -17,5 +17,5 @@ class TestServices:
 
   def test_generated_header(self):
     with tempfile.NamedTemporaryFile(suffix=".h") as f:
-      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
+      ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
       assert ret == 0, "generated services header is not valid C"

cereal/messaging/tests/validate_sp_cereal_upstream.py

@@ -1,251 +0,0 @@
-#!/usr/bin/env python3
-"""Schema-level cereal compat check between sunnypilot and upstream openpilot.
-
-Rules (per struct matched across sides by typeId):
-  R1  shared ordinal must reference the same type.
-  R2  sunnypilot-only ordinal in a union -> FAIL (unknown discriminant upstream).
-  R3  sunnypilot-only ordinal on a regular field -> OK (additive struct evolution).
-  R4  upstream-only ordinal -> OK.
-  R5  sunnypilot-only struct referenced via an upstream-shared field -> FAIL.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import sys
-from typing import Any
-
-NO_DISCRIMINANT = 0xFFFF
-
-
-def hex_id(value: int) -> str:
-  return f"0x{value:016x}"
-
-
-def encode_type(type_node: Any) -> dict:
-  which = type_node.which()
-  if which == "struct":
-    return {"kind": "struct", "typeId": hex_id(type_node.struct.typeId)}
-  if which == "enum":
-    return {"kind": "enum", "typeId": hex_id(type_node.enum.typeId)}
-  if which == "interface":
-    return {"kind": "interface", "typeId": hex_id(type_node.interface.typeId)}
-  if which == "list":
-    return {"kind": "list", "element": encode_type(type_node.list.elementType)}
-  if which == "anyPointer":
-    return {"kind": "anyPointer"}
-  return {"kind": which}
-
-
-def encode_field(name: str, field: Any) -> dict:
-  proto = field.proto
-  ordinal = proto.ordinal.explicit if proto.ordinal.which() == "explicit" else None
-  discriminant = proto.discriminantValue if proto.discriminantValue != NO_DISCRIMINANT else None
-
-  if proto.which() == "group":
-    type_desc = {"kind": "group", "typeId": hex_id(proto.group.typeId)}
-  else:
-    type_desc = encode_type(proto.slot.type)
-
-  return {
-    "name": name,
-    "ordinal": ordinal,
-    "discriminant": discriminant,
-    "type": type_desc,
-  }
-
-
-def encode_struct(schema: Any) -> dict:
-  node = schema.node
-  return {
-    "typeId": hex_id(node.id),
-    "displayName": node.displayName,
-    "hasUnion": node.struct.discriminantCount > 0,
-    "fields": [encode_field(name, field) for name, field in schema.fields.items()],
-  }
-
-
-def _child_struct_schema(field: Any) -> Any:
-  proto = field.proto
-  if proto.which() == "group":
-    return field.schema
-  type_node = proto.slot.type
-  which = type_node.which()
-  if which == "struct":
-    return field.schema
-  if which == "list":
-    container = field.schema
-    element_type = type_node.list.elementType
-    while element_type.which() == "list":
-      container = container.elementType
-      element_type = element_type.list.elementType
-    if element_type.which() == "struct":
-      return container.elementType
-  return None
-
-
-def collect_schema(root: Any) -> dict[str, dict]:
-  structs: dict[str, dict] = {}
-  stack = [root]
-  while stack:
-    schema = stack.pop()
-    type_id = hex_id(schema.node.id)
-    if type_id in structs:
-      continue
-    structs[type_id] = encode_struct(schema)
-    for _name, field in schema.fields.items():
-      try:
-        child = _child_struct_schema(field)
-      except Exception:
-        child = None
-      if child is not None:
-        stack.append(child)
-  return structs
-
-
-def dump_schema(path: str) -> None:
-  from cereal import log
-  payload = {
-    "root": hex_id(log.Event.schema.node.id),
-    "structs": collect_schema(log.Event.schema),
-  }
-  with open(path, "w", encoding="utf-8") as handle:
-    json.dump(payload, handle, indent=2, sort_keys=True)
-  print(f"wrote schema dump with {len(payload['structs'])} structs to {path}")
-
-
-def types_equal(a: dict, b: dict) -> bool:
-  if a.get("kind") != b.get("kind"):
-    return False
-  kind = a["kind"]
-  if kind in ("struct", "enum", "interface", "group"):
-    return a.get("typeId") == b.get("typeId")
-  if kind == "list":
-    return types_equal(a["element"], b["element"])
-  return True
-
-
-def type_repr(t: dict) -> str:
-  kind = t.get("kind", "?")
-  if kind in ("struct", "enum", "interface", "group"):
-    return f"{kind}({t.get('typeId')})"
-  if kind == "list":
-    return f"list<{type_repr(t['element'])}>"
-  return kind
-
-
-def field_is_union_variant(field: dict) -> bool:
-  return field.get("discriminant") is not None
-
-
-def index_fields_by_ordinal(struct: dict) -> dict[int, dict]:
-  indexed: dict[int, dict] = {}
-  for field in struct["fields"]:
-    ordinal = field.get("ordinal")
-    if ordinal is None:
-      continue
-    indexed[ordinal] = field
-  return indexed
-
-
-def compare(sunnypilot_dump: dict, upstream_dump: dict) -> list[str]:
-  violations: list[str] = []
-  sunnypilot_structs: dict[str, dict] = sunnypilot_dump["structs"]
-  upstream_structs: dict[str, dict] = upstream_dump["structs"]
-
-  sunnypilot_struct_referenced_from_shared: set[str] = set()
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    upstream_struct = upstream_structs.get(type_id)
-    if upstream_struct is None:
-      continue
-
-    sunnypilot_fields = index_fields_by_ordinal(sunnypilot_struct)
-    upstream_fields = index_fields_by_ordinal(upstream_struct)
-    display = sunnypilot_struct["displayName"]
-
-    for ordinal, sunnypilot_field in sunnypilot_fields.items():
-      upstream_field = upstream_fields.get(ordinal)
-      if upstream_field is None:
-        if field_is_union_variant(sunnypilot_field):
-          violations.append(
-            f"[R2] {display} @{ordinal} ('{sunnypilot_field['name']}', {type_repr(sunnypilot_field['type'])}): "
-            f"union variant not present upstream. upstream cannot parse this discriminant."
-          )
-        continue
-
-      if not types_equal(sunnypilot_field["type"], upstream_field["type"]):
-        violations.append(
-          f"[R1] {display} @{ordinal}: type mismatch. "
-          f"sunnypilot='{sunnypilot_field['name']}' {type_repr(sunnypilot_field['type'])} vs "
-          f"upstream='{upstream_field['name']}' {type_repr(upstream_field['type'])}."
-        )
-        continue
-
-      cursor = sunnypilot_field["type"]
-      while cursor.get("kind") == "list":
-        cursor = cursor["element"]
-      if cursor.get("kind") in ("struct", "group", "interface") and cursor.get("typeId"):
-        sunnypilot_struct_referenced_from_shared.add(cursor["typeId"])
-
-  for type_id, sunnypilot_struct in sunnypilot_structs.items():
-    if type_id in upstream_structs:
-      continue
-    if type_id in sunnypilot_struct_referenced_from_shared:
-      violations.append(
-        f"[R5] struct {sunnypilot_struct['displayName']} ({type_id}) exists only on sunnypilot "
-        f"but is referenced from an upstream-shared field. upstream cannot resolve this type."
-      )
-
-  return violations
-
-
-def load_peer(path: str) -> dict:
-  with open(path, "r", encoding="utf-8") as handle:
-    return json.load(handle)
-
-
-def run_read(peer_path: str) -> int:
-  from cereal import log
-  peer_dump = load_peer(peer_path)
-  local_dump = {
-    "root": hex_id(log.Event.schema.node.id),
-    "structs": collect_schema(log.Event.schema),
-  }
-  violations = compare(sunnypilot_dump=peer_dump, upstream_dump=local_dump)
-
-  if not violations:
-    print("cereal compat OK: upstream openpilot can parse sunnypilot routes "
-          "(no leaked structs, no ordinal collisions).")
-    return 0
-
-  print(f"cereal compat FAIL: upstream openpilot would misparse sunnypilot routes "
-        f"({len(violations)} violation(s)):")
-  for v in violations:
-    print(f"  {v}")
-  return 1
-
-
-def main() -> int:
-  parser = argparse.ArgumentParser(
-    description="sunnypilot <-> upstream cereal compatibility validator (schema-level)."
-  )
-  mode = parser.add_mutually_exclusive_group(required=True)
-  mode.add_argument("-g", "--generate", action="store_true", help="dump local schema to JSON")
-  mode.add_argument("-r", "--read", action="store_true", help="load peer JSON and diff against local")
-  parser.add_argument("-f", "--file", default="schema.json", help="JSON file path (default: schema.json)")
-  args = parser.parse_args()
-
-  try:
-    if args.generate:
-      dump_schema(args.file)
-      return 0
-    return run_read(args.file)
-  except ImportError as exc:
-    print(f"error: cannot import cereal ({exc}). did scons build cereal?")
-    return 2
-
-
-if __name__ == "__main__":
-  sys.exit(main())

cereal/services.py

@@ -1,64 +1,58 @@
 #!/usr/bin/env python3
-from enum import IntEnum
 from typing import Optional
 
 
-# TODO: this should be automatically determined using the capnp schema
-class QueueSize(IntEnum):
-  BIG = 10 * 1024 * 1024      # 10MB - video frames, large AI outputs
-  MEDIUM = 2 * 1024 * 1024    # 2MB - high freq (CAN), livestream
-  SMALL = 250 * 1024          # 250KB - most services
-
-
 class Service:
-  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
-               queue_size: QueueSize = QueueSize.SMALL):
+  def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None):
     self.should_log = should_log
     self.frequency = frequency
     self.decimation = decimation
-    self.queue_size = queue_size
 
 
 _services: dict[str, tuple] = {
   # service: (should_log, frequency, qlog decimation (optional))
   # note: the "EncodeIdx" packets will still be in the log
   "gyroscope": (True, 104., 104),
+  "gyroscope2": (True, 100., 100),
   "accelerometer": (True, 104., 104),
+  "accelerometer2": (True, 100., 100),
+  "magnetometer": (True, 25.),
+  "lightSensor": (True, 100., 100),
   "temperatureSensor": (True, 2., 200),
+  "temperatureSensor2": (True, 2., 200),
+  "gpsNMEA": (True, 9.),
   "deviceState": (True, 2., 1),
-  "touch": (True, 20., 1),
-  "can": (True, 100., 2053, QueueSize.BIG),  # decimation gives ~3 msgs in a full segment
-  "controlsState": (True, 100., 10, QueueSize.MEDIUM),
+  "can": (True, 100., 2053),  # decimation gives ~3 msgs in a full segment
+  "controlsState": (True, 100., 10),
   "selfdriveState": (True, 100., 10),
   "pandaStates": (True, 10., 1),
   "peripheralState": (True, 2., 1),
   "radarState": (True, 20., 5),
   "roadEncodeIdx": (False, 20., 1),
   "liveTracks": (True, 20.),
-  "sendcan": (True, 100., 139, QueueSize.MEDIUM),
-  "logMessage": (True, 0., None, QueueSize.BIG),
-  "errorLogMessage": (True, 0., 1, QueueSize.BIG),
+  "sendcan": (True, 100., 139),
+  "logMessage": (True, 0.),
+  "errorLogMessage": (True, 0., 1),
   "liveCalibration": (True, 4., 4),
   "liveTorqueParameters": (True, 4., 1),
-  "liveDelay": (True, 4., 1),
   "androidLog": (True, 0.),
   "carState": (True, 100., 10),
   "carControl": (True, 100., 10),
   "carOutput": (True, 100., 10),
   "longitudinalPlan": (True, 20., 10),
-  "lateralManeuverPlan": (True, 20.),
   "driverAssistance": (True, 20., 20),
-  "procLog": (True, 0.5, 15, QueueSize.BIG),
+  "procLog": (True, 0.5, 15),
   "gpsLocationExternal": (True, 10., 10),
   "gpsLocation": (True, 1., 1),
   "ubloxGnss": (True, 10.),
   "qcomGnss": (True, 2.),
+  "gnssMeasurements": (True, 10., 10),
   "clocks": (True, 0.1, 1),
   "ubloxRaw": (True, 20.),
   "livePose": (True, 20., 4),
   "liveParameters": (True, 20., 5),
   "cameraOdometry": (True, 20., 10),
-  "thumbnail": (True, 1 / 60., 1),
+  "thumbnail": (True, 0.2, 1),
   "onroadEvents": (True, 1., 1),
   "carParams": (True, 0.02, 1),
   "roadCameraState": (True, 20., 20),
@@ -69,43 +63,33 @@ _services: dict[str, tuple] = {
   "wideRoadEncodeIdx": (False, 20., 1),
   "wideRoadCameraState": (True, 20., 20),
   "drivingModelData": (True, 20., 10),
-  "modelV2": (True, 20., None, QueueSize.BIG),
+  "modelV2": (True, 20.),
   "managerState": (True, 2., 1),
+  "uploaderState": (True, 0., 1),
+  "navInstruction": (True, 1., 10),
+  "navRoute": (True, 0.),
+  "navThumbnail": (True, 0.),
   "qRoadEncodeIdx": (False, 20.),
-  "userBookmark": (True, 0., 1),
-  "soundPressure": (True, 10., 10),
-  "rawAudioData": (False, 20.),
-  "bookmarkButton": (True, 0., 1),
-  "audioFeedback": (True, 0., 1),
-  "roadEncodeData": (False, 20., None, QueueSize.BIG),
-  "driverEncodeData": (False, 20., None, QueueSize.BIG),
-  "wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
-  "qRoadEncodeData": (False, 20., None, QueueSize.BIG),
-
-  # sunnypilot
-  "modelManagerSP": (False, 1., 1, QueueSize.BIG),
-  "backupManagerSP": (False, 1., 1, QueueSize.BIG),
-  "selfdriveStateSP": (True, 100., 10),
-  "longitudinalPlanSP": (True, 20., 10),
-  "onroadEventsSP": (True, 1., 1),
-  "carParamsSP": (True, 0.02, 1),
-  "carControlSP": (True, 100., 10),
-  "carStateSP": (True, 100., 10),
-  "liveMapDataSP": (True, 1., 1),
-  "modelDataV2SP": (True, 20., None, QueueSize.BIG),
-  "liveLocationKalman": (True, 20.),
+  "userFlag": (True, 0., 1),
+  "microphone": (True, 10., 10),
 
   # debug
   "uiDebug": (True, 0., 1),
   "testJoystick": (True, 0.),
   "alertDebug": (True, 20., 5),
+  "roadEncodeData": (False, 20.),
+  "driverEncodeData": (False, 20.),
+  "wideRoadEncodeData": (False, 20.),
+  "qRoadEncodeData": (False, 20.),
   "livestreamWideRoadEncodeIdx": (False, 20.),
   "livestreamRoadEncodeIdx": (False, 20.),
   "livestreamDriverEncodeIdx": (False, 20.),
-  "livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
-  "livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
+  "livestreamWideRoadEncodeData": (False, 20.),
+  "livestreamRoadEncodeData": (False, 20.),
+  "livestreamDriverEncodeData": (False, 20.),
   "customReservedRawData0": (True, 0.),
+  "customReservedRawData1": (True, 0.),
+  "customReservedRawData2": (True, 0.),
 }
 SERVICE_LIST = {name: Service(*vals) for
                 idx, (name, vals) in enumerate(_services.items())}
@@ -120,13 +104,13 @@ def build_header():
   h += "#include <map>\n"
   h += "#include <string>\n"
 
-  h += "struct service { std::string name; bool should_log; float frequency; int decimation; size_t queue_size; };\n"
+  h += "struct service { std::string name; bool should_log; int frequency; int decimation; };\n"
   h += "static std::map<std::string, service> services = {\n"
   for k, v in SERVICE_LIST.items():
     should_log = "true" if v.should_log else "false"
     decimation = -1 if v.decimation is None else v.decimation
-    h += '  { "%s", {"%s", %s, %f, %d, %d}},\n' % \
-         (k, k, should_log, v.frequency, decimation, v.queue_size)
+    h += '  { "%s", {"%s", %s, %d, %d}},\n' % \
+         (k, k, should_log, v.frequency, decimation)
   h += "};\n"
 
   h += "#endif\n"

codecov.yml

@@ -0,0 +1,13 @@
+comment: false
+coverage:
+  status:
+    project:
+      default:
+        informational: true
+    patch: off
+
+ignore:
+  - "**/test_*.py"
+  - "selfdrive/test/**"
+  - "system/version.py" # codecov changes depending on if we are in a branch or not
+  - "tools"

common/.gitignore

@@ -0,0 +1 @@
+*.cpp

common/SConscript

@@ -1,14 +1,25 @@
-Import('env', 'envCython')
+Import('env', 'envCython', 'arch')
 
 common_libs = [
   'params.cc',
   'swaglog.cc',
   'util.cc',
-  'ratekeeper.cc',
+  'i2c.cc',
+  'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
+if arch != "Darwin":
+  common_libs.append('gpio.cc')
+
 _common = env.Library('common', common_libs, LIBS="json11")
-Export('_common')
+
+files = [
+  'clutil.cc',
+]
+
+_gpucommon = env.Library('gpucommon', files)
+Export('_common', '_gpucommon')
 
 if GetOption('extras'):
   env.Program('tests/test_common',
@@ -18,6 +29,11 @@ if GetOption('extras'):
 # Cython bindings
 params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
 
-common_python = [params_python]
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
 
 Export('common_python')

common/api/__init__.py

@@ -1,26 +1,46 @@
-from openpilot.common.api.comma_connect import CommaConnectApi
+import jwt
+import os
+import requests
+from datetime import datetime, timedelta, UTC
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
 
+API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
 class Api:
   def __init__(self, dongle_id):
-    self.service = CommaConnectApi(dongle_id)
-
-  def request(self, method, endpoint, **params):
-    return self.service.request(method, endpoint, **params)
+    self.dongle_id = dongle_id
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
+      self.private_key = f.read()
 
   def get(self, *args, **kwargs):
-    return self.service.get(*args, **kwargs)
+    return self.request('GET', *args, **kwargs)
 
   def post(self, *args, **kwargs):
-    return self.service.post(*args, **kwargs)
+    return self.request('POST', *args, **kwargs)
 
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self.service.get_token(payload_extra, expiry_hours)
+  def request(self, method, endpoint, timeout=None, access_token=None, **params):
+    return api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
+
+  def get_token(self, expiry_hours=1):
+    now = datetime.now(UTC).replace(tzinfo=None)
+    payload = {
+      'identity': self.dongle_id,
+      'nbf': now,
+      'iat': now,
+      'exp': now + timedelta(hours=expiry_hours)
+    }
+    token = jwt.encode(payload, self.private_key, algorithm='RS256')
+    if isinstance(token, bytes):
+      token = token.decode('utf8')
+    return token
 
 
-def api_get(endpoint, method='GET', timeout=None, access_token=None, session=None, **params):
-  return CommaConnectApi(None).api_get(endpoint, method, timeout, access_token, session, **params)
+def api_get(endpoint, method='GET', timeout=None, access_token=None, **params):
+  headers = {}
+  if access_token is not None:
+    headers['Authorization'] = "JWT " + access_token
 
+  headers['User-Agent'] = "openpilot-" + get_version()
 
-def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-  return CommaConnectApi(None).get_key_pair()
+  return requests.request(method, API_HOST + "/" + endpoint, timeout=timeout, headers=headers, params=params)

common/api/base.py

@@ -1,72 +0,0 @@
-import jwt
-import os
-import requests
-import unicodedata
-from datetime import datetime, timedelta, UTC
-from openpilot.system.hardware.hw import Paths
-from openpilot.system.version import get_version
-
-# name: jwt signature algorithm
-KEYS = {"id_rsa": "RS256",
-        "id_ecdsa": "ES256"}
-
-
-class BaseApi:
-  def __init__(self, dongle_id, api_host, user_agent="openpilot-"):
-    self.dongle_id = dongle_id
-    self.api_host = api_host
-    self.user_agent = user_agent
-    self.jwt_algorithm, self.private_key, _ = self.get_key_pair()
-
-  def get(self, *args, **kwargs):
-    return self.request('GET', *args, **kwargs)
-
-  def post(self, *args, **kwargs):
-    return self.request('POST', *args, **kwargs)
-
-  def request(self, method, endpoint, timeout=None, access_token=None, **params):
-    return self.api_get(endpoint, method=method, timeout=timeout, access_token=access_token, **params)
-
-  def _get_token(self, payload_extra=None, expiry_hours=1, **extra_payload):
-    now = datetime.now(UTC).replace(tzinfo=None)
-    payload = {
-      'identity': self.dongle_id,
-      'nbf': now,
-      'iat': now,
-      'exp': now + timedelta(hours=expiry_hours),
-      **extra_payload
-    }
-    if payload_extra is not None:
-      payload.update(payload_extra)
-    token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
-    if isinstance(token, bytes):
-      token = token.decode('utf8')
-    return token
-
-  def get_token(self, payload_extra=None, expiry_hours=1):
-    return self._get_token(payload_extra, expiry_hours)
-
-  def remove_non_ascii_chars(self, text):
-    normalized_text = unicodedata.normalize('NFD', text)
-    ascii_encoded_text = normalized_text.encode('ascii', 'ignore')
-    return ascii_encoded_text.decode()
-
-  def api_get(self, endpoint, method='GET', timeout=None, access_token=None, session=None, json=None, **params):
-    headers = {}
-    if access_token is not None:
-      headers['Authorization'] = "JWT " + access_token
-
-    version = self.remove_non_ascii_chars(get_version())
-    headers['User-Agent'] = self.user_agent + version
-
-    # TODO: add session to Api
-    req = requests if session is None else session
-    return req.request(method, f"{self.api_host}/{endpoint}", timeout=timeout, headers=headers, json=json, params=params)
-
-  @staticmethod
-  def get_key_pair() -> tuple[str, str, str] | tuple[None, None, None]:
-    for key in KEYS:
-      if os.path.isfile(Paths.persist_root() + f'/comma/{key}') and os.path.isfile(Paths.persist_root() + f'/comma/{key}.pub'):
-        with open(Paths.persist_root() + f'/comma/{key}') as private, open(Paths.persist_root() + f'/comma/{key}.pub') as public:
-          return KEYS[key], private.read(), public.read()
-    return None, None, None