Updating ESCC guide to use relay box

ios shortcut support

.devcontainer/.gitignore

@@ -0,0 +1,3 @@
+.Xauthority
+.env
+.host/

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+FROM ghcr.io/commaai/openpilot-base:latest
+
+RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa libegl1-mesa
+RUN pip install ipython jupyter jupyterlab
+
+RUN cd /tmp && \
+    ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
+    curl -L -o virtualgl.deb "https://downloads.sourceforge.net/project/virtualgl/3.1/virtualgl_3.1_$ARCH.deb" && \
+    dpkg -i virtualgl.deb
+
+USER batman
+
+RUN cd $HOME && \
+    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
+    curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc

.devcontainer/container_post_create.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+TARGET_USER=batman
+source .devcontainer/.host/.env
+
+# override display flag for mac hosts
+if [[ $HOST_OS == darwin ]]; then
+  echo "Setting up DISPLAY override for macOS..."
+  cat <<EOF >> /home/$TARGET_USER/.bashrc
+source .devcontainer/.host/.env
+if [ -n "\$HOST_DISPLAY" ]; then
+  DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}')
+  export DISPLAY=host.docker.internal:\$DISPLAY_NUM
+fi
+EOF
+fi
+
+# setup virtualgl for mac hosts
+if [[ $HOST_OS == darwin ]]; then
+  echo "Setting up virtualgl for macOS..."
+  cat <<EOF >> /home/$TARGET_USER/.bashrc
+if [ -n "\$HOST_DISPLAY" ]; then
+  export VGL_PORT=10000
+  export VGL_CLIENT=host.docker.internal
+  export VGL_COMPRESS=rgb
+  export VGL_DISPLAY=:99
+  export VGL_FPS=60
+  # prevent vglrun from running exec
+  alias exec=:; source vglrun :; unalias exec
+fi
+EOF
+fi
+
+# These lines are temporary, to remain backwards compatible with old devcontainers
+# that were running as root and therefore had their caches written as root
+sudo chown -R $TARGET_USER: /tmp/scons_cache
+sudo chown -R $TARGET_USER: /tmp/comma_download_cache
+sudo chown -R $TARGET_USER: /home/batman/.comma

.devcontainer/container_post_start.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+source .devcontainer/.host/.env
+
+# setup safe directories for submodules
+SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
+for DIR in $SUBMODULE_DIRS; do 
+  git config --global --add safe.directory "$PWD/$DIR"
+done
+
+# virtual display for virtualgl
+if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then
+  echo "Starting virtual display at :99 ..."
+  tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24
+fi

.devcontainer/devcontainer.json

@@ -0,0 +1,47 @@
+{
+  "name": "openpilot devcontainer",
+  "build": {
+    "dockerfile": "Dockerfile"
+  },
+  "postCreateCommand": ".devcontainer/container_post_create.sh",
+  "postStartCommand": ".devcontainer/container_post_start.sh",
+  "initializeCommand": [".devcontainer/host_setup"],
+  "privileged": true,
+  "containerEnv": {
+    "DISPLAY": "${localEnv:DISPLAY}",
+    "PYTHONPATH": "${containerWorkspaceFolder}",
+    "TERM": "xterm-256color",
+    "force_color_prompt": "1"
+  },
+  "runArgs": [
+    "--volume=/tmp/.X11-unix:/tmp/.X11-unix",
+    "--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority",
+    "--volume=${localEnv:HOME}/.comma:/home/batman/.comma",
+    "--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
+    "--volume=/tmp/devcontainer_scons_cache:/tmp/scons_cache",
+    "--shm-size=1G",
+    "--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux
+    "--publish=0.0.0.0:8070-8079:8070-8079"  // body ZMQ services
+  ],
+  "features": {
+    "ghcr.io/devcontainers/features/common-utils:2": {
+      "installZsh": false,
+      "installOhMyZsh": false,
+      "upgradePackages": false,
+      "username": "batman"
+    }
+  },
+  "containerUser": "batman",
+  "remoteUser": "batman",
+  "customizations": {
+    "vscode": {
+      "extensions": [
+        "ms-python.python",
+        "ms-vscode.cpptools",
+        "ms-toolsai.jupyter",
+        "guyskk.language-cython",
+        "lharri73.dbc"
+      ]
+    }
+  }
+}

.devcontainer/host_setup

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# pull base image
+if [[ -z $USE_LOCAL_IMAGE ]]; then
+  echo "Updating openpilot_base image if needed..."
+  docker pull ghcr.io/commaai/openpilot-base:latest
+fi
+
+# setup .host dir
+mkdir -p .devcontainer/.host
+
+# setup links to Xauthority
+XAUTHORITY_LINK=".devcontainer/.host/.Xauthority"
+rm -f $XAUTHORITY_LINK
+if [[ -z $XAUTHORITY ]]; then
+  echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
+  if ! [[ -f $HOME/.Xauthority ]]; then
+    echo "~/.XAuthority file does not exist. GUI tools may not work properly."
+    touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
+  else
+    ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
+  fi
+else
+    ln -sf $XAUTHORITY $XAUTHORITY_LINK
+fi
+
+# setup host env file
+HOST_INFO_FILE=".devcontainer/.host/.env"
+SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
+echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE
+echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE
+
+# run virtualgl if macos
+if [[ $SYSTEM == "darwin" ]]; then
+  echo
+  if [[ -f /opt/VirtualGL/bin/vglclient ]]; then
+    echo "Starting VirtualGL client at port 10000..."
+    VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log"
+    mkdir -p "$(dirname $VGL_LOG_FILE)"
+    /opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach
+  else
+    echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:"
+    echo
+    echo "  brew install --cask virtualgl"
+    echo
+  fi
+fi

.devcontainer/host_setup.cmd

@@ -0,0 +1,10 @@
+:: pull base image
+IF NOT DEFINED USE_LOCAL_IMAGE ^
+echo "Updating openpilot_base image if needed..." && ^
+docker pull ghcr.io/commaai/openpilot-base:latest
+
+:: setup .host dir
+mkdir .devcontainer\.host
+
+:: setup host env file
+echo "" > .devcontainer\.host\.env

.dir-locals.el

@@ -1,3 +0,0 @@
-((c++-mode (flycheck-gcc-language-standard . "c++11")
-           (flycheck-clang-language-standard . "c++11")
-           ))

.dockerignore

@@ -16,6 +16,9 @@
 *.so
 *.a
 
+venv/
+.venv/
+
 notebooks
 phone
 massivemap
@@ -25,10 +28,8 @@ chffr/app2
 chffr/backend/env
 selfdrive/nav
 selfdrive/baseui
-chffr/lib/vidindex/vidindex
 selfdrive/test/simulator2
 **/cache_data
-xx/chffr/lib/vidindex/vidindex
 xx/plus
 xx/community
 xx/projects

.gitattributes

@@ -1,2 +1,24 @@
+* text=auto
+
+# to move existing files into LFS:
+# git add --renormalize .
 *.dlc filter=lfs diff=lfs merge=lfs -text
 *.onnx filter=lfs diff=lfs merge=lfs -text
+*.svg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.ttf filter=lfs diff=lfs merge=lfs -text
+*.wav filter=lfs diff=lfs merge=lfs -text
+
+selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
+system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
+selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
+third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
+third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
+third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
+third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text

.github/FUNDING.yml

@@ -0,0 +1,12 @@
+# These are supported funding model platforms
+
+github: [sunnyhaibin] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: sunnyhaibin # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: ['https://paypal.me/sunnyhaibin0850'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/PULL_REQUEST_TEMPLATE/car_port.md

@@ -8,7 +8,7 @@ assignees: ''
 
 **Checklist**
 
-- [ ] added to README
+- [ ] added entry to CarInfo in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
 - [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
 - [ ] route with openpilot:
 - [ ] route with stock system:

.github/PULL_REQUEST_TEMPLATE/fingerprint.md

@@ -6,6 +6,8 @@ labels: 'fingerprint'
 assignees: ''
 ---
 
-Discord username: []
+**Car**
+Which car (make, model, year) this fingerprint is for
 
-Route: []
+**Route**
+A route with the fingerprint

.github/PULL_REQUEST_TEMPLATE/tuning.md

@@ -28,5 +28,4 @@ Longitudinal:
 Lateral:
 * Straight driving at ~25, ~45 and ~65mph
 * Turns driving at ~25, ~45 and ~65mph
-
--->
+-->

.github/build.py

@@ -0,0 +1,30 @@
+import pathlib
+
+GITHUB_FOLDER = pathlib.Path(__file__).parent
+
+PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
+
+order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
+
+def create_pull_request_template():
+  with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
+    f.write("<!-- Please copy and paste the relevant template -->\n\n")
+
+    for t in order:
+      template = PULL_REQUEST_TEMPLATES / f"{t}.md"
+      text = template.read_text()
+
+      # Remove metadata for GitHub
+      start = text.find("---")
+      end = text.find("---", start+1)
+      text = text[end + 4:]
+
+      # Remove comments
+      text = text.replace("<!-- ", "").replace("-->", "")
+
+      f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
+      f.write(text)
+      f.write("\n\n")
+      f.write("-->\n\n")
+
+create_pull_request_template()

.github/labeler.yaml

@@ -0,0 +1,79 @@
+CI / testing:
+  - changed-files:
+    - any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
+
+car:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/**'
+
+body:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/body/*'
+
+chrysler:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/chrysler/*'
+
+ford:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/ford/*'
+
+gm:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/gm/*'
+
+honda:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/honda/*'
+
+hyundai:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/hyundai/*'
+
+mazda:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/mazda/*'
+
+nissan:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/nissan/*'
+
+subaru:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/subaru/*'
+
+tesla:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/telsa/*'
+
+toyota:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/toyota/*'
+
+volkswagen:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/volkswagen/*'
+
+fingerprint:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/car/*/fingerprints.py'
+
+simulation:
+  - changed-files:
+    - any-glob-to-all-files: 'tools/sim/**'
+
+ui:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/ui/**'
+
+tools:
+  - changed-files:
+    - any-glob-to-all-files: 'tools/**'
+
+multilanguage:
+  - changed-files:
+    - any-glob-to-all-files: 'selfdrive/ui/translations/**'
+
+research:
+  - changed-files:
+    - any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"

.github/pull_request_template.md

@@ -1,38 +1,68 @@
 <!-- Please copy and paste the relevant template -->
 
-<!--- ***** Template: Car bug fix *****
+<!--- ***** Template: Fingerprint *****
 
-**Description** [](A description of the bug and the fix. Also link any relevant issues.)
-
-**Verification** [](Explain how you tested this bug fix.)
+**Car**
+Which car (make, model, year) this fingerprint is for
 
 **Route**
+A route with the fingerprint
+
+-->
+
+<!--- ***** Template: Car Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+**Route**
+
 Route: [a route with the bug fix]
 
--->
-
-<!--- ***** Template: Bug fix *****
-
-**Description** [](A description of the bug and the fix. Also link any relevant issues.)
-
-**Verification** [](Explain how you tested this bug fix.)
 
 -->
 
-<!--- ***** Template: Car port *****
+<!--- ***** Template: Bugfix *****
+
+**Description**
+
+A description of the bug and the fix. Also link the issue if it exists. 
+
+**Verification**
+
+Explain how you tested this bug fix. 
+
+
+-->
+
+<!--- ***** Template: Car Port *****
 
 **Checklist**
+
 - [ ] added entry to CarInfo in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
 - [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
 - [ ] route with openpilot:
 - [ ] route with stock system:
+- [ ] car harness used (if comma doesn't sell it, put N/A):
+
 
 -->
 
 <!--- ***** Template: Refactor *****
 
-**Description** [](A description of the refactor, including the goals it accomplishes.)
+**Description**
+
+A description of the refactor, including the goals it accomplishes. 
+
+**Verification**
+
+Explain how you tested the refactor for regressions. 
 
-**Verification** [](Explain how you tested the refactor for regressions.)
 
 -->
+

.github/workflows/auto-cache/action.yaml

@@ -0,0 +1,49 @@
+name: 'automatically cache based on current runner'
+
+inputs:
+  path:
+    description: 'path to cache'
+    required: true
+  key:
+    description: 'key'
+    required: true
+  restore-keys:
+    description: 'restore-keys'
+    required: true
+  save:
+    description: 'whether to save the cache'
+    default: 'false'
+    required: false
+
+runs:
+  using: "composite"
+  steps:
+    - name: setup namespace cache
+      if: ${{ contains(runner.name, 'nsc') }} 
+      uses: namespacelabs/nscloud-cache-action@v1
+      with:
+        path: ${{ inputs.path }}
+
+    - name: setup github cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
+      uses: 'actions/cache@v3'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    - name: setup github cache
+      if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
+      uses: 'actions/cache/restore@v3'
+      with:
+        path: ${{ inputs.path }}
+        key: ${{ inputs.key }}
+        restore-keys: ${{ inputs.restore-keys }}
+
+    # make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
+    - id: scons-cache-setup
+      shell: bash
+      run: |
+        mkdir -p ${{ inputs.path }}
+        sudo chmod -R 777 ${{ inputs.path }}
+        sudo chown -R $USER ${{ inputs.path }}

.github/workflows/auto_pr_review.yaml

@@ -0,0 +1,207 @@
+name: "PR review"
+on:
+  pull_request_target:
+    types: [opened, reopened, synchronize, edited, edited]
+
+jobs:
+  labeler:
+    name: apply labels
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - uses: actions/labeler@v5.0.0
+      with:
+        dot: true
+        configuration-path: .github/labeler.yaml
+
+  pr_branch_check:
+    name: check branch
+    runs-on: ubuntu-latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+    - uses: Vankka/pr-target-branch-action@69ab6dd5c221de3548b3b6c4d102c1f4913d3baa
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      with:
+        target: /^(?!master$).*/
+        exclude: /commaai:.*/
+        change-to: ${{ github.base_ref }}
+        already-exists-action: close_this
+        already-exists-comment: "Your PR should be made against the `master` branch"
+
+  check-pr-template:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      issues: write
+      pull-requests: write
+      actions: read
+    if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
+    steps:
+      - uses: actions/github-script@v7
+        with:
+          script: |
+            // Comment to add to the PR if no template has been used
+            const NO_TEMPLATE_MESSAGE =
+              "It looks like you didn't use one of the Pull Request templates. Please check [the contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md). \
+            Also make sure that you didn't modify any of the checkboxes or headings within the template.";
+            // body data for future requests
+            const body_data = {
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+            };
+
+            // Utility function to extract all headings
+            const extractHeadings = (markdown) => {
+              const headingRegex = /^(#{1,6})\s+(.+)$/gm;
+              const boldTextRegex = /^(?:\*\*|__)(.+?)(?:\*\*|__)\s*$/gm;
+              const headings = [];
+              let headingMatch;
+              while ((headingMatch = headingRegex.exec(markdown))) {
+                headings.push(headingMatch[2].trim());
+              }
+              let boldMatch;
+              while ((boldMatch = boldTextRegex.exec(markdown))) {
+                headings.push(boldMatch[1].trim());
+              }
+              return headings;
+            };
+
+            // Utility function to extract all check box descriptions
+            const extractCheckBoxTexts = (markdown) => {
+              const checkboxRegex = /^\s*-\s*\[( |x)\]\s+(.+)$/gm;
+              const checkboxes = [];
+              let match;
+              while ((match = checkboxRegex.exec(markdown))) {
+                checkboxes.push(match[2].trim());
+              }
+              return checkboxes;
+            };
+
+            // Utility function to check if a list is a subset of another list
+            isSubset = (subset, superset) => {
+              return subset.every((item) => superset.includes(item));
+            };
+
+            // Utility function to check if a list of checkboxes is a subset of another list of checkboxes
+            isCheckboxSubset = (templateCheckBoxTexts, prTextCheckBoxTexts) => {
+              // Check if each template checkbox text is a substring of at least one PR checkbox text 
+              // (user should be allowed to add additional text)
+              return templateCheckBoxTexts.every((item) => prTextCheckBoxTexts.some((element) => element.includes(item)))
+            }
+
+            // Get filenames of all currently checked-in PR templates
+            const template_contents = await github.rest.repos.getContent({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              path: ".github/PULL_REQUEST_TEMPLATE",
+            });
+            var template_filenames = [];
+            for (const content of template_contents.data) {
+              template_filenames.push(content.path);
+            }
+            console.debug("Received template filenames: " + template_filenames);
+            // Retrieve templates
+            var templates = [];
+            for (const template_filename of template_filenames) {
+              const template_response = await github.rest.repos.getContent({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                path: template_filename,
+              });
+              // Convert Base64 content back
+              const decoded_template = atob(template_response.data.content);
+              const headings = extractHeadings(decoded_template);
+              const checkboxes = extractCheckBoxTexts(decoded_template);
+              if (!headings.length && !checkboxes.length) {
+                console.warn(
+                  "Invalid template! Contains neither headings nor checkboxes, ignoring it: \n" +
+                    decoded_template
+                );
+              } else {
+                templates.push({ headings: headings, checkboxes: checkboxes });
+              }
+            }
+            // Retrieve the PR Body
+            const pull_request = await github.rest.issues.get({
+              ...body_data,
+            });
+            const pull_request_text = pull_request.data.body;
+            console.debug("Received Pull Request body: \n" + pull_request_text);
+
+            /* Check if the PR Body matches one of the templates
+            A template is defined by all headings and checkboxes it contains
+            We extract all Headings and Checkboxes from the PR text and check if any of the templates is a subset of that
+            */
+            const pr_headings = extractHeadings(pull_request_text);
+            const pr_checkboxes = extractCheckBoxTexts(pull_request_text);
+            console.debug("Found Headings in PR body:\n" + pr_headings);
+            console.debug("Found Checkboxes in PR body:\n" + pr_checkboxes);
+            var template_found = false;
+            // Iterate over each template to check if it applies
+            for (const template of templates) {
+              console.log(
+                "Checking for headings: [" +
+                  template.headings +
+                  "] and checkboxes: [" +
+                  template.checkboxes + "]"
+              );
+              if (
+                isCheckboxSubset(template.checkboxes, pr_checkboxes) &&
+                isSubset(template.headings, pr_headings)
+              ) {
+                console.debug("Found matching template!");
+                template_found = true;
+              }
+            }
+
+            // List comments from previous runs
+            var existing_comments = [];
+            const comments = await github.rest.issues.listComments({
+              ...body_data,
+            });
+            for (const comment of comments.data) {
+              if (comment.body === NO_TEMPLATE_MESSAGE) {
+                existing_comments.push(comment);
+              }
+            }
+
+            // Add a comment to the PR that it is not using a the template (but only if this comment does not exist already)
+            if (!template_found) {
+              var comment_already_sent = false;
+
+              // Add an 'in-bot-review' label since this PR doesn't have the template
+              github.rest.issues.addLabels({
+                ...body_data,
+                labels: ["in-bot-review"],
+              });
+
+              if (existing_comments.length < 1) {
+                github.rest.issues.createComment({
+                  ...body_data,
+                  body: NO_TEMPLATE_MESSAGE,
+                });
+              }
+            } else {
+              // If template has been found, delete any old comment about missing template
+              for (const existing_comment of existing_comments) {
+                github.rest.issues.deleteComment({
+                  ...body_data,
+                  comment_id: existing_comment.id,
+                });
+              }
+              // Remove the 'in-bot-review' label after the review is done and the PR has passed
+              github.rest.issues.removeLabel({
+                ...body_data,
+                name: "in-bot-review",
+              }).catch((error) => {
+                console.log("Label 'in-bot-review' not found, ignoring");
+              });
+            }
+

.github/workflows/badges.yaml

@@ -7,22 +7,26 @@ on:
 env:
   BASE_IMAGE: openpilot-base
   DOCKER_REGISTRY: ghcr.io/commaai
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
 
 jobs:
   badges:
     name: create badges
     runs-on: ubuntu-20.04
     if: github.repository == 'commaai/openpilot'
+    permissions:
+      contents: write
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Push badges
       run: |
         ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
 
+        rm .gitattributes
+
         git checkout --orphan badges
         git rm -rf --cached .
         git config user.email "badge-researcher@comma.ai"

.github/workflows/compile-openpilot/action.yaml

@@ -0,0 +1,21 @@
+name: 'compile openpilot'
+
+runs:
+  using: "composite"
+  steps:
+    - shell: bash
+      name: Build openpilot with all flags
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+        ${{ env.RUN }} "release/check-dirty.sh"
+    - shell: bash
+      name: Cleanup scons cache and rebuild
+      run: |
+          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
+                          scons -j$(nproc) --cache-populate"
+    - name: Save scons cache
+      uses: actions/cache/save@v3
+      if: github.ref == 'refs/heads/master'
+      with:
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

.github/workflows/docs.yaml

@@ -0,0 +1,64 @@
+name: docs
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+
+concurrency:
+  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
+  cancel-in-progress: true
+
+env:
+  BASE_IMAGE: openpilot-base
+
+  BUILD: selfdrive/test/docker_build.sh base
+
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
+
+jobs:
+  docs:
+    name: build docs
+    runs-on: ubuntu-20.04
+    timeout-minutes: 45
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Build docs
+      run: |
+        ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html"
+
+    - uses: actions/checkout@v4
+      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      with:
+        path: openpilot-docs
+        ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
+        repository: commaai/openpilot-docs
+    - name: Push
+      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      run: |
+        set -x
+
+        source release/identity.sh
+
+        cd openpilot-docs
+
+        git checkout --orphan tmp
+        git rm -rf .
+
+        cp -r ../build/docs/html/ docs/
+        cp -r ../docs/README.md .
+        touch docs/.nojekyll
+        echo -n docs.comma.ai > docs/CNAME
+        git add -f .
+
+        git commit -m "build docs"
+
+        # docs live in different repo to not bloat openpilot's full clone size
+        git push -f origin tmp:gh-pages

.github/workflows/prebuilt.yaml

@@ -5,37 +5,35 @@ on:
   workflow_dispatch:
 
 env:
-  BASE_IMAGE: openpilot-base
-  DOCKER_REGISTRY: ghcr.io/commaai
-
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh prebuilt
 
 jobs:
   build_prebuilt:
     name: build prebuilt
     runs-on: ubuntu-20.04
-    timeout-minutes: 60
     if: github.repository == 'commaai/openpilot'
     env:
-      IMAGE_NAME: openpilot-prebuilt
+      PUSH_IMAGE: true
+    permissions:
+      checks: read
+      contents: read
+      packages: write
     steps:
     - name: Wait for green check mark
-      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
+      if: ${{ github.event_name != 'workflow_dispatch' }}
+      uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build prebuilt'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build master-ci).*).)*$
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Build Docker image
-      run: |
-        eval "$BUILD"
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f Dockerfile.openpilot .
-    - name: Push to container registry
+    - run: git lfs pull
+    - name: Build and Push docker image
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+        eval "$BUILD"

.github/workflows/release.yaml

@@ -1,29 +1,54 @@
 name: release
 on:
   schedule:
-    - cron: '0 * * * *'
+    - cron: '0 10 * * *'
   workflow_dispatch:
 
 jobs:
   build_masterci:
     name: build master-ci
+    env:
+      TARGET_DIR: /tmp/openpilot
+      ImageOS: ubuntu20
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
     runs-on: ubuntu-20.04
-    timeout-minutes: 60
     if: github.repository == 'commaai/openpilot'
+    permissions:
+      checks: read
+      contents: write
     steps:
+    - name: Install wait-on-check-action dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y libyaml-dev
     - name: Wait for green check mark
-      uses: lewagon/wait-on-check-action@e2558238c09778af25867eb5de5a3ce4bbae3dcd
+      if: ${{ github.event_name != 'workflow_dispatch' }}
+      uses: lewagon/wait-on-check-action@595dabb3acf442d47e29c9ec9ba44db0c6bdd18f
       with:
         ref: master
         wait-interval: 30
         running-workflow-name: 'build master-ci'
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
         check-regexp: ^((?!.*(build prebuilt).*).)*$
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
         fetch-depth: 0
     - name: Pull LFS
-      run: git lfs pull
+      run: |
+        git config --global --add safe.directory '*'
+        git lfs pull
     - name: Build master-ci
       run: |
+        release/build_devel.sh
+    - name: Run tests
+      run: |
+        export PYTHONPATH=$TARGET_DIR
+        cd $TARGET_DIR
+        scons -j$(nproc)
+        pytest -n logical selfdrive/car/tests/test_car_interfaces.py
+    - name: Push master-ci
+      run: |
+        unset TARGET_DIR
         BRANCH=master-ci release/build_devel.sh

.github/workflows/repo-maintenance.yaml

@@ -0,0 +1,61 @@
+name: repo maintenance
+
+on:
+  schedule:
+    - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
+  workflow_dispatch:
+
+jobs:
+  bump_submodules:
+    name: bump_submodules
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - name: bump submodules
+      run: |
+        git config --global --add safe.directory '*'
+        git -c submodule."tinygrad".update=none submodule update --remote
+        git add .
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
+      with:
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
+        commit-message: bump submodules
+        title: '[bot] Bump submodules'
+        branch: auto-bump-submodules
+        base: master
+        delete-branch: true
+        body: 'Automatic PR from repo-maintenance -> bump_submodules'
+        labels: bot
+  package_updates:
+    name: package_updates
+    runs-on: ubuntu-20.04
+    container:
+      image: ghcr.io/commaai/openpilot-base:latest
+    if: github.repository == 'commaai/openpilot'
+    steps:
+    - uses: actions/checkout@v4
+    - name: poetry lock
+      run: |
+        pip install poetry
+        poetry lock
+    - name: pre-commit autoupdate
+      run: |
+        git config --global --add safe.directory '*'
+        pre-commit autoupdate
+    - name: Create Pull Request
+      uses: peter-evans/create-pull-request@5b4a9f6a9e2af26e5f02351490b90d01eb8ec1e5
+      with:
+        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
+        commit-message: Update Python packages and pre-commit hooks
+        title: '[bot] Update Python packages and pre-commit hooks'
+        branch: auto-package-updates
+        base: master
+        delete-branch: true
+        body: 'Automatic PR from repo-maintenance -> package_updates'
+        labels: bot

.github/workflows/selfdrive_tests.yaml

@@ -2,193 +2,150 @@ name: selfdrive
 
 on:
   push:
-    branches-ignore:
-      - 'testing-closet*'
+    branches:
+      - master
   pull_request:
+  workflow_dispatch:
 
 concurrency:
   group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
   cancel-in-progress: true
 
 env:
+  PYTHONWARNINGS: error
   BASE_IMAGE: openpilot-base
-  CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
 
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
-  BUILD_CL: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
-
-  UNIT_TEST: coverage run --append -m unittest discover
+  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
 
 jobs:
   build_release:
     name: build release
     runs-on: ubuntu-20.04
-    timeout-minutes: 30
     env:
       STRIPPED_DIR: /tmp/releasepilot
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
+    - run: git lfs pull
     - name: Build devel
+      timeout-minutes: 1
       run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-pre-commit
+    - uses: ./.github/workflows/setup-with-retry
     - name: Check submodules
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
+      timeout-minutes: 1
       run: release/check-submodules.sh
     - name: Build openpilot and run checks
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: |
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "CI=1 python selfdrive/manager/build.py"
+        ${{ env.RUN }} "python selfdrive/manager/build.py"
     - name: Run tests
+      timeout-minutes: 3
       run: |
         cd $STRIPPED_DIR
         ${{ env.RUN }} "release/check-dirty.sh && \
-                        python -m unittest discover selfdrive/car"
+                        MAX_EXAMPLES=5 $PYTEST selfdrive/car"
     - name: pre-commit
+      timeout-minutes: 3
       run: |
         cd $GITHUB_WORKSPACE
         cp .pre-commit-config.yaml $STRIPPED_DIR
-        cp .pylintrc $STRIPPED_DIR
-        cp mypy.ini $STRIPPED_DIR
+        cp pyproject.toml $STRIPPED_DIR
+        cp poetry.lock $STRIPPED_DIR
         cd $STRIPPED_DIR
-        ${{ env.RUN }} "SKIP=test_translations pre-commit run --all"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
-  build_all:
-    name: build all
-    runs-on: ubuntu-20.04
-    timeout-minutes: 30
+  build:
+    strategy:
+      matrix:
+        arch: ${{ fromJson(
+           ((github.repository == 'commaai/openpilot') &&
+              ((github.event_name != 'pull_request') || 
+               (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
       with:
-        save-cache: true
-    - name: Build openpilot with all flags
-      run: ${{ env.RUN }} "scons -j$(nproc) --extras && release/check-dirty.sh"
-    - name: Cleanup scons cache
-      run: |
-          ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
-                          scons -j$(nproc) --cache-populate"
-
-  #build_mac:
-  #  name: build macos
-  #  runs-on: macos-latest
-  #  timeout-minutes: 60
-  #  steps:
-  #  - uses: actions/checkout@v3
-  #    with:
-  #      submodules: true
-  #  - name: Determine pre-existing Homebrew packages
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      echo 'EXISTING_CELLAR<<EOF' >> $GITHUB_ENV
-  #      ls -1 /usr/local/Cellar >> $GITHUB_ENV
-  #      echo 'EOF' >> $GITHUB_ENV
-  #  - name: Cache dependencies
-  #    id: dependency-cache
-  #    uses: actions/cache@v2
-  #    with:
-  #      path: |
-  #        ~/.pyenv
-  #        ~/.local/share/virtualenvs/
-  #        /usr/local/Cellar
-  #        ~/github_brew_cache_entries.txt
-  #        /tmp/scons_cache
-  #      key: macos-${{ hashFiles('tools/mac_setup.sh', 'update_requirements.sh', 'poetry.lock') }}
-  #      restore-keys: macos-
-  #  - name: Brew link restored dependencies
-  #    run: |
-  #      if [ -f ~/github_brew_cache_entries.txt ]; then
-  #        while read pkg; do
-  #          brew link --force "$pkg" # `--force` for keg-only packages
-  #        done < ~/github_brew_cache_entries.txt
-  #      else
-  #        echo "Cache entries not found"
-  #      fi
-  #  - name: Install dependencies
-  #    run: ./tools/mac_setup.sh
-  #  - name: Build openpilot
-  #    run: |
-  #      source tools/openpilot_env.sh
-  #      poetry run selfdrive/manager/build.py
-  #
-  #      # cleanup scons cache
-  #      rm -rf /tmp/scons_cache/
-  #      poetry run scons -j$(nproc) --cache-populate
-  #  - name: Remove pre-existing Homebrew packages for caching
-  #    if: steps.dependency-cache.outputs.cache-hit != 'true'
-  #    run: |
-  #      cd /usr/local/Cellar
-  #      new_cellar=$(ls -1)
-  #      comm -12 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | while read pkg; do
-  #        if [[ $pkg != "zstd" ]]; then # caching step needs zstd
-  #          rm -rf "$pkg"
-  #        fi
-  #      done
-  #      comm -13 <(echo "$EXISTING_CELLAR") <(echo "$new_cellar") | tee ~/github_brew_cache_entries.txt
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+    - uses: ./.github/workflows/compile-openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
 
   docker_push:
     name: docker push
-    runs-on: ubuntu-20.04
-    timeout-minutes: 22
+    strategy:
+      matrix:
+        arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
+    runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-20.04' }}
     if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
-    needs: static_analysis # hack to ensure slow tests run first since this and static_analysis are fast
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Build Docker image
-      run: eval "$BUILD"
-      timeout-minutes: 13
-    - name: Push to container registry
+    - name: Setup to push to repo
+      run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
+        echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
+        $DOCKER_LOGIN
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
+
+  docker_push_multiarch:
+    name: docker push multiarch tag
+    runs-on: ubuntu-20.04
+    if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+    needs: [docker_push]
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: false
+    - name: Setup docker
       run: |
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$BASE_IMAGE:latest
-    - name: Build CL Docker image
-      run: eval "$BUILD_CL"
-      timeout-minutes: 4
-    - name: Push to container registry
+    - name: Merge x64 and arm64 tags
       run: |
-        $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest
+        export PUSH_IMAGE=true
+        scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
 
   static_analysis:
     name: static analysis
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Build Docker image
-      run: eval "$BUILD"
+    - uses: ./.github/workflows/setup-pre-commit
+    - uses: ./.github/workflows/setup-with-retry
     - name: pre-commit
-      timeout-minutes: 5
-      run: ${{ env.RUN }} "pre-commit run --all"
+      timeout-minutes: 4
+      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"
 
   valgrind:
     name: valgrind
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run valgrind
+      timeout-minutes: 1
       run: |
         ${{ env.RUN }} "python selfdrive/test/test_valgrind_replay.py"
     - name: Print logs
@@ -197,197 +154,205 @@ jobs:
 
   unit_tests:
     name: unit tests
-    runs-on: ubuntu-20.04
-    timeout-minutes: 30
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
     - name: Build openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run unit tests
       timeout-minutes: 15
       run: |
-        ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \
-                        $UNIT_TEST common && \
-                        $UNIT_TEST opendbc/can && \
-                        $UNIT_TEST selfdrive/boardd && \
-                        $UNIT_TEST selfdrive/controls && \
-                        $UNIT_TEST selfdrive/monitoring && \
-                        $UNIT_TEST selfdrive/loggerd && \
-                        $UNIT_TEST selfdrive/car && \
-                        $UNIT_TEST selfdrive/locationd && \
-                        selfdrive/locationd/test/_test_locationd_lib.py && \
-                        $UNIT_TEST selfdrive/athena && \
-                        $UNIT_TEST selfdrive/thermald && \
-                        $UNIT_TEST system/hardware/tici && \
-                        $UNIT_TEST tools/lib/tests && \
+        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
+                        export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
+                        $PYTEST --timeout 60 -m 'not slow' && \
                         ./selfdrive/ui/tests/create_test_translations.sh && \
                         QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
-                        ./selfdrive/ui/tests/test_translations.py && \
-                        ./common/tests/test_util && \
-                        ./common/tests/test_swaglog && \
-                        ./selfdrive/boardd/tests/test_boardd_usbprotocol && \
-                        ./selfdrive/loggerd/tests/test_logger &&\
-                        ./system/proclogd/tests/test_proclog && \
-                        ./tools/replay/tests/test_replay && \
-                        ./tools/cabana/tests/test_cabana && \
-                        ./system/camerad/test/ae_gray_test && \
-                        coverage xml"
+                        ./selfdrive/ui/tests/test_translations.py"
     - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v3
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   process_replay:
     name: process replay
-    runs-on: ubuntu-20.04
-    timeout-minutes: 25
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
+      with:
+        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
     - name: Cache test routes
       id: dependency-cache
       uses: actions/cache@v3
       with:
-        path: /tmp/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit') }}
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
     - name: Run replay
+      timeout-minutes: 30
       run: |
-        ${{ env.RUN }} "CI=1 coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
+                        chmod -R 777 /tmp/comma_download_cache && \
+                        coverage combine && \
                         coverage xml"
     - name: Print diff
+      id: print-diff
       if: always()
       run: cat selfdrive/test/process_replay/diff.txt
-    - uses: actions/upload-artifact@v2
+    - uses: actions/upload-artifact@v3
       if: always()
       continue-on-error: true
       with:
         name: process_replay_diff.txt
         path: selfdrive/test/process_replay/diff.txt
     - name: Upload reference logs
-      if: ${{ failure() && github.event_name == 'pull_request' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
+      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
       run: |
-        ${{ env.RUN }} "CI=1 AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
     - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v3
+      with:
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  regen:
+    name: regen
+    runs-on: 'ubuntu-20.04'
+    steps: 
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Cache test routes
+      id: dependency-cache
+      uses: actions/cache@v3
+      with:
+        path: .ci_cache/comma_download_cache
+        key: regen-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/test_regen.py') }}
+    - name: Build base Docker image
+      run: eval "$BUILD"
+    - name: Build openpilot
+      run: |
+        ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Run regen
+      timeout-minutes: 30
+      run: |
+        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
+                           chmod -R 777 /tmp/comma_download_cache"
 
   test_modeld:
     name: model tests
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
-    - name: Build Docker image
-      # Sim docker is needed to get the OpenCL drivers
-      run: eval "$BUILD_CL"
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build base Docker image
+      run: eval "$BUILD"
     - name: Build openpilot
       run: |
         ${{ env.RUN }} "scons -j$(nproc)"
+    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
     - name: Run model replay with ONNX
-      timeout-minutes: 2
+      timeout-minutes: 4
       run: |
-        ${{ env.RUN_CL }} "ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+        ${{ env.RUN }} "unset PYTHONWARNINGS && \
+                           ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
+                           coverage combine && \
                            coverage xml"
     - name: Run unit tests
-      timeout-minutes: 5
+      timeout-minutes: 4
       run: |
-        ${{ env.RUN_CL }} "$UNIT_TEST selfdrive/modeld && \
-                           coverage xml"
+        ${{ env.RUN }} "unset PYTHONWARNINGS && \
+                           $PYTEST selfdrive/modeld"
     - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v2
-
-  test_longitudinal:
-    name: longitudinal
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
-    steps:
-    - uses: actions/checkout@v3
+      uses: codecov/codecov-action@v3
       with:
-        submodules: true
-    - uses: ./.github/workflows/setup
-    - name: Build openpilot
-      run: |
-        ${{ env.RUN }} "scons -j$(nproc)"
-    - name: Test longitudinal
-      run: |
-        ${{ env.RUN }} "mkdir -p selfdrive/test/out && \
-                        cd selfdrive/test/longitudinal_maneuvers && \
-                        coverage run ./test_longitudinal.py && \
-                        coverage xml"
-      timeout-minutes: 2
-    - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v2
-    - uses: actions/upload-artifact@v2
-      if: always()
-      continue-on-error: true
-      with:
-        name: longitudinal
-        path: selfdrive/test/longitudinal_maneuvers/out/longitudinal/
+        name: ${{ github.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   test_cars:
     name: cars
-    runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
+                   ((github.event_name != 'pull_request') || 
+                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-20.04' }}
     strategy:
       fail-fast: false
       matrix:
         job: [0, 1, 2, 3, 4]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - uses: ./.github/workflows/setup
+    - uses: ./.github/workflows/setup-with-retry
     - name: Cache test routes
       id: dependency-cache
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      uses: ./.github/workflows/auto-cache
       with:
-        path: /tmp/comma_download_cache
+        path: .ci_cache/comma_download_cache
         key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
     - name: Build openpilot
       run: ${{ env.RUN }} "scons -j$(nproc)"
     - name: Test car models
-      timeout-minutes: 12
+      timeout-minutes: 10
       run: |
-        ${{ env.RUN }} "coverage run -m pytest selfdrive/car/tests/test_models.py && \
-                        coverage xml && \
+        ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                         chmod -R 777 /tmp/comma_download_cache"
       env:
         NUM_JOBS: 5
         JOB_ID: ${{ matrix.job }}
     - name: "Upload coverage to Codecov"
-      uses: codecov/codecov-action@v2
+      uses: codecov/codecov-action@v3
+      with:
+        name: ${{ github.job }}-${{ matrix.job }}
+      env:
+        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
 
   car_docs_diff:
     name: PR comments
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
     if: github.event_name == 'pull_request'
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
           ref: ${{ github.event.pull_request.base.ref }}
-      - uses: ./.github/workflows/setup
+      - run: git lfs pull
+      - uses: ./.github/workflows/setup-with-retry
       - name: Get base car info
         run: |
           ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_info.py --path /tmp/openpilot_cache/base_car_info"
           sudo chown -R $USER:$USER ${{ github.workspace }}
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: true
+          path: current
+      - run: cd current && git lfs pull
       - name: Save car docs diff
         id: save_diff
         run: |
+          cd current
           ${{ env.RUN }} "scons -j$(nproc)"
-          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info") || true
+          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_info")
           output="${output//$'\n'/'%0A'}"
           echo "::set-output name=diff::$output"
       - name: Find comment
@@ -415,3 +380,25 @@ jobs:
               repo: context.repo.repo,
               comment_id: ${{ steps.fc.outputs.comment-id }}
             })
+
+  create_ui_report:
+    name: Create UI Report
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          submodules: true
+      - uses: ./.github/workflows/setup-with-retry
+      - name: Build openpilot
+        run: ${{ env.RUN }} "scons -j$(nproc)"
+      - name: Create Test Report
+        run: >
+            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
+                            source selfdrive/test/setup_xvfb.sh &&
+                            export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && 
+                            python selfdrive/ui/tests/test_ui/run.py"
+      - name: Upload Test Report
+        uses: actions/upload-artifact@v2
+        with:
+          name: report
+          path: selfdrive/ui/tests/test_ui/report

.github/workflows/setup-pre-commit/action.yaml

@@ -0,0 +1,12 @@
+name: 'set up pre-commit environment'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: ./.github/workflows/auto-cache
+      with: 
+        path: .ci_cache/pre-commit
+        key: pre-commit-${{ hashFiles('**/.pre-commit-config.yaml') }}
+        restore-keys: |
+          pre-commit-
+        save: ${{ github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' }}

.github/workflows/setup-with-retry/action.yaml

@@ -0,0 +1,40 @@
+name: 'openpilot env setup, with retry on failure'
+
+inputs:
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: false
+    default: ''
+  sleep_time:
+    description: 'Time to sleep between retries'
+    required: false
+    default: 30
+
+runs:
+  using: "composite"
+  steps:
+    - id: setup1
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true
+    - if: steps.setup1.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup2
+      if: steps.setup1.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      continue-on-error: true
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true
+    - if: steps.setup2.outcome == 'failure'
+      shell: bash
+      run: sleep ${{ inputs.sleep_time }}
+    - id: setup3
+      if: steps.setup2.outcome == 'failure'
+      uses: ./.github/workflows/setup
+      with:
+        docker_hub_pat: ${{ inputs.docker_hub_pat }}
+        is_retried: true

.github/workflows/setup/action.yaml

@@ -1,37 +1,63 @@
 name: 'openpilot env setup'
 
 inputs:
-  save-cache:
-    default: false
+  docker_hub_pat:
+    description: 'Auth token for Docker Hub, required for BuildJet jobs'
+    required: true
+    default: ''
+  is_retried:
+    description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
     required: false
+    default: 'false'
 
 runs:
   using: "composite"
   steps:
+    # assert that this action is retried using the setup-with-retry
+    - shell: bash
+      if: ${{ inputs.is_retried == 'false' }}
+      run: |
+        echo "You should not run this action directly. Use setup-with-retry instead"
+        exit 1
+
     # do this after checkout to ensure our custom LFS config is used to pull from GitLab
     - shell: bash
       run: git lfs pull
 
+    # on BuildJet runners, must be logged into DockerHub to avoid rate limiting
+    # https://buildjet.com/for-github-actions/docs/guides/docker
+    - shell: bash
+      if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }}
+      run: |
+        echo "Need to set the Docker Hub PAT secret as an input to this action"
+        exit 1
+    - name: Login to Docker Hub
+      if: contains(runner.name, 'buildjet')
+      shell: bash
+      run: |
+        docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }}
+
     # build cache
     - id: date
       shell: bash
       run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
     - shell: bash
       run: echo "$CACHE_COMMIT_DATE"
-    - shell: bash
-      run: echo "CACHE_SKIP_SAVE=true" >> $GITHUB_ENV
-      if: github.ref != 'refs/heads/master' || inputs.save-cache == 'false'
     - id: scons-cache
-      # TODO: change the version to the released version
-      # when https://github.com/actions/cache/pull/489 (or 571) is merged.
-      uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
+      uses: ./.github/workflows/auto-cache
       with:
-        path: /tmp/scons_cache
-        key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
+        path: .ci_cache/scons_cache
+        key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
         restore-keys: |
-          scons-${{ env.CACHE_COMMIT_DATE }}-
-          scons-
-
+          scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
+          scons-${{ runner.arch }}
+    # as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
+    - id: normalize-file-permissions
+      shell: bash
+      name: Normalize file permissions to ensure a consistent docker build cache
+      run: |
+        find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
+        find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
     # build our docker image
     - shell: bash
-      run: eval ${{ env.BUILD }}
+      run: eval ${{ env.BUILD }}

.github/workflows/stale.yaml

@@ -0,0 +1,28 @@
+name: stale
+on:
+  schedule:
+    - cron: '30 1 * * *'
+  workflow_dispatch:
+
+env:
+  DAYS_BEFORE_PR_CLOSE: 7
+  DAYS_BEFORE_PR_STALE: 30
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/stale@v8
+        with:
+          exempt-milestones: true
+
+          # pull request config
+          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
+          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
+          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
+          exempt-pr-labels: "ignore stale,needs testing,car port" # if wip or it needs testing from the community, don't mark as stale
+          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
+          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
+
+          # issue config
+          days-before-issue-stale: -1 # ignore issues for now

.github/workflows/tools_tests.yaml

@@ -2,8 +2,8 @@ name: tools
 
 on:
   push:
-    branches-ignore:
-      - 'testing-closet*'
+    branches:
+      - master
   pull_request:
 
 concurrency:
@@ -12,80 +12,91 @@ concurrency:
 
 env:
   BASE_IMAGE: openpilot-base
-  CL_BASE_IMAGE: openpilot-base-cl
-  DOCKER_REGISTRY: ghcr.io/commaai
   DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
 
-  BUILD: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $DOCKER_REGISTRY/$BASE_IMAGE:latest -t $BASE_IMAGE:latest -f Dockerfile.openpilot_base .
+  BUILD: selfdrive/test/docker_build.sh base
 
-  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c
-
-  BUILD_CL: |
-      DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $DOCKER_REGISTRY/$CL_BASE_IMAGE:latest -t $CL_BASE_IMAGE:latest -f Dockerfile.openpilot_base_cl .
-  RUN_CL: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v /tmp/scons_cache:/tmp/scons_cache -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/openpilot_cache:/tmp/openpilot_cache $CL_BASE_IMAGE /bin/sh -c
+  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
 
 
 jobs:
   plotjuggler:
     name: plotjuggler
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 45
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Build Docker image
-      run: eval "$BUILD"
-    - name: Unit test
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      timeout-minutes: 5
+      run: ${{ env.RUN }} "scons -j$(nproc) cereal/ common/ --minimal"
+    - name: Test PlotJuggler
       timeout-minutes: 2
       run: |
-        ${{ env.RUN }} "scons -j$(nproc) --directory=/tmp/openpilot/cereal && \
-                        apt-get update && \
-                        apt-get install -y libdw-dev libqt5svg5-dev libqt5x11extras5-dev && \
-                        cd /tmp/openpilot/tools/plotjuggler && \
-                        ./test_plotjuggler.py"
+        ${{ env.RUN }} "pytest tools/plotjuggler/"
 
   simulator:
     name: simulator
     runs-on: ubuntu-20.04
-    timeout-minutes: 30
-    env:
-      IMAGE_NAME: openpilot-sim
     if: github.repository == 'commaai/openpilot'
+    timeout-minutes: 45
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Pull LFS
-      run: git lfs pull
-    - name: Build base image
-      run: eval "$BUILD"
-    - name: Build base cl image
-      run: eval "$BUILD_CL"
-    - name: Build simulator image
-      run: DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/$IMAGE_NAME:latest -t $DOCKER_REGISTRY/$IMAGE_NAME:latest -f tools/sim/Dockerfile.sim .
-    - name: Push to container registry
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Setup to push to repo
       if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
       run: |
+        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
         $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/$IMAGE_NAME:latest
+    - name: Build and push sim image
+      run: |
+        selfdrive/test/docker_build.sh sim
 
-  docs:
-    name: build docs
-    runs-on: ubuntu-20.04
-    timeout-minutes: 25
+  devcontainer:
+    name: devcontainer
+    runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
       with:
         submodules: true
-    - name: Build docker container
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Use local image for testing devcontainer with latest base image
       run: |
-        DOCKER_BUILDKIT=1 docker build --pull --build-arg BUILDKIT_INLINE_CACHE=1 --cache-from $DOCKER_REGISTRY/openpilot-docs:latest -t $DOCKER_REGISTRY/openpilot-docs:latest -f docs/docker/Dockerfile .
-    - name: Push docker container
-      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
+        echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
+    - name: Setup Dev Container CLI
+      run: npm install -g @devcontainers/cli
+    - name: Build dev container image
+      run: devcontainer build --workspace-folder .
+    - name: Run dev container
       run: |
-        $DOCKER_LOGIN
-        docker push $DOCKER_REGISTRY/openpilot-docs:latest
-
+        mkdir -p /tmp/devcontainer_scons_cache/
+        cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
+        devcontainer up --workspace-folder .
+    - name: Test environment
+      run: |
+        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
+        devcontainer exec --workspace-folder . pip install pip-install-test
+        devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
+        devcontainer exec --workspace-folder . sudo touch /root/test.txt
 
+  notebooks:
+    name: notebooks
+    runs-on: ubuntu-20.04
+    if: github.repository == 'commaai/openpilot'
+    timeout-minutes: 45
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        submodules: true
+    - uses: ./.github/workflows/setup-with-retry
+    - name: Build openpilot
+      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
+      run: ${{ env.RUN }} "scons -j$(nproc)"
+    - name: Test notebooks
+      timeout-minutes: 3
+      run: |
+        ${{ env.RUN }} "pip install nbmake && pytest --nbmake tools/car_porting/examples/"

.gitignore

@@ -1,4 +1,6 @@
 venv/
+.venv/
+.ci_cache
 .env
 .clang-format
 .DS_Store
@@ -8,7 +10,6 @@ venv/
 .overlay_init
 .overlay_consistent
 .sconsign.dblite
-.vscode*
 model2.png
 a.out
 .hypothesis
@@ -44,11 +45,10 @@ selfdrive/boardd/boardd
 selfdrive/logcatd/logcatd
 selfdrive/mapd/default_speeds_by_region.json
 system/proclogd/proclogd
-selfdrive/ui/_ui
+selfdrive/ui/translations/alerts_generated.h
+selfdrive/ui/translations/tmp
 selfdrive/test/longitudinal_maneuvers/out
-selfdrive/visiond/visiond
-selfdrive/sensord/_gpsd
-selfdrive/sensord/_sensord
+selfdrive/car/tests/cars_dump
 system/camerad/camerad
 system/camerad/test/ae_gray_test
 selfdrive/modeld/_modeld
@@ -57,7 +57,6 @@ selfdrive/modeld/_dmonitoringmodeld
 /src/
 
 one
-openpilot
 notebooks
 xx
 yy
@@ -78,6 +77,7 @@ comma*.sh
 
 selfdrive/modeld/thneed/compile
 selfdrive/modeld/models/*.thneed
+selfdrive/modeld/models/*.pkl
 
 *.bz2
 
@@ -86,3 +86,4 @@ build/
 !**/.gitkeep
 
 poetry.toml
+Pipfile

.gitmodules

@@ -1,21 +1,21 @@
 [submodule "panda"]
   path = panda
-  url = ../../commaai/panda.git
+  url = ../../sunnyhaibin/panda.git
 [submodule "opendbc"]
   path = opendbc
-  url = ../../commaai/opendbc.git
-[submodule "laika_repo"]
-  path = laika_repo
-  url = ../../commaai/laika.git
+  url = ../../sunnyhaibin/opendbc.git
 [submodule "cereal"]
   path = cereal
-  url = ../../commaai/cereal.git
+  url = ../../sunnyhaibin/cereal.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = ../../commaai/rednose.git
 [submodule "body"]
   path = body
   url = ../../commaai/body.git
+[submodule "teleoprtc_repo"]
+  path = teleoprtc_repo
+  url = ../../commaai/teleoprtc
 [submodule "tinygrad"]
   path = tinygrad_repo
   url = https://github.com/geohot/tinygrad.git

.pre-commit-config.yaml

@@ -1,29 +1,42 @@
+exclude: '^(tinygrad_repo)'
 repos:
 -   repo: meta
     hooks:
     -   id: check-hooks-apply
     -   id: check-useless-excludes
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.1.0
+    rev: v4.5.0
     hooks:
     -   id: check-ast
         exclude: '^(third_party)/'
     -   id: check-json
+        exclude: '.devcontainer/devcontainer.json|.vscode/'  # these support JSON with comments
+    -   id: check-toml
     -   id: check-xml
     -   id: check-yaml
     -   id: check-merge-conflict
     -   id: check-symlinks
+    -   id: check-executables-have-shebangs
+    -   id: check-shebang-scripts-are-executable
     -   id: check-added-large-files
-        args: ['--maxkb=100']
+        exclude: '(docs/CARS.md)|(poetry.lock)|(third_party/acados/include/blasfeo/include/blasfeo_d_kernel.h)'
+        args:
+        - --maxkb=120
+        - --enforce-all
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.1
+    rev: v2.2.6
     hooks:
     -   id: codespell
-        exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)'
+        exclude: '^(third_party/)|(body/)|(cereal/)|(panda/)|(opendbc/)|(rednose/)|(rednose_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)'
         args:
         # if you've got a short variable name that's getting flagged, add it here
-        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup
+        - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie,preints
         - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US
+-   repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.2.2
+    hooks:
+    -   id: ruff
+        exclude: '^(third_party/)|(cereal/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)'
 -   repo: local
     hooks:
     -   id: mypy
@@ -31,32 +44,10 @@ repos:
         entry: mypy
         language: system
         types: [python]
-        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)'
--   repo: https://github.com/PyCQA/flake8
-    rev: 4.0.1
-    hooks:
-    -   id: flake8
-        exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(selfdrive/debug/)/'
-        additional_dependencies: ['flake8-no-implicit-concat']
         args:
-        - --indent-size=2
-        - --enable-extensions=NIC
-        - --select=F,E112,E113,E304,E502,E701,E702,E703,E71,E72,E731,W191,W6
-        - --statistics
-        - -j4
--   repo: local
-    hooks:
-    -   id: pylint
-        name: pylint
-        entry: pylint
-        language: system
-        types: [python]
-        exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)'
-        args:
-        - -j0
-        - -rn
-        - -sn
-        - --rcfile=.pylintrc
+        - --local-partial-types
+        - --explicit-package-bases
+        exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(teleoprtc/)|(teleoprtc_repo/)|(xx/)'
 -   repo: local
     hooks:
     -   id: cppcheck
@@ -71,6 +62,26 @@ repos:
         - --quiet
         - --force
         - -j8
+-   repo: https://github.com/cpplint/cpplint
+    rev: 1.6.1
+    hooks:
+    -   id: cpplint
+        exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)'
+        args:
+        - --quiet
+        - --counting=total
+        - --linelength=240
+        # https://google.github.io/styleguide/cppguide.html
+        # relevant rules are whitelisted, see all options with: cpplint --filter=
+        - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces
+-   repo: https://github.com/MarcoGorelli/cython-lint
+    rev: v0.16.0
+    hooks:
+    -   id: cython-lint
+        exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)'
+        args:
+        - --max-line-length=240
+        - --ignore=E111, E302, E305
 -   repo: local
     hooks:
     -   id: test_translations
@@ -78,7 +89,15 @@ repos:
         entry: selfdrive/ui/tests/test_translations.py
         language: script
         pass_filenames: false
+        files: 'selfdrive/ui/translations/*'
 -   repo: https://github.com/python-poetry/poetry
-    rev: '1.2.2'
+    rev: '1.8.0'
     hooks:
     -   id: poetry-check
+        name: validate poetry lock
+        args:
+        - --lock
+-   repo: https://github.com/python-jsonschema/check-jsonschema
+    rev: 0.28.0
+    hooks:
+    -   id: check-github-workflows

.pylintrc

@@ -1,469 +0,0 @@
-[MASTER]
-
-# A comma-separated list of package or module names from where C extensions may
-# be loaded. Extensions are loading into the active Python interpreter and may
-# run arbitrary code
-extension-pkg-whitelist=scipy,cereal.messaging.messaging_pyx,PyQt5,av
-
-# Add files or directories to the blacklist. They should be base names, not
-# paths.
-ignore=CVS
-
-# Add files or directories matching the regex patterns to the blacklist. The
-# regex matches against base names, not paths.
-ignore-patterns=
-
-# Python code to execute, usually for sys.path manipulation such as
-# pygtk.require().
-#init-hook=
-
-# Use multiple processes to speed up Pylint.
-jobs=4
-
-# List of plugins (as comma separated values of python modules names) to load,
-# usually to register additional checkers.
-load-plugins=
-
-# Pickle collected data for later comparisons.
-persistent=yes
-
-# Specify a configuration file.
-#rcfile=
-
-# When enabled, pylint would attempt to guess common misconfiguration and emit
-# user-friendly hints instead of false-positive error messages
-suggestion-mode=yes
-
-# Allow loading of arbitrary C extensions. Extensions are imported into the
-# active Python interpreter and may run arbitrary code.
-unsafe-load-any-extension=no
-
-
-[MESSAGES CONTROL]
-
-# Only show warnings with the listed confidence levels. Leave empty to show
-# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED
-confidence=
-
-# Disable the message, report, category or checker with the given id(s). You
-# can either give multiple identifiers separated by comma (,) or put this
-# option multiple times (only on the command line, not in the configuration
-# file where it should appear only once).You can also use "--disable=all" to
-# disable everything first and then reenable specific checks. For example, if
-# you want to run only the similarities checker, you can use "--disable=all
-# --enable=similarities". If you want to run only the classes checker, but have
-# no Warning level messages displayed, use"--disable=all --enable=classes
-# --disable=W"
-disable=C,R,W0613,W0511,W0212,W0201,W0106,W0603,W0621,W0703,W1201,W1203,E1136,W1514
-
-
-# Enable the message, report, category or checker with the given id(s). You can
-# either give multiple identifier separated by comma (,) or put this option
-# multiple time (only on the command line, not in the configuration file where
-# it should appear only once). See also the "--disable" option for examples.
-enable=c-extension-no-member
-
-
-[REPORTS]
-
-# Python expression which should return a note less than 10 (10 is the highest
-# note). You have access to the variables errors warning, statement which
-# respectively contain the number of errors / warnings messages and the total
-# number of statements analyzed. This is used by the global evaluation report
-# (RP0004).
-evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)
-
-# Template used to display messages. This is a python new-style format string
-# used to format the message information. See doc for all details
-#msg-template=
-
-# Set the output format. Available formats are text, parseable, colorized, json
-# and msvs (visual studio).You can also give a reporter class, eg
-# mypackage.mymodule.MyReporterClass.
-output-format=text
-
-# Tells whether to display a full report or only the messages
-reports=no
-
-# Activate the evaluation score.
-score=yes
-
-
-[REFACTORING]
-
-# Maximum number of nested blocks for function / method body
-max-nested-blocks=5
-
-# Complete name of functions that never returns. When checking for
-# inconsistent-return-statements if a never returning function is called then
-# it will be considered as an explicit return statement and no message will be
-# printed.
-never-returning-functions=optparse.Values,sys.exit
-
-
-[LOGGING]
-
-# Logging modules to check that the string format arguments are in logging
-# function parameter format
-logging-modules=logging
-
-
-[SPELLING]
-
-# Limits count of emitted suggestions for spelling mistakes
-max-spelling-suggestions=4
-
-# Spelling dictionary name. Available dictionaries: none. To make it working
-# install python-enchant package.
-spelling-dict=
-
-# List of comma separated words that should not be checked.
-spelling-ignore-words=
-
-# A path to a file that contains private dictionary; one word per line.
-spelling-private-dict-file=
-
-# Tells whether to store unknown words to indicated private dictionary in
-# --spelling-private-dict-file option instead of raising a message.
-spelling-store-unknown-words=no
-
-
-[MISCELLANEOUS]
-
-# List of note tags to take in consideration, separated by a comma.
-notes=FIXME,
-      XXX,
-      TODO
-
-
-[SIMILARITIES]
-
-# Ignore comments when computing similarities.
-ignore-comments=yes
-
-# Ignore docstrings when computing similarities.
-ignore-docstrings=yes
-
-# Ignore imports when computing similarities.
-ignore-imports=no
-
-# Minimum lines number of a similarity.
-min-similarity-lines=4
-
-
-[TYPECHECK]
-
-# List of decorators that produce context managers, such as
-# contextlib.contextmanager. Add to this list to register other decorators that
-# produce valid context managers.
-contextmanager-decorators=contextlib.contextmanager
-
-# List of members which are set dynamically and missed by pylint inference
-# system, and so shouldn't trigger E1101 when accessed. Python regular
-# expressions are accepted.
-generated-members=capnp.* cereal.* pygame.* zmq.* setproctitle.* smbus2.* usb1.* serial.* cv2.* ft4222.* carla.*
-
-# Tells whether missing members accessed in mixin class should be ignored. A
-# mixin class is detected if its name ends with "mixin" (case insensitive).
-ignore-mixin-members=yes
-
-# This flag controls whether pylint should warn about no-member and similar
-# checks whenever an opaque object is returned when inferring. The inference
-# can return multiple potential results while evaluating a Python object, but
-# some branches might not be evaluated, which results in partial inference. In
-# that case, it might be useful to still emit no-member and other checks for
-# the rest of the inferred objects.
-ignore-on-opaque-inference=yes
-
-# List of class names for which member attributes should not be checked (useful
-# for classes with dynamically set attributes). This supports the use of
-# qualified names.
-ignored-classes=optparse.Values,thread._local,_thread._local
-
-# List of module names for which member attributes should not be checked
-# (useful for modules/projects where namespaces are manipulated during runtime
-# and thus existing member attributes cannot be deduced by static analysis. It
-# supports qualified module names, as well as Unix pattern matching.
-ignored-modules=flask setproctitle usb1 flask.ext.socketio smbus2 usb1.*
-
-# Show a hint with possible names when a member name was not found. The aspect
-# of finding the hint is based on edit distance.
-missing-member-hint=yes
-
-# The minimum edit distance a name should have in order to be considered a
-# similar match for a missing member name.
-missing-member-hint-distance=1
-
-# The total number of similar names that should be taken in consideration when
-# showing a hint for a missing member.
-missing-member-max-choices=1
-
-
-[VARIABLES]
-
-# List of additional names supposed to be defined in builtins. Remember that
-# you should avoid to define new builtins when possible.
-additional-builtins=
-
-# Tells whether unused global variables should be treated as a violation.
-allow-global-unused-variables=yes
-
-# List of strings which can identify a callback function by name. A callback
-# name must start or end with one of those strings.
-callbacks=cb_,
-          _cb
-
-# A regular expression matching the name of dummy variables (i.e. expectedly
-# not used).
-dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_
-
-# Argument names that match this expression will be ignored. Default to name
-# with leading underscore
-ignored-argument-names=_.*|^ignored_|^unused_
-
-# Tells whether we should check for unused import in __init__ files.
-init-import=no
-
-# List of qualified module names which can have objects that can redefine
-# builtins.
-redefining-builtins-modules=six.moves,past.builtins,future.builtins
-
-
-[FORMAT]
-
-# Expected format of line ending, e.g. empty (any line ending), LF or CRLF.
-expected-line-ending-format=
-
-# Regexp for a line that is allowed to be longer than the limit.
-ignore-long-lines=^\s*(# )?<?https?://\S+>?$
-
-# Number of spaces of indent required inside a hanging  or continued line.
-indent-after-paren=4
-
-# String used as indentation unit. This is usually "    " (4 spaces) or "\t" (1
-# tab).
-indent-string='  '
-
-# Maximum number of characters on a single line.
-max-line-length=100
-
-# Maximum number of lines in a module
-max-module-lines=1000
-
-# Allow the body of a class to be on the same line as the declaration if body
-# contains single statement.
-single-line-class-stmt=no
-
-# Allow the body of an if to be on the same line as the test if there is no
-# else.
-single-line-if-stmt=no
-
-
-[BASIC]
-
-# Naming style matching correct argument names
-argument-naming-style=snake_case
-
-# Regular expression matching correct argument names. Overrides argument-
-# naming-style
-#argument-rgx=
-
-# Naming style matching correct attribute names
-attr-naming-style=snake_case
-
-# Regular expression matching correct attribute names. Overrides attr-naming-
-# style
-#attr-rgx=
-
-# Bad variable names which should always be refused, separated by a comma
-bad-names=foo,
-          bar,
-          baz,
-          toto,
-          tutu,
-          tata
-
-# Naming style matching correct class attribute names
-class-attribute-naming-style=any
-
-# Regular expression matching correct class attribute names. Overrides class-
-# attribute-naming-style
-#class-attribute-rgx=
-
-# Naming style matching correct class names
-class-naming-style=PascalCase
-
-# Regular expression matching correct class names. Overrides class-naming-style
-#class-rgx=
-
-# Naming style matching correct constant names
-const-naming-style=UPPER_CASE
-
-# Regular expression matching correct constant names. Overrides const-naming-
-# style
-#const-rgx=
-
-# Minimum line length for functions/classes that require docstrings, shorter
-# ones are exempt.
-docstring-min-length=-1
-
-# Naming style matching correct function names
-function-naming-style=snake_case
-
-# Regular expression matching correct function names. Overrides function-
-# naming-style
-#function-rgx=
-
-# Good variable names which should always be accepted, separated by a comma
-good-names=i,
-           j,
-           k,
-           ex,
-           Run,
-           _
-
-# Include a hint for the correct naming format with invalid-name
-include-naming-hint=no
-
-# Naming style matching correct inline iteration names
-inlinevar-naming-style=any
-
-# Regular expression matching correct inline iteration names. Overrides
-# inlinevar-naming-style
-#inlinevar-rgx=
-
-# Naming style matching correct method names
-method-naming-style=snake_case
-
-# Regular expression matching correct method names. Overrides method-naming-
-# style
-#method-rgx=
-
-# Naming style matching correct module names
-module-naming-style=snake_case
-
-# Regular expression matching correct module names. Overrides module-naming-
-# style
-#module-rgx=
-
-# Colon-delimited sets of names that determine each other's naming style when
-# the name regexes allow several styles.
-name-group=
-
-# Regular expression which should only match function or class names that do
-# not require a docstring.
-no-docstring-rgx=^_
-
-# List of decorators that produce properties, such as abc.abstractproperty. Add
-# to this list to register other decorators that produce valid properties.
-property-classes=abc.abstractproperty
-
-# Naming style matching correct variable names
-variable-naming-style=snake_case
-
-# Regular expression matching correct variable names. Overrides variable-
-# naming-style
-#variable-rgx=
-
-
-[DESIGN]
-
-# Maximum number of arguments for function / method
-max-args=5
-
-# Maximum number of attributes for a class (see R0902).
-max-attributes=7
-
-# Maximum number of boolean expressions in a if statement
-max-bool-expr=5
-
-# Maximum number of branch for function / method body
-max-branches=12
-
-# Maximum number of locals for function / method body
-max-locals=15
-
-# Maximum number of parents for a class (see R0901).
-max-parents=7
-
-# Maximum number of public methods for a class (see R0904).
-max-public-methods=20
-
-# Maximum number of return / yield for function / method body
-max-returns=6
-
-# Maximum number of statements in function / method body
-max-statements=50
-
-# Minimum number of public methods for a class (see R0903).
-min-public-methods=2
-
-
-[CLASSES]
-
-# List of method names used to declare (i.e. assign) instance attributes.
-defining-attr-methods=__init__,
-                      __new__,
-                      setUp
-
-# List of member names, which should be excluded from the protected access
-# warning.
-exclude-protected=_asdict,
-                  _fields,
-                  _replace,
-                  _source,
-                  _make
-
-# List of valid names for the first argument in a class method.
-valid-classmethod-first-arg=cls
-
-# List of valid names for the first argument in a metaclass class method.
-valid-metaclass-classmethod-first-arg=mcs
-
-
-[IMPORTS]
-
-# Allow wildcard imports from modules that define __all__.
-allow-wildcard-with-all=no
-
-# Analyse import fallback blocks. This can be used to support both Python 2 and
-# 3 compatible code, which means that the block might have code that exists
-# only in one or another interpreter, leading to false positives when analysed.
-analyse-fallback-blocks=no
-
-# Deprecated modules which should not be used, separated by a comma
-deprecated-modules=regsub,
-                   TERMIOS,
-                   Bastion,
-                   rexec
-
-# Create a graph of external dependencies in the given file (report RP0402 must
-# not be disabled)
-ext-import-graph=
-
-# Create a graph of every (i.e. internal and external) dependencies in the
-# given file (report RP0402 must not be disabled)
-import-graph=
-
-# Create a graph of internal dependencies in the given file (report RP0402 must
-# not be disabled)
-int-import-graph=
-
-# Force import order to recognize a module as part of the standard
-# compatibility libraries.
-known-standard-library=
-
-# Force import order to recognize a module as part of a third party library.
-known-third-party=enchant
-
-[STRING]
-
-# This flag controls whether the implicit-str-concat should generate a warning
-# on implicit string concatenation in sequences defined over several lines.
-check-str-concat-over-line-jumps=yes
-
-[EXCEPTIONS]
-
-# Exceptions that will emit a warning when being caught. Defaults to
-# "Exception"
-overgeneral-exceptions=Exception

.python-version

@@ -1 +1 @@
-3.8.10
+3.11.4

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+  "recommendations": [
+    "ms-python.python",
+    "ms-vscode.cpptools",
+    "elagil.pre-commit-helper",
+    "charliermarsh.ruff",
+  ]
+}

.vscode/launch.json

@@ -0,0 +1,47 @@
+{
+  "version": "0.2.0",
+  "inputs": [
+    {
+      "id": "python_process",
+      "type": "pickString",
+      "description": "Select the process to debug",
+      "options": [
+        "selfdrive/controls/controlsd.py",
+        "selfdrive/navd/navd.py",
+        "system/timed/timed.py",
+        "tools/sim/run_bridge.py"
+      ]
+    },
+    {
+      "id": "cpp_process",
+      "type": "pickString",
+      "description": "Select the process to debug",
+      "options": [
+        "selfdrive/ui/ui"
+      ]
+    },
+    {
+      "id": "args",
+      "description": "Arguments to pass to the process",
+      "type": "promptString"
+    }
+  ],
+  "configurations": [
+    {
+      "name": "Python: openpilot Process",
+      "type": "debugpy",
+      "request": "launch",
+      "program": "${input:python_process}",
+      "console": "integratedTerminal",
+      "justMyCode": true,
+      "args": "${input:args}"
+    },
+    {
+      "name": "C++: openpilot Process",
+      "type": "cppdbg",
+      "request": "launch",
+      "program": "${workspaceFolder}/${input:cpp_process}",
+      "cwd": "${workspaceFolder}",
+    }
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,16 @@
+{
+  "editor.tabSize": 2,
+  "editor.insertSpaces": true,
+  "editor.renderWhitespace": "trailing",
+  "files.trimTrailingWhitespace": true,
+  "search.exclude": {
+    "**/.git": true,
+    "**/.venv": true,
+    "**/__pycache__": true
+  },
+  "files.exclude": {
+    "**/.git": true,
+    "**/.venv": true,
+    "**/__pycache__": true
+  }
+}

CHANGELOGS.md

@@ -0,0 +1,855 @@
+sunnypilot - 0.9.7.0 (2024-xx-xx)
+========================
+* New driving model
+* Support for many hybrid Ford models
+************************
+* UPDATED: Synced with commaai's openpilot
+  * master commit 56e343b (February 27, 2024)
+* NEW❗: iOS Siri Shortcuts Navigation support thanks to twilsonco and mike86437!
+  * iOS and macOS Shortcuts to quickly set navigation destinations from your iOS device
+  * comma Prime support
+  * Personal Mapbox/Amap/Google Maps token support
+  * Instructions on how to set up your iOS Siri Shortcuts: https://routinehub.co/shortcut/17677/
+
+sunnypilot - 0.9.6.1 (2024-02-27)
+========================
+* New driving model
+  * Vision model trained on more data
+  * Improved driving performance
+  * Directly outputs curvature for lateral control
+* New driver monitoring model
+  * Trained on larger dataset
+* AGNOS 9
+* comma body streaming and controls over WebRTC
+* Improved fuzzy fingerprinting for many makes and models
+* Alpha longitudinal support for new Toyota models
+* Chevrolet Equinox 2019-22 support thanks to JasonJShuler and nworb-cire!
+* Dodge Durango 2020-21 support
+* Hyundai Staria 2023 support thanks to sunnyhaibin!
+* Kia Niro Plug-in Hybrid 2022 support thanks to sunnyhaibin!
+* Lexus LC 2024 support thanks to nelsonjchen!
+* Toyota RAV4 2023-24 support
+* Toyota RAV4 Hybrid 2023-24 support
+************************
+* UPDATED: Synced with commaai's openpilot
+  * master commit db57a21 (February 22, 2024)
+  * v0.9.6 release (February 27, 2024)
+* UPDATED: Dynamic Experimental Control (DEC)
+  * Synced with dragonpilot-community/dragonpilot:beta3 commit f4ee52f
+* NEW❗: Default Driving Model: Certified Herbalist v2 (February 13, 2024)
+* UPDATED: Driving Model Selector v3
+  * NEW❗: Driving Model additions
+    * Certified Herbalist v2 (February 13, 2024) - CHv2
+    * Certified Herbalist (February 5, 2024) - CH
+    * Los Angeles v2 (January 24, 2024) - LAv2
+    * Los Angeles (January 22, 2024) - LAv1
+  * NEW❗: Model Caching thanks to DevTekVE!
+    * Model caching allows the selection of previously downloaded Driving Model
+    * Users can now access cached versions of selected models, eliminating redundant downloads for previously fetched models
+  * Legacy Driving Models support
+    * New Delhi (December 21, 2023) - ND
+    * Blue Diamond v2 (December 11, 2023) - BDv2
+    * Blue Diamond (November 18, 2023) - BDv1
+    * Farmville (November 7, 2023) - FV
+    * Night Strike (October 3, 2023) - NS
+  * Certain features are deprecated with newer Driving Models
+    * Dynamic Lane Profile (DLP)
+    * Custom Offsets
+* UPDATED: Dynamic Lane Profile (DLP)
+  * Continued support for Legacy Driving Models (e.g., ND, BDv2, BDv1, FV, NS)
+  * Deprecated support for newer Driving Models (e.g., CHv2, CH, LAv2, LAv1)
+* UPDATED: Custom Offsets
+  * Continued support for Legacy Driving Models (e.g., ND, BDv2, BDv1, FV, NS)
+  * Deprecated support for newer Driving Models (e.g., CHv2, CH, LAv2, LAv1)
+* UPDATED: Hyundai/Kia/Genesis - ESCC Radar Interceptor
+  * Message parsing improvements with the latest firmware update: https://github.com/sunnypilot/panda/tree/test-escc-smdps
+* UI Updates
+  * NEW❗: Visuals: Display Feature Status toggle
+    * Display the statuses of certain features on the driving screen
+  * NEW❗: Visuals: Enable Onroad Settings toggle
+    * Display the Onroad Settings button on the driving screen to adjust feature options on the driving screen, without navigating into the settings menu
+  * REMOVED: "Device ambient" temperature option on the sidebar
+* FIXED: New comma 3X support
+* FIXED: New comma eSIM support
+* Bug fixes and performance improvements
+
+sunnypilot - 0.9.5.3 (2023-12-24)
+========================
+* UPDATED: Dynamic Experimental Control (DEC)
+  * Synced with dragonpilot-community/dragonpilot:lp-dp-beta2 commit 578d38b
+* UPDATED: Driving Model Selector v2
+  * Driving models sort in descending order based on availability date
+  * Experimental/unmerged driving models are only available in "dev-c3" branch
+    * To select and use experimental driving models, navigate to "Software" panel, select the "dev-c3" branch, and check for update
+* UPDATED: Vision-based Turn Speed Control (V-TSC) implementation
+  * Refactored implementation thanks to pfeiferj!
+  * More accurate and consistent velocity calculation to achieve smoother longitudinal control in curves
+* NEW❗: Speed Limit Warning
+  * Display alert and/or chime to warn the driver when the cruising speed is faster than the speed limit plus the Warning Offset
+  * Customizable Warning Offset, independent of Speed Limit Control (SLC)'s Limit Offset
+* UPDATED: Speed Limit Source Policy
+  * Selectable speed limit source for Speed Limit Control and Speed Limit Warning
+  * Applicable to: Speed Limit Control, Speed Limit Warning
+* UPDATED: Speed Limit Control (SLC)
+  * Engage Mode: Removed "Warning Only" mode - this has been replaced by the new Speed Limit Warning sub-menu
+* UPDATED: OpenStreetMap (OSM) implementation
+  * Refactored implementation thanks to pfeiferj!
+    * Less resource impact
+    * Significantly smaller sizes with databases
+    * All regions are available to download
+    * Weekly map updates thanks to pfeiferj!
+    * Increased the font size of the road name
+  * C3X-specific changes
+    * Altitude (ALT.) display on Developer UI
+    * Current street name on top of driving screen when "OSM Debug UI" is enabled
+* UPDATED: Map-based Turn Speed Control (M-TSC) implementation
+  * Only available in "staging-c3" and "dev-c3" branches. If you are using "release-c3" branch, navigate to "Software" panel, select the desired target branch, and check for update
+  * Refactored implementation thanks to pfeiferj!
+  * Based on the new OpenStreetMap implementation
+  * Improved predicted curvature calculations from OpenStreetMap data
+* UI updates
+  * RE-ENABLED: Navigation: Full screen support
+    * Display the map view in full screen
+    * To switch back to driving view, tap on the border edge
+* Hyundai Bayon Non-SCC 2019 support thanks to polein78!
+
+sunnypilot - 0.9.5.2 (2023-12-07)
+========================
+* NEW❗: MADS: Allow Navigate on openpilot in Chill Mode
+  * Allow navigation to feed map view into the driving model while using Chill Mode
+  * Support all platforms, including platforms that do not support openpilot longitudinal control & Experimental Mode
+* NEW❗: Neural Network Lateral Controller
+  * Formerly known as "NNFF", this replaces the lateral "torque" controller with one using a neural network trained on each car's (actually, each separate EPS firmware) driving data for increased controls accuracy
+  * Contact @twilsonco in the sunnypilot Discord server with feedback, or to provide log data for your car if your car is currently unsupported
+* NEW❗: Driving Model Selector
+  * Easily switch between driving models without reinstalling branches. Offering immediate access to the latest models upon release
+    * An internet connection is required for downloading models. Each model switch currently involves downloading the model again. Future updates may allow for offline switching
+  * Warning is displayed for metered connections to avoid unexpected data usage if on cellular data
+  * Change driving models via **Settings -> Software -> Current Driving Model**.
+* NEW❗: Hyundai CAN longitudinal:
+  * NEW❗: Enable radar tracks for certain Santa Fe platforms
+    * Internal Combustion Engine (ICE) 2021-23
+    * Hybrid 2022-23
+    * Plug-in Hybrid 2022-23
+* NEW❗: Lane Change: When manually braking with steering engaged, turning on the turn signal will default to Nudge mode
+* Volkswagen MQB CC only platforms (radar or no radar) support thanks to jyoung8607!
+
+sunnypilot - 0.9.5.1 (2023-11-17)
+========================
+* UPDATED: Synced with commaai's master commit e94c3c5
+* NEW❗: Farmville driving model
+* NEW❗: Onroad Settings Panel
+  * Onroad buttons (i.e., DLP, GAC) moved to its dedicated panel
+    * Driving Personality
+    * Dynamic Lane Profile (DLP)
+    * Dynamic Experimental Control (DEC)
+    * Speed Limit Control (SLC)
+* NEW❗: Display main feature status on onroad view in real-time
+  * GAP - Driving Personality
+  * DLP - Dynamic Lane Profile
+  * DEC - Dynamic Experimental Control
+  * SLC - Speed Limit Control
+* NEW❗: Dynamic Experimental Control (DEC) thanks to dragonpilot-community!
+  * Automatically determines and selects between openpilot ACC and openpilot End to End longitudinal based on conditions for a more natural drive
+  * Dynamic Experimental Control is only active while in Experimental Mode
+  * When Dynamic Experimental Control is ON, initially setting cruise speed will set to the vehicle's current speed
+* NEW❗: Hyundai CAN longitudinal:
+  * NEW❗: Parse lead info for camera-based SCC platforms
+    * Improve lead tracking when using openpilot longitudinal
+  * NEW❗: Parse lead distance to display on car cluster
+    * Introduced better lead distance calculation to display on the car's cluster, replacing the binary "lead visible" indication on the SCC cluster
+    * Lead distance is now categorized into different ranges for more detailed and comprehensive information to the driver similar to how stock ACC does it
+  * NEW❗: Parse speed limit sign recognition from camera for certain supported platforms
+* NEW❗: Subaru - Stop and Go auto-resume support thanks to martinl!
+  * Global (excluding Gen 2 and Hybrid) and Pre-Global support
+* NEW❗: Toyota - Stop and Go hack
+  * Allow some Toyota/Lexus cars to auto resume during stop and go traffic
+  * Only applicable to certain models and model years
+* NEW❗: Toyota: ZSS support thanks to dragonpilot-community and ErichMoraga!
+* NEW❗: MSPA (Cereal structs refactor)
+  * Make sunnypilot Parsable Again - @sshane
+  * sunnypilot is now parsable with stock openpilot tools
+* NEW❗: Display 3D buildings on map thanks to jakethesnake420!
+* openpilot Longitudianl Control capable cars only
+  * UPDATED: Gap Adjust Cruise is now a part of Driving Personality
+    * [DISTANCE/FOLLOW DISTANCE/GAP DISTANCE] physical button on the steering wheel to select Driving Personality on by default
+    * Status now viewable in onroad view or Onroad Settings Panel
+    * REMOVED: Gap Adjust Cruise toggle
+* UPDATED: Speed Limit Control (SLC)
+  * NEW❗: Speed Limit Engage Mode
+    * Select the desired mode to set the cruising speed to the speed limit
+      * Warning Only: Warn the driver when the vehicle is driven faster than the speed limit
+      * Auto: Automatic speed adjustment on motorways based on speed limit data
+      * User Confirm: Inform the driver to change set speed of Adaptive Cruise Control to help the driver stay within the speed limit
+    * Supported platforms
+      * openpilot Longitudinal Control available cars (Excluding certain Toyota/Lexus, Ford, explained below)
+      * Custom Stock Longitudinal Control available cars
+    * Unsupported platforms
+      * Toyota/Lexus and Ford - most platforms do not allow us to control the PCM's set speed, requires testers to verify
+  * NEW❗: Speed limit source selector
+    * Select the desired precedence order of sources used to adapt cruise speed to road limits
+* UPDATED: Custom Stock Longitudinal Control
+  * RE-ENABLED: Hyundai/Kia/Genesis CAN-FD platforms
+* UPDATED: Custom Offsets reimplementation
+  * Camera Offset only works in Laneful (Laneful Only or Laneful in Auto mode when using Dynamic Lane Profile)
+  * Path Offset can be applied to both Laneless and Laneful
+* UPDATED: Refactored Torque Lateral Control custom tuning menu
+  * NEW❗: Less Restrict Settings for Self-Tune (Beta)
+  * NEW❗: Custom Tuning for setting offline and live values in real-time
+* UPDATED: Auto-detect custom Mapbox token if a personal Mapbox token is provided
+  * REMOVED: "Enable Mapbox Navigation" toggle
+* UI updates
+  * New Settings menu redesign and improved interactions
+* FIXED: Retain hotspot/tethering state was not consistently saved
+* FIXED: Map stuck in "Map Loading" if comma Prime is active
+* FIXED: OpenStreetMap implementation on C3X devices
+  * M-TSC
+  * Altitude (ALT.) display on Developer UI
+  * Current street name on top of driving screen when "OSM Debug UI" is enabled
+* Hyundai Kona Non-SCC 2019 support thanks to Quex!
+* Kia Seltos Non-SCC 2023-24 support thanks to Moodkiller and jeroid_!
+
+sunnypilot - 0.9.4.1 (2023-08-11)
+========================
+* UPDATED: Synced with commaai's 0.9.4 release
+* NEW❗: Moonrise driving model
+* NEW❗: Ford upstream models support
+* UPDATED: Dynamic Lane Profile selector in the "SP - Controls" menu
+* REMOVED: Dynamic Lane Profile driving screen UI button
+* FIXED: Disallow torque lateral control for angle control platforms (e.g. Ford, Nissan, Tesla)
+  * Torque lateral control cannot be used by angle control platforms, and would cause a "Controls Unresponsive" error if Torque lateral control is enforced in settings
+* REMOVED: Speed Limit Style override
+* Honda Accord 2016-17 support thanks to mlocoteta!
+  * Serial Steering hardware required. For more information, see https://github.com/mlocoteta/serialSteeringHardware
+* mapd: utilize advisory speed limit in curves (#142) thanks to pfeiferj!
+
+sunnypilot - 0.9.3.1 (2023-07-09)
+========================
+* UPDATED: Synced with commaai's 0.9.3 release
+* NEW❗: Display Temperature on Sidebar toggle
+  * Display Ambient temperature, memory temperature, CPU core with the highest temperature, GPU temperature, or max of Memory/CPU/GPU on the sidebar
+  * Replace "Display CPU Temperature on Sidebar" toggle
+* NEW❗: Hot Coffee driving model
+* NEW❗: HKG CAN: Smoother Stopping Performance (Beta) toggle
+  * Smoother stopping behind a stopped car or desired stopping event.
+  * This is only applicable to HKG CAN platforms using openpilot longitudinal control
+* NEW❗: Toyota: TSS2 longitudinal: Custom Tuning
+  * Smoother longitudinal performance for Toyota/Lexus TSS2/LSS2 cars thanks to dragonpilot-community!
+* NEW❗: Enable Screen Recorder toggle
+  * Enable this will display a button on the onroad screen to toggle on or off real-time screen recording with UI elements.
+* IMPROVED: Dynamic Lane Profile: when using Laneline planner via Laneline Mode or Auto Mode, enforce Laneless planner while traveling below 10 MPH or 16 km/h
+* REMOVED: Display CPU Temperature on Sidebar
+
+sunnypilot - 0.9.2.3 (2023-06-18)
+========================
+* NEW❗: Auto Lane Change: Delay with Blind Spot
+  * Toggle to enable a delay timer for seamless lane changes when blind spot monitoring (BSM) detects an obstructing vehicle, ensuring safe maneuvering
+* NEW❗: Driving Screen Off: Wake with Non-Critical Events
+  * When Driving Screen Off Timer is not set to "Always On":
+    * Enabled: Wake the brightness of the screen to display all events
+    * Disabled: Wake the brightness of the screen to display critical events
+  * Currently, all non-nudge modes are default to continue lane change after 1 seconds of blind spot detection
+* NEW❗: Fleet Manager PIN Requirement toggle
+  * User can now enable or disable PIN requirement on the comma device before accessing Fleet Manager
+* NEW❗: Reset all sunnypilot settings toggle
+* NEW❗: Turn signals display on screen when blinker is used
+  * Green: Blinker is on
+  * Red: Blinker is on, car detected in the adjacent blind spot or road edge detected
+* IMPROVED: mapd: better exceptions handling when loading dependencies
+* UPDATED: Green Traffic Light Chime no longer displays an orange border when executed
+* FIXED: mapd: Road name flashing caused by desync with last GPS timestamp
+* FIXED: Ram HD (2500/3500): Ignore paramsd sanity check
+  * Live parameters have trouble with self-tuning on this platform with upstream openpilot 0.9.2
+* Hyundai: Longitudinal support for CAN-based Camera SCC cars thanks to Zack1010OP's Patreon sponsor!
+
+sunnypilot - 0.9.2.2 (2023-06-13)
+========================
+* NEW❗: Toyota: Allow M.A.D.S. toggling with LKAS Button (Beta)
+* IMPROVED: Ram: cruise button handling
+
+sunnypilot - 0.9.2.1 (2023-06-10)
+========================
+* UPDATED: Synced with commaai's 0.9.2 release
+* UPDATED: feature revamp with better stability
+* UPDATED:
+  * M.A.D.S.
+    * Path color becomes LIGHT ORANGE during Driver Steering Override
+  * Gap Adjust Cruise (now known as Driving Personality in upstream openpilot 0.9.3):
+    * Updated profiles and jerk changes
+    * Experimental Mode support
+    * Three settings: Stock, Aggressive, and Maniac
+    * Stock is recommended and the default
+    * In Aggressive/Maniac mode, lead follow distance is shorter and quicker gas/brake response
+  * Dynamic Lane Profile
+    * Display blue borders on both sides of the driving path when Laneline mode is being used in the planner
+    * Auto Mode optimization
+      * Permanent: Laneless during Auto Lane Change execution
+  * Mapd
+    * OpenStreetMap Database: new regions added
+  * Developer UI (Dev UI)
+    * REMOVED: 2-column design
+    * NEW❗: 1-column + 1-row design
+  * Custom Stock Longitudinal Control
+    * NEW❗: Chrysler/Jeep/Ram support
+    * NEW❗: Mazda support
+    * NEW❗: Volkswagen PQ support
+    * DISABLED: Hyundai/Kia/Genesis CAN-FD platforms
+* NEW❗: Switch between Chill (openpilot ACC) and Experimental (E2E longitudinal) with DISTANCE button on the steering wheel
+  * To switch between Chill and Experimental Mode: press and hold the DISTANCE button on the steering wheel for over 0.5 second
+  * All openpilot longitudinal capable cars support
+* NEW❗: Nicki Minaj driving model
+* NEW❗: Nissan and Mazda upstream models support
+* NEW❗: Pre-Global Subaru upstream models support
+* NEW❗: Display End-to-end Longitudinal Status (Beta)
+  * Display an icon that appears when the End-to-end model decides to start or stop
+* NEW❗: Green Traffic Light Chime (Beta)
+  * A chime will play when the traffic light you are waiting for turns green, and you have no vehicle in front of you.
+* NEW❗: Lead Vehicle Departure Alert
+  * Notify when the leading vehicle drives away
+* NEW❗: Speedometer: Display True Speed
+  * Display the true vehicle current speed from wheel speed sensors.
+* NEW❗: Speedometer: Hide from Onroad Screen
+* NEW❗: Auto-Hide UI Buttons
+  * Hide UI buttons on driving screen after a 30-second timeout. Tap on the screen at anytime to reveal the UI buttons
+  * Applicable to Dynamic Lane Profile (DLP) and Gap Adjust Cruise (GAC)
+* NEW❗: Display DM Camera in Reverse Gear
+  * Show Driver Monitoring camera while the car is in reverse gear
+* NEW❗: Block Lane Change: Road Edge Detection (Beta)
+  * Block lane change when road edge is detected on the stalk actuated side
+* NEW❗: Display CPU Temperature on Sidebar
+  * Display the CPU core with the highest temperature on the sidebar
+* NEW❗: Display current driving model in Software settings
+* NEW❗: HKG: smartMDPS automatic detection (installed with applicable firmware)
+* FIXED: Unintended siren/alarm from the comma device if the vehicle is turned off too quickly in PARK gear
+* FIXED: mapd: Exception handling for loading dependencies
+* Fleet Manager via Browser support thanks to actuallylemoncurd, AlexandreSato, ntegan1, and royjr!
+  * Access your dashcam footage, screen recordings, and error logs when the car is turned off
+  * Connect to the device via Wi-Fi, mobile hotspot, or tethering on the comma device, then navigate to http://ipAddress:5050 to access.
+* Honda Clarity 2018-22 support thanks to mcallbosco, vanillagorillaa and wirelessnet2!
+* Ram: Steer to 0/7 MPH support thanks to vincentw56!
+* Retain hotspot/tethering state across reboots thanks to rogerioaguas!
+
+sunnypilot - Version Latest (2023-02-22)
+========================
+* UPDATED: Synced with commaai's master branch - 2023.02.19-04:52:00:GMT - 0.9.2
+* Refactor sunnypilot features to be more stable
+
+sunnypilot - Version Latest (2022-12-16)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.12.16-06:31:00:GMT - 0.9.1
+* NEW❗: GM:
+    * NEW❗: Gap Adjust Cruise support - Chill, Normal, Aggressive
+    * NEW❗: Experimental Mode: Hold DISTANCE button on the steering wheel for 0.5 second to switch between Experimental Mode and Chill Mode
+* REMOVED❌: Toytoa: SnG Hack
+    * This method is not recommended and may cause some cars to not behave as expected
+    * SDSU is strongly recommended to enable SnG for Toyota vehicles without SnG from factory
+* commaai: radard: add missing accel data for vision-only leads (commaai/openpilot#26619) - pending PR
+    * VOACC performance is drastically improved when using Chill Mode
+* IMPROVED: M.A.D.S. events handling
+* IMPROVED: UI: screen recorder button change
+* IMPROVED: OpenStreetMap Offline Database optimization
+* FIXED: Toyota: vehicles' LKAS button no longer has a delay with toggling M.A.D.S.
+* FIXED: Toyota: brake pedal press at standstill causing Cruise Fault
+* FIXED: Volkswagen MQB: reduce Camera Malfunction occurrences (requires testing)
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-12-10)
+========================
+* IMPROVED: NEW❗ Developer UI design
+    * Second column metrics is now moved to the bottom of the screen
+        * ACC. = Acceleration
+        * L.S. = Lead Speed
+        * E.T. = EPS Torque
+        * B.D. = Bearing Degree
+        * FRI. = Friction
+        * L.A. = Lateral Acceleration
+        * ALT. = Altitude
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-12-07)
+========================
+* NEW❗: Screen Recorder support thanks to neokii and Kumar!
+* NEW❗: End-to-end longitudinal start/stop status icon
+    * Only appears when Experimental Mode is enabled
+* NEW❗: End-to-end longitudinal car chime when starting
+    * Hyundai/Kia/Genesis CAN platform, Honda/Acura Bosch/Nidec, Toyota/Lexus
+    * i.e. Traffic light turns green, stop sign ready to go, etc.
+    * Only appears when Experimental Mode is enabled AND longitudinal control is disengaged
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-12-05)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.12.04-22:46:00:GMT - 0.9.1
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-11-12)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.11.12-10:02:00:GMT - 0.8.17
+* FIXED: CAN Error for CAN HKG cars that do not have navigation from the factory
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-11-11)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.11.11-21:22:00:GMT - 0.8.17
+* commaai: AGNOS 6.2 (commaai/openpilot#26441)
+* NEW❗: Speed Limit Control - HKG - add speed limit from car's navigation head unit
+    * Compatible with certain models, trims, and model years
+* DISABLED: FCA: RAM HD - steer down to 0
+* FIXED: UI: End-to-end longitudinal button on driving screen synchronization
+* FIXED: Honda: Longitudinal status with set cruise speed now displays properly in the car's dashboard
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-11-08)
+========================
+* ADDED: New Zealand offline OpenStreetMap database
+
+sunnypilot - Version Latest (2022-11-04)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.11.05-01:44:00:GMT - 0.8.17
+* RE-ENABLED: Dynamic Lane Profile - preserves lanelines
+    * Can be found in "SP - Controls" menu
+* NEW❗: DLP: switch to laneless for current/future curves thanks to @twilsonco!
+    * Can be found in "SP - Controls" menu
+* NEW❗: UI: Road Camera Selector
+    * Enable this will display a button on the driving screen to select the driving camera
+    * Can be found in "SP - Visuals" menu
+* NEW❗: Controls: Camera & Path Custom Offsets
+    * Only applicable to laneline mode when using Dynamic Lane Profile
+* NEW❗: Buttons on driving screen are now sorted based on priority and availability
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-28)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.28-03:53:00:GMT - 0.8.17
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-26)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.26-06:20:00:GMT - 0.8.17
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-25)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.25-23:53:00:GMT - 0.8.17
+* Pre-Global Subaru support thanks to @martinl!
+* NEW❗: Speed Limit values turn red when current speed is higher than posted speed limit
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-23)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.22-23:15:00:GMT - 0.8.17
+* IMPROVED: Custom Stock Longitudinal Control - HKG - only allow engagement on user button press
+* IMPROVED: Custom Stock Longitudinal Control - Volkswagen MQB & PQ - more consistent set speed change
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-21)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.21-17:33:00:GMT - 0.8.17
+* IMPROVED: Custom Stock Longitudinal Control - Volkswagen MQB & PQ - more predictable button send logic
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-20)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.20-20:25:00:GMT - 0.8.17
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-19)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.19-08:31:00:GMT - 0.8.17
+* IMPROVED: Controls: Speed Limit Control - accelerator press only disengage if "Disengage on Accelerator Pedal" is enabled
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-18)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.18-04:44:00:GMT - 0.8.17
+* RE-ENABLED: Volkswagen MQB & PQ with Custom Stock Longitudinal Control
+* NEW❗: Steering Rate Cost Live Tune
+    * Enables live tune for Steering Rate Cost. Lower value allows steering wheel to move more freely at low speed
+    * Can be found in "SP - Controls" menu
+* FIXED: MADS: GM - include Regen Paddle logic thanks to @twilsonco!
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-17)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.17-23:54:00:GMT+1 - 0.8.17
+* ENABLED: "Custom Stock Longitudinal Control" toggle for CAN-FD cars
+* FIXED: HKG CAN-FD: Could not engage when openpilot longitudinal is enabled
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-13)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.13-19:43:00:GMT+1 - 0.8.17
+* ADDED: Live Tmux toggle
+    * Can be found in "SP - General" menu
+* IMPROVED: OpenStreetMap Database Update - only check for database update with explicit user decision
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-11)
+========================
+* ADDED: Hyundai openpilot longitudinal improvements - huge thanks to @aragon7777!
+* ADDED: Check for OpenStreetMap Database Update button
+* UPDATED: commaai: Low speed lateral control improvements (commaai:openpilot#26022, bbcd448) - pending PR
+* FIXED: MUTCD speed limit spacing adjusts dynamically when no subtext is shown (i.e., speed limit offset, distance to next speed limit)
+* FIXED: MADS: Intermittent CAN Error when engaging for Toyota Prius TSS-P
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-09)
+========================
+* ADDED: commaai: Low speed lateral control improvements (commaai:openpilot#26022, bca288bb) - pending PR
+* FIXED: MADS: Intermittent CAN Error when engaging for Toyota Prius TSS-P
+* IMPROVED: mapd: stop signs and other supported traffic_calming tags are now slowing/stopping as expected
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-08)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.08-12:07:00:GMT+1 - 0.8.17
+* FIXED: MADS: Intermittent CAN Error when engaging for Toyota Prius TSS-P
+* IMPROVED: mapd: Speed Humps are now set at 20 MPH or 32 km/h
+* IMPROVED: OpenStreetMap Offline Database download experience
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-10-07)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.10.07-08:16:00:GMT - 0.8.17
+* NEW❗: OpenStreetMap database can now be downloaded locally for offline use
+    * Now offering US South, US West, US Northeast, US Florida, Taiwan, and South Africa
+    * Databases updated - 2022.10.05-03:30:00:GMT
+* NEW❗: mapd: Stop Sign, Yield, Speed Bump, Speed Hump, Sharp Curve support - huge thanks to @move-fast and @dragonpilot-community!
+    * Go to https://openstreetmap.org and start mapping out your area!
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-09-30)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.30-22:43:00:GMT - 0.8.17
+* RE-ADDED: Torque Lateral Controller Live Tune Menu
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-09-23)
+========================
+* ADDED: Developer UI: latAccelFactorFiltered & frictionCoefficientFiltered values displays in green if Torque is using live params
+* Bug fixes and performance improvements
+
+sunnypilot - Version Latest (2022-09-22)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.19-22:19:00:GMT - 0.8.17
+* NEW❗: Toggle to explicitly enable Custom Stock Longitudinal Control
+    * Applicable cars only: Honda, Hyundai/Kia/Genesis
+    * Settings -> Toggles menu
+
+sunnypilot - Version Latest (2022-09-21)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.19-22:19:00:GMT - 0.8.17
+* ADDED: Toggle to enable Live Torque (self/auto tune) with Torque lateral controller
+    * To enable, first enable "Enforce Torque Lateral Controller" toggle
+* UPDATED: New metrics in Developer UI (when Live Torque is enabled)
+    * REMOVED: latAccelFactorRaw & frictionCoefficientRaw from torqued
+    * ADDED: latAccelFactorFiltered & frictionCoefficientFiltered from torqued
+* REMOVED: Temporary remove Torque Lateral Controller Live Tune Menu
+
+sunnypilot - Version Latest (2022-09-20)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.19-22:19:00:GMT - 0.8.17
+* ADDED: Toggle to enable Live Torque (self/auto tune) with Torque lateral controller
+    * To enable, first enable "Enforce Torque Lateral Controller" toggle
+* REMOVED: Temporary remove Torque Lateral Controller Live Tune Menu
+
+sunnypilot - Version Latest (2022-09-18)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.17-11:23:00:GMT - 0.8.17
+* ADDED: Kia Forte Non-SCC 2019 support for @askalice
+* FIXED: Torque Lateral Control Live Tune now syncs with commaai:openpilot#25822
+* FIXED: mapd dependencies no longer need to be re-downloaded after unknown reboots
+
+sunnypilot - Version Latest (2022-09-17)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.17-11:23:00:GMT - 0.8.17
+* NEW❗: Non SCC HKG support
+    * Custom Stock Longitudinal Control
+    * ❗No❗ openpilot longitudinal control
+* FIXED: Honda Bosch random low-value set speed changes
+
+sunnypilot - Version Latest (2022-09-16)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.16-20:23:00:GMT - 0.8.17
+
+sunnypilot - Version Latest (2022-09-15)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.16-02:00:00:GMT - 0.8.17
+* FIXED: Block additional auto lane change actions if blinker stays on after the first lane change
+* REVERTED: Some Toyota with LKAS button no longer requires double press to engage/disengage M.A.D.S.
+
+sunnypilot - Version Latest (2022-09-14)u
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.11-02:47:00:GMT - 0.8.17
+* NEW❗: GM models supported in Force Car Recognition (FCR)
+    * Under "SP - Vehicles"
+* NEW❗: Prompt to select car in "SP - Vehicles" if car unrecognized on startup
+* FIXED: Some Toyota with LKAS button no longer requires double press to engage/disengage M.A.D.S.
+* UPDATED: ESCC: Use radar tracks from radar if available
+
+sunnypilot - Version Latest (2022-09-13)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.11-02:47:00:GMT - 0.8.17
+* NEW❗: New metric in Developer UI
+    * Actual Lateral Acceleration (Roll Compensated)
+
+sunnypilot - Version Latest (2022-09-12)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.11-02:47:00:GMT - 0.8.17
+* FIXED: Honda Nidec models not gaining speed when longitudinal engaged
+
+sunnypilot - Version Latest (2022-09-11)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.11-02:47:00:GMT - 0.8.17
+* NEW❗: Hyundai Enhanced SCC now forwards FCW and AEB signals and commands from radar to car
+* RE-ENABLED: MADS Status Icon toggle
+
+sunnypilot - Version Latest (2022-09-10)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.11-02:47:00:GMT - 0.8.17
+* NEW❗: RAM improvement implementation thanks to realfast!
+* DISABLED: Chrysler/Jeep/Ram with Custom Stock Longitudinal Control
+* DISABLED: Volkswagen MQB & PQ with Custom Stock Longitudinal Control
+
+sunnypilot - Version Latest (2022-09-09)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.09-07:35:00:GMT - 0.8.17
+* NEW❗: MADS now supporting General Motors (GM)
+* ADDED: Custom Stock Longitudinal Control - Volkswagen
+    * MQB & PQ
+* ADDED: Reverse ACC Change
+    * ACC +/-: Short=5, Long=1
+* ADDED: Custom Stock Longitudinal Control
+    * Hyundai/Kia/Genesis
+    * Honda Bosch
+* ADDED: Hyundai: 2015-16 Genesis resume from standstill fix (commaai:openpilot#25579) - pending PR
+* Vision Turn Speed Control re-enabled
+* Disable Onroad Uploads toggle re-enabled
+
+sunnypilot - Version Latest (2022-09-08)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.08-04:05:00:GMT - 0.8.17
+* NEW❗: Block lane change initiation while brake is pressed
+
+sunnypilot - Version Latest (2022-09-07)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.08-04:05:00:GMT - 0.8.17
+* NEW❗: Display End-to-end longitudinal 🌮 on screen
+    * NEW❗: Hold DISTANCE button on the steering wheel for 1 second to switch between E2E Long and ACC mode
+    * Enable toggle on the driving screen to switch between modes with End-to-end longitudinal
+    * Only applicable to cars with openpilot longitudinal control
+* NEW❗: Block lane change initiation while brake is pressed
+* REMOVED: Dynamic Lane Profile - upstream laneless model is now on by default
+* REMOVED: hyundai: consistent start from stop (commaai:openpilot#25672) - pending PR
+
+sunnypilot - Version Latest (2022-09-06)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.06 - 0.8.17
+* NEW❗: Display useful metrics above the chevron that tracks the lead car
+    * Under "SP - Visuals" menu
+    * Only applicable to cars with openpilot longitudinal control
+* ADDED: hyundai: consistent start from stop (commaai:openpilot#25672) - pending PR
+* FIXED: Vienna speed limit interface now scales properly with the outer box
+* REMOVED: Hyundai long improvements (commaai:openpilot#25604) - closed PR
+
+sunnypilot - Version Latest (2022-09-05)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.03 - 0.8.17
+* NEW❗: Speed Limit Control (SLC) interface integrated with upstream
+* NEW❗: Speed limit from active navigation is now prioritized for Speed Limit Control
+* NEW❗: MUTCD (U.S.) or Vienna (E.U.) speed limit interfaces can now be selected under "SP - Controls"
+
+sunnypilot - Version Latest (2022-09-04)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.03 - 0.8.17
+* FIXED: Gap Adjust Cruise status now displays properly on screen
+* FIXED: mapd - missing index in list caused mapd to crash
+* REMOVED: Temporary removed Vision Turn Speed Control
+
+sunnypilot - Version Latest (2022-09-03)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.09.03 - 0.8.17
+* ADDED: New border colors for different operation engagements
+* ADDED: UI: Show barrier when car detected in blind spot
+    * Only applicable to cars that have BSM detection with openpilot
+* FIXED: Cruise Cancel button no longer display prompt if cruise not engaged
+* TWEAKED: Update changelogs on startup in Settings -> Software -> Version
+* REMOVED: Upload Raw Logs and Full Resolution Videos toggles
+
+sunnypilot - Version Latest (2022-08-31)
+========================
+* UPDATED: Synced with commaai's master branch - 2022.08.31 - 0.8.17
+* ADDED: New border colors for different operation engagements
+* ADDED: UI: Show barrier when car detected in blind spot
+    * Only applicable to cars that have BSM detection with openpilot
+* FIXED: Cruise Cancel button no longer display prompt if cruise not engaged
+* REMOVED: Upload Raw Logs and Full Resolution Videos toggles
+
+sunnypilot - Version 0.8.16 (2022-07-16)
+========================
+* Sync with commaai's master branches
+* NEW❗: Add toggle to pause lateral actuation below 30 MPH / 50 KM/H
+* IMPROVED: Better controls mismatch handling
+* IMPROVED: Less frequent Low Memory alert
+* IMPROVED: Only allow lateral control when in forward gears
+* IMPROVED: Better alerts handling on gear changes
+
+sunnypilot - Version 0.8.14-1.3 (2022-06-29)
+========================
+* Hyundai/Kia/Genesis
+    * NEW❗: MADS: Add GAP/Distance button on the steering wheel to engage/disengage
+        * To engage/disengage MADS: Hold the button for 0.5 second
+* NEW❗: Dynamic Lane Profile: Add toggle to enable "Laneless for Curves in Auto Lane"
+* HOTFIX🛠: Improve Torque lateral control and reduce ping pong for some Toyota cars
+    * Torque control: higher low speed gains and better steering angle deadzone logic
+* Developer UI: Remove Distance Traveled, replace with Memory Usage %
+    * This may have a potential to fix the Low Memory alert that may appear
+
+sunnypilot - Version 0.8.14-1 (2022-06-27)
+========================
+* HOTFIX🛠: Honda, Toyota, Volkswagen now initialized correctly with Torque Lateral Live Tune
+
+sunnypilot - Version 0.8.14-1 (2022-06-27)
+========================
+* NEW❗: Added toggle to enable updates for sunnypilot
+* HOTFIX🛠: Volkswagen car list now displays properly in Force Car Recognition menu
+* REVERTED: Honda - temporary removes CRUISE (MAIN) for MADS engagement
+    * LKAS button continues to be used for MADS engagement/disengagement
+
+sunnypilot - Version 0.8.14-1 (2022-06-26)
+========================
+Visit https://bit.ly/sunnyreadme for more details
+* sunnypilot 0.8.14 release - based on openpilot 0.8.14 devel
+* "0.8.14-prod-c3" branch only supports comma three
+    * If you have a comma two, EON, or other devices than a comma three, visit sunnyhaibin's discord server for more details: https://discord.gg/wRW3meAgtx
+* Mono-branch support
+    * Honda/Acura
+    * Hyundai/Kia/Genesis
+    * Toyota/Lexus
+    * Volkswagen MQB
+* Modified Assistive Driving Safety (MADS) Mode
+    * NEW❗: CRUISE (MAIN) now engages MADS for all supported car makes
+    * NEW❗: Added toggle to disable disengaging Automatic Lane Centering (ALC) on the brake pedal
+* Dynamic Lane Profile (DLP)
+* NEW❗: Gap Adjust Cruise (GAC)
+    * openpilot longitudinal cars can now adjust between the lead car's following distance gap via 3 modes:
+        * Steering Wheel (SW) | User Interface (UI) | Steering Wheel + User Interface (SW+UI)
+* NEW❗: Custom Camera & Path Offsets
+* NEW❗: Torque Lateral Control from openpilot 0.8.15 master (as of 2022-06-15)
+* NEW❗: Torque Lateral Control Live Tune Menu
+* NEW❗: Speed Limit Sign from openpilot 0.8.15 master (as of 2022-06-22)
+* NEW❗: Mapbox Speed Limit data will now be utilized in Speed Limit Control (SLC)
+    * Speed limit data will be utilized in the following availability:
+        * Mapbox (active navigation) -> OpenStreetMap -> Car Interface (Toyota's TSR)
+* Custom Stock Longitudinal Control
+    * NEW❗: Volkswagen MQB
+    * Honda
+    * Hyundai/Kia/Genesis
+* NEW❗: Mapbox navigation support for non-Prime users
+    * Visit sunnyhaibin's discord server for more details: https://discord.gg/wRW3meAgtx
+* Hyundai/Kia/Genesis
+    * NEW❗: Enhanced SCC (ESCC) Support
+        * Requires hardware modification. Visit sunnyhaibin's discord server for more details: https://discord.gg/wRW3meAgtx
+    * NEW❗: Smart MDPS (SMDPS) Support - Auto-detection
+        * Requires hardware modification and custom firmware for the SMDPS. Visit sunnyhaibin's discord server for more details: https://discord.gg/wRW3meAgtx
+* Toyota/Lexus
+    * NEW❗: Added toggle to enforce stock longitudinal control
+
+sunnypilot - Version 0.8.12-4
+========================
+* NEW❗: Custom Stock Longitudinal Control by setting the target speed via openpilot's "MAX" speed thanks to multikyd!
+    * Speed Limit Control
+    * Vision-based Turn Control
+    * Map-based Turn Control
+* NEW❗: HDA status integration with Custom Stock Longitudinal Control on applicable HKG cars only
+* NEW❗: Roll Compensation and SteerRatio fix from comma's 0.8.13
+* NEW❗: Dev UI to display different metrics on screen
+    * Click on the "MAX" box on the top left of the openpilot display to toggle different metrics display
+    * Lead car relative distance; Lead car relative speed; Actual steering degree; Desired steering degree; Engine RPM; Longitudinal acceleration; Lead car actual speed; EPS torque; Current altitude; Compass direction
+* NEW❗: Stand Still Timer to display time spent at a stop with M.A.D.S engaged (i.e., stop lights, stop signs, traffic congestions)
+* NEW❗: Current car speed text turns red when the car is braking
+* NEW❗: Export GPS tracks into GPX files and upload to OSM thanks to eFini!
+* NEW❗: Enable ACC and M.A.D.S with a single press of the RES+/SET- button
+* NEW❗: ACC +/-: Short=5, Long=1
+    * Change the ACC +/- buttons behavior with cruise speed change in openpilot
+    * Disabled (Stock):  Short=1, Long=5
+    * Enabled:  Short=5, Long=1
+* NEW❗: Speed Limit Value Offset (not %)*
+    * Set speed limit higher or lower than actual speed limit for a more personalized drive.
+    * *To use this feature, turn off "Enable Speed Limit % Offset"*
+* NEW❗: Dedicated icon to show the status of M.A.D.S.
+* NEW❗: No Offroad Fix for non-official devices that cannot shut down after the car is turned off
+* NEW❗: Stop N' Go Resume Alternative
+    * Offer alternative behavior to auto resume when stopped behind a lead car using stock SCC/ACC. This feature removes the repeating prompt chime when stopped and/or allows some cars to use auto resume (i.e., Genesis)
+* IMPROVED: Show the lead car icon in the car's dashboard when a lead car is detected by openpilot's camera vision
+* FIXED: MADS button unintentionally set MAX when using stock longitudinal control thanks to Spektor56!
+
+sunnypilot - Version 0.8.12-3
+========================
+* NEW❗: Bypass "System Malfunction" alert toggle
+    * Prevent openpilot from returning the "System Malfunction" alert that hinders the ability use openpilot
+* FIXED: Hyundai/Kia/Genesis Brake Hold Active now outputs the correct events on screen with M.A.D.S. engaged
+
+sunnypilot - Version 0.8.12-2
+========================
+* NEW❗: Disable M.A.D.S. toggle to disable the beloved M.A.D.S. feature
+    * Enable Stock openpilot engagement/disengagement
+* ADJUST: Initialize Driving Screen Off Brightness at 50%
+
+sunnypilot - Version 0.8.12-1
+========================
+* sunnypilot 0.8.12 release - based on openpilot 0.8.12 devel
+* Dedicated Hyundai/Kia/Genesis branch support
+* NEW❗: OpenStreetMap integration thanks to the Move Fast team!
+    * NEW❗: Vision-based Turn Control
+    * NEW❗: Map-Data-based Turn Control
+    * NEW❗: Speed Limit Control w/ optional Speed Limit Offset
+    * NEW❗: OpenStreetMap integration debug UI
+    * Only available to openpilot longitudinal enabled cars
+* NEW❗: Hands on Wheel Monitoring according to EU r079r4e regulation
+* NEW❗: Disable Onroad Uploads for data-limited Wi-Fi hotspots when using OpenStreetMap related features
+* NEW❗: Fast Boot (Prebuilt)
+* NEW❗: Auto Lane Change Timer
+* NEW❗: Screen Brightness Control (Global)
+* NEW❗: Driving Screen Off Timer
+* NEW❗: Driving Screen Off Brightness (%)
+* NEW❗: Max Time Offroad
+* Improved user feedback with M.A.D.S. operations thanks to Spektor56!
+    * Lane Path
+        * Green🟢 (Laneful), Red🔴 (Laneless): M.A.D.S. engaged
+        * White⚪: M.A.D.S. suspended or disengaged
+        * Black⚫: M.A.D.S. engaged, steering is being manually override by user
+    * Screen border now only illuminates Green when SCC/ACC is engaged
+
+sunnypilot - Version 0.8.10-1 (Unreleased)
+========================
+* sunnypilot 0.8.10 release - based on openpilot 0.8.10 `devel`
+* Add Toyota cars to Force Car Recognition
+
+sunnypilot - Version 0.8.9-4
+========================
+* Hyundai: Fix Ioniq Hybrid signals
+
+sunnypilot - Version 0.8.9-3
+========================
+* Update home screen brand and version structure
+
+sunnypilot - Version 0.8.9-2
+========================
+* Added additional Sonata Hybrid Firmware Versions
+* Features
+    * Modified Assistive Driving Safety (MADS) Mode
+    * Dynamic Lane Profile (DLP)
+    * Quiet Drive 🤫
+    * Force Car Recognition (FCR)
+    * PID Controller: add kd into the stock PID controller
+
+sunnypilot - Version 0.8.9-1
+========================
+* First changelog!
+* Features
+    * Modified Assistive Driving Safety (MADS) Mode
+    * Dynamic Lane Profile (DLP)
+    * Quiet Drive 🤫
+    * Force Car Recognition (FCR)
+    * PID Controller: add kd into the stock PID controller

Dockerfile.openpilot

@@ -2,7 +2,7 @@ FROM ghcr.io/commaai/openpilot-base:latest
 
 ENV PYTHONUNBUFFERED 1
 
-ENV OPENPILOT_PATH /home/batman/openpilot/
+ENV OPENPILOT_PATH /home/batman/openpilot
 ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
 
 RUN mkdir -p ${OPENPILOT_PATH}
@@ -10,11 +10,11 @@ WORKDIR ${OPENPILOT_PATH}
 
 COPY SConstruct ${OPENPILOT_PATH}
 
+COPY ./openpilot ${OPENPILOT_PATH}/openpilot
 COPY ./third_party ${OPENPILOT_PATH}/third_party
 COPY ./site_scons ${OPENPILOT_PATH}/site_scons
-COPY ./laika ${OPENPILOT_PATH}/laika
-COPY ./laika_repo ${OPENPILOT_PATH}/laika_repo
 COPY ./rednose ${OPENPILOT_PATH}/rednose
+COPY ./rednose_repo/site_scons ${OPENPILOT_PATH}/rednose_repo/site_scons
 COPY ./tools ${OPENPILOT_PATH}/tools
 COPY ./release ${OPENPILOT_PATH}/release
 COPY ./common ${OPENPILOT_PATH}/common
@@ -23,5 +23,6 @@ COPY ./cereal ${OPENPILOT_PATH}/cereal
 COPY ./panda ${OPENPILOT_PATH}/panda
 COPY ./selfdrive ${OPENPILOT_PATH}/selfdrive
 COPY ./system ${OPENPILOT_PATH}/system
+COPY ./body ${OPENPILOT_PATH}/body
 
 RUN scons --cache-readonly -j$(nproc)

Dockerfile.openpilot_base

@@ -4,7 +4,7 @@ ENV PYTHONUNBUFFERED 1
 
 ENV DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends sudo tzdata locales ssh && \
+    apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
     rm -rf /var/lib/apt/lists/*
 
 RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
@@ -12,22 +12,75 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-ENV POETRY_VIRTUALENVS_CREATE=false
-ENV PYENV_VERSION=3.8.10
-ENV PYENV_ROOT="/root/.pyenv"
-ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
-
-COPY pyproject.toml poetry.lock .python-version update_requirements.sh /tmp/
-COPY tools/ubuntu_setup.sh /tmp/tools/
+COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
 RUN cd /tmp && \
-    tools/ubuntu_setup.sh && \
+    tools/install_ubuntu_dependencies.sh && \
     rm -rf /var/lib/apt/lists/* && \
     rm -rf /tmp/* && \
-    rm -rf /root/.cache && \
-    pip uninstall -y poetry && \
     # remove unused architectures from gcc for panda
     cd /usr/lib/gcc/arm-none-eabi/9.2.1 && \
     rm -rf arm/ && \
     rm -rf thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
 
-RUN sudo git config --global --add safe.directory /tmp/openpilot
+# Add OpenCL
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    apt-utils \
+    alien \
+    unzip \
+    tar \
+    curl \
+    xz-utils \
+    dbus \
+    gcc-arm-none-eabi \
+    tmux \
+    vim \
+    lsb-core \
+    libx11-6 \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG INTEL_DRIVER=l_opencl_p_18.1.0.015.tgz
+ARG INTEL_DRIVER_URL=https://registrationcenter-download.intel.com/akdlm/irc_nas/vcp/15532
+RUN mkdir -p /tmp/opencl-driver-intel
+
+RUN cd /tmp/opencl-driver-intel && \
+    echo INTEL_DRIVER is $INTEL_DRIVER && \
+    curl -O $INTEL_DRIVER_URL/$INTEL_DRIVER && \
+    tar -xzf $INTEL_DRIVER && \
+    for i in $(basename $INTEL_DRIVER .tgz)/rpm/*.rpm; do alien --to-deb $i; done && \
+    dpkg -i *.deb && \
+    rm -rf $INTEL_DRIVER $(basename $INTEL_DRIVER .tgz) *.deb && \
+    mkdir -p /etc/OpenCL/vendors && \
+    echo /opt/intel/opencl_compilers_and_libraries_18.1.0.015/linux/compiler/lib/intel64_lin/libintelocl.so > /etc/OpenCL/vendors/intel.icd && \
+    cd / && \
+    rm -rf /tmp/opencl-driver-intel
+
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
+ENV QTWEBENGINE_DISABLE_SANDBOX 1
+
+RUN dbus-uuidgen > /etc/machine-id
+
+ARG USER=batman
+ARG USER_UID=1000
+RUN useradd -m -s /bin/bash -u $USER_UID $USER
+RUN usermod -aG sudo $USER
+RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
+USER $USER
+
+ENV POETRY_VIRTUALENVS_CREATE=false
+ENV PYENV_VERSION=3.11.4
+ENV PYENV_ROOT="/home/$USER/pyenv"
+ENV PATH="$PYENV_ROOT/bin:$PYENV_ROOT/shims:$PATH"
+
+COPY --chown=$USER pyproject.toml poetry.lock .python-version /tmp/
+COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
+
+RUN cd /tmp && \
+    tools/install_python_dependencies.sh && \
+    rm -rf /tmp/* && \
+    rm -rf /home/$USER/.cache && \
+    find /home/$USER/pyenv -type d -name ".git" | xargs rm -rf && \
+    rm -rf /home/$USER/pyenv/versions/3.11.4/lib/python3.11/test
+
+USER root
+RUN sudo git config --global --add safe.directory /tmp/openpilot

Dockerfile.openpilot_base_cl

@@ -1,37 +0,0 @@
-FROM ghcr.io/commaai/openpilot-base:latest
-
-RUN apt-get update && apt-get install -y --no-install-recommends\
-    apt-utils \
-    alien \
-    unzip \
-    tar \
-    curl \
-    xz-utils \
-    dbus \
-    gcc-arm-none-eabi \
-    tmux \
-    vim \
-    lsb-core \
-    libx11-6 \
-  && rm -rf /var/lib/apt/lists/*
-
-# Intel OpenCL driver
-ARG INTEL_DRIVER=l_opencl_p_18.1.0.015.tgz
-ARG INTEL_DRIVER_URL=https://registrationcenter-download.intel.com/akdlm/irc_nas/vcp/15532
-RUN mkdir -p /tmp/opencl-driver-intel
-WORKDIR /tmp/opencl-driver-intel
-RUN echo INTEL_DRIVER is $INTEL_DRIVER && \
-    curl -O $INTEL_DRIVER_URL/$INTEL_DRIVER && \
-    tar -xzf $INTEL_DRIVER && \
-    for i in $(basename $INTEL_DRIVER .tgz)/rpm/*.rpm; do alien --to-deb $i; done && \
-    dpkg -i *.deb && \
-    rm -rf $INTEL_DRIVER $(basename $INTEL_DRIVER .tgz) *.deb && \
-    mkdir -p /etc/OpenCL/vendors && \
-    echo /opt/intel/opencl_compilers_and_libraries_18.1.0.015/linux/compiler/lib/intel64_lin/libintelocl.so > /etc/OpenCL/vendors/intel.icd && \
-    rm -rf /tmp/opencl-driver-intel
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
-ENV QTWEBENGINE_DISABLE_SANDBOX 1
-
-RUN dbus-uuidgen > /etc/machine-id
-

Jenkinsfile

@@ -1,21 +1,62 @@
-def phone(String ip, String step_label, String cmd) {
+def retryWithDelay(int maxRetries, int delay, Closure body) {
+  for (int i = 0; i < maxRetries; i++) {
+    try {
+      return body()
+    } catch (Exception e) {
+      sleep(delay)
+    }
+  }
+  throw Exception("Failed after ${maxRetries} retries")
+}
+
+def device(String ip, String step_label, String cmd) {
   withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
     def ssh_cmd = """
-ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END'
+ssh -tt -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END'
 
 set -e
 
+shopt -s huponexit # kill all child processes when the shell exits
+
 export CI=1
+export PYTHONWARNINGS=error
+export LOGPRINT=debug
 export TEST_DIR=${env.TEST_DIR}
 export SOURCE_DIR=${env.SOURCE_DIR}
 export GIT_BRANCH=${env.GIT_BRANCH}
 export GIT_COMMIT=${env.GIT_COMMIT}
 export AZURE_TOKEN='${env.AZURE_TOKEN}'
 export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}'
+# only use 1 thread for tici tests since most require HIL
+export PYTEST_ADDOPTS="-n 0"
+
+
+export GIT_SSH_COMMAND="ssh -i /data/gitkey"
 
 source ~/.bash_profile
 if [ -f /TICI ]; then
   source /etc/profile
+
+  rm -rf /tmp/tmp*
+  rm -rf ~/.commacache
+  rm -rf /dev/shm/*
+
+  if ! systemctl is-active --quiet systemd-resolved; then
+    echo "restarting resolved"
+    sudo systemctl start systemd-resolved
+    sleep 3
+  fi
+
+  # restart aux USB
+  if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
+    echo "restarting aux usb"
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
+    sleep 0.5
+    echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
+  fi
+fi
+if [ -f /data/openpilot/launch_env.sh ]; then
+  source /data/openpilot/launch_env.sh
 fi
 
 ln -snf ${env.TEST_DIR} /data/pythonpath
@@ -30,172 +71,182 @@ END"""
   }
 }
 
-def phone_steps(String device_type, steps) {
-  lock(resource: "", label: device_type, inversePrecedence: true, variable: 'device_ip', quantity: 1) {
-    timeout(time: 20, unit: 'MINUTES') {
-      phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),)
-      steps.each { item ->
-        phone(device_ip, item[0], item[1])
+def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
+  stage(stageName) {
+    if (currentBuild.result != null) {
+        return
+    }
+
+    def extra = extra_env.collect { "export ${it}" }.join('\n');
+    def branch = env.BRANCH_NAME ?: 'master';
+
+    lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
+      docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
+        timeout(time: 20, unit: 'MINUTES') {
+          retry (3) {
+            device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
+          }
+          steps.each { item ->
+            device(device_ip, item[0], item[1])
+          }
+        }
       }
     }
   }
 }
 
-pipeline {
-  agent none
-  environment {
-    CI = "1"
-    TEST_DIR = "/data/openpilot"
-    SOURCE_DIR = "/data/openpilot_source/"
-    AZURE_TOKEN = credentials('azure_token')
-    MAPBOX_TOKEN = credentials('mapbox_token')
-  }
-  options {
-    timeout(time: 3, unit: 'HOURS')
-    disableConcurrentBuilds(abortPrevious: env.BRANCH_NAME != 'master')
-  }
-
-  stages {
-    stage('build release3') {
-      agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-      when {
-        branch 'devel-staging'
-      }
-      steps {
-        phone_steps("tici-needs-can", [
-          ["build release3-staging & dashcam3-staging", "PUSH=1 $SOURCE_DIR/release/build_release.sh"],
-        ])
-      }
+def pcStage(String stageName, Closure body) {
+  node {
+  stage(stageName) {
+    if (currentBuild.result != null) {
+        return
     }
 
-    stage('openpilot tests') {
-      when {
-        not {
-          anyOf {
-            branch 'master-ci'; branch 'devel'; branch 'devel-staging';
-            branch 'release3'; branch 'release3-staging'; branch 'dashcam3'; branch 'dashcam3-staging';
-            branch 'testing-closet*'; branch 'hotfix-*'
-          }
-        }
-      }
+    checkout scm
 
-      parallel {
+    def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'";
 
-        stage('simulator') {
-          agent {
-            dockerfile {
-              filename 'Dockerfile.sim_nvidia'
-              dir 'tools/sim'
-              args '--user=root'
+    def openpilot_base = retryWithDelay (3, 15) {
+      return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .")
+    }
+
+    lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) {
+      openpilot_base.inside(dockerArgs) {
+        timeout(time: 20, unit: 'MINUTES') {
+          try {
+            retryWithDelay (3, 15) {
+              sh "git config --global --add safe.directory '*'"
+              sh "git submodule update --init --recursive"
+              sh "git lfs pull"
             }
-          }
-          steps {
-            sh "git config --global --add safe.directory ${WORKSPACE}"
-            sh "git lfs pull"
-            lock(resource: "", label: "simulator", inversePrecedence: true, quantity: 1) {
-              sh "${WORKSPACE}/tools/sim/build_container.sh"
-              sh "DETACH=1 ${WORKSPACE}/tools/sim/start_carla.sh"
-              sh "${WORKSPACE}/tools/sim/start_openpilot_docker.sh"
-            }
-          }
-
-          post {
-            always {
-              sh "docker kill carla_sim || true"
-              sh "rm -rf ${WORKSPACE}/* || true"
+            body()
+          } finally {
+              sh "rm -rf ${env.WORKSPACE}/* || true"
               sh "rm -rf .* || true"
             }
           }
         }
-
-        stage('build') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          environment {
-            R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}"
-          }
-          steps {
-            phone_steps("tici-needs-can", [
-              ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR EXTRA_FILES='tools/' ./build_devel.sh"],
-              ["build openpilot", "cd selfdrive/manager && ./build.py"],
-              ["check dirty", "release/check-dirty.sh"],
-              ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"],
-              ["test car interfaces", "cd selfdrive/car/tests/ && ./test_car_interfaces.py"],
-            ])
-          }
-        }
-
-        stage('loopback-tests') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-loopback", [
-              ["build openpilot", "cd selfdrive/manager && ./build.py"],
-              ["test boardd loopback", "python selfdrive/boardd/tests/test_boardd_loopback.py"],
-            ])
-          }
-        }
-
-        stage('HW + Unit Tests') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-common", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["test power draw", "python system/hardware/tici/test_power_draw.py"],
-              ["test loggerd", "python selfdrive/loggerd/tests/test_loggerd.py"],
-              ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib python selfdrive/loggerd/tests/test_encoder.py"],
-              ["test pigeond", "python selfdrive/sensord/tests/test_pigeond.py"],
-              ["test manager", "python selfdrive/manager/test/test_manager.py"],
-            ])
-          }
-        }
-
-        stage('camerad-ar') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-ar0321", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["test camerad", "python system/camerad/test/test_camerad.py"],
-              ["test exposure", "python system/camerad/test/test_exposure.py"],
-            ])
-          }
-        }
-
-        stage('camerad-ox') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-ox03c10", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["test camerad", "python system/camerad/test/test_camerad.py"],
-              ["test exposure", "python system/camerad/test/test_exposure.py"],
-            ])
-          }
-        }
-
-        stage('sensord') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-lsmc", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["test sensord", "cd selfdrive/sensord/tests && python -m unittest test_sensord.py"],
-            ])
-            phone_steps("tici-bmx-lsm", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["test sensord", "cd selfdrive/sensord/tests && python -m unittest test_sensord.py"],
-            ])
-          }
-        }
-
-        stage('replay') {
-          agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } }
-          steps {
-            phone_steps("tici-common", [
-              ["build", "cd selfdrive/manager && ./build.py"],
-              ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"],
-            ])
-          }
-        }
-
       }
     }
-
   }
 }
+
+def setupCredentials() {
+  withCredentials([
+    string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
+    string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN')
+  ]) {
+    env.AZURE_TOKEN = "${AZURE_TOKEN}"
+    env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}"
+  }
+}
+
+
+node {
+  env.CI = "1"
+  env.PYTHONWARNINGS = "error"
+  env.TEST_DIR = "/data/openpilot"
+  env.SOURCE_DIR = "/data/openpilot_source/"
+  setupCredentials()
+
+  env.GIT_BRANCH = checkout(scm).GIT_BRANCH
+  env.GIT_COMMIT = checkout(scm).GIT_COMMIT
+
+  def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
+                         'testing-closet*', 'hotfix-*']
+  def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
+
+  if (env.BRANCH_NAME != 'master') {
+    properties([
+        disableConcurrentBuilds(abortPrevious: true)
+    ])
+  }
+
+  try {
+    if (env.BRANCH_NAME == 'devel-staging') {
+      deviceStage("build release3-staging", "tici-needs-can", [], [
+        ["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"],
+      ])
+    }
+
+    if (env.BRANCH_NAME == 'master-ci') {
+      deviceStage("build nightly", "tici-needs-can", [], [
+        ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"],
+      ])
+    }
+
+    if (!env.BRANCH_NAME.matches(excludeRegex)) {
+    parallel (
+      // tici tests
+      'onroad tests': {
+        deviceStage("onroad", "tici-needs-can", [], [
+          // TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
+          //["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
+          ["build openpilot", "cd selfdrive/manager && ./build.py"],
+          ["check dirty", "release/check-dirty.sh"],
+          ["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
+          ["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
+        ])
+      },
+      'HW + Unit Tests': {
+        deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
+          ["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"],
+          ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
+          ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"],
+          ["test manager", "pytest selfdrive/manager/test/test_manager.py"],
+        ])
+      },
+      'loopback': {
+        deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
+          ["build openpilot", "cd selfdrive/manager && ./build.py"],
+          ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
+        ])
+      },
+      'camerad': {
+        deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["test camerad", "pytest system/camerad/test/test_camerad.py"],
+          ["test exposure", "pytest system/camerad/test/test_exposure.py"],
+        ])
+        deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["test camerad", "pytest system/camerad/test/test_camerad.py"],
+          ["test exposure", "pytest system/camerad/test/test_exposure.py"],
+        ])
+      },
+      'sensord': {
+        deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["test sensord", "pytest system/sensord/tests/test_sensord.py"],
+        ])
+        deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["test sensord", "pytest system/sensord/tests/test_sensord.py"],
+        ])
+      },
+      'replay': {
+        deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
+          ["build", "cd selfdrive/manager && ./build.py"],
+          ["model replay", "selfdrive/test/process_replay/model_replay.py"],
+        ])
+      },
+      'tizi': {
+        deviceStage("tizi", "tizi", ["UNSAFE=1"], [
+          ["build openpilot", "cd selfdrive/manager && ./build.py"],
+          ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"],
+          ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"],
+          ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
+          ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
+          ["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
+        ])
+      },
+
+    )
+    }
+  } catch (Exception e) {
+    currentBuild.result = 'FAILED'
+    throw e
+  }
+}

README.md

@@ -1,145 +1,460 @@
-![](https://i.imgur.com/b0ZyIx5.jpg)
+![](https://user-images.githubusercontent.com/47793918/233812617-beab2e71-57b9-479e-8bff-c3931347ca40.png)
 
 Table of Contents
 =======================
 
-* [What is openpilot?](#what-is-openpilot)
-* [Running in a car](#running-on-a-dedicated-device-in-a-car)
-* [Running on PC](#running-on-pc)
-* [Community and Contributing](#community-and-contributing)
-* [User Data and comma Account](#user-data-and-comma-account)
-* [Safety and Testing](#safety-and-testing)
-* [Directory Structure](#directory-structure)
+* [Join our Discord](#-join-our-discord)
+* [What is sunnypilot?](#-what-is-sunnypilot)
+* [Running in a car](#-running-on-a-dedicated-device-in-a-car)
+* [Read Before Installing](#-read-before-installing)
+* [Prohibited Safety Modifications](#-prohibited-safety-modifications)
+* [Installation](#-installation)
+* [Highlight Features](#-highlight-features)
+* [Driving Enhancements](#-driving-enhancements)
+* [Branch Definitions](#-branch-definitions)
+* [Recommended Branches](#-recommended-branches)
+* [How-To's](#-How-Tos)
+* [Pull Requests](#-Pull-Requests)
+* [Special Thanks](#-special-thanks)
+* [User Data](#-user-data)
 * [Licensing](#licensing)
+* [Donate](#-support-sunnypilot)
 
 ---
 
-What is openpilot?
+<details><summary><h3>💭 Join our Discord</h3></summary>
+
+---
+
+Join the official sunnypilot Discord server to stay up to date with all the latest features and be a part of shaping the future of sunnypilot!
+* https://discord.gg/sunnypilot
+
+  ![](https://dcbadge.vercel.app/api/server/wRW3meAgtx?style=flat) ![Discord Shield](https://discordapp.com/api/guilds/880416502577266699/widget.png?style=shield)
+
+</details>
+
+<details><summary><h3>🌞 What is sunnypilot?</h3></summary>
+
+---
+
+[sunnypilot](https://github.com/sunnyhaibin/sunnypilot) is a fork of comma.ai's openpilot, an open source driver assistance system. sunnypilot offers the user a unique driving experience for over 250+ supported car makes and models with modified behaviors of driving assist engagements. sunnypilot complies with comma.ai's safety rules as accurately as possible.
+
+</details>
+
+<details><summary><h3>🚘 Running on a dedicated device in a car</h3></summary>
+
+---
+
+To use sunnypilot in a car, you need the following:
+* A supported device to run this software
+    * a [comma three](https://comma.ai/shop/products/three), or
+    * a comma two (only with older versions below 0.8.13)
+* This software
+* One of [the 250+ supported cars](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
+* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
+
+Detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
+
+</details>
+
+<details><summary><h3>🚨 Read Before Installing</h3></summary>
+
+---
+
+It is recommended to read this **entire page** before proceeding. This will ensure that you fully understand each added feature on sunnypilot, and you are selecting the right branch for your car to have the best driving experience.
+
+This is a fork of [comma.ai's openpilot](https://github.com/commaai/openpilot). By installing this software, you accept all responsibility for anything that might occur while you use it. All contributors to sunnypilot are not liable. ❗<ins>**Use at your own risk.**</ins>❗
+
+</details>
+
+<details><summary><h3>⛔ Prohibited Safety Modifications</h3></summary>
+
+---
+
+All [official sunnypilot branches](https://github.com/sunnyhaibin/sunnypilot/branches) strictly adhere to [comma.ai's safety policy](https://github.com/commaai/openpilot/blob/master/docs/SAFETY.md). Any changes that go against this policy will result in your fork and your device being banned from both comma.ai and sunnypilot channels.
+
+The following changes are a **VIOLATION** of this policy and **ARE NOT** included in any sunnypilot branches:
+* Driver Monitoring:
+    * ❌ "Nerfing" or reducing monitoring parameters.
+* Panda safety:
+    * ❌ No preventing disengaging of <ins>**LONGITUDINAL CONTROL**</ins> (acceleration/brake) on brake pedal press.
+    * ❌ No auto re-engaging of <ins>**LONGITUDINAL CONTROL**</ins> (acceleration/brake) on brake pedal release.
+    * ❌ No disengaging on ACC MAIN in OFF state.
+
+</details>
+
+
+<details><summary><h3>⚒ Installation</h3></summary>
+
+---
+
+  <details><summary>URL (Easy)</summary>
+
+comma three
 ------
 
-[openpilot](http://github.com/commaai/openpilot) is an open source driver assistance system. Currently, openpilot performs the functions of Adaptive Cruise Control (ACC), Automated Lane Centering (ALC), Forward Collision Warning (FCW), and Lane Departure Warning (LDW) for a growing variety of [supported car makes, models, and model years](docs/CARS.md). In addition, while openpilot is engaged, a camera-based Driver Monitoring (DM) feature alerts distracted and asleep drivers. See more about [the vehicle integration](docs/INTEGRATION.md) and [limitations](docs/LIMITATIONS.md).
+Please refer to [Recommended Branches](#-recommended-branches) to find your preferred/supported branch. This guide will assume you want to install the latest `release-c3` branch.
 
-<table>
-  <tr>
-    <td><a href="https://youtu.be/NmBfgOanCyk" title="Video By Greer Viau"><img src="https://i.imgur.com/1w8c6d2.jpg"></a></td>
-    <td><a href="https://youtu.be/VHKyqZ7t8Gw" title="Video By Logan LeGrand"><img src="https://i.imgur.com/LnBucik.jpg"></a></td>
-    <td><a href="https://youtu.be/VxiR4iyBruo" title="Video By Charlie Kim"><img src="https://i.imgur.com/4Qoy48c.jpg"></a></td>
-    <td><a href="https://youtu.be/-IkImTe1NYE" title="Video By Aragon"><img src="https://i.imgur.com/04VNzPf.jpg"></a></td>
-  </tr>
-  <tr>
-    <td><a href="https://youtu.be/iIUICQkdwFQ" title="Video By Logan LeGrand"><img src="https://i.imgur.com/b1LHQTy.jpg"></a></td>
-    <td><a href="https://youtu.be/XOsa0FsVIsg" title="Video By PinoyDrives"><img src="https://i.imgur.com/6FG0Bd8.jpg"></a></td>
-    <td><a href="https://youtu.be/bCwcJ98R_Xw" title="Video By JS"><img src="https://i.imgur.com/zO18CbW.jpg"></a></td>
-    <td><a href="https://youtu.be/BQ0tF3MTyyc" title="Video By Tsai-Fi"><img src="https://i.imgur.com/eZzelq3.jpg"></a></td>
-  </tr>
-</table>
+* sunnypilot not installed or you installed a version before 0.8.17?
+  1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+  2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+  3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```release-c3.sunnypilot.ai``` [^4] (note: `https://` is not requirement on the comma three)
+  4. Complete the rest of the installation following the onscreen instructions.
 
+* sunnypilot already installed and you installed a version after 0.8.17?
+  1. On the comma three, go to `Settings` ▶️ `Software`.
+  2. At the `Download` option, press `CHECK`. This will fetch the list of latest branches from sunnypilot.
+  3. At the `Target Branch` option, press `SELECT` to open the Target Branch selector.
+  4. Scroll to select the desired branch per [Recommended Branches](#-recommended-branches). Example: `release-c3`
 
-Running on a dedicated device in a car
+|    Branch    |         Installation URL         |
+|:------------:|:--------------------------------:|
+| `release-c3` | https://release-c3.sunnypilot.ai |
+| `staging-c3` | https://staging-c3.sunnypilot.ai |
+|   `dev-c3`   | https://dev-c3.sunnypilot.ai     |
+
+Requires further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
+
+comma two
 ------
 
-To use openpilot in a car, you need four things
-* A supported device to run this software: a [comma three](https://comma.ai/shop/products/three).
-* This software. The setup procedure of the comma three allows the user to enter a URL for custom software.
-The URL, openpilot.comma.ai will install the release version of openpilot. To install openpilot master, you can use installer.comma.ai/commaai/master, and replacing commaai with another GitHub username can install a fork.
-* One of [the 200+ supported cars](docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run openpilot.
-* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car.
+1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
+2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
+3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```https://smiskol.com/fork/sunnyhaibin/0.8.12-4-prod```
+4. Complete the rest of the installation following the onscreen instructions.
 
-We have detailed instructions for [how to mount the device in a car](https://comma.ai/setup).
+Requires further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
 
-Running on PC
+  </details>
+
+  <details>
+  <summary>SSH (More Versatile)</summary>
+  <br>
+
+Prerequisites: [How to SSH](https://github.com/commaai/openpilot/wiki/SSH)
+
+If you are looking to install sunnypilot via SSH, run the following command in an SSH terminal after connecting to your device:
+
+comma three:
 ------
+* [`release-c3`](https://github.com/sunnyhaibin/openpilot/tree/release-c3):
 
-All openpilot services can run as usual on a PC without requiring special hardware or a car. You can also run openpilot on recorded or simulated data to develop or experiment with openpilot.
+  ```
+  cd /data; rm -rf ./openpilot; git clone -b release-c3 --recurse-submodules https://github.com/sunnyhaibin/sunnypilot.git openpilot; cd openpilot; sudo reboot
+  ```
 
-With openpilot's tools, you can plot logs, replay drives, and watch the full-res camera streams. See [the tools README](tools/README.md) for more information.
-
-You can also run openpilot in simulation [with the CARLA simulator](tools/sim/README.md). This allows openpilot to drive around a virtual car on your Ubuntu machine. The whole setup should only take a few minutes but does require a decent GPU.
-
-A PC running openpilot can also control your vehicle if it is connected to a [webcam](https://github.com/commaai/openpilot/tree/master/tools/webcam), a [black panda](https://comma.ai/shop/products/panda), and a [harness](https://comma.ai/shop/products/car-harness).
-
-Community and Contributing
+comma two:
 ------
+* [`0.8.12-prod-personal-hkg`](https://github.com/sunnyhaibin/openpilot/tree/0.8.12-prod-personal-hkg):
 
-openpilot is developed by [comma](https://comma.ai/) and by users like you. We welcome both pull requests and issues on [GitHub](http://github.com/commaai/openpilot). Bug fixes and new car ports are encouraged. Check out [the contributing docs](docs/CONTRIBUTING.md).
+  ```
+  cd /data; rm -rf ./openpilot; git clone -b 0.8.12-prod-personal-hkg --recurse-submodules https://github.com/sunnyhaibin/sunnypilot.git openpilot; cd openpilot; sudo reboot
+  ```
 
-Documentation related to openpilot development can be found on [docs.comma.ai](https://docs.comma.ai). Information about running openpilot (e.g. FAQ, fingerprinting, troubleshooting, custom forks, community hardware) should go on the [wiki](https://github.com/commaai/openpilot/wiki).
+After running the command to install the desired branch, your comma device should reboot.
+  </details>
 
-You can add support for your car by following guides we have written for [Brand](https://blog.comma.ai/how-to-write-a-car-port-for-openpilot/) and [Model](https://blog.comma.ai/openpilot-port-guide-for-toyota-models/) ports. Generally, a car with adaptive cruise control and lane keep assist is a good candidate. [Join our Discord](https://discord.comma.ai) to discuss car ports: most car makes have a dedicated channel.
+</details>
 
-Want to get paid to work on openpilot? [comma is hiring](https://comma.ai/jobs/).
 
-And [follow us on Twitter](https://twitter.com/comma_ai).
+<details><summary><h3>🚗 Highlight Features</h3></summary>
 
-User Data and comma Account
-------
+---
 
-By default, openpilot uploads the driving data to our servers. You can also access your data through [comma connect](https://connect.comma.ai/). We use your data to train better models and improve openpilot for everyone.
+### Quality of Life Enhancements
+- [**Modified Assistive Driving Safety (MADS)**](#modified-assistive-driving-safety-mads) - Automatic Lane Centering (ALC) / Lane Keep Assist System (LKAS) and Adaptive Cruise Control (ACC) / Smart Cruise Control (SCC) can be engaged independently of each other
+- [**Dynamic Lane Profile (DLP)**](#dynamic-lane-profile-dlp) - Dynamically switch lane profile (between Laneful and Laneless) based on lane recognition confidence
+- [**Enhanced Speed Control**](#enhanced-speed-control) - Automatically adjust cruise control speed using vision model, OpenStreetMap (OSM) data, and/or Speed Limit control (SLC) without user interaction
+    * Vision-based Turn Speed Control (V-TSC) - lower speed when going around corners using vision model
+    * Map-Data-based Turn Speed Control (M-TSC) - lower speed when going around corners using OSM data[^1]
+    * Speed Limit Control (SLC) - Set speed limit based on map data or car interface (if applicable)
+    * HKG only: Highway Driving Assist (HDA) status integration - Use cars native speed sign detection to set desired speed (on applicable HKG cars only)
+- [**Gap Adjust Cruise (GAC)**](#gap-adjust-cruise) - Allow `GAP`/`INTERVAL`/`DISTANCE` button on the steering wheel or on-screen button to adjust the follow distance from the lead car. See table below for options
+    - [**Quiet Drive 🤫**](#-quiet-drive) - Toggle to mute all notification sounds (excluding driver safety warnings)
+    - [**Auto Lane Change Timer**](#Auto-Lane-Change-Timer) - Set a timer to delay the auto lane change operation when the blinker is used. No nudge on the steering wheel is required to auto lane change if a timer is set
+    - [**Force Car Recognition (FCR)**](#Force-Car-Recognition-) - Use a selector to force your car to be recognized by sunnypilot
+    - [**Fix sunnypilot No Offroad**](#Fix-sunnypilot-No-Offroad) - Enforce sunnypilot to go offroad and turns off after shutting down the car. This feature fixes non-official devices running sunnypilot without comma power
+    - [**Enable ACC+MADS with RES+/SET-**](#Enable-ACC+MADS-with-RES+/SET-) - Engage both ACC and MADS with a single press of RES+ or SET- button
+    - [**Offline OSM Maps**](#Offline-OSM-Maps) - OSM database can now be downloaded locally for offline use[^2]. This enables offline SLC, V-TSC and M-TSC. Currently available for US South, US West, US Northeast, Florida, Taiwan, South Africa and New Zealand
+    - [**Various Live Tuning**](#Various-Live-Tuning) - Ability to tailor your driving experience on the fly:
+        * Enforce Torque Lateral Control - Use the newest [torque controller](https://blog.comma.ai/0815release/#torque-controller) for all vehicles.
+        * Torque Lateral Control Live Tune - Ability to adjust the torque controller’s `FRICTION` and `LAT_ACCEL_FACTOR` values to suit your vehicle.
+        * Torque Lateral Controller Self-Tune - Enable automatic turning for the Torque controller.
 
-openpilot is open source software: the user is free to disable data collection if they wish to do so.
+### Visual Enhancements
+* **M.A.D.S Status Icon** - Dedicated icon to display M.A.D.S. engagement status
+    * Green🟢: M.A.D.S. engaged
+    * White⚪: M.A.D.S. suspended or disengaged
+* **Lane Path Color** - Various lane path colors to display real-time Lane Model and M.A.D.S. engagement status
+    * 0.8.14 and later:
+        * Blue🔵: Laneful mode & M.A.D.S. engaged
+        * Green🟢: Laneless mode & M.A.D.S. engaged
+        * Yellow🟡: Experimental e2e & M.A.D.S. engaged
+    * Pre 0.8.14:
+        * Green🟢: Laneful mode & M.A.D.S. engaged
+        * Red🔴: Laneless mode & M.A.D.S. engaged
+    * White⚪: M.A.D.S. suspended or disengaged
+    * Black⚫: M.A.D.S. engaged, steering is being manually overridden by user
+* **Developer (Dev) UI** - Display various real-time metrics on screen while driving
+* **Stand Still Timer** - Display time spent at a stop with M.A.D.S engaged (i.e., at traffic lights, stop signs, traffic congestions)
+* **Braking Status** - Current car speed text turns red when the car is braking by the driver or ACC/SCC
 
-openpilot logs the road-facing cameras, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
+### Operational Enhancements
+* **Fast Boot** - sunnypilot will fast boot by creating a Prebuilt file
+* **Disable Onroad Uploads** - Disable uploads completely when onroad. Necessary to avoid high data usage when connected to Wi-Fi hotspot
+* **Brightness Control (Global)** - Manually adjusts the global brightness of the screen
+* **Driving Screen Off Timer** - Turns off the device screen or reduces brightness to protect the screen after car starts
+* **Driving Screen Off Brightness (%)** - When using the Driving Screen Off feature, the brightness is reduced according to the automatic brightness ratio
+* **Max Time Offroad** - Device is automatically turned off after a set time when the engine is turned off (off-road) after driving (on-road)
+
+</details>
+
+<details><summary><h3>🚗 Driving Enhancements</h3></summary>
+
+---
+
+### Modified Assistive Driving Safety (MADS)
+The goal of Modified Assistive Driving Safety (MADS) is to enhance the user driving experience with modified behaviors of driving assist engagements. This feature complies with comma.ai's safety rules as accurately as possible with the following changes:
+* sunnypilot Automatic Lane Centering (ALC) and ACC/SCC can be engaged independently of each other
+* Dedicated button to toggle sunnypilot ALC:
+    * `CRUISE (MAIN)` button: All supported cars on sunnypilot
+        * `LFA` button: Newer HKG cars with `LFA` button
+        * `LKAS` button: Honda, Toyota, Global Subaru
+* `SET-` button enables ACC/SCC
+* `CANCEL` button only disables ACC/SCC
+* `CRUISE (MAIN)` must be `ON` to use ACC/SCC
+* `CRUISE (MAIN)` button disables sunnypilot completely when `OFF` **(strictly enforced in panda safety code)**
+
+### Disengage Lateral ALC on Brake Press Mode toggle
+Dedicated toggle to handle Lateral state on brake pedal press and release:
+1. `ON`: `BRAKE pedal` press will pause Automatic Lane Centering; `BRAKE pedal` release will resume Automatic Lane Centering. Note: `BRAKE pedal` release will NOT resume ACC/SCC/Long control without explicit user engagement **(strictly enforced in panda safety code)**
+2. `OFF`: `BRAKE pedal` press will NOT pause Automatic Lane Centering; `BRAKE pedal` release will NOT resume ACC/SCC/Long control without explicit user engagement **(strictly enforced in panda safety code)**
+
+### Miscellaneous
+* `TURN SIGNALS` (`Left` or `Right`) will pause Automatic Lane Centering if the vehicle speed is below the [threshold](https://github.com/commaai/openpilot/blob/master/selfdrive/controls/lib/desire_helper.py#L8) for Automatic Lane Change
+* Event audible alerts are more relaxed to match manufacturer's stock behavior
+* Critical events trigger disengagement of Automatic Lane Centering completely. The disengagement is enforced in sunnypilot and panda safety
+
+### Dynamic Lane Profile (DLP)
+
+Dynamic Lane Profile (DLP) aims to provide the best driving experience at staying within a lane confidently. Dynamic Lane Profile allows sunnypilot to dynamically switch between lane profiles based on lane recognition confidence level on road.
+
+There are 3 modes to select on the onroad camera screen:
+* **Auto Lane**: sunnypilot dynamically chooses between `Laneline` or `Laneless` model
+* **Laneline**: sunnypilot uses Laneline model only.
+* **Laneless**: sunnypilot uses Laneless model only.
+
+To use Dynamic Lane Profile, do the following:
+```
+1. sunnypilot Settings -> `SP - Controls` -> Enable Dynamic Lane Profile -> ON toggle
+2. Reboot.
+3. Before driving, on the onroad camera screen, toggle between the 3 modes by pressing on the button.
+4. Drive.
+```
+
+### Enhanced Speed Control
+This fork now allows supported cars to dynamically adjust the longitudinal plan based on the fetched map data. Big thanks to the Move Fast team for the amazing implementation!
+
+**Supported cars:**
+* sunnypilot Longitudinal Control capable
+* Stock Longitudinal Control
+    * Hyundai/Kia/Genesis (non CAN-FD)
+    * Honda Bosch
+    * Volkswagen MQB
+
+Certain features are only available with an active data connection, via:
+* [comma Prime](https://comma.ai/prime) - Intuitive service provided directly by comma, or
+* Personal Hotspot - From your mobile device, or a dedicated hotspot from a cellular carrier.
+
+**Features:**
+* Vision-based Turn Speed Control (VTSC) - Use vision path predictions to estimate the appropriate speed to drive through turns ahead - i.e. slowing down for curves
+* Map-Data-based Turn Speed Control (MTSC) - Use curvature information from map data to define speed limits to take turns ahead - i.e. slowing down for curves[^1]
+* Speed Limit Control (SLC) - Use speed limit signs information from map data and car interface to automatically adapt cruise speed to road limits
+    * HKG only: Highway Driving Assist (HDA) status integration - on applicable HKG cars only[^1]
+    * Speed Limit Offset - When Speed Limit Control is enabled, set speed limit slightly higher than the actual speed limit for a more natural drive[^1]
+* Toggle Hands on Wheel Monitoring - Monitors and alerts the driver when their hands have not been on the steering wheel for an extended time
+
+### Custom Stock Longitudinal Control
+While using stock Adaptive/Smart Cruise Control, Custom Stock Longitudinal Control in sunnypilot allows sunnypilot to manipulate and take over the set speed on the car's dashboard.
+
+**Supported Cars:**
+* Hyundai/Kia/Genesis
+    * CAN platform
+    * CAN-FD platform with 0x1CF broadcasted in CAN traffic
+* Honda Bosch
+* Volkswagen MQB
+
+**Instruction**
+
+**📗 How to use Custom Longitudinal Control on sunnypilot **
+
+When using Speed Limit, Vision, or Map based Turn control, you will be setting the "MAX" ACC speed on the sunnypilot display instead of the one in the dashboard. The car will then set the ACC setting in the dashboard to the targeted speed, but will never exceed the max speed set on the sunnypilot display. A quick press of the RES+ or SET- buttons will change this speed by 5 MPH or KM/H on the sunnypilot display, while a long deliberate press (about a 1/2 second press) changes it by 1 MPH or KM/H. DO NOT hold the RES+ or SET- buttons for longer that a 1 second. Either make quick or long deliberate presses only.
+
+**‼ Where to look when setting ACC speed ‼**
+
+Do not look at the dashboard when setting your ACC max speed. Instead, only look at the one on the sunnypilot display, "MAX". The reason you need to look at the sunnypilot display is because sunnypilot will be changing the one in the dashboard. It will be adjusting it as needed, never raising it above the one set on the sunnypilot display. ONLY look at the MAX speed on the sunnypilot display when setting the ACC speed instead of the dashboard!
+
+(Courtesy instructions from John, author of jvePilot)
+
+### Gap Adjust Cruise
+This fork now allows supported openpilot longitudinal cars to adjust the cruise gap between the car and the lead car.
+
+**Supported cars:**
+* sunnypilot Longitudinal Control capable
+
+🚨**PROCEED WITH EXTREME CAUTION AND BE READY TO MANUALLY TAKE OVER AT ALL TIMES**
+
+There are 4 modes to select on the steering wheel and/or the onroad camera screen:
+* **Stock Gap**: Stock sunnypilot distance - 1.45 second profile
+* **Mild Gap**: Semi-aggressive distance - 1.25 second profile
+* 🚨**Aggro Gap**🚨: Aggressive distance - 1.0 second profile
+
+**Availability**
+
+|      Car Make       | Stock Gap | Mild Gap | Aggro Gap |
+|:-------------------:|:---------:|:--------:|:---------:|
+|     Honda/Acura     |     ✅     |    ✅     |     ✅     |
+| Hyundai/Kia/Genesis |     ✅     |    ✅     |     ✅     |
+|    Toyota/Lexus     |     ✅     |    ✅     |     ✅     |
+|  Volkswagen MQB/PQ  |     ✅     |    ✅     |     ✅     |
+
+</details>
+
+
+<details><summary><h3>⚒ Branch Definitions</h3></summary>
+
+---
+
+|    Tag    | Definition           | Description                                                                                                                                                                                 |
+|:---------:|----------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `release` | Release branches     | Include features that are **verified** by trusted testers and the community. Ready to use. ✅                                                                                                |
+| `staging` | Staging branches     | Include new features that are **tested** by trusted testers and the community. Stability may vary. ⚠                                                                                        |
+|   `dev`   | Development branches | All features are gathered in respective versions. Reviewed and merged features will be committed to `dev`. Stability may vary. ⚠                                                            |
+| `master`  | Main branch          | Syncs with [commaai's openpilot `master`](https://github.com/commaai/openpilot) upstream branch. Accepts all pull requests. Does not include all sunnypilot features. Stability may vary. ⚠ |
+
+Example:
+* [`release-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/release-c3): Latest release branch for comma three that are verified by trusted testers and the community. Ready to use.
+* [`staging-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/staging-c3): Latest staging branch for comma three that are tested by trusted testers and the community. Verification required.
+* [`dev-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/dev-c3): Latest development branch for comma three that include all sunnypilot features. Testing required.
+
+</details>
+
+<details><summary><h3>✅ Recommended Branches</h3></summary>
+
+---
+
+| Branch                                                                              | Definition                                              | Compatible Device | Changelogs                                                                                 |
+|:------------------------------------------------------------------------------------|---------------------------------------------------------|-------------------|--------------------------------------------------------------------------------------------|
+| [`release-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/release-c3)           | • Latest release/stable branch                          | comma three       | [`CHANGELOGS.md`](https://github.com/sunnyhaibin/sunnypilot/blob/release-c3/CHANGELOGS.md) |
+| [`staging-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/staging-c3)           | • Latest staging branch                                 | comma three       | [`CHANGELOGS.md`](https://github.com/sunnyhaibin/sunnypilot/blob/staging-c3/CHANGELOGS.md) |
+| [`dev-c3`](https://github.com/sunnyhaibin/sunnypilot/tree/dev-c3)                   | • Latest development branch with experimental features  | comma three       | [`CHANGELOGS.md`](https://github.com/sunnyhaibin/sunnypilot/blob/dev-c3/CHANGELOGS.md)     |
+
+</details>
+
+<details><summary><h3>📗 How To's</h3></summary>
+
+---
+
+How-To instructions can be found in [HOW-TOS.md](https://github.com/sunnyhaibin/openpilot/blob/(!)README/HOW-TOS.md).
+
+</details>
+
+
+<details><summary><h3>🎆 Pull Requests</h3></summary>
+
+---
+
+We welcome both pull requests and issues on GitHub. Bug fixes are encouraged.
+
+Pull requests should be against the most current `master` branch.
+
+</details>
+
+<details><summary><h3>🏆 Special Thanks</h3></summary>
+
+---
+
+* [spektor56](https://github.com/spektor56/openpilot)
+* [rav4kumar](https://github.com/rav4kumar/openpilot)
+* [mob9221](https://github.com/mob9221/opendbc)
+* [briantran33](https://github.com/briantran33/openpilot)
+* [Aragon7777](https://github.com/aragon7777/openpilot)
+* [sshane](https://github.com/sshane/openpilot-installer-generator)
+* [jung](https://github.com/chanhojung/openpilot)
+* [dri94](https://github.com/dri94/openpilot)
+* [FrogAi](https://github.com/frogAi/FrogPilot/)
+* [twilsonco](https://github.com/twilsonco/openpilot)
+* [martinl](https://github.com/martinl/openpilot)
+* [multikyd](https://github.com/openpilotkr)
+* [Move Fast GmbH](https://github.com/move-fast/openpilot)
+* [dragonpilot](https://github.com/dragonpilot-community/dragonpilot)
+* [neokii](https://github.com/neokii/openpilot)
+* [AlexandreSato](https://github.com/AlexandreSato/openpilot)
+* [Moodkiller](https://github.com/moodkiller)
+
+</details>
+
+<details><summary><h3>📊 User Data</h3></summary>
+
+---
+
+By default, sunnypilot uploads the driving data to comma servers. You can also access your data through [comma connect](https://connect.comma.ai/).
+
+sunnypilot is open source software. The user is free to disable data collection if they wish to do so.
+
+sunnypilot logs the road-facing camera, CAN, GPS, IMU, magnetometer, thermal sensors, crashes, and operating system logs.
 The driver-facing camera is only logged if you explicitly opt-in in settings. The microphone is not recorded.
 
-By using openpilot, you agree to [our Privacy Policy](https://comma.ai/privacy). You understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
+By using this software, you understand that use of this software or its related services will generate certain types of user data, which may be logged and stored at the sole discretion of comma. By accepting this agreement, you grant an irrevocable, perpetual, worldwide right to comma for the use of this data.
 
-Safety and Testing
-----
+</details>
 
-* openpilot observes ISO26262 guidelines, see [SAFETY.md](docs/SAFETY.md) for more details.
-* openpilot has software-in-the-loop [tests](.github/workflows/selfdrive_tests.yaml) that run on every commit.
-* The code enforcing the safety model lives in panda and is written in C, see [code rigor](https://github.com/commaai/panda#code-rigor) for more details.
-* panda has software-in-the-loop [safety tests](https://github.com/commaai/panda/tree/master/tests/safety).
-* Internally, we have a hardware-in-the-loop Jenkins test suite that builds and unit tests the various processes.
-* panda has additional hardware-in-the-loop [tests](https://github.com/commaai/panda/blob/master/Jenkinsfile).
-* We run the latest openpilot in a testing closet containing 10 comma devices continuously replaying routes.
-
-Directory Structure
-------
-    .
-    ├── cereal              # The messaging spec and libs used for all logs
-    ├── common              # Library like functionality we've developed here
-    ├── docs                # Documentation
-    ├── opendbc             # Files showing how to interpret data from cars
-    ├── panda               # Code used to communicate on CAN
-    ├── third_party         # External libraries
-    └── system              # Generic services
-        ├── camerad         # Driver to capture images from the camera sensors
-        ├── clocksd         # Broadcasts current time
-        ├── hardware        # Hardware abstraction classes
-        ├── logcatd         # systemd journal as a service
-        └── proclogd        # Logs information from /proc
-    └── selfdrive           # Code needed to drive the car
-        ├── assets          # Fonts, images, and sounds for UI
-        ├── athena          # Allows communication with the app
-        ├── boardd          # Daemon to talk to the board
-        ├── car             # Car specific code to read states and control actuators
-        ├── controls        # Planning and controls
-        ├── debug           # Tools to help you debug and do car ports
-        ├── locationd       # Precise localization and vehicle parameter estimation
-        ├── loggerd         # Logger and uploader of car data
-        ├── manager         # Daemon that starts/stops all other daemons as needed
-        ├── modeld          # Driving and monitoring model runners
-        ├── monitoring      # Daemon to determine driver attention
-        ├── navd            # Turn-by-turn navigation
-        ├── sensord         # IMU interface code
-        ├── test            # Unit tests, system tests, and a car simulator
-        └── ui              # The UI
-
-Licensing
-------
+<details><summary><h3>Licensing</h3></summary>
 
 openpilot is released under the MIT license. Some parts of the software are released under other licenses as specified.
 
-Any user of this software shall indemnify and hold harmless Comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
+Any user of this software shall indemnify and hold harmless comma.ai, Inc. and its directors, officers, employees, agents, stockholders, affiliates, subcontractors and customers from and against all allegations, claims, actions, suits, demands, damages, liabilities, obligations, losses, settlements, judgments, costs and expenses (including without limitation attorneys’ fees and costs) which arise out of, relate to or result from any use of this software by user.
 
 **THIS IS ALPHA QUALITY SOFTWARE FOR RESEARCH PURPOSES ONLY. THIS IS NOT A PRODUCT.
 YOU ARE RESPONSIBLE FOR COMPLYING WITH LOCAL LAWS AND REGULATIONS.
 NO WARRANTY EXPRESSED OR IMPLIED.**
 
+</details>
+
+<h3>💰 Support sunnypilot</h3>
+
 ---
 
-<img src="https://d1qb2nb5cznatu.cloudfront.net/startups/i/1061157-bc7e9bf3b246ece7322e6ffe653f6af8-medium_jpg.jpg?buster=1458363130" width="75"></img> <img src="https://cdn-images-1.medium.com/max/1600/1*C87EjxGeMPrkTuVRVWVg4w.png" width="225"></img>
+If you find any of the features useful, consider becoming a [patron on Patreon](https://www.patreon.com/sunnyhaibin) or a [sponsor on GitHub](https://github.com/sponsors/sunnyhaibin) to support future feature development and improvements.
 
-[![openpilot tests](https://github.com/commaai/openpilot/workflows/openpilot%20tests/badge.svg?event=push)](https://github.com/commaai/openpilot/actions)
-[![codecov](https://codecov.io/gh/commaai/openpilot/branch/master/graph/badge.svg)](https://codecov.io/gh/commaai/openpilot)
+
+By becoming a patron/sponsor, you will gain access to exclusive content, early access to new features, and the opportunity to directly influence the project's development.
+
+<h3>Patreon</h3>
+
+<a href="https://www.patreon.com/sunnyhaibin">
+  <img src="https://user-images.githubusercontent.com/47793918/244128051-bc7e913e-a196-4455-926e-23aec9a4bd3b.png" alt="Become a Patron" width="300" style="max-width: 100%; height: auto;">
+</a>
+<br>
+
+<h3>GitHub Sponsor</h3>
+
+<a href="https://github.com/sponsors/sunnyhaibin">
+  <img src="https://user-images.githubusercontent.com/47793918/244135584-9800acbd-69fd-4b2b-bec9-e5fa2d85c817.png" alt="Become a Sponsor" width="300" style="max-width: 100%; height: auto;">
+</a>
+<br>
+
+<h3>PayPal</h3>
+
+<a href="https://paypal.me/sunnyhaibin0850" target="_blank">
+<img src="https://www.paypalobjects.com/en_US/i/btn/btn_donateCC_LG.gif" alt="PayPal this" title="PayPal - The safer, easier way to pay online!" border="0" />
+</a>
+<br></br>
+
+Your continuous love and support are greatly appreciated! Enjoy 🥰
+
+<span>-</span> Jason, Founder of sunnypilot
+
+[^1]:Requires data connection if not using Offline Maps data
+[^2]:At least 50 GB of storage space is required. If you have the 32 GB version of comma three, upgrading with a compatible 250 GB or 1 TB SSD is strongly recommended
+[^4]:Shortened URL for convenience. Full URL is ```smiskol.com/fork/sunnyhaibin/release-c3```

RELEASES.md

@@ -1,12 +1,122 @@
-Version 0.9.1 (2022-12-XX)
+Version 0.9.7 (2024-XX-XX)
 ========================
-* Adjust alert volume using ambient noise level
-* Removed driver monitoring timer resetting on interaction if face detected and distracted
-* New German translation thanks to Vrabetz and CzokNorris!
+* New driving model
+* Support for many hybrid Ford models
+
+Version 0.9.6 (2024-02-27)
+========================
+* New driving model
+  * Vision model trained on more data
+  * Improved driving performance
+  * Directly outputs curvature for lateral control
+* New driver monitoring model
+  * Trained on larger dataset
+* AGNOS 9
+* comma body streaming and controls over WebRTC
+* Improved fuzzy fingerprinting for many makes and models
+* Alpha longitudinal support for new Toyota models
+* Chevrolet Equinox 2019-22 support thanks to JasonJShuler and nworb-cire!
+* Dodge Durango 2020-21 support
+* Hyundai Staria 2023 support thanks to sunnyhaibin!
+* Kia Niro Plug-in Hybrid 2022 support thanks to sunnyhaibin!
+* Lexus LC 2024 support thanks to nelsonjchen!
+* Toyota RAV4 2023-24 support
+* Toyota RAV4 Hybrid 2023-24 support
+
+Version 0.9.5 (2023-11-17)
+========================
+* New driving model
+  * Improved navigate on openpilot performance using navigation instructions as an additional model input
+  * Do lateral planning inside the model
+  * New vision transformer architecture
+* Cadillac Escalade ESV 2019 support thanks to twilsonco!
+* Hyundai Azera 2022 support thanks to sunnyhaibin!
+* Hyundai Azera Hybrid 2020 support thanks to chanhojung and haram-KONA!
+* Hyundai Custin 2023 support thanks to sunnyhaibin and Saber422!
+* Hyundai Ioniq 6 2023 support thanks to sunnyhaibin and alamo3!
+* Hyundai Kona Electric 2023 (Korean version) support thanks to sunnyhaibin and haram-KONA!
+* Kia K8 Hybrid (with HDA II) 2023 support thanks to sunnyhaibin!
+* Kia Optima Hybrid 2019 support
+* Kia Sorento Hybrid 2023 support thanks to sunnyhaibin!
+* Lexus GS F 2016 support thanks to snyperifle!
+* Lexus IS 2023 support thanks to L3R5!
+
+Version 0.9.4 (2023-07-27)
+========================
+* comma 3X support
+* Navigate on openpilot in Experimental mode
+  * When navigation has a destination, openpilot will input the map information into the model, which provides useful context to help the model understand the scene
+  * When navigating on openpilot, openpilot will keep left or right appropriately at forks and exits
+  * When navigating on openpilot, lane change behavior is unchanged and still activated by the driver
+  * When navigate on openpilot is active, the path on the map is green
+* UI updates
+  * Navigation settings moved to home screen and map
+  * Border color always shows engagement status. Blue means disengaged, green means engaged, and grey means engaged with human overriding
+  * Alerts are shown inside the border. Black means info, orange means warning, and red means critical alert
+* Bookmarked segments are preserved on the device's storage
+* Ford Focus 2018 support
+* Kia Carnival 2023 support thanks to sunnyhaibin!
+
+Version 0.9.3 (2023-06-29)
+========================
+* New driving model
+  * Improved height estimation and added height tracking in liveCalibration
+  * Model inputs refactor
+* New driving personality setting
+  * Three settings: aggressive, standard, and relaxed
+  * Standard is recommended and the default
+  * In aggressive mode, lead follow distance is shorter and acceleration response is quicker
+  * In relaxed mode, lead follow distance is longer
+* Improved fuzzy fingerprinting for Hyundai, Kia, and Genesis
+* Improved thermal management logic
+
+Version 0.9.2 (2023-05-22)
+========================
+* New driving model
+  * Reduced turn diving
+  * Trained on a new dataset
+* UI updates
+  * New experimental mode visualization
+  * Draw MPC path instead of model-predicted path
+* AGNOS 7
+  * Faster boot time
+  * Fixes rare no sounds bug
+  * Fixes bootsplash bug at extreme temperatures
+* Buick LaCrosse 2017-19 support thanks to koch-cf!
+* Chevrolet Trailblazer 2021-22 support thanks to TurboCE!
+* Ford Bronco Sport 2021-22 support
+* Ford Escape 2020-22 support
+* Ford Explorer 2020-22 support
+* Ford Kuga 2020-22 support
+* Ford Maverick 2022-23 support
+* Genesis GV80 2023 support thanks to JWingate80!
+* Honda HR-V 2023 support thanks to AlexandreSato and galegozi!
+* Kia Niro EV 2023 support thanks to JosselinLecocq!
+* Lexus ES 2017-18 support
+* Lincoln Aviator 2021 support
+* Škoda Fabia 2022-23 support thanks to jyoung8607!
+
+
+Version 0.9.1 (2023-02-28)
+========================
+* New driving model
+  * 30% improved height estimation resulting in better driving performance for tall cars
+* Driver monitoring: removed timer resetting on user interaction if distracted
+* UI updates
+  * Adjust alert volume using ambient noise level
+  * Driver monitoring icon shows driver's head pose
+  * German translation thanks to Vrabetz and CzokNorris!
+* Cadillac Escalade 2017 support thanks to rickygilleland!
 * Chevrolet Bolt EV 2022-23 support thanks to JasonJShuler!
 * Genesis GV60 2023 support thanks to sunnyhaibin!
 * Hyundai Tucson 2022-23 support
+* Kia K5 Hybrid 2020 support thanks to sunnyhaibin!
+* Kia Niro Hybrid 2023 support thanks to sunnyhaibin!
+* Kia Sorento 2022-23 support thanks to sunnyhaibin!
 * Kia Sorento Plug-in Hybrid 2022 support thanks to sunnyhaibin!
+* Toyota C-HR 2021 support thanks to eFiniLan!
+* Toyota C-HR Hybrid 2022 support thanks to Korben00!
+* Volkswagen Crafter and MAN TGE 2017-23 support thanks to jyoung8607!
 
 Version 0.9.0 (2022-11-21)
 ========================

SConstruct

@@ -5,14 +5,19 @@ import sysconfig
 import platform
 import numpy as np
 
+import SCons.Errors
+
+SCons.Warnings.warningAsException(True)
+
+# pending upstream fix - https://github.com/SCons/scons/issues/4461
+#SetOption('warn', 'all')
+
 TICI = os.path.isfile('/TICI')
 AGNOS = TICI
 
 Decider('MD5-timestamp')
 
-AddOption('--extras',
-          action='store_true',
-          help='build misc extras, like setup and installer files')
+SetOption('num_jobs', int(os.cpu_count()/2))
 
 AddOption('--kaitai',
           action='store_true',
@@ -26,6 +31,10 @@ AddOption('--ubsan',
           action='store_true',
           help='turn on UBSan')
 
+AddOption('--coverage',
+          action='store_true',
+          help='build with test coverage options')
+
 AddOption('--clazy',
           action='store_true',
           help='build with clazy')
@@ -34,6 +43,12 @@ AddOption('--compile_db',
           action='store_true',
           help='build clang compilation database')
 
+AddOption('--ccflags',
+          action='store',
+          type='string',
+          default='',
+          help='pass arbitrary flags over the command line')
+
 AddOption('--snpe',
           action='store_true',
           help='use SNPE on PC')
@@ -44,36 +59,36 @@ AddOption('--external-sconscript',
           dest='external_sconscript',
           help='add an external SConscript to the build')
 
-AddOption('--no-thneed',
-          action='store_true',
-          dest='no_thneed',
-          help='avoid using thneed')
-
 AddOption('--pc-thneed',
           action='store_true',
           dest='pc_thneed',
           help='use thneed on pc')
 
-AddOption('--no-test',
+AddOption('--minimal',
           action='store_false',
-          dest='test',
-          default=os.path.islink(Dir('#laika/').abspath),
-          help='skip building test files')
+          dest='extras',
+          default=os.path.islink(Dir('#rednose/').abspath), # minimal by default on release branch (where rednose is not a link)
+          help='the minimum build to run openpilot. no tests, tools, etc.')
 
+## Architecture name breakdown (arch)
+## - larch64: linux tici aarch64
+## - aarch64: linux pc aarch64
+## - x86_64:  linux pc x64
+## - Darwin:  mac x64 or arm64
 real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
 if platform.system() == "Darwin":
   arch = "Darwin"
-
-if arch == "aarch64" and AGNOS:
+  brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
+elif arch == "aarch64" and AGNOS:
   arch = "larch64"
-
+assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
 
 lenv = {
   "PATH": os.environ['PATH'],
   "LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
-  "PYTHONPATH": Dir("#").abspath,
+  "PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
 
-  "ACADOS_SOURCE_DIR": Dir("#third_party/acados/include/acados").abspath,
+  "ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
   "ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
   "TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
 }
@@ -81,8 +96,6 @@ lenv = {
 rpath = lenv["LD_LIBRARY_PATH"].copy()
 
 if arch == "larch64":
-  lenv["LD_LIBRARY_PATH"] += ['/data/data/com.termux/files/usr/lib']
-
   cpppath = [
     "#third_party/opencl/include",
   ]
@@ -106,51 +119,41 @@ else:
   cflags = []
   cxxflags = []
   cpppath = []
+  rpath += []
 
   # MacOS
   if arch == "Darwin":
-    if real_arch == "x86_64":
-      lenv["TERA_PATH"] = Dir("#").abspath + f"/third_party/acados/Darwin_x86_64/t_renderer"
-
-    brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
-    yuv_dir = "mac" if real_arch != "arm64" else "mac_arm64"
     libpath = [
-      f"#third_party/libyuv/{yuv_dir}/lib",
+      f"#third_party/libyuv/{arch}/lib",
+      f"#third_party/acados/{arch}/lib",
       f"{brew_prefix}/lib",
-      f"{brew_prefix}/Library",
-      f"{brew_prefix}/opt/openssl/lib",
-      f"{brew_prefix}/Cellar",
+      f"{brew_prefix}/opt/openssl@3.0/lib",
       "/System/Library/Frameworks/OpenGL.framework/Libraries",
     ]
-    if real_arch == "x86_64":
-      libpath.append(f"#third_party/acados/Darwin_x86_64/lib")
-    else:
-      libpath.append(f"#third_party/acados/{arch}/lib")
 
     cflags += ["-DGL_SILENCE_DEPRECATION"]
     cxxflags += ["-DGL_SILENCE_DEPRECATION"]
     cpppath += [
       f"{brew_prefix}/include",
-      f"{brew_prefix}/opt/openssl/include",
+      f"{brew_prefix}/opt/openssl@3.0/include",
     ]
-  # Linux 86_64
+    lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
+  # Linux
   else:
     libpath = [
-      "#third_party/acados/x86_64/lib",
-      "#third_party/snpe/x86_64-linux-clang",
-      "#third_party/libyuv/x64/lib",
-      "#third_party/mapbox-gl-native-qt/x86_64",
-      "#cereal",
-      "#common",
+      f"#third_party/acados/{arch}/lib",
+      f"#third_party/libyuv/{arch}/lib",
       "/usr/lib",
       "/usr/local/lib",
     ]
 
-  rpath += [
-    Dir("#third_party/snpe/x86_64-linux-clang").abspath,
-    Dir("#cereal").abspath,
-    Dir("#common").abspath
-  ]
+    if arch == "x86_64":
+      libpath += [
+        f"#third_party/snpe/{arch}"
+      ]
+      rpath += [
+        Dir(f"#third_party/snpe/{arch}").abspath,
+      ]
 
 if GetOption('asan'):
   ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
@@ -170,6 +173,10 @@ if arch != "Darwin":
 cflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
 
+ccflags_option = GetOption('ccflags')
+if ccflags_option:
+  ccflags += ccflags_option.split(' ')
+
 env = Environment(
   ENV=lenv,
   CCFLAGS=[
@@ -196,18 +203,14 @@ env = Environment(
     "#third_party/catch2/include",
     "#third_party/libyuv/include",
     "#third_party/json11",
-    "#third_party/curl/include",
-    "#third_party/libgralloc/include",
-    "#third_party/android_frameworks_native/include",
-    "#third_party/android_hardware_libhardware/include",
-    "#third_party/android_system_core/include",
     "#third_party/linux/include",
     "#third_party/snpe/include",
-    "#third_party/mapbox-gl-native-qt/include",
     "#third_party/qrcode",
     "#third_party",
     "#cereal",
     "#opendbc/can",
+    "#third_party/maplibre-native-qt/include",
+    f"#third_party/maplibre-native-qt/{arch}/include"
   ],
 
   CC='clang',
@@ -224,14 +227,19 @@ env = Environment(
     "#opendbc/can",
     "#selfdrive/boardd",
     "#common",
+    "#rednose/helpers",
   ],
   CYTHONCFILESUFFIX=".cpp",
   COMPILATIONDB_USE_ABSPATH=True,
-  tools=["default", "cython", "compilation_db"],
+  REDNOSE_ROOT="#",
+  tools=["default", "cython", "compilation_db", "rednose_filter"],
+  toolpath=["#rednose_repo/site_scons/site_tools"],
 )
 
 if arch == "Darwin":
-  env['RPATHPREFIX'] = "-rpath "
+  # RPATH is not supported on macOS, instead use the linker flags
+  darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
+  env["LINKFLAGS"] += darwin_rpath_link_flags
 
 if GetOption('compile_db'):
   env.CompilationDatabase('compile_commands.json')
@@ -251,30 +259,16 @@ def progress_function(node):
 if os.environ.get('SCONS_PROGRESS'):
   Progress(progress_function, interval=node_interval)
 
-SHARED = False
-
-# TODO: this can probably be removed
-def abspath(x):
-  if arch == 'aarch64':
-    pth = os.path.join("/data/pythonpath", x[0].path)
-    env.Depends(pth, x)
-    return File(pth)
-  else:
-    # rpath works elsewhere
-    return x[0].path.rsplit("/", 1)[1][:-3]
-
 # Cython build environment
 py_include = sysconfig.get_paths()['include']
 envCython = env.Clone()
 envCython["CPPPATH"] += [py_include, np.get_include()]
 envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
+envCython["CCFLAGS"].remove("-Werror")
 
 envCython["LIBS"] = []
 if arch == "Darwin":
-  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"]
-elif arch == "aarch64":
-  envCython["LINKFLAGS"] = ["-shared"]
-  envCython["LIBS"] = [os.path.basename(py_include)]
+  envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
 else:
   envCython["LINKFLAGS"] = ["-pthread", "-shared"]
 
@@ -282,14 +276,11 @@ Export('envCython')
 
 # Qt build environment
 qt_env = env.Clone()
-qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus"]
+qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"]
 
 qt_libs = []
 if arch == "Darwin":
-  if real_arch == "arm64":
-    qt_env['QTDIR'] = "/opt/homebrew/opt/qt@5"
-  else:
-    qt_env['QTDIR'] = "/usr/local/opt/qt@5"
+  qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
   qt_dirs = [
     os.path.join(qt_env['QTDIR'], "include"),
   ]
@@ -304,18 +295,28 @@ else:
   qt_env['QTDIR'] = qt_install_prefix
   qt_dirs = [
     f"{qt_install_headers}",
-    f"{qt_install_headers}/QtGui/5.12.8/QtGui",
   ]
+
+  qt_gui_path = os.path.join(qt_install_headers, "QtGui")
+  qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
+  qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
   qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
 
   qt_libs = [f"Qt5{m}" for m in qt_modules]
   if arch == "larch64":
     qt_libs += ["GLESv2", "wayland-client"]
+    qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
   elif arch != "Darwin":
     qt_libs += ["GL"]
+qt_env['QT3DIR'] = qt_env['QTDIR']
 
-qt_env.Tool('qt')
-qt_env['CPPPATH'] += qt_dirs + ["#selfdrive/ui/qt/"]
+# compatibility for older SCons versions
+try:
+  qt_env.Tool('qt3')
+except SCons.Errors.UserError:
+  qt_env.Tool('qt')
+
+qt_env['CPPPATH'] += qt_dirs# + ["#selfdrive/ui/qt/"]
 qt_flags = [
   "-D_REENTRANT",
   "-DQT_NO_DEBUG",
@@ -328,9 +329,9 @@ qt_flags = [
   "-DQT_MESSAGELOGCONTEXT",
 ]
 qt_env['CXXFLAGS'] += qt_flags
-qt_env['LIBPATH'] += ['#selfdrive/ui']
+qt_env['LIBPATH'] += ['#selfdrive/ui', f"#third_party/maplibre-native-qt/{arch}/lib"]
+qt_env['RPATH'] += [Dir(f"#third_party/maplibre-native-qt/{arch}/lib").srcnode().abspath]
 qt_env['LIBS'] = qt_libs
-qt_env['QT_MOCHPREFIX'] = cache_dir + '/moc_files/moc_'
 
 if GetOption("clazy"):
   checks = [
@@ -343,102 +344,56 @@ if GetOption("clazy"):
   qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
   qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
 
-Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED')
+Export('env', 'qt_env', 'arch', 'real_arch')
 
+# Build common module
 SConscript(['common/SConscript'])
 Import('_common', '_gpucommon')
 
-if SHARED:
-  common, gpucommon = abspath(common), abspath(gpucommon)
-else:
-  common = [_common, 'json11']
-  gpucommon = [_gpucommon]
+common = [_common, 'json11']
+gpucommon = [_gpucommon]
 
 Export('common', 'gpucommon')
 
-# cereal and messaging are shared with the system
+# Build cereal and messaging
 SConscript(['cereal/SConscript'])
-if SHARED:
-  cereal = abspath([File('cereal/libcereal_shared.so')])
-  messaging = abspath([File('cereal/libmessaging_shared.so')])
-else:
-  cereal = [File('#cereal/libcereal.a')]
-  messaging = [File('#cereal/libmessaging.a')]
-  visionipc = [File('#cereal/libvisionipc.a')]
 
-Export('cereal', 'messaging', 'visionipc')
-
-# Build rednose library and ekf models
-
-rednose_deps = [
-  "#selfdrive/locationd/models/constants.py",
-  "#selfdrive/locationd/models/gnss_helpers.py",
-]
-
-rednose_config = {
-  'generated_folder': '#selfdrive/locationd/models/generated',
-  'to_build': {
-    'gnss': ('#selfdrive/locationd/models/gnss_kf.py', True, [], rednose_deps),
-    'live': ('#selfdrive/locationd/models/live_kf.py', True, ['live_kf_constants.h'], rednose_deps),
-    'car': ('#selfdrive/locationd/models/car_kf.py', True, [], rednose_deps),
-  },
-}
-
-if arch != "larch64":
-  rednose_config['to_build'].update({
-    'loc_4': ('#selfdrive/locationd/models/loc_kf.py', True, [], rednose_deps),
-    'pos_computer_4': ('#rednose/helpers/lst_sq_computer.py', False, [], []),
-    'pos_computer_5': ('#rednose/helpers/lst_sq_computer.py', False, [], []),
-    'feature_handler_5': ('#rednose/helpers/feature_handler.py', False, [], []),
-    'lane': ('#xx/pipeline/lib/ekf/lane_kf.py', True, [], rednose_deps),
-  })
-
-Export('rednose_config')
-SConscript(['rednose/SConscript'])
-
-# Build system services
-SConscript([
-  'system/camerad/SConscript',
-  'system/clocksd/SConscript',
-  'system/proclogd/SConscript',
-])
-if arch != "Darwin":
-  SConscript(['system/logcatd/SConscript'])
-
-# Build openpilot
-
-# build submodules
+# Build other submodules
 SConscript([
   'body/board/SConscript',
-  'cereal/SConscript',
   'opendbc/can/SConscript',
   'panda/SConscript',
 ])
 
+# Build rednose library
+SConscript(['rednose/SConscript'])
+
+# Build system services
+SConscript([
+  'system/proclogd/SConscript',
+  'system/ubloxd/SConscript',
+  'system/loggerd/SConscript',
+])
+if arch != "Darwin":
+  SConscript([
+    'system/camerad/SConscript',
+    'system/sensord/SConscript',
+    'system/logcatd/SConscript',
+  ])
+
+# Build openpilot
 SConscript(['third_party/SConscript'])
 
-SConscript(['common/kalman/SConscript'])
-SConscript(['common/transformations/SConscript'])
-
-SConscript(['selfdrive/modeld/SConscript'])
-
+SConscript(['selfdrive/boardd/SConscript'])
 SConscript(['selfdrive/controls/lib/lateral_mpc_lib/SConscript'])
 SConscript(['selfdrive/controls/lib/longitudinal_mpc_lib/SConscript'])
-
-SConscript(['selfdrive/boardd/SConscript'])
-
-SConscript(['selfdrive/loggerd/SConscript'])
-
 SConscript(['selfdrive/locationd/SConscript'])
-SConscript(['selfdrive/sensord/SConscript'])
-SConscript(['selfdrive/ui/SConscript'])
 SConscript(['selfdrive/navd/SConscript'])
+SConscript(['selfdrive/modeld/SConscript'])
+SConscript(['selfdrive/ui/SConscript'])
 
-if arch in ['x86_64', 'Darwin'] or GetOption('extras'):
+if arch in ['x86_64', 'aarch64', 'Darwin'] and Dir('#tools/cabana/').exists() and GetOption('extras'):
   SConscript(['tools/replay/SConscript'])
-
-  opendbc = abspath([File('opendbc/can/libdbc.so')])
-  Export('opendbc')
   SConscript(['tools/cabana/SConscript'])
 
 external_sconscript = GetOption('external_sconscript')

codecov.yml

@@ -6,3 +6,8 @@ coverage:
         informational: true
     patch: off
 
+ignore:
+  - "**/test_*.py"
+  - "selfdrive/test/**"
+  - "system/version.py" # codecov changes depending on if we are in a branch or not
+  - "tools"

common/SConscript

@@ -1,33 +1,39 @@
-Import('env', 'envCython', 'arch', 'SHARED')
-
-if SHARED:
-  fxn = env.SharedLibrary
-else:
-  fxn = env.Library
+Import('env', 'envCython', 'arch')
 
 common_libs = [
   'params.cc',
-  'statlog.cc',
   'swaglog.cc',
   'util.cc',
-  'gpio.cc',
   'i2c.cc',
   'watchdog.cc',
+  'ratekeeper.cc'
 ]
 
-_common = fxn('common', common_libs, LIBS="json11")
+if arch != "Darwin":
+  common_libs.append('gpio.cc')
+
+_common = env.Library('common', common_libs, LIBS="json11")
 
 files = [
   'clutil.cc',
 ]
 
-_gpucommon = fxn('gpucommon', files)
+_gpucommon = env.Library('gpucommon', files)
 Export('_common', '_gpucommon')
 
-if GetOption('test'):
-  env.Program('tests/test_util', ['tests/test_util.cc'], LIBS=[_common])
-  env.Program('tests/test_swaglog', ['tests/test_swaglog.cc'], LIBS=[_common, 'json11', 'zmq', 'pthread'])
+if GetOption('extras'):
+  env.Program('tests/test_common',
+              ['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
+              LIBS=[_common, 'json11', 'zmq', 'pthread'])
 
-# Cython
-envCython.Program('clock.so', 'clock.pyx')
-envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+# Cython bindings
+params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
+
+SConscript([
+  'transformations/SConscript',
+])
+
+Import('transformations_python')
+common_python = [params_python, transformations_python]
+
+Export('common_python')

common/api/__init__.py

@@ -2,15 +2,15 @@ import jwt
 import os
 import requests
 from datetime import datetime, timedelta
-from common.basedir import PERSIST
-from system.version import get_version
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.version import get_version
 
 API_HOST = os.getenv('API_HOST', 'https://api.commadotai.com')
 
 class Api():
   def __init__(self, dongle_id):
     self.dongle_id = dongle_id
-    with open(PERSIST+'/comma/id_rsa') as f:
+    with open(Paths.persist_root()+'/comma/id_rsa') as f:
       self.private_key = f.read()
 
   def get(self, *args, **kwargs):

common/basedir.py

@@ -1,11 +1,4 @@
 import os
-from pathlib import Path
 
-from system.hardware import PC
 
 BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
-
-if PC:
-  PERSIST = os.path.join(str(Path.home()), ".comma", "persist")
-else:
-  PERSIST = "/persist"

common/clock.pyx

@@ -1,24 +0,0 @@
-# distutils: language = c++
-# cython: language_level = 3
-from posix.time cimport clock_gettime, timespec, CLOCK_MONOTONIC_RAW, clockid_t
-
-IF UNAME_SYSNAME == "Darwin":
-  # Darwin doesn't have a CLOCK_BOOTTIME
-  CLOCK_BOOTTIME = CLOCK_MONOTONIC_RAW
-ELSE:
-  from posix.time cimport CLOCK_BOOTTIME
-
-cdef double readclock(clockid_t clock_id):
-  cdef timespec ts
-  cdef double current
-
-  clock_gettime(clock_id, &ts)
-  current = ts.tv_sec + (ts.tv_nsec / 1000000000.)
-  return current
-
-def monotonic_time():
-  return readclock(CLOCK_MONOTONIC_RAW)
-
-def sec_since_boot():
-  return readclock(CLOCK_BOOTTIME)
-

common/clutil.cc

@@ -5,6 +5,7 @@
 #include <memory>
 
 #include "common/util.h"
+#include "common/swaglog.h"
 
 namespace {  // helper functions
 
@@ -31,14 +32,14 @@ void cl_print_info(cl_platform_id platform, cl_device_id device) {
     case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
   }
 
-  std::cout << "vendor: " << get_platform_info(platform, CL_PLATFORM_VENDOR) << std::endl
-            << "platform version: " << get_platform_info(platform, CL_PLATFORM_VERSION) << std::endl
-            << "profile: " << get_platform_info(platform, CL_PLATFORM_PROFILE) << std::endl
-            << "extensions: " << get_platform_info(platform, CL_PLATFORM_EXTENSIONS) << std::endl
-            << "name :" << get_device_info(device, CL_DEVICE_NAME) << std::endl
-            << "device version :" << get_device_info(device, CL_DEVICE_VERSION) << std::endl
-            << "max work group size :" << work_group_size << std::endl
-            << "type = " << device_type << " = " << type_str << std::endl;
+  LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
+  LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
+  LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
+  LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
+  LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
+  LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
+  LOGD("max work group size: %zu", work_group_size);
+  LOGD("type = %d, %s", (int)device_type, type_str);
 }
 
 void cl_print_build_errors(cl_program program, cl_device_id device) {
@@ -49,7 +50,7 @@ void cl_print_build_errors(cl_program program, cl_device_id device) {
   std::string log(log_size, '\0');
   clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
 
-  std::cout << "build failed; status=" << status << ", log:" << std::endl << log << std::endl;
+  LOGE("build failed; status=%d, log: %s", status, log.c_str());
 }
 
 }  // namespace
@@ -61,18 +62,23 @@ cl_device_id cl_get_device_id(cl_device_type device_type) {
   CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
 
   for (size_t i = 0; i < num_platforms; ++i) {
-    std::cout << "platform[" << i << "] CL_PLATFORM_NAME: " << get_platform_info(platform_ids[i], CL_PLATFORM_NAME) << std::endl;
+    LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
+
     // Get first device
     if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
       cl_print_info(platform_ids[i], device_id);
       return device_id;
     }
   }
-  std::cout << "No valid openCL platform found" << std::endl;
+  LOGE("No valid openCL platform found");
   assert(0);
   return nullptr;
 }
 
+cl_context cl_create_context(cl_device_id device_id) {
+  return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
+}
+
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
   return cl_program_from_source(ctx, device_id, util::read_file(path), args);
 }

common/clutil.h

@@ -22,6 +22,7 @@
   })
 
 cl_device_id cl_get_device_id(cl_device_type device_type);
+cl_context cl_create_context(cl_device_id device_id);
 cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args = nullptr);
 cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args = nullptr);
 cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args);

common/ffi_wrapper.py

@@ -1,55 +1,8 @@
-import os
-import sys
-import fcntl
-import hashlib
 import platform
-from cffi import FFI
+
 
 def suffix():
   if platform.system() == "Darwin":
     return ".dylib"
   else:
     return ".so"
-
-def ffi_wrap(name, c_code, c_header, tmpdir="/tmp/ccache", cflags="", libraries=None):
-  if libraries is None:
-    libraries = []
-
-  cache = name + "_" + hashlib.sha1(c_code.encode('utf-8')).hexdigest()
-  try:
-    os.mkdir(tmpdir)
-  except OSError:
-    pass
-
-  fd = os.open(tmpdir, 0)
-  fcntl.flock(fd, fcntl.LOCK_EX)
-  try:
-    sys.path.append(tmpdir)
-    try:
-      mod = __import__(cache)
-    except Exception:
-      print(f"cache miss {cache}")
-      compile_code(cache, c_code, c_header, tmpdir, cflags, libraries)
-      mod = __import__(cache)
-  finally:
-    os.close(fd)
-
-  return mod.ffi, mod.lib
-
-
-def compile_code(name, c_code, c_header, directory, cflags="", libraries=None):
-  if libraries is None:
-    libraries = []
-
-  ffibuilder = FFI()
-  ffibuilder.set_source(name, c_code, source_extension='.cpp', libraries=libraries)
-  ffibuilder.cdef(c_header)
-  os.environ['OPT'] = "-fwrapv -O2 -DNDEBUG -std=c++1z"
-  os.environ['CFLAGS'] = cflags
-  ffibuilder.compile(verbose=True, debug=False, tmpdir=directory)
-
-
-def wrap_compiled(name, directory):
-  sys.path.append(directory)
-  mod = __import__(name)
-  return mod.ffi, mod.lib

common/file_helpers.py

@@ -1,60 +1,6 @@
 import os
-import shutil
 import tempfile
-from atomicwrites import AtomicWriter
-
-
-def mkdirs_exists_ok(path):
-  if path.startswith('http://') or path.startswith('https://'):
-    raise ValueError('URL path')
-  try:
-    os.makedirs(path)
-  except OSError:
-    if not os.path.isdir(path):
-      raise
-
-
-def rm_not_exists_ok(path):
-  try:
-    os.remove(path)
-  except OSError:
-    if os.path.exists(path):
-      raise
-
-
-def rm_tree_or_link(path):
-  if os.path.islink(path):
-    os.unlink(path)
-  elif os.path.isdir(path):
-    shutil.rmtree(path)
-
-
-def get_tmpdir_on_same_filesystem(path):
-  normpath = os.path.normpath(path)
-  parts = normpath.split("/")
-  if len(parts) > 1 and parts[1] == "scratch":
-    return "/scratch/tmp"
-  elif len(parts) > 2 and parts[2] == "runner":
-    return f"/{parts[1]}/runner/tmp"
-  return "/tmp"
-
-
-class NamedTemporaryDir():
-  def __init__(self, temp_dir=None):
-    self._path = tempfile.mkdtemp(dir=temp_dir)
-
-  @property
-  def name(self):
-    return self._path
-
-  def close(self):
-    shutil.rmtree(self._path)
-
-  def __enter__(self):
-    return self
-
-  def __exit__(self, exc_type, exc_value, traceback):
-    self.close()
+import contextlib
 
 
 class CallbackReader:
@@ -76,24 +22,16 @@ class CallbackReader:
     return chunk
 
 
-def _get_fileobject_func(writer, temp_dir):
-  def _get_fileobject():
-    return writer.get_fileobject(dir=temp_dir)
-  return _get_fileobject
+@contextlib.contextmanager
+def atomic_write_in_dir(path: str, mode: str = 'w', buffering: int = -1, encoding: str = None, newline: str = None,
+                        overwrite: bool = False):
+  """Write to a file atomically using a temporary file in the same directory as the destination file."""
+  dir_name = os.path.dirname(path)
 
-def atomic_write_on_fs_tmp(path, **kwargs):
-  """Creates an atomic writer using a temporary file in a temporary directory
-     on the same filesystem as path.
-  """
-  # TODO(mgraczyk): This use of AtomicWriter relies on implementation details to set the temp
-  #                 directory.
-  writer = AtomicWriter(path, **kwargs)
-  return writer._open(_get_fileobject_func(writer, get_tmpdir_on_same_filesystem(path)))
+  if not overwrite and os.path.exists(path):
+    raise FileExistsError(f"File '{path}' already exists. To overwrite it, set 'overwrite' to True.")
 
-
-def atomic_write_in_dir(path, **kwargs):
-  """Creates an atomic writer using a temporary file in the same directory
-     as the destination file.
-  """
-  writer = AtomicWriter(path, **kwargs)
-  return writer._open(_get_fileobject_func(writer, os.path.dirname(path)))
+  with tempfile.NamedTemporaryFile(mode=mode, buffering=buffering, encoding=encoding, newline=newline, dir=dir_name, delete=False) as tmp_file:
+    yield tmp_file
+    tmp_file_name = tmp_file.name
+  os.replace(tmp_file_name, path)

common/gpio.cc

@@ -1,5 +1,22 @@
 #include "common/gpio.h"
 
+#include <string>
+
+#ifdef __APPLE__
+int gpio_init(int pin_nr, bool output) {
+  return 0;
+}
+
+int gpio_set(int pin_nr, bool high) {
+  return 0;
+}
+
+int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
+  return 0;
+}
+
+#else
+
 #include <fcntl.h>
 #include <unistd.h>
 
@@ -14,7 +31,7 @@ int gpio_init(int pin_nr, bool output) {
   char pin_dir_path[50];
   int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
                            "/sys/class/gpio/gpio%d/direction", pin_nr);
-  if(pin_dir_path_len <= 0) {
+  if (pin_dir_path_len <= 0) {
     return -1;
   }
   const char *value = output ? "out" : "in";
@@ -25,7 +42,7 @@ int gpio_set(int pin_nr, bool high) {
   char pin_val_path[50];
   int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
                            "/sys/class/gpio/gpio%d/value", pin_nr);
-  if(pin_val_path_len <= 0) {
+  if (pin_val_path_len <= 0) {
     return -1;
   }
   return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
@@ -38,7 +55,7 @@ int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int p
   std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
   int fd = open(gpiochip_path.c_str(), O_RDONLY);
   if (fd < 0) {
-    LOGE("Error opening gpiochip0 fd")
+    LOGE("Error opening gpiochip0 fd");
     return -1;
   }
 
@@ -53,7 +70,7 @@ int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int p
   rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
 
   strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
-  int ret = ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
+  int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
   if (ret == -1) {
     LOGE("Unable to get line event from ioctl : %s", strerror(errno));
     close(fd);
@@ -63,3 +80,5 @@ int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int p
   close(fd);
   return rq.fd;
 }
+
+#endif

common/gpio.h

@@ -5,7 +5,7 @@
   #define GPIO_HUB_RST_N        30
   #define GPIO_UBLOX_RST_N      32
   #define GPIO_UBLOX_SAFEBOOT_N 33
-  #define GPIO_UBLOX_PWR_EN     34
+  #define GPIO_GNSS_PWR_EN      34 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
   #define GPIO_STM_RST_N        124
   #define GPIO_STM_BOOT0        134
   #define GPIO_BMX_ACCEL_INT    21
@@ -17,7 +17,7 @@
   #define GPIO_HUB_RST_N        0
   #define GPIO_UBLOX_RST_N      0
   #define GPIO_UBLOX_SAFEBOOT_N 0
-  #define GPIO_UBLOX_PWR_EN     0
+  #define GPIO_GNSS_PWR_EN      0 /* SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN */
   #define GPIO_STM_RST_N        0
   #define GPIO_STM_BOOT0        0
   #define GPIO_BMX_ACCEL_INT    0

common/gpio.py

@@ -1,4 +1,5 @@
-from typing import Optional
+import os
+from functools import lru_cache
 
 def gpio_init(pin: int, output: bool) -> None:
   try:
@@ -14,7 +15,7 @@ def gpio_set(pin: int, high: bool) -> None:
   except Exception as e:
     print(f"Failed to set gpio {pin} value: {e}")
 
-def gpio_read(pin: int) -> Optional[bool]:
+def gpio_read(pin: int) -> bool | None:
   val = None
   try:
     with open(f"/sys/class/gpio/gpio{pin}/value", 'rb') as f:
@@ -23,3 +24,31 @@ def gpio_read(pin: int) -> Optional[bool]:
     print(f"Failed to set gpio {pin} value: {e}")
 
   return val
+
+def gpio_export(pin: int) -> None:
+  if os.path.isdir(f"/sys/class/gpio/gpio{pin}"):
+    return
+
+  try:
+    with open("/sys/class/gpio/export", 'w') as f:
+      f.write(str(pin))
+  except Exception:
+    print(f"Failed to export gpio {pin}")
+
+@lru_cache(maxsize=None)
+def get_irq_action(irq: int) -> list[str]:
+  try:
+    with open(f"/sys/kernel/irq/{irq}/actions") as f:
+      actions = f.read().strip().split(',')
+      return actions
+  except FileNotFoundError:
+    return []
+
+def get_irqs_for_action(action: str) -> list[str]:
+  ret = []
+  with open("/proc/interrupts") as f:
+    for l in f.readlines():
+      irq = l.split(':')[0].strip()
+      if irq.isdigit() and action in get_irq_action(irq):
+        ret.append(irq)
+  return ret

common/i2c.cc

@@ -8,14 +8,13 @@
 #include <cstdio>
 #include <stdexcept>
 
-#include "common/util.h"
 #include "common/swaglog.h"
 #include "common/util.h"
 
 #define UNUSED(x) (void)(x)
 
 #ifdef QCOM2
-// TODO: decide if we want to isntall libi2c-dev everywhere
+// TODO: decide if we want to install libi2c-dev everywhere
 extern "C" {
   #include <linux/i2c-dev.h>
   #include <i2c/smbus.h>
@@ -26,36 +25,42 @@ I2CBus::I2CBus(uint8_t bus_id) {
   snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
 
   i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
-  if(i2c_fd < 0) {
+  if (i2c_fd < 0) {
     throw std::runtime_error("Failed to open I2C bus");
   }
 }
 
 I2CBus::~I2CBus() {
-  if(i2c_fd >= 0) { close(i2c_fd); }
+  if (i2c_fd >= 0) {
+    close(i2c_fd);
+  }
 }
 
 int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
+  std::lock_guard lk(m);
+
   int ret = 0;
 
   ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
   ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
-  if((ret < 0) || (ret != len)) { goto fail; }
+  if ((ret < 0) || (ret != len)) { goto fail; }
 
 fail:
   return ret;
 }
 
 int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
+  std::lock_guard lk(m);
+
   int ret = 0;
 
   ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
   ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
-  if(ret < 0) { goto fail; }
+  if (ret < 0) { goto fail; }
 
 fail:
   return ret;

common/i2c.h

@@ -1,12 +1,14 @@
 #pragma once
 
 #include <cstdint>
+#include <mutex>
 
 #include <sys/types.h>
 
 class I2CBus {
   private:
     int i2c_fd;
+    std::mutex m;
 
   public:
     I2CBus(uint8_t bus_id);

common/kalman/.gitignore

@@ -1 +0,0 @@
-simple_kalman_impl.c

common/kalman/SConscript

@@ -1,3 +0,0 @@
-Import('envCython')
-
-envCython.Program('simple_kalman_impl.so', 'simple_kalman_impl.pyx')

common/kalman/simple_kalman.py

@@ -1,3 +0,0 @@
-# pylint: skip-file
-from common.kalman.simple_kalman_impl import KF1D as KF1D
-assert KF1D

common/kalman/simple_kalman_impl.pxd

@@ -1,18 +0,0 @@
-# cython: language_level = 3
-
-cdef class KF1D:
-  cdef public:
-    double x0_0
-    double x1_0
-    double K0_0
-    double K1_0
-    double A0_0
-    double A0_1
-    double A1_0
-    double A1_1
-    double C0_0
-    double C0_1
-    double A_K_0
-    double A_K_1
-    double A_K_2
-    double A_K_3

common/kalman/simple_kalman_impl.pyx

@@ -1,37 +0,0 @@
-# distutils: language = c++
-# cython: language_level=3
-
-cdef class KF1D:
-  def __init__(self, x0, A, C, K):
-    self.x0_0 = x0[0][0]
-    self.x1_0 = x0[1][0]
-    self.A0_0 = A[0][0]
-    self.A0_1 = A[0][1]
-    self.A1_0 = A[1][0]
-    self.A1_1 = A[1][1]
-    self.C0_0 = C[0]
-    self.C0_1 = C[1]
-    self.K0_0 = K[0][0]
-    self.K1_0 = K[1][0]
-
-    self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
-    self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
-    self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
-    self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
-
-  def update(self, meas):
-    cdef double x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
-    cdef double x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
-    self.x0_0 = x0_0
-    self.x1_0 = x1_0
-
-    return [self.x0_0, self.x1_0]
-
-  @property
-  def x(self):
-    return [[self.x0_0], [self.x1_0]]
-
-  @x.setter
-  def x(self, x):
-    self.x0_0 = x[0][0]
-    self.x1_0 = x[1][0]

common/kalman/simple_kalman_old.py

@@ -1,23 +0,0 @@
-import numpy as np
-
-
-class KF1D:
-  # this EKF assumes constant covariance matrix, so calculations are much simpler
-  # the Kalman gain also needs to be precomputed using the control module
-
-  def __init__(self, x0, A, C, K):
-    self.x = x0
-    self.A = A
-    self.C = np.atleast_2d(C)
-    self.K = K
-
-    self.A_K = self.A - np.dot(self.K, self.C)
-
-    # K matrix needs to  be pre-computed as follow:
-    # import control
-    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
-    # self.K = np.transpose(K)
-
-  def update(self, meas):
-    self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
-    return self.x

common/kalman/tests/test_simple_kalman.py

@@ -1,87 +0,0 @@
-import unittest
-import random
-import timeit
-import numpy as np
-
-from common.kalman.simple_kalman import KF1D
-from common.kalman.simple_kalman_old import KF1D as KF1D_old
-
-
-class TestSimpleKalman(unittest.TestCase):
-  def setUp(self):
-    dt = 0.01
-    x0_0 = 0.0
-    x1_0 = 0.0
-    A0_0 = 1.0
-    A0_1 = dt
-    A1_0 = 0.0
-    A1_1 = 1.0
-    C0_0 = 1.0
-    C0_1 = 0.0
-    K0_0 = 0.12287673
-    K1_0 = 0.29666309
-
-    self.kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
-                           A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
-                           C=np.array([C0_0, C0_1]),
-                           K=np.array([[K0_0], [K1_0]]))
-
-    self.kf = KF1D(x0=[[x0_0], [x1_0]],
-                   A=[[A0_0, A0_1], [A1_0, A1_1]],
-                   C=[C0_0, C0_1],
-                   K=[[K0_0], [K1_0]])
-
-  def test_getter_setter(self):
-    self.kf.x = [[1.0], [1.0]]
-    self.assertEqual(self.kf.x, [[1.0], [1.0]])
-
-  def update_returns_state(self):
-    x = self.kf.update(100)
-    self.assertEqual(x, self.kf.x)
-
-  def test_old_equal_new(self):
-    for _ in range(1000):
-      v_wheel = random.uniform(0, 200)
-
-      x_old = self.kf_old.update(v_wheel)
-      x = self.kf.update(v_wheel)
-
-      # Compare the output x, verify that the error is less than 1e-4
-      np.testing.assert_almost_equal(x_old[0], x[0])
-      np.testing.assert_almost_equal(x_old[1], x[1])
-
-  def test_new_is_faster(self):
-    setup = """
-import numpy as np
-
-from common.kalman.simple_kalman import KF1D
-from common.kalman.simple_kalman_old import KF1D as KF1D_old
-
-dt = 0.01
-x0_0 = 0.0
-x1_0 = 0.0
-A0_0 = 1.0
-A0_1 = dt
-A1_0 = 0.0
-A1_1 = 1.0
-C0_0 = 1.0
-C0_1 = 0.0
-K0_0 = 0.12287673
-K1_0 = 0.29666309
-
-kf_old = KF1D_old(x0=np.array([[x0_0], [x1_0]]),
-                  A=np.array([[A0_0, A0_1], [A1_0, A1_1]]),
-                  C=np.array([C0_0, C0_1]),
-                  K=np.array([[K0_0], [K1_0]]))
-
-kf = KF1D(x0=[[x0_0], [x1_0]],
-          A=[[A0_0, A0_1], [A1_0, A1_1]],
-          C=[C0_0, C0_1],
-          K=[[K0_0], [K1_0]])
-    """
-    kf_speed = timeit.timeit("kf.update(1234)", setup=setup, number=10000)
-    kf_old_speed = timeit.timeit("kf_old.update(1234)", setup=setup, number=10000)
-    self.assertTrue(kf_speed < kf_old_speed / 4)
-
-if __name__ == "__main__":
-  unittest.main()

common/lazy_property.py

@@ -1,12 +0,0 @@
-class lazy_property():
-  """Defines a property whose value will be computed only once and as needed.
-
-     This can only be used on instance methods.
-  """
-  def __init__(self, func):
-    self._func = func
-
-  def __get__(self, obj_self, cls):
-    value = self._func(obj_self)
-    setattr(obj_self, self._func.__name__, value)
-    return value

common/logging_extra.py

@@ -153,9 +153,9 @@ class SwagLogger(logging.Logger):
   def bind_global(self, **kwargs):
     self.global_ctx.update(kwargs)
 
-  def event(self, event_name, *args, **kwargs):
+  def event(self, event, *args, **kwargs):
     evt = NiceOrderedDict()
-    evt['event'] = event_name
+    evt['event'] = event
     if args:
       evt['args'] = args
     evt.update(kwargs)
@@ -197,7 +197,7 @@ class SwagLogger(logging.Logger):
       filename = os.path.normcase(co.co_filename)
 
       # TODO: is this pylint exception correct?
-      if filename == _srcfile:  # pylint: disable=comparison-with-callable
+      if filename == _srcfile:
         f = f.f_back
         continue
       sinfo = None

common/mat.h

@@ -1,7 +1,7 @@
 #pragma once
 
 typedef struct vec3 {
-	float v[3];
+  float v[3];
 } vec3;
 
 typedef struct vec4 {
@@ -9,7 +9,7 @@ typedef struct vec4 {
 } vec4;
 
 typedef struct mat3 {
-	float v[3*3];
+  float v[3*3];
 } mat3;
 
 typedef struct mat4 {

common/mock/__init__.py

@@ -0,0 +1,50 @@
+"""
+Utilities for generating mock messages for testing.
+example in common/tests/test_mock.py
+"""
+
+
+import functools
+import threading
+from cereal.messaging import PubMaster
+from cereal.services import SERVICE_LIST
+from openpilot.common.mock.generators import generate_liveLocationKalman
+from openpilot.common.realtime import Ratekeeper
+
+
+MOCK_GENERATOR = {
+  "liveLocationKalman": generate_liveLocationKalman
+}
+
+
+def generate_messages_loop(services: list[str], done: threading.Event):
+  pm = PubMaster(services)
+  rk = Ratekeeper(100)
+  i = 0
+  while not done.is_set():
+    for s in services:
+      should_send = i % (100/SERVICE_LIST[s].frequency) == 0
+      if should_send:
+        message = MOCK_GENERATOR[s]()
+        pm.send(s, message)
+    i += 1
+    rk.keep_time()
+
+
+def mock_messages(services: list[str] | str):
+  if isinstance(services, str):
+    services = [services]
+
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      done = threading.Event()
+      t = threading.Thread(target=generate_messages_loop, args=(services, done))
+      t.start()
+      try:
+        return func(*args, **kwargs)
+      finally:
+        done.set()
+        t.join()
+    return wrapper
+  return decorator

common/mock/generators.py

@@ -0,0 +1,20 @@
+from cereal import messaging
+
+
+LOCATION1 = (32.7174, -117.16277)
+LOCATION2 = (32.7558, -117.2037)
+
+LLK_DECIMATION = 10
+RENDER_FRAMES = 15
+DEFAULT_ITERATIONS = RENDER_FRAMES * LLK_DECIMATION
+
+
+def generate_liveLocationKalman(location=LOCATION1):
+  msg = messaging.new_message('liveLocationKalman')
+  msg.liveLocationKalman.positionGeodetic = {'value': [*location, 0], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.positionECEF = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.calibratedOrientationNED = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.velocityCalibrated = {'value': [0., 0., 0.], 'std': [0., 0., 0.], 'valid': True}
+  msg.liveLocationKalman.status = 'valid'
+  msg.liveLocationKalman.gpsOK = True
+  return msg

common/modeldata.h

@@ -1,46 +0,0 @@
-#pragma once
-
-#include <array>
-#include "common/mat.h"
-#include "system/hardware/hw.h"
-
-const int TRAJECTORY_SIZE = 33;
-const int LAT_MPC_N = 16;
-const int LON_MPC_N = 32;
-const float MIN_DRAW_DISTANCE = 10.0;
-const float MAX_DRAW_DISTANCE = 100.0;
-
-template <typename T, size_t size>
-constexpr std::array<T, size> build_idxs(float max_val) {
-  std::array<T, size> result{};
-  for (int i = 0; i < size; ++i) {
-    result[i] = max_val * ((i / (double)(size - 1)) * (i / (double)(size - 1)));
-  }
-  return result;
-}
-
-constexpr auto T_IDXS = build_idxs<double, TRAJECTORY_SIZE>(10.0);
-constexpr auto T_IDXS_FLOAT = build_idxs<float, TRAJECTORY_SIZE>(10.0);
-constexpr auto X_IDXS = build_idxs<double, TRAJECTORY_SIZE>(192.0);
-constexpr auto X_IDXS_FLOAT = build_idxs<float, TRAJECTORY_SIZE>(192.0);
-
-const mat3 fcam_intrinsic_matrix = (mat3){{2648.0, 0.0, 1928.0 / 2,
-                                           0.0, 2648.0, 1208.0 / 2,
-                                           0.0, 0.0, 1.0}};
-
-// tici ecam focal probably wrong? magnification is not consistent across frame
-// Need to retrain model before this can be changed
-const mat3 ecam_intrinsic_matrix = (mat3){{567.0, 0.0, 1928.0 / 2,
-                                           0.0, 567.0, 1208.0 / 2,
-                                           0.0, 0.0, 1.0}};
-
-static inline mat3 get_model_yuv_transform() {
-  float db_s = 1.0;
-  const mat3 transform = (mat3){{
-    1.0, 0.0, 0.0,
-    0.0, 1.0, 0.0,
-    0.0, 0.0, 1.0
-  }};
-  // Can this be removed since scale is 1?
-  return transform_scale_buffer(transform, db_s);
-}

common/numpy_helpers.py

@@ -1,22 +0,0 @@
-import numpy as np
-
-
-def deep_interp_np(x, xp, fp, axis=None):
-  if axis is not None:
-    fp = fp.swapaxes(0,axis)
-  x = np.atleast_1d(x)
-  xp = np.array(xp)
-  if len(xp) < 2:
-    return np.repeat(fp, len(x), axis=0)
-  if min(np.diff(xp)) < 0:
-    raise RuntimeError('Bad x array for interpolation')
-  j = np.searchsorted(xp, x) - 1
-  j = np.clip(j, 0, len(xp)-2)
-  d = np.divide(x - xp[j], xp[j + 1] - xp[j], out=np.ones_like(x, dtype=np.float64), where=xp[j + 1] - xp[j] != 0)
-  vals_interp = (fp[j].T*(1 - d)).T + (fp[j + 1].T*d).T
-  if axis is not None:
-    vals_interp = vals_interp.swapaxes(0,axis)
-  if len(vals_interp) == 1:
-    return vals_interp[0]
-  else:
-    return vals_interp

common/params.cc

@@ -4,9 +4,11 @@
 #include <sys/file.h>
 
 #include <algorithm>
+#include <cassert>
 #include <csignal>
 #include <unordered_map>
 
+#include "common/queue.h"
 #include "common/swaglog.h"
 #include "common/util.h"
 #include "system/hardware/hw.h"
@@ -64,7 +66,9 @@ bool create_params_path(const std::string &param_path, const std::string &key_pa
 std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
   std::string params_path = path.empty() ? Path::params() : path;
   if (!create_params_path(params_path, params_path + prefix)) {
-    throw std::runtime_error(util::string_format("Failed to ensure params path, errno=%d", errno));
+    throw std::runtime_error(util::string_format(
+        "Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
+        errno, params_path.c_str(), prefix.c_str()));
   }
   return params_path;
 }
@@ -85,35 +89,43 @@ private:
 
 std::unordered_map<std::string, uint32_t> keys = {
     {"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
+    {"ApiCache_Device", PERSISTENT},
+    {"ApiCache_NavDestinations", PERSISTENT},
     {"AssistNowToken", PERSISTENT},
     {"AthenadPid", PERSISTENT},
     {"AthenadUploadQueue", PERSISTENT},
+    {"AthenadRecentlyViewedRoutes", PERSISTENT},
+    {"BootCount", PERSISTENT},
     {"CalibrationParams", PERSISTENT},
     {"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
     {"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
     {"CarBatteryCapacity", PERSISTENT},
-    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
+    {"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"CarParamsCache", CLEAR_ON_MANAGER_START},
     {"CarParamsPersistent", PERSISTENT},
-    {"CarVin", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
+    {"CarParamsPrevRoute", PERSISTENT},
+    {"CarVin", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"CompletedTrainingVersion", PERSISTENT},
-    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
-    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
-    {"DashcamOverride", PERSISTENT},
-    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
+    {"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"CurrentBootlog", PERSISTENT},
+    {"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"DisablePowerDown", PERSISTENT},
-    {"ExperimentalMode", PERSISTENT},
-    {"ExperimentalModeConfirmed", PERSISTENT},
-    {"ExperimentalLongitudinalEnabled", PERSISTENT}, // WARNING: THIS MAY DISABLE AEB
     {"DisableUpdates", PERSISTENT},
     {"DisengageOnAccelerator", PERSISTENT},
+    {"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
     {"DongleId", PERSISTENT},
     {"DoReboot", CLEAR_ON_MANAGER_START},
     {"DoShutdown", CLEAR_ON_MANAGER_START},
     {"DoUninstall", CLEAR_ON_MANAGER_START},
-    {"ForcePowerDown", CLEAR_ON_MANAGER_START},
+    {"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY},
+    {"ExperimentalMode", PERSISTENT},
+    {"ExperimentalModeConfirmed", PERSISTENT},
+    {"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ForcePowerDown", PERSISTENT},
     {"GitBranch", PERSISTENT},
     {"GitCommit", PERSISTENT},
+    {"GitCommitDate", PERSISTENT},
     {"GitDiff", PERSISTENT},
     {"GithubSshKeys", PERSISTENT},
     {"GithubUsername", PERSISTENT},
@@ -132,80 +144,178 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"IsOffroad", CLEAR_ON_MANAGER_START},
     {"IsOnroad", PERSISTENT},
     {"IsRhdDetected", PERSISTENT},
+    {"IsReleaseBranch", CLEAR_ON_MANAGER_START},
     {"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
     {"IsTestedBranch", CLEAR_ON_MANAGER_START},
-    {"IsUpdateAvailable", CLEAR_ON_MANAGER_START},
-    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},
-    {"LaikadEphemeris", PERSISTENT | DONT_LOG},
+    {"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"LanguageSetting", PERSISTENT},
     {"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
     {"LastGPSPosition", PERSISTENT},
     {"LastManagerExitReason", CLEAR_ON_MANAGER_START},
+    {"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
     {"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
-    {"LastSystemShutdown", CLEAR_ON_MANAGER_START},
     {"LastUpdateException", CLEAR_ON_MANAGER_START},
     {"LastUpdateTime", PERSISTENT},
     {"LiveParameters", PERSISTENT},
-    {"LiveTorqueCarParams", PERSISTENT},
     {"LiveTorqueParameters", PERSISTENT | DONT_LOG},
-    {"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},
-    {"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},
-    {"NavSettingTime24h", PERSISTENT},
+    {"LongitudinalPersonality", PERSISTENT},
+    {"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"NavPastDestinations", PERSISTENT},
     {"NavSettingLeftSide", PERSISTENT},
-    {"NavdRender", PERSISTENT},
-    {"OpenpilotEnabledToggle", PERSISTENT},
-    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},
-    {"PandaSignatures", CLEAR_ON_MANAGER_START},
-    {"Passive", PERSISTENT},
-    {"PrimeType", PERSISTENT},
-    {"RecordFront", PERSISTENT},
-    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
-    {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
-    {"ShouldDoUpdate", CLEAR_ON_MANAGER_START},
-    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_OFF},
-    {"SshEnabled", PERSISTENT},
-    {"SubscriberInfo", PERSISTENT},
-    {"TermsVersion", PERSISTENT},
-    {"Timezone", PERSISTENT},
-    {"TrainingVersion", PERSISTENT},
-    {"UbloxAvailable", PERSISTENT},
-    {"UpdateAvailable", CLEAR_ON_MANAGER_START},
-    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
-    {"UpdaterState", CLEAR_ON_MANAGER_START},
-    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
-    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
-    {"UpdaterAvailableBranches", CLEAR_ON_MANAGER_START},
-    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
-    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
-    {"Version", PERSISTENT},
-    {"VisionRadarToggle", PERSISTENT},
-    {"WideCameraOnly", PERSISTENT},
-    {"ApiCache_Device", PERSISTENT},
-    {"ApiCache_DriveStats", PERSISTENT},
-    {"ApiCache_NavDestinations", PERSISTENT},
-    {"ApiCache_Owner", PERSISTENT},
+    {"NavSettingTime24h", PERSISTENT},
+    {"NetworkMetered", PERSISTENT},
+    {"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
-    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
+    {"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
     {"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
     {"Offroad_InvalidTime", CLEAR_ON_MANAGER_START},
     {"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
     {"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
-    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON},
+    {"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
     {"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
     {"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
     {"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
+    {"OpenpilotEnabledToggle", PERSISTENT},
+    {"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"PandaSignatures", CLEAR_ON_MANAGER_START},
+    {"PrimeType", PERSISTENT},
+    {"RecordFront", PERSISTENT},
+    {"RecordFrontLock", PERSISTENT},  // for the internal fleet
+    {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SshEnabled", PERSISTENT},
+    {"TermsVersion", PERSISTENT},
+    {"Timezone", PERSISTENT},
+    {"TrainingVersion", PERSISTENT},
+    {"UbloxAvailable", PERSISTENT},
+    {"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
+    {"UpdateFailedCount", CLEAR_ON_MANAGER_START},
+    {"UpdaterAvailableBranches", PERSISTENT},
+    {"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
+    {"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
+    {"UpdaterState", CLEAR_ON_MANAGER_START},
+    {"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
+    {"UpdaterLastFetchTime", PERSISTENT},
+    {"Version", PERSISTENT},
+    {"VisionRadarToggle", PERSISTENT},
+
+    {"AccMadsCombo", PERSISTENT},
+    {"AmapKey1", PERSISTENT},
+    {"AmapKey2", PERSISTENT},
+    {"AutoLaneChangeTimer", PERSISTENT},
+    {"AutoLaneChangeBsmDelay", PERSISTENT},
+    {"BelowSpeedPause", PERSISTENT},
+    {"BrakeLights", PERSISTENT},
+    {"BrightnessControl", PERSISTENT},
+    {"ButtonAutoHide", PERSISTENT},
+    {"CameraControl", PERSISTENT},
+    {"CameraControlToggle", PERSISTENT},
+    {"CameraOffset", PERSISTENT},
+    {"CarModel", PERSISTENT},
+    {"CarModelText", PERSISTENT},
+    {"ChevronInfo", PERSISTENT},
+    {"CustomBootScreen", PERSISTENT},
+    {"CustomMapboxTokenPk", PERSISTENT},
+    {"CustomMapboxTokenSk", PERSISTENT},
+    {"CustomOffsets", PERSISTENT},
+    {"CustomStockLong", PERSISTENT},
+    {"CustomTorqueLateral", PERSISTENT},
+    {"DevUI", PERSISTENT},
+    {"DevUIInfo", PERSISTENT},
+    {"DisableOnroadUploads", PERSISTENT},
+    {"DisengageLateralOnBrake", PERSISTENT},
+    {"DynamicExperimentalControl", PERSISTENT},
+    {"DynamicExperimentalControlToggle", PERSISTENT},
+    {"DynamicLaneProfile", PERSISTENT},
+    {"DynamicLaneProfileToggle", PERSISTENT},
+    {"EnableAmap", PERSISTENT},
+    {"EnableDebugSnapshot", PERSISTENT},
+    {"EnableGmap", PERSISTENT},
+    {"EnableMads", PERSISTENT},
+    {"EndToEndLongAlertLead", PERSISTENT},
+    {"EndToEndLongAlertLight", PERSISTENT},
+    {"EndToEndLongAlertUI", PERSISTENT},
+    {"EndToEndLongToggle", PERSISTENT},
+    {"EnforceTorqueLateral", PERSISTENT},
+    {"EnhancedScc", PERSISTENT},
+    {"FleetManagerPin", PERSISTENT},
+    {"GapAdjustCruise", PERSISTENT},
+    {"GapAdjustCruiseMax", PERSISTENT},
+    {"GapAdjustCruiseMin", PERSISTENT},
+    {"GapAdjustCruiseMode", PERSISTENT},
+    {"GapAdjustCruiseTr", PERSISTENT},
+    {"GmapKey", PERSISTENT},
+    {"HandsOnWheelMonitoring", PERSISTENT},
+    {"HideVEgoUi", PERSISTENT},
+    {"HkgSmoothStop", PERSISTENT},
+    {"HotspotOnBoot", PERSISTENT},
+    {"LastCarModel", PERSISTENT},
+    {"LastSpeedLimitSignTap", PERSISTENT},
+    {"LiveTorque", PERSISTENT},
+    {"LkasToggle", PERSISTENT},
+    {"MadsCruiseMain", PERSISTENT},
+    {"MadsIconToggle", PERSISTENT},
+    {"MapboxFullScreen", PERSISTENT},
+    {"MaxTimeOffroad", PERSISTENT},
+    {"OnroadScreenOff", PERSISTENT},
+    {"OnroadScreenOffBrightness", PERSISTENT},
+    {"OnroadScreenOffEvent", PERSISTENT},
+    {"OsmDbUpdatesCheck", PERSISTENT},
+    {"OsmLocal", PERSISTENT},
+    {"OsmLocalDb", PERSISTENT},
+    {"OsmLocationName", PERSISTENT},
+    {"OsmLocationTitle", PERSISTENT},
+    {"OsmLocationUrl", PERSISTENT},
+    {"OsmWayTest", PERSISTENT},
+    {"PathOffset", PERSISTENT},
+    {"QuietDrive", PERSISTENT},
+    {"RoadEdge", PERSISTENT},
+    {"ReverseAccChange", PERSISTENT},
+    {"ReverseDmCam", PERSISTENT},
+    {"ScreenRecorder", PERSISTENT},
+    {"ShowDebugUI", PERSISTENT},
+    {"SidebarTemperature", PERSISTENT},
+    {"SidebarTemperatureOptions", PERSISTENT},
+    {"SpeedLimitControl", PERSISTENT},
+    {"SpeedLimitPercOffset", PERSISTENT},
+    {"SpeedLimitValueOffset", PERSISTENT},
+    {"SpeedLimitOffsetType", PERSISTENT},
+    {"StandStillTimer", PERSISTENT},
+    {"StockLongToyota", PERSISTENT},
+    {"TorqueDeadzoneDeg", PERSISTENT},
+    {"TorqueFriction", PERSISTENT},
+    {"TorqueMaxLatAccel", PERSISTENT},
+    {"ToyotaTSS2Long", PERSISTENT},
+    {"TrueVEgoUi", PERSISTENT},
+    {"TurnSpeedControl", PERSISTENT},
+    {"TurnVisionControl", PERSISTENT},
+    {"VisionCurveLaneless", PERSISTENT},
+    {"VwAccType", PERSISTENT},
+    {"Offroad_SupersededUpdate", PERSISTENT},
 };
 
 } // namespace
 
 
 Params::Params(const std::string &path) {
-  prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
-  params_path = ensure_params_path(prefix, path);
+  params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
+  params_path = ensure_params_path(params_prefix, path);
+}
+
+Params::~Params() {
+  if (future.valid()) {
+    future.wait();
+  }
+  assert(queue.empty());
 }
 
 std::vector<std::string> Params::allKeys() const {
@@ -301,15 +411,37 @@ std::map<std::string, std::string> Params::readAll() {
 void Params::clearAll(ParamKeyType key_type) {
   FileLock file_lock(params_path + "/.lock");
 
-  if (key_type == ALL) {
-    util::remove_files_in_dir(getParamPath());
-  } else {
-    for (auto &[key, type] : keys) {
-      if (type & key_type) {
-        unlink(getParamPath(key).c_str());
+  // 1) delete params of key_type
+  // 2) delete files that are not defined in the keys.
+  if (DIR *d = opendir(getParamPath().c_str())) {
+    struct dirent *de = NULL;
+    while ((de = readdir(d))) {
+      if (de->d_type != DT_DIR) {
+        auto it = keys.find(de->d_name);
+        if (it == keys.end() || (it->second & key_type)) {
+          unlink(getParamPath(de->d_name).c_str());
+        }
       }
     }
+    closedir(d);
   }
 
   fsync_dir(getParamPath());
 }
+
+void Params::putNonBlocking(const std::string &key, const std::string &val) {
+   queue.push(std::make_pair(key, val));
+  // start thread on demand
+  if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
+    future = std::async(std::launch::async, &Params::asyncWriteThread, this);
+  }
+}
+
+void Params::asyncWriteThread() {
+  // TODO: write the latest one if a key has multiple values in the queue.
+  std::pair<std::string, std::string> p;
+  while (queue.try_pop(p, 0)) {
+    // Params::put is Thread-Safe
+    put(p.first, p.second);
+  }
+}

common/params.h

@@ -1,26 +1,37 @@
 #pragma once
 
+#include <future>
 #include <map>
 #include <string>
+#include <tuple>
+#include <utility>
 #include <vector>
 
+#include "common/queue.h"
+
 enum ParamKeyType {
   PERSISTENT = 0x02,
   CLEAR_ON_MANAGER_START = 0x04,
-  CLEAR_ON_IGNITION_ON = 0x08,
-  CLEAR_ON_IGNITION_OFF = 0x10,
+  CLEAR_ON_ONROAD_TRANSITION = 0x08,
+  CLEAR_ON_OFFROAD_TRANSITION = 0x10,
   DONT_LOG = 0x20,
+  DEVELOPMENT_ONLY = 0x40,
   ALL = 0xFFFFFFFF
 };
 
 class Params {
 public:
-  Params(const std::string &path = {});
+  explicit Params(const std::string &path = {});
+  ~Params();
+  // Not copyable.
+  Params(const Params&) = delete;
+  Params& operator=(const Params&) = delete;
+
   std::vector<std::string> allKeys() const;
   bool checkKey(const std::string &key);
   ParamKeyType getKeyType(const std::string &key);
   inline std::string getParamPath(const std::string &key = {}) {
-    return params_path + prefix + (key.empty() ? "" : "/" + key);
+    return params_path + params_prefix + (key.empty() ? "" : "/" + key);
   }
 
   // Delete a value
@@ -42,8 +53,18 @@ public:
   inline int putBool(const std::string &key, bool val) {
     return put(key.c_str(), val ? "1" : "0", 1);
   }
+  void putNonBlocking(const std::string &key, const std::string &val);
+  inline void putBoolNonBlocking(const std::string &key, bool val) {
+    putNonBlocking(key, val ? "1" : "0");
+  }
 
 private:
+  void asyncWriteThread();
+
   std::string params_path;
-  std::string prefix;
+  std::string params_prefix;
+
+  // for nonblocking write
+  std::future<void> future;
+  SafeQueue<std::pair<std::string, std::string>> queue;
 };

common/params.py

@@ -1,9 +1,7 @@
-from common.params_pyx import Params, ParamKeyType, UnknownKeyName, put_nonblocking, put_bool_nonblocking # pylint: disable=no-name-in-module, import-error
+from openpilot.common.params_pyx import Params, ParamKeyType, UnknownKeyName
 assert Params
 assert ParamKeyType
 assert UnknownKeyName
-assert put_nonblocking
-assert put_bool_nonblocking
 
 if __name__ == "__main__":
   import sys

common/params_pyx.pyx

@@ -3,22 +3,24 @@
 from libcpp cimport bool
 from libcpp.string cimport string
 from libcpp.vector cimport vector
-import threading
 
 cdef extern from "common/params.h":
   cpdef enum ParamKeyType:
     PERSISTENT
     CLEAR_ON_MANAGER_START
-    CLEAR_ON_IGNITION_ON
-    CLEAR_ON_IGNITION_OFF
+    CLEAR_ON_ONROAD_TRANSITION
+    CLEAR_ON_OFFROAD_TRANSITION
+    DEVELOPMENT_ONLY
     ALL
 
   cdef cppclass c_Params "Params":
-    c_Params(string) nogil
+    c_Params(string) except + nogil
     string get(string, bool) nogil
     bool getBool(string, bool) nogil
     int remove(string) nogil
     int put(string, string) nogil
+    void putNonBlocking(string, string) nogil
+    void putBoolNonBlocking(string, bool) nogil
     int putBool(string, bool) nogil
     bool checkKey(string) nogil
     string getParamPath(string) nogil
@@ -27,7 +29,7 @@ cdef extern from "common/params.h":
 
 
 def ensure_bytes(v):
-  return v.encode() if isinstance(v, str) else v;
+  return v.encode() if isinstance(v, str) else v
 
 class UnknownKeyName(Exception):
   pass
@@ -79,7 +81,7 @@ cdef class Params:
     """
     Warning: This function blocks until the param is written to disk!
     In very rare cases this can take over a second, and your code will hang.
-    Use the put_nonblocking helper function in time sensitive code, but
+    Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
     in general try to avoid writing params as much as possible.
     """
     cdef string k = self.check_key(key)
@@ -92,6 +94,17 @@ cdef class Params:
     with nogil:
       self.p.putBool(k, val)
 
+  def put_nonblocking(self, key, dat):
+    cdef string k = self.check_key(key)
+    cdef string dat_bytes = ensure_bytes(dat)
+    with nogil:
+      self.p.putNonBlocking(k, dat_bytes)
+
+  def put_bool_nonblocking(self, key, bool val):
+    cdef string k = self.check_key(key)
+    with nogil:
+      self.p.putBoolNonBlocking(k, val)
+
   def remove(self, key):
     cdef string k = self.check_key(key)
     with nogil:
@@ -103,9 +116,3 @@ cdef class Params:
 
   def all_keys(self):
     return self.p.allKeys()
-
-def put_nonblocking(key, val, d=""):
-  threading.Thread(target=lambda: Params(d).put(key, val)).start()
-
-def put_bool_nonblocking(key, bool val, d=""):
-  threading.Thread(target=lambda: Params(d).put_bool(key, val)).start()

common/prefix.py

@@ -0,0 +1,51 @@
+import os
+import shutil
+import uuid
+
+
+from openpilot.common.params import Params
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
+
+class OpenpilotPrefix:
+  def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
+    self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
+    self.msgq_path = os.path.join('/dev/shm', self.prefix)
+    self.clean_dirs_on_exit = clean_dirs_on_exit
+    self.shared_download_cache = shared_download_cache
+
+  def __enter__(self):
+    self.original_prefix = os.environ.get('OPENPILOT_PREFIX', None)
+    os.environ['OPENPILOT_PREFIX'] = self.prefix
+    try:
+      os.mkdir(self.msgq_path)
+    except FileExistsError:
+      pass
+    os.makedirs(Paths.log_root(), exist_ok=True)
+
+    if self.shared_download_cache:
+      os.environ["COMMA_CACHE"] = DEFAULT_DOWNLOAD_CACHE_ROOT
+
+    return self
+
+  def __exit__(self, exc_type, exc_obj, exc_tb):
+    if self.clean_dirs_on_exit:
+      self.clean_dirs()
+    try:
+      del os.environ['OPENPILOT_PREFIX']
+      if self.original_prefix is not None:
+        os.environ['OPENPILOT_PREFIX'] = self.original_prefix
+    except KeyError:
+      pass
+    return False
+
+  def clean_dirs(self):
+    symlink_path = Params().get_param_path()
+    if os.path.exists(symlink_path):
+      shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True)
+      os.remove(symlink_path)
+    shutil.rmtree(self.msgq_path, ignore_errors=True)
+    shutil.rmtree(Paths.log_root(), ignore_errors=True)
+    if not os.environ.get("COMMA_CACHE", False):
+      shutil.rmtree(Paths.download_cache_root(), ignore_errors=True)
+    shutil.rmtree(Paths.comma_home(), ignore_errors=True)

common/profiler.py

@@ -1,45 +0,0 @@
-import time
-
-class Profiler():
-  def __init__(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.time()
-    self.last_time = self.start_time
-    self.tot = 0.
-
-  def reset(self, enabled=False):
-    self.enabled = enabled
-    self.cp = {}
-    self.cp_ignored = []
-    self.iter = 0
-    self.start_time = time.time()
-    self.last_time = self.start_time
-
-  def checkpoint(self, name, ignore=False):
-    # ignore flag needed when benchmarking threads with ratekeeper
-    if not self.enabled:
-      return
-    tt = time.time()
-    if name not in self.cp:
-      self.cp[name] = 0.
-      if ignore:
-        self.cp_ignored.append(name)
-    self.cp[name] += tt - self.last_time
-    if not ignore:
-      self.tot += tt - self.last_time
-    self.last_time = tt
-
-  def display(self):
-    if not self.enabled:
-      return
-    self.iter += 1
-    print("******* Profiling %d *******" % self.iter)
-    for n, ms in sorted(self.cp.items(), key=lambda x: -x[1]):
-      if n in self.cp_ignored:
-        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f   IGNORED" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
-      else:
-        print("%30s: %9.2f  avg: %7.2f  percent: %3.0f" % (n, ms*1000.0, ms*1000.0/self.iter, ms/self.tot*100))
-    print(f"Iter clock: {self.tot / self.iter:2.6f}   TOTAL: {self.tot:2.2f}")

common/ratekeeper.cc

@@ -0,0 +1,40 @@
+#include "common/ratekeeper.h"
+
+#include <algorithm>
+
+#include "common/swaglog.h"
+#include "common/timing.h"
+#include "common/util.h"
+
+RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
+    : name(name),
+      print_delay_threshold(std::max(0.f, print_delay_threshold)) {
+  interval = 1 / rate;
+  last_monitor_time = seconds_since_boot();
+  next_frame_time = last_monitor_time + interval;
+}
+
+bool RateKeeper::keepTime() {
+  bool lagged = monitorTime();
+  if (remaining_ > 0) {
+    util::sleep_for(remaining_ * 1000);
+  }
+  return lagged;
+}
+
+bool RateKeeper::monitorTime() {
+  ++frame_;
+  last_monitor_time = seconds_since_boot();
+  remaining_ = next_frame_time - last_monitor_time;
+
+  bool lagged = remaining_ < 0;
+  if (lagged) {
+    if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
+      LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
+    }
+    next_frame_time = last_monitor_time + interval;
+  } else {
+    next_frame_time += interval;
+  }
+  return lagged;
+}

common/ratekeeper.h

@@ -0,0 +1,23 @@
+#pragma once
+
+#include <cstdint>
+#include <string>
+
+class RateKeeper {
+public:
+  RateKeeper(const std::string &name, float rate, float print_delay_threshold = 0);
+  ~RateKeeper() {}
+  bool keepTime();
+  bool monitorTime();
+  inline double frame() const { return frame_; }
+  inline double remaining() const { return remaining_; }
+
+private:
+  double interval;
+  double next_frame_time;
+  double last_monitor_time;
+  double remaining_ = 0;
+  float print_delay_threshold = 0;
+  uint64_t frame_ = 0;
+  std::string name;
+};

common/realtime.py

@@ -3,12 +3,10 @@ import gc
 import os
 import time
 from collections import deque
-from typing import Optional, List, Union
 
-from setproctitle import getproctitle  # pylint: disable=no-name-in-module
+from setproctitle import getproctitle
 
-from common.clock import sec_since_boot  # pylint: disable=no-name-in-module, import-error
-from system.hardware import PC
+from openpilot.system.hardware import PC
 
 
 # time step for each process
@@ -31,15 +29,15 @@ class Priority:
 
 def set_realtime_priority(level: int) -> None:
   if not PC:
-    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))  # pylint: disable=no-member
+    os.sched_setscheduler(0, os.SCHED_FIFO, os.sched_param(level))
 
 
-def set_core_affinity(cores: List[int]) -> None:
+def set_core_affinity(cores: list[int]) -> None:
   if not PC:
-    os.sched_setaffinity(0, cores)  # pylint: disable=no-member
+    os.sched_setaffinity(0, cores)
 
 
-def config_realtime_process(cores: Union[int, List[int]], priority: int) -> None:
+def config_realtime_process(cores: int | list[int], priority: int) -> None:
   gc.disable()
   set_realtime_priority(priority)
   c = cores if isinstance(cores, list) else [cores, ]
@@ -47,16 +45,16 @@ def config_realtime_process(cores: Union[int, List[int]], priority: int) -> None
 
 
 class Ratekeeper:
-  def __init__(self, rate: float, print_delay_threshold: Optional[float] = 0.0) -> None:
+  def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
     """Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
     self._interval = 1. / rate
-    self._next_frame_time = sec_since_boot() + self._interval
+    self._next_frame_time = time.monotonic() + self._interval
     self._print_delay_threshold = print_delay_threshold
     self._frame = 0
     self._remaining = 0.0
     self._process_name = getproctitle()
     self._dts = deque([self._interval], maxlen=100)
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
 
   @property
   def frame(self) -> int:
@@ -79,14 +77,14 @@ class Ratekeeper:
       time.sleep(self._remaining)
     return lagged
 
-  # this only monitor the cumulative lag, but does not enforce a rate
+  # Monitors the cumulative lag, but does not enforce a rate
   def monitor_time(self) -> bool:
     prev = self._last_monitor_time
-    self._last_monitor_time = sec_since_boot()
+    self._last_monitor_time = time.monotonic()
     self._dts.append(self._last_monitor_time - prev)
 
     lagged = False
-    remaining = self._next_frame_time - sec_since_boot()
+    remaining = self._next_frame_time - time.monotonic()
     self._next_frame_time += self._interval
     if self._print_delay_threshold is not None and remaining < -self._print_delay_threshold:
       print(f"{self._process_name} lagging by {-remaining * 1000:.2f} ms")

common/retry.py

@@ -0,0 +1,30 @@
+import time
+import functools
+
+from openpilot.common.swaglog import cloudlog
+
+
+def retry(attempts=3, delay=1.0, ignore_failure=False):
+  def decorator(func):
+    @functools.wraps(func)
+    def wrapper(*args, **kwargs):
+      for _ in range(attempts):
+        try:
+          return func(*args, **kwargs)
+        except Exception:
+          cloudlog.exception(f"{func.__name__} failed, trying again")
+          time.sleep(delay)
+
+      if ignore_failure:
+        cloudlog.error(f"{func.__name__} failed after retry")
+      else:
+        raise Exception(f"{func.__name__} failed after retry")
+    return wrapper
+  return decorator
+
+
+if __name__ == "__main__":
+  @retry(attempts=10)
+  def abc():
+    raise ValueError("abc failed :(")
+  abc()

common/simple_kalman.py

@@ -0,0 +1,54 @@
+import numpy as np
+
+
+def get_kalman_gain(dt, A, C, Q, R, iterations=100):
+  P = np.zeros_like(Q)
+  for _ in range(iterations):
+    P = A.dot(P).dot(A.T) + dt * Q
+    S = C.dot(P).dot(C.T) + R
+    K = P.dot(C.T).dot(np.linalg.inv(S))
+    P = (np.eye(len(P)) - K.dot(C)).dot(P)
+  return K
+
+
+class KF1D:
+  # this EKF assumes constant covariance matrix, so calculations are much simpler
+  # the Kalman gain also needs to be precomputed using the control module
+
+  def __init__(self, x0, A, C, K):
+    self.x0_0 = x0[0][0]
+    self.x1_0 = x0[1][0]
+    self.A0_0 = A[0][0]
+    self.A0_1 = A[0][1]
+    self.A1_0 = A[1][0]
+    self.A1_1 = A[1][1]
+    self.C0_0 = C[0]
+    self.C0_1 = C[1]
+    self.K0_0 = K[0][0]
+    self.K1_0 = K[1][0]
+
+    self.A_K_0 = self.A0_0 - self.K0_0 * self.C0_0
+    self.A_K_1 = self.A0_1 - self.K0_0 * self.C0_1
+    self.A_K_2 = self.A1_0 - self.K1_0 * self.C0_0
+    self.A_K_3 = self.A1_1 - self.K1_0 * self.C0_1
+
+    # K matrix needs to  be pre-computed as follow:
+    # import control
+    # (x, l, K) = control.dare(np.transpose(self.A), np.transpose(self.C), Q, R)
+    # self.K = np.transpose(K)
+
+  def update(self, meas):
+    #self.x = np.dot(self.A_K, self.x) + np.dot(self.K, meas)
+    x0_0 = self.A_K_0 * self.x0_0 + self.A_K_1 * self.x1_0 + self.K0_0 * meas
+    x1_0 = self.A_K_2 * self.x0_0 + self.A_K_3 * self.x1_0 + self.K1_0 * meas
+    self.x0_0 = x0_0
+    self.x1_0 = x1_0
+    return [self.x0_0, self.x1_0]
+
+  @property
+  def x(self):
+    return [[self.x0_0], [self.x1_0]]
+
+  def set_x(self, x):
+    self.x0_0 = x[0][0]
+    self.x1_0 = x[1][0]

common/spinner.py

@@ -1,6 +1,6 @@
 import os
 import subprocess
-from common.basedir import BASEDIR
+from openpilot.common.basedir import BASEDIR
 
 
 class Spinner():
@@ -29,11 +29,11 @@ class Spinner():
 
   def close(self):
     if self.spinner_proc is not None:
+      self.spinner_proc.kill()
       try:
-        self.spinner_proc.stdin.close()
-      except BrokenPipeError:
-        pass
-      self.spinner_proc.terminate()
+        self.spinner_proc.communicate(timeout=2.)
+      except subprocess.TimeoutExpired:
+        print("WARNING: failed to kill spinner")
       self.spinner_proc = None
 
   def __del__(self):

common/statlog.cc

@@ -1,46 +0,0 @@
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include "common/statlog.h"
-#include "common/util.h"
-
-#include <stdio.h>
-#include <mutex>
-#include <zmq.h>
-
-class StatlogState : public LogState {
-  public:
-    StatlogState() : LogState("ipc:///tmp/stats") {}
-};
-
-static StatlogState s = {};
-
-static void log(const char* metric_type, const char* metric, const char* fmt, ...) {
-  std::lock_guard lk(s.lock);
-  if (!s.initialized) s.initialize();
-
-  char* value_buf = nullptr;
-  va_list args;
-  va_start(args, fmt);
-  int ret = vasprintf(&value_buf, fmt, args);
-  va_end(args);
-
-  if (ret > 0 && value_buf) {
-    char* line_buf = nullptr;
-    ret = asprintf(&line_buf, "%s:%s|%s", metric, value_buf, metric_type);
-    if (ret > 0 && line_buf) {
-      zmq_send(s.sock, line_buf, ret, ZMQ_NOBLOCK);
-      free(line_buf);
-    }
-    free(value_buf);
-  }
-}
-
-void statlog_log(const char* metric_type, const char* metric, int value) {
-  log(metric_type, metric, "%d", value);
-}
-
-void statlog_log(const char* metric_type, const char* metric, float value) {
-  log(metric_type, metric, "%f", value);
-}

common/statlog.h

@@ -1,10 +0,0 @@
-#pragma once
-
-#define STATLOG_GAUGE "g"
-#define STATLOG_SAMPLE "sa"
-
-void statlog_log(const char* metric_type, const char* metric, int value);
-void statlog_log(const char* metric_type, const char* metric, float value);
-
-#define statlog_gauge(metric, value) statlog_log(STATLOG_GAUGE, metric, value)
-#define statlog_sample(metric, value) statlog_log(STATLOG_SAMPLE, metric, value)

common/swaglog.cc

@@ -5,29 +5,29 @@
 #include "common/swaglog.h"
 
 #include <cassert>
-#include <cstring>
 #include <limits>
 #include <mutex>
 #include <string>
 
 #include <zmq.h>
-#include "json11.hpp"
-
-#include "common/util.h"
+#include <stdarg.h>
+#include "third_party/json11/json11.hpp"
 #include "common/version.h"
 #include "system/hardware/hw.h"
 
-class SwaglogState : public LogState {
- public:
-  SwaglogState() : LogState("ipc:///tmp/logmessage") {}
+class SwaglogState {
+public:
+  SwaglogState() {
+    zctx = zmq_ctx_new();
+    sock = zmq_socket(zctx, ZMQ_PUSH);
 
-  json11::Json::object ctx_j;
+    // Timeout on shutdown for messages to be received by the logging process
+    int timeout = 100;
+    zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
+    zmq_connect(sock, Path::swaglog_ipc().c_str());
 
-  inline void initialize() {
-    ctx_j = json11::Json::object {};
     print_level = CLOUDLOG_WARNING;
-    const char* print_lvl = getenv("LOGPRINT");
-    if (print_lvl) {
+    if (const char* print_lvl = getenv("LOGPRINT")) {
       if (strcmp(print_lvl, "debug") == 0) {
         print_level = CLOUDLOG_DEBUG;
       } else if (strcmp(print_lvl, "info") == 0) {
@@ -37,40 +37,53 @@ class SwaglogState : public LogState {
       }
     }
 
-    // openpilot bindings
-    char* dongle_id = getenv("DONGLE_ID");
-    if (dongle_id) {
+    ctx_j = json11::Json::object{};
+    if (char* dongle_id = getenv("DONGLE_ID")) {
       ctx_j["dongle_id"] = dongle_id;
     }
-    char* daemon_name = getenv("MANAGER_DAEMON");
-    if (daemon_name) {
+    if (char* git_origin = getenv("GIT_ORIGIN")) {
+      ctx_j["origin"] = git_origin;
+    }
+    if (char* git_branch = getenv("GIT_BRANCH")) {
+      ctx_j["branch"] = git_branch;
+    }
+    if (char* git_commit = getenv("GIT_COMMIT")) {
+      ctx_j["commit"] = git_commit;
+    }
+    if (char* daemon_name = getenv("MANAGER_DAEMON")) {
       ctx_j["daemon"] = daemon_name;
     }
     ctx_j["version"] = COMMA_VERSION;
     ctx_j["dirty"] = !getenv("CLEAN");
-
-    // device type
     ctx_j["device"] = Hardware::get_name();
-    LogState::initialize();
   }
+
+  ~SwaglogState() {
+    zmq_close(sock);
+    zmq_ctx_destroy(zctx);
+  }
+
+  void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
+    std::lock_guard lk(lock);
+    if (levelnum >= print_level) {
+      printf("%s: %s\n", filename, msg);
+    }
+    zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
+  }
+
+  std::mutex lock;
+  void* zctx = nullptr;
+  void* sock = nullptr;
+  int print_level;
+  json11::Json::object ctx_j;
 };
 
-static SwaglogState s = {};
 bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
 uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
 
-static void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
-  if (levelnum >= s.print_level) {
-    printf("%s: %s\n", filename, msg);
-  }
-  char levelnum_c = levelnum;
-  zmq_send(s.sock, (levelnum_c + log_s).c_str(), log_s.length() + 1, ZMQ_NOBLOCK);
-}
-
 static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
                             char* msg_buf, const json11::Json::object &msg_j={}) {
-  std::lock_guard lk(s.lock);
-  if (!s.initialized) s.initialize();
+  static SwaglogState s;
 
   json11::Json::object log_j = json11::Json::object {
     {"ctx", s.ctx_j},
@@ -86,8 +99,11 @@ static void cloudlog_common(int levelnum, const char* filename, int lineno, cons
     log_j["msg"] = msg_j;
   }
 
-  std::string log_s = ((json11::Json)log_j).dump();
-  log(levelnum, filename, lineno, func, msg_buf, log_s);
+  std::string log_s;
+  log_s += (char)levelnum;
+  ((json11::Json)log_j).dump(log_s);
+  s.log(levelnum, filename, lineno, func, msg_buf, log_s);
+
   free(msg_buf);
 }
 
@@ -134,4 +150,3 @@ void cloudlog_te(int levelnum, const char* filename, int lineno, const char* fun
   cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
   va_end(args);
 }
-

common/swaglog.h

@@ -9,23 +9,29 @@
 #define CLOUDLOG_CRITICAL 50
 
 
+#ifdef __GNUC__
+#define SWAG_LOG_CHECK_FMT(a, b) __attribute__ ((format (printf, a, b)))
+#else
+#define SWAG_LOG_CHECK_FMT(a, b)
+#endif
+
 void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
-                const char* fmt, ...) /*__attribute__ ((format (printf, 6, 7)))*/;
+                const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
 
 void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
-                 const char* fmt, ...) /*__attribute__ ((format (printf, 6, 7)))*/;
+                 const char* fmt, ...) SWAG_LOG_CHECK_FMT(5, 6);
 
 void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
-                 uint32_t frame_id, const char* fmt, ...) /*__attribute__ ((format (printf, 6, 7)))*/;
+                 uint32_t frame_id, const char* fmt, ...) SWAG_LOG_CHECK_FMT(6, 7);
 
 
 #define cloudlog(lvl, fmt, ...) cloudlog_e(lvl, __FILE__, __LINE__, \
                                            __func__, \
-                                           fmt, ## __VA_ARGS__);
- 
+                                           fmt, ## __VA_ARGS__)
+
 #define cloudlog_t(lvl, ...) cloudlog_te(lvl, __FILE__, __LINE__, \
                                           __func__, \
-                                          __VA_ARGS__);
+                                          __VA_ARGS__)
 
 
 #define cloudlog_rl(burst, millis, lvl, fmt, ...)   \
@@ -38,7 +44,7 @@ void cloudlog_te(int levelnum, const char* filename, int lineno, const char* fun
   int __millis = (millis);                          \
   uint64_t __ts = nanos_since_boot();               \
                                                     \
-  if (!__begin) __begin = __ts;                     \
+  if (!__begin) { __begin = __ts; }                 \
                                                     \
   if (__begin + __millis*1000000ULL < __ts) {       \
     if (__missed) {                                 \

common/swaglog.py

@@ -1,22 +1,19 @@
 import logging
 import os
 import time
+import warnings
 from pathlib import Path
 from logging.handlers import BaseRotatingHandler
 
 import zmq
 
-from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from system.hardware import PC
+from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
+from openpilot.system.hardware.hw import Paths
 
-if PC:
-  SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
-else:
-  SWAGLOG_DIR = "/data/log/"
 
 def get_file_handler():
-  Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
-  base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
+  Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
+  base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
   handler = SwaglogRotatingFileHandler(base_filename)
   return handler
 
@@ -72,15 +69,29 @@ class UnixDomainSocketHandler(logging.Handler):
     self.setFormatter(formatter)
     self.pid = None
 
+    self.zctx = None
+    self.sock = None
+
+  def __del__(self):
+    self.close()
+
+  def close(self):
+    if self.sock is not None:
+      self.sock.close()
+    if self.zctx is not None:
+      self.zctx.term()
+
   def connect(self):
     self.zctx = zmq.Context()
     self.sock = self.zctx.socket(zmq.PUSH)
     self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect("ipc:///tmp/logmessage")
+    self.sock.connect(Paths.swaglog_ipc())
     self.pid = os.getpid()
 
   def emit(self, record):
     if os.getpid() != self.pid:
+      # TODO suppresses warning about forking proc with zmq socket, fix root cause
+      warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
       self.connect()
 
     msg = self.format(record).rstrip('\n')
@@ -117,6 +128,8 @@ elif print_level == 'info':
 elif print_level == 'warning':
   outhandler.setLevel(logging.WARNING)
 
+ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
+
 log.addHandler(outhandler)
 # logs are sent through IPC before writing to disk to prevent disk I/O blocking
-log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))
+log.addHandler(ipchandler)

common/tests/.gitignore

@@ -1,2 +1 @@
-test_util
-test_swaglog
+test_common

common/tests/test_file_helpers.py

@@ -2,8 +2,7 @@ import os
 import unittest
 from uuid import uuid4
 
-from common.file_helpers import atomic_write_on_fs_tmp
-from common.file_helpers import atomic_write_in_dir
+from openpilot.common.file_helpers import atomic_write_in_dir
 
 
 class TestFileHelpers(unittest.TestCase):
@@ -11,14 +10,12 @@ class TestFileHelpers(unittest.TestCase):
     path = f"/tmp/tmp{uuid4()}"
     with atomic_write_func(path) as f:
       f.write("test")
+      assert not os.path.exists(path)
 
     with open(path) as f:
       self.assertEqual(f.read(), "test")
     os.remove(path)
 
-  def test_atomic_write_on_fs_tmp(self):
-    self.run_atomic_write_func(atomic_write_on_fs_tmp)
-
   def test_atomic_write_in_dir(self):
     self.run_atomic_write_func(atomic_write_in_dir)
 

common/tests/test_numpy_fast.py

@@ -1,7 +1,7 @@
 import numpy as np
 import unittest
 
-from common.numpy_fast import interp
+from openpilot.common.numpy_fast import interp
 
 
 class InterpTest(unittest.TestCase):

common/tests/test_params.cc

@@ -0,0 +1,27 @@
+#include "catch2/catch.hpp"
+#define private public
+#include "common/params.h"
+#include "common/util.h"
+
+TEST_CASE("params_nonblocking_put") {
+  char tmp_path[] = "/tmp/asyncWriter_XXXXXX";
+  const std::string param_path = mkdtemp(tmp_path);
+  auto param_names = {"CarParams", "IsMetric"};
+  {
+    Params params(param_path);
+    for (const auto &name : param_names) {
+      params.putNonBlocking(name, "1");
+      // param is empty
+      REQUIRE(params.get(name).empty());
+    }
+
+    // check if thread is running
+    REQUIRE(params.future.valid());
+    REQUIRE(params.future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout);
+  }
+  // check results
+  Params p(param_path);
+  for (const auto &name : param_names) {
+    REQUIRE(p.get(name) == "1");
+  }
+}

common/tests/test_params.py

@@ -1,19 +1,14 @@
+import os
 import threading
 import time
-import tempfile
-import shutil
+import uuid
 import unittest
 
-from common.params import Params, ParamKeyType, UnknownKeyName, put_nonblocking, put_bool_nonblocking
+from openpilot.common.params import Params, ParamKeyType, UnknownKeyName
 
 class TestParams(unittest.TestCase):
   def setUp(self):
-    self.tmpdir = tempfile.mkdtemp()
-    print("using", self.tmpdir)
-    self.params = Params(self.tmpdir)
-
-  def tearDown(self):
-    shutil.rmtree(self.tmpdir)
+    self.params = Params()
 
   def test_params_put_and_get(self):
     self.params.put("DongleId", "cb38263377b873ee")
@@ -28,9 +23,16 @@ class TestParams(unittest.TestCase):
     self.params.put("CarParams", "test")
     self.params.put("DongleId", "cb38263377b873ee")
     assert self.params.get("CarParams") == b"test"
+
+    undefined_param = self.params.get_param_path(uuid.uuid4().hex)
+    with open(undefined_param, "w") as f:
+      f.write("test")
+    assert os.path.isfile(undefined_param)
+
     self.params.clear_all(ParamKeyType.CLEAR_ON_MANAGER_START)
     assert self.params.get("CarParams") is None
     assert self.params.get("DongleId") is not None
+    assert not os.path.isfile(undefined_param)
 
   def test_params_two_things(self):
     self.params.put("DongleId", "bob")
@@ -81,19 +83,19 @@ class TestParams(unittest.TestCase):
     self.assertFalse(self.params.get_bool("IsMetric"))
 
   def test_put_non_blocking_with_get_block(self):
-    q = Params(self.tmpdir)
+    q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      put_nonblocking("CarParams", "test", self.tmpdir)
+      Params().put_nonblocking("CarParams", "test")
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"test"
 
   def test_put_bool_non_blocking_with_get_block(self):
-    q = Params(self.tmpdir)
+    q = Params()
     def _delayed_writer():
       time.sleep(0.1)
-      put_bool_nonblocking("CarParams", True, self.tmpdir)
+      Params().put_bool_nonblocking("CarParams", True)
     threading.Thread(target=_delayed_writer).start()
     assert q.get("CarParams") is None
     assert q.get("CarParams", True) == b"1"