Compare commits

..

9 Commits

Author SHA1 Message Date
Jason Wen b24e762592 more 2024-11-21 10:40:58 -05:00
Jason Wen ed8812e730 FCA: S20 init 2024-11-21 10:18:40 -05:00
DevTekVE 36f8192612 ci: sunnypilot UI GH Action fix ensure push to master-ui artifact (#453)
Fixes the GitHub action that was failing when the pipeline was running as a result of a push to master-new because it was not properly recognized as "master"
2024-11-17 09:49:37 +01:00
DevTekVE 4b66cd0577 ci: fix repo permissions in ui preview (#450)
* more fix

* forcing a change on UI to test

* undo ui change for test

* temp add branch

* adding more ways to set the master uo

* more bruteforce

* undoing force
2024-11-16 22:27:29 -05:00
DevTekVE ad742515f1 ci: sunnypilot show UI diff on PRs (#449)
add master-new as well to the triggers for the diff ui
2024-11-16 23:45:12 +01:00
Jason Wen 2426354d72 bump opendbc (#448)
* bump opendbc

* Bump opendbc to our changed car params

* New Int16 test

* spFlags

* original name?

* Testing more carparams combinations

* Another crazy idea

* typo

* use this field

* bump

* bump opendbc

---------

Co-authored-by: DevTekVE <devtekve@gmail.com>
2024-11-16 17:04:12 -05:00
DevTekVE cf55992c26 CI: sunnypilot configuration (#447)
* initial updates to the actions to run for sp

* ignore license file

* more updates

* undoing some of the changes because I was blocking the runs on

* allowing the submodule check as well

* Allowing macos builds

* test adding cache key

* don't attempt build_release for selfdrive for the time being.

* Blocking macos builds as well since they have a 10x miltiplier for GH aciton minutes, waaaay too much!

* lol nice typo codespell

* change ref commit id to check if replay passes
2024-11-16 10:24:47 -05:00
DevTekVE 3da0dfd47f Updating submodules 2024-11-15 08:16:52 +01:00
Jason Wen 6edaf619bf Add Custom MIT License (#438) 2024-11-15 08:11:51 +01:00
200 changed files with 3210 additions and 4702 deletions
+3
View File
@@ -0,0 +1,3 @@
.Xauthority
.env
.host/
+18
View File
@@ -0,0 +1,18 @@
FROM ghcr.io/commaai/openpilot-base:latest
RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
RUN python3 -m ensurepip --upgrade
RUN pip3 install ipython jupyter jupyterlab
RUN cd /tmp && \
ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
curl -L -o virtualgl.deb "https://github.com/VirtualGL/virtualgl/releases/download/3.1.1/virtualgl_3.1.1_$ARCH.deb" && \
dpkg -i virtualgl.deb
RUN usermod -aG video batman
USER batman
RUN cd $HOME && \
curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc
+38
View File
@@ -0,0 +1,38 @@
#!/usr/bin/env bash
TARGET_USER=batman
source .devcontainer/.host/.env
# override display flag for mac hosts
if [[ $HOST_OS == darwin ]]; then
echo "Setting up DISPLAY override for macOS..."
cat <<EOF >> /home/$TARGET_USER/.bashrc
source .devcontainer/.host/.env
if [ -n "\$HOST_DISPLAY" ]; then
DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}')
export DISPLAY=host.docker.internal:\$DISPLAY_NUM
fi
EOF
fi
# setup virtualgl for mac hosts
if [[ $HOST_OS == darwin ]]; then
echo "Setting up virtualgl for macOS..."
cat <<EOF >> /home/$TARGET_USER/.bashrc
if [ -n "\$HOST_DISPLAY" ]; then
export VGL_PORT=10000
export VGL_CLIENT=host.docker.internal
export VGL_COMPRESS=rgb
export VGL_DISPLAY=:99
export VGL_FPS=60
# prevent vglrun from running exec
alias exec=:; source vglrun :; unalias exec
fi
EOF
fi
# These lines are temporary, to remain backwards compatible with old devcontainers
# that were running as root and therefore had their caches written as root
sudo chown -R $TARGET_USER: /tmp/scons_cache
sudo chown -R $TARGET_USER: /tmp/comma_download_cache
sudo chown -R $TARGET_USER: /home/batman/.comma
+15
View File
@@ -0,0 +1,15 @@
#!/usr/bin/env bash
source .devcontainer/.host/.env
# setup safe directories for submodules
SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
for DIR in $SUBMODULE_DIRS; do
git config --global --add safe.directory "$PWD/$DIR"
done
# virtual display for virtualgl
if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then
echo "Starting virtual display at :99 ..."
tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24
fi
+53
View File
@@ -0,0 +1,53 @@
{
"name": "openpilot devcontainer",
"build": {
"dockerfile": "Dockerfile"
},
"postCreateCommand": ".devcontainer/container_post_create.sh",
"postStartCommand": ".devcontainer/container_post_start.sh",
"initializeCommand": [".devcontainer/host_setup"],
"privileged": true,
"containerEnv": {
"DISPLAY": "${localEnv:DISPLAY}",
"PYTHONPATH": "${containerWorkspaceFolder}",
"TERM": "xterm-256color",
"force_color_prompt": "1"
},
"runArgs": [
"--volume=/dev:/dev",
"--volume=/tmp/.X11-unix:/tmp/.X11-unix",
"--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority",
"--volume=${localEnv:HOME}/.comma:/home/batman/.comma",
"--volume=${localEnv:HOME}/.azure:/home/batman/.azure",
"--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
"--shm-size=1G",
"--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux
"--publish=0.0.0.0:8070-8079:8070-8079" // body ZMQ services
],
"features": {
"ghcr.io/devcontainers/features/common-utils:2": {
"installZsh": false,
"installOhMyZsh": false,
"upgradePackages": false,
"username": "batman"
},
"ghcr.io/devcontainers-contrib/features/gh-cli:1": {},
"ghcr.io/devcontainers/features/azure-cli:1": {}
},
"containerUser": "batman",
"remoteUser": "batman",
"customizations": {
"vscode": {
"extensions": [
"ms-python.python",
"ms-vscode.cpptools",
"ms-toolsai.jupyter",
"guyskk.language-cython",
"lharri73.dbc"
]
}
},
"mounts": [
"type=volume,source=scons_cache,target=/tmp/scons_cache"
]
}
+47
View File
@@ -0,0 +1,47 @@
#!/usr/bin/env bash
# pull base image
if [[ -z $USE_LOCAL_IMAGE ]]; then
echo "Updating openpilot_base image if needed..."
docker pull ghcr.io/commaai/openpilot-base:latest
fi
# setup .host dir
mkdir -p .devcontainer/.host
# setup links to Xauthority
XAUTHORITY_LINK=".devcontainer/.host/.Xauthority"
rm -f $XAUTHORITY_LINK
if [[ -z $XAUTHORITY ]]; then
echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
if ! [[ -f $HOME/.Xauthority ]]; then
echo "~/.XAuthority file does not exist. GUI tools may not work properly."
touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
else
ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
fi
else
ln -sf $XAUTHORITY $XAUTHORITY_LINK
fi
# setup host env file
HOST_INFO_FILE=".devcontainer/.host/.env"
SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE
echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE
# run virtualgl if macos
if [[ $SYSTEM == "darwin" ]]; then
echo
if [[ -f /opt/VirtualGL/bin/vglclient ]]; then
echo "Starting VirtualGL client at port 10000..."
VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log"
mkdir -p "$(dirname $VGL_LOG_FILE)"
/opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach
else
echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:"
echo
echo " brew install --cask virtualgl"
echo
fi
fi
+10
View File
@@ -0,0 +1,10 @@
:: pull base image
IF NOT DEFINED USE_LOCAL_IMAGE ^
echo "Updating openpilot_base image if needed..." && ^
docker pull ghcr.io/commaai/openpilot-base:latest
:: setup .host dir
mkdir .devcontainer\.host
:: setup host env file
echo "" > .devcontainer\.host\.env
+1 -1
View File
@@ -8,7 +8,7 @@ assignees: ''
**Checklist**
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
- [ ] route with openpilot:
- [ ] route with stock system:
+1 -1
View File
@@ -44,7 +44,7 @@ Explain how you tested this bug fix.
**Checklist**
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/opcar/docs.py` to generate new docs
- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
- [ ] route with openpilot:
- [ ] route with stock system:
-9
View File
@@ -14,25 +14,17 @@ inputs:
description: 'whether to save the cache'
default: 'false'
required: false
outputs:
cache-hit:
description: 'cache hit occurred'
value: ${{ (contains(runner.name, 'nsc') && steps.ns-cache.outputs.cache-hit) ||
(!contains(runner.name, 'nsc') && inputs.save != 'false' && steps.gha-cache.outputs.cache-hit) ||
(!contains(runner.name, 'nsc') && inputs.save == 'false' && steps.gha-cache-ro.outputs.cache-hit) }}
runs:
using: "composite"
steps:
- name: setup namespace cache
id: ns-cache
if: ${{ contains(runner.name, 'nsc') }}
uses: namespacelabs/nscloud-cache-action@v1
with:
path: ${{ inputs.path }}
- name: setup github cache
id: gha-cache
if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
uses: 'actions/cache@v4'
with:
@@ -41,7 +33,6 @@ runs:
restore-keys: ${{ inputs.restore-keys }}
- name: setup github cache
id: gha-cache-ro
if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
uses: 'actions/cache/restore@v4'
with:
+5 -5
View File
@@ -24,7 +24,7 @@ jobs:
# Check PR target branch
- name: check branch
uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
@@ -37,17 +37,17 @@ jobs:
# Welcome comment
- name: "First timers PR"
uses: actions/first-interaction@v1
if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
if: github.event.pull_request.head.repo.full_name != 'sunnypilot/sunnypilot'
with:
repo-token: ${{ secrets.GITHUB_TOKEN }}
pr-message: |
<!-- _(run_id **${{ github.run_id }}**)_ -->
Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
* Convert your PR to a draft unless it's ready to review
* Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
* Read the [contributing docs](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md)
* Before marking as "ready for review", ensure:
* the goal is clearly stated in the description
* all the tests are passing
* the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
* the change is [something we merge](https://github.com/sunnypilot/sunnypilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
* include a route or your device' dongle ID if relevant
+2 -2
View File
@@ -13,7 +13,7 @@ jobs:
badges:
name: create badges
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
permissions:
contents: write
steps:
@@ -29,7 +29,7 @@ jobs:
git checkout --orphan badges
git rm -rf --cached .
git config user.email "badge-researcher@comma.ai"
git config user.email "badge-researcher@sunnypilot.ai"
git config user.name "Badge Researcher"
git add translation_badge.svg
+2 -2
View File
@@ -15,7 +15,7 @@ env:
jobs:
setup:
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
runs-on: ubuntu-latest
outputs:
ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
@@ -31,7 +31,7 @@ jobs:
strategy:
fail-fast: false
matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
uses: sunnypilot/sunnypilot/.github/workflows/ci_weekly_run.yaml@master
with:
run_number: ${{ matrix.run_number }}
+5 -1
View File
@@ -12,6 +12,10 @@ concurrency:
jobs:
selfdrive_tests:
uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
uses: sunnypilot/sunnypilot/.github/workflows/selfdrive_tests.yaml@master
with:
run_number: ${{ inputs.run_number }}
tools_tests:
uses: sunnypilot/sunnypilot/.github/workflows/tools_tests.yaml@master
with:
run_number: ${{ inputs.run_number }}
+5 -6
View File
@@ -18,10 +18,9 @@ concurrency:
jobs:
docs:
name: build docs
runs-on: ubuntu-24.04
runs-on: ubuntu-latest
timeout-minutes: 1
steps:
- uses: commaai/timeout@v1
- uses: actions/checkout@v4
with:
submodules: true
@@ -35,13 +34,13 @@ jobs:
# Push to docs.comma.ai
- uses: actions/checkout@v4
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
with:
path: openpilot-docs
ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
repository: commaai/openpilot-docs
repository: sunnypilot/sunnypilot-docs
- name: Push
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
run: |
set -x
+1 -1
View File
@@ -12,7 +12,7 @@ jobs:
build_prebuilt:
name: build prebuilt
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
env:
PUSH_IMAGE: true
permissions:
+1 -1
View File
@@ -13,7 +13,7 @@ jobs:
container:
image: ghcr.io/commaai/openpilot-base:latest
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
permissions:
checks: read
contents: write
+2 -2
View File
@@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
container:
image: ghcr.io/commaai/openpilot-base:latest
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
steps:
- uses: actions/checkout@v4
with:
@@ -28,7 +28,7 @@ jobs:
git add .
- name: update car docs
run: |
scons -j$(nproc) --minimal opendbc_repo
scons -j$(nproc) --minimal opendbc
PYTHONPATH=. python selfdrive/car/docs.py
git add docs/CARS.md
- name: Create Pull Request
+80 -37
View File
@@ -4,6 +4,7 @@ on:
push:
branches:
- master
- master-new
pull_request:
workflow_dispatch:
workflow_call:
@@ -14,10 +15,11 @@ on:
type: string
concurrency:
group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
REPORT_NAME: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
PYTHONWARNINGS: error
BASE_IMAGE: openpilot-base
AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
@@ -31,10 +33,11 @@ env:
jobs:
build_release:
if: github.repository == 'commaai/openpilot' # build_release blocked for the time being to only comma as we may have a different process.
name: build release
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
env:
STRIPPED_DIR: /tmp/releasepilot
steps:
@@ -52,7 +55,7 @@ jobs:
run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
- uses: ./.github/workflows/setup-with-retry
- name: Check submodules
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
timeout-minutes: 3
run: release/check-submodules.sh
- name: Build openpilot and run checks
@@ -75,9 +78,15 @@ jobs:
${{ env.RUN }} "scripts/lint/lint.sh --skip check_added_large_files"
build:
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
strategy:
matrix:
arch: ${{ fromJson(
((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
runs-on: ${{ ((matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8') ||
((matrix.arch == 'x86_64') && ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16') ||
'ubuntu-latest'}}
steps:
- uses: actions/checkout@v4
with:
@@ -86,14 +95,20 @@ jobs:
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
run: |
echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
$DOCKER_LOGIN
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- uses: ./.github/workflows/compile-openpilot
timeout-minutes: 30
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
build_mac:
if: github.repository == 'commaai/openpilot' # Blocking macos builds as well since they have a 10x miltiplier for GH action minutes, waaaay too much!
name: build macOS
runs-on: ${{ github.repository == 'commaai/openpilot' && 'namespace-profile-macos-8x14' || 'macos-latest' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
steps:
- uses: actions/checkout@v4
with:
@@ -102,6 +117,7 @@ jobs:
uses: ./.github/workflows/auto-cache
with:
path: ~/Library/Caches/Homebrew
key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
- name: Install dependencies
run: ./tools/mac_setup.sh
env:
@@ -113,14 +129,32 @@ jobs:
uses: ./.github/workflows/auto-cache
with:
path: /tmp/scons_cache
key: build_macos_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
- name: Building openpilot
run: . .venv/bin/activate && scons -j$(nproc)
docker_push_multiarch:
name: docker push multiarch tag
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: [build]
steps:
- uses: actions/checkout@v4
with:
submodules: false
- name: Setup docker
run: |
$DOCKER_LOGIN
- name: Merge x64 and arm64 tags
run: |
export PUSH_IMAGE=true
scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
static_analysis:
name: static analysis
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
env:
PYTHONWARNINGS: default
steps:
@@ -134,16 +168,24 @@ jobs:
unit_tests:
name: unit tests
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- name: Build openpilot
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Setup cache
uses: ./.github/workflows/auto-cache
with:
path: .ci_cache/comma_download_cache
key: unit_tests_${{ hashFiles('.github/workflows/selfdrive_tests.yaml') }}
- name: Run unit tests
timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
run: |
@@ -161,25 +203,27 @@ jobs:
process_replay:
name: process replay
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- name: Cache test routes
id: dependency-cache
uses: actions/cache@v4
with:
path: .ci_cache/comma_download_cache
key: proc-replay-${{ hashFiles('selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_processes.py') }}
key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.dependency-cache.outputs.cache-hit == 'true') && 1 || 20 }}
timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
run: |
${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache && \
@@ -214,9 +258,9 @@ jobs:
test_cars:
name: cars
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
strategy:
fail-fast: false
matrix:
@@ -227,17 +271,17 @@ jobs:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Cache test routes
id: routes-cache
uses: actions/cache@v4
id: dependency-cache
uses: ./.github/workflows/auto-cache
with:
path: .ci_cache/comma_download_cache
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'opendbc/car/tests/routes.py') }}-${{ matrix.job }}
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Test car models
timeout-minutes: ${{ contains(runner.name, 'nsc') && (steps.routes-cache.outputs.cache-hit == 'true') && 1 || 20 }}
timeout-minutes: ${{ contains(runner.name, 'nsc') && 1 || 20 }}
run: |
${{ env.RUN }} "MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
${{ env.RUN }} "FILEREADER_CACHE=1 MAX_EXAMPLES=1 $PYTEST selfdrive/car/tests/test_models.py && \
chmod -R 777 /tmp/comma_download_cache"
env:
NUM_JOBS: 4
@@ -306,10 +350,9 @@ jobs:
simulator_driving:
name: simulator driving
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
if: (github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
steps:
- uses: actions/checkout@v4
with:
@@ -328,9 +371,9 @@ jobs:
create_ui_report:
# This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
name: Create UI Report
runs-on:
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
- ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-experiments:docker.builds.local-cache=separate' || 'ubuntu-24.04' }}
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-24.04' }}
steps:
- uses: actions/checkout@v4
with:
@@ -354,5 +397,5 @@ jobs:
- name: Upload Test Report
uses: actions/upload-artifact@v4
with:
name: report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
name: ${{ env.REPORT_NAME }}
path: selfdrive/ui/tests/test_ui/report_1/screenshots
@@ -17,6 +17,7 @@ runs:
uses: ./.github/workflows/setup
continue-on-error: true
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
- if: steps.setup1.outcome == 'failure'
shell: bash
@@ -26,6 +27,7 @@ runs:
uses: ./.github/workflows/setup
continue-on-error: true
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
- if: steps.setup2.outcome == 'failure'
shell: bash
@@ -34,4 +36,5 @@ runs:
if: steps.setup2.outcome == 'failure'
uses: ./.github/workflows/setup
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
+22 -3
View File
@@ -1,6 +1,10 @@
name: 'openpilot env setup'
inputs:
docker_hub_pat:
description: 'Auth token for Docker Hub, required for BuildJet jobs'
required: true
default: ''
is_retried:
description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
required: false
@@ -20,9 +24,11 @@ runs:
name: No retries!
run: |
if [ "${{ github.run_attempt }}" -gt 1 ]; then
echo -e "\033[0;31m##################################################"
echo -e "\033[0;31m Retries not allowed! Fix the flaky test! "
echo -e "\033[0;31m##################################################\033[0m"
echo -e "\033[31m"
echo "##################################################"
echo " Retries not allowed! Fix the flaky test! "
echo "##################################################"
echo -e "\033[0m"
exit 1
fi
@@ -30,6 +36,19 @@ runs:
- shell: bash
run: git lfs pull
# on BuildJet runners, must be logged into DockerHub to avoid rate limiting
# https://buildjet.com/for-github-actions/docs/guides/docker
- shell: bash
if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }}
run: |
echo "Need to set the Docker Hub PAT secret as an input to this action"
exit 1
- name: Login to Docker Hub
if: contains(runner.name, 'buildjet')
shell: bash
run: |
docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }}
# build cache
- id: date
shell: bash
+1 -1
View File
@@ -20,7 +20,7 @@ jobs:
stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
stale-pr-label: stale
delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'sunnypilot/sunnypilot' }} # only delete branches on the main repo
exempt-pr-labels: "ignore stale,needs testing" # if wip or it needs testing from the community, don't mark as stale
days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
+55
View File
@@ -0,0 +1,55 @@
name: tools
on:
push:
branches:
- master
- master-new
pull_request:
workflow_call:
inputs:
run_number:
default: '1'
required: true
type: string
concurrency:
group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
BASE_IMAGE: openpilot-base
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh base
RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
jobs:
devcontainer:
name: devcontainer
runs-on: ubuntu-latest
if: false # we can re-enable once this is faster
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Use local image for testing devcontainer with latest base image
run: |
echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
- name: Setup Dev Container CLI
run: npm install -g @devcontainers/cli
- name: Build dev container image
run: ./scripts/retry.sh devcontainer build --workspace-folder .
- name: Run dev container
run: |
mkdir -p /tmp/devcontainer_scons_cache/
cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
devcontainer up --workspace-folder .
- name: Test environment
run: |
devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
devcontainer exec --workspace-folder . pip3 install pip-install-test
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
devcontainer exec --workspace-folder . sudo touch /root/test.txt
+12 -10
View File
@@ -3,23 +3,25 @@ on:
push:
branches:
- master
- master-new
pull_request_target:
types: [assigned, opened, synchronize, reopened, edited]
branches:
- 'master'
- 'master-new'
paths:
- 'selfdrive/ui/**'
workflow_dispatch:
env:
UI_JOB_NAME: "Create UI Report"
REPORT_NAME: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
SHA: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.sha || github.event.pull_request.head.sha }}
REPORT_NAME: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && 'master' || github.event.number }}
SHA: ${{ github.event_name == 'push' && (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.sha || github.event.pull_request.head.sha }}
BRANCH_NAME: "openpilot/pr-${{ github.event.number }}"
jobs:
preview:
if: github.repository == 'commaai/openpilot'
if: github.repository == 'sunnypilot/sunnypilot'
name: preview
runs-on: ubuntu-latest
timeout-minutes: 20
@@ -58,13 +60,13 @@ jobs:
- name: Getting master ui
uses: actions/checkout@v4
with:
repository: commaai/ci-artifacts
repository: sunnypilot/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
path: ${{ github.workspace }}/master_ui
ref: openpilot_master_ui
- name: Saving new master ui
if: github.ref == 'refs/heads/master' && github.event_name == 'push'
if: (github.ref == 'refs/heads/master' || github.ref == 'refs/heads/master-new') && github.event_name == 'push'
working-directory: ${{ github.workspace }}/master_ui
run: |
git checkout --orphan=new_master_ui
@@ -106,13 +108,13 @@ jobs:
DIFF="${DIFF}<table>"
DIFF="${DIFF}<tr>"
DIFF="${DIFF} <td> master <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
DIFF="${DIFF} <td> proposed <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
DIFF="${DIFF} <td> master <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_master_ref.png\"> </td>"
DIFF="${DIFF} <td> proposed <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
DIFF="${DIFF}</tr>"
DIFF="${DIFF}<tr>"
DIFF="${DIFF} <td> diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
DIFF="${DIFF} <td> composite diff <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
DIFF="${DIFF} <td> diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.png\"> </td>"
DIFF="${DIFF} <td> composite diff <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}_diff.gif\"> </td>"
DIFF="${DIFF}</tr>"
DIFF="${DIFF}</table>"
@@ -125,7 +127,7 @@ jobs:
if [[ $INDEX -eq 0 ]]; then
TABLE="${TABLE}<tr>"
fi
TABLE="${TABLE} <td> <img src=\"https://raw.githubusercontent.com/commaai/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
TABLE="${TABLE} <td> <img src=\"https://raw.githubusercontent.com/sunnypilot/ci-artifacts/${{ env.BRANCH_NAME }}/${A[$i]}.png\"> </td>"
if [[ $INDEX -eq 1 || $(($i + 1)) -eq ${#A[*]} ]]; then
TABLE="${TABLE}</tr>"
fi
+6 -6
View File
@@ -1,18 +1,18 @@
[submodule "panda"]
path = panda
url = ../../commaai/panda.git
url = https://github.com/sunnyhaibin/panda.git
[submodule "opendbc"]
path = opendbc_repo
url = ../../commaai/opendbc.git
url = https://github.com/sunnypilot/opendbc.git
[submodule "msgq"]
path = msgq_repo
url = ../../commaai/msgq.git
url = https://github.com/sunnypilot/msgq.git
[submodule "rednose_repo"]
path = rednose_repo
url = ../../commaai/rednose.git
url = https://github.com/commaai/rednose.git
[submodule "teleoprtc_repo"]
path = teleoprtc_repo
url = ../../commaai/teleoprtc
url = https://github.com/commaai/teleoprtc
[submodule "tinygrad"]
path = tinygrad_repo
url = https://github.com/commaai/tinygrad.git
url = https://github.com/tinygrad/tinygrad.git
+3 -3
View File
@@ -1,9 +1,9 @@
FROM ghcr.io/commaai/openpilot-base:latest
ENV PYTHONUNBUFFERED=1
ENV PYTHONUNBUFFERED 1
ENV OPENPILOT_PATH=/home/batman/openpilot
ENV PYTHONPATH=${OPENPILOT_PATH}:${PYTHONPATH}
ENV OPENPILOT_PATH /home/batman/openpilot
ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
RUN mkdir -p ${OPENPILOT_PATH}
WORKDIR ${OPENPILOT_PATH}
+14 -11
View File
@@ -1,6 +1,6 @@
FROM ubuntu:24.04
ENV PYTHONUNBUFFERED=1
ENV PYTHONUNBUFFERED 1
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
@@ -8,9 +8,9 @@ RUN apt-get update && \
rm -rf /var/lib/apt/lists/*
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
RUN /tmp/tools/install_ubuntu_dependencies.sh && \
@@ -55,9 +55,9 @@ RUN mkdir -p /tmp/opencl-driver-intel && \
cd / && \
rm -rf /tmp/opencl-driver-intel
ENV NVIDIA_VISIBLE_DEVICES=all
ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
ENV QTWEBENGINE_DISABLE_SANDBOX=1
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
ENV QTWEBENGINE_DISABLE_SANDBOX 1
RUN dbus-uuidgen > /etc/machine-id
@@ -68,14 +68,17 @@ RUN usermod -aG sudo $USER
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER $USER
COPY --chown=$USER pyproject.toml uv.lock /home/$USER
COPY --chown=$USER tools/install_python_dependencies.sh /home/$USER/tools/
COPY --chown=$USER pyproject.toml uv.lock /tmp/
COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
ENV VIRTUAL_ENV=/home/$USER/.venv
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN cd /home/$USER && \
RUN cd /tmp && \
tools/install_python_dependencies.sh && \
rm -rf tools/ pyproject.toml uv.lock .cache
mkdir -p $VIRTUAL_ENV && \
cp -r /tmp/.venv/* $VIRTUAL_ENV && \
rm -rf /tmp/* && \
rm -rf /home/$USER/.cache
USER root
RUN sudo git config --global --add safe.directory /tmp/openpilot
Vendored
+4 -13
View File
@@ -79,10 +79,6 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
return
}
if (isReplay()) {
error("REPLAYING TESTS IS NOT ALLOWED. FIX THEM INSTEAD.")
}
def extra = extra_env.collect { "export ${it}" }.join('\n');
def branch = env.BRANCH_NAME ?: 'master';
def gitDiff = sh returnStdout: true, script: 'curl -s -H "Authorization: Bearer ${GITHUB_COMMENTS_TOKEN}" https://api.github.com/repos/commaai/openpilot/compare/master...${GIT_BRANCH} | jq .files[].filename || echo "/"', label: 'Getting changes'
@@ -127,11 +123,6 @@ def hasPathChanged(String gitDiff, List<String> paths) {
return false
}
def isReplay() {
def replayClass = "org.jenkinsci.plugins.workflow.cps.replay.ReplayCause"
return currentBuild.rawBuild.getCauses().any{ cause -> cause.toString().contains(replayClass) }
}
def setupCredentials() {
withCredentials([
string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
@@ -216,8 +207,8 @@ node {
step("build", "cd system/manager && ./build.py"),
step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py", [diffPaths: ["system/ubloxd/"]]),
step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"),
step("test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"),
step("test manager", "pytest system/manager/test/test_manager.py"),
])
},
@@ -252,7 +243,7 @@ node {
'replay': {
deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/"]]),
])
},
'tizi': {
@@ -262,7 +253,7 @@ node {
step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"),
])
},
+21
View File
@@ -0,0 +1,21 @@
# Custom MIT License
Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
"This software is licensed under a custom license requiring permission for use."
3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
"This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
---
Haibin Wen, SUNNYPILOT LLC
+1 -10
View File
@@ -38,8 +38,7 @@ Quick start: `bash <(curl -fsSL openpilot.comma.ai)`
</tr>
</table>
Using openpilot in a car
To start using openpilot in a car
------
To use openpilot in a car, you need four things:
@@ -50,14 +49,6 @@ To use openpilot in a car, you need four things:
We have detailed instructions for [how to install the harness and device in a car](https://comma.ai/setup). Note that it's possible to run openpilot on [other hardware](https://blog.comma.ai/self-driving-car-for-free/), although it's not plug-and-play.
### Branches
| branch | URL | description |
|------------------|----------------------------------------|-------------------------------------------------------------------------------------|
| `release3` | openpilot.comma.ai | This is openpilot's release branch. |
| `release3-staging` | openpilot-test.comma.ai | This is the staging branch for releases. Use it to get new releases slightly early. |
| `nightly` | openpilot-nightly.comma.ai | This is the bleeding edge development branch. Do not expect this to be stable. |
| `nightly-dev` | installer.comma.ai/commaai/nightly-dev | Same as nightly, but includes experimental development features for some cars. |
To start developing openpilot
------
+1 -2
View File
@@ -349,7 +349,7 @@ Export('common', 'gpucommon')
env_swaglog = env.Clone()
env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
SConscript(['opendbc/can/SConscript'], exports={'env': env_swaglog})
SConscript(['cereal/SConscript'])
@@ -366,7 +366,6 @@ SConscript(['rednose/SConscript'])
# Build system services
SConscript([
'system/ui/SConscript',
'system/proclogd/SConscript',
'system/ubloxd/SConscript',
'system/loggerd/SConscript',
-14
View File
@@ -486,9 +486,6 @@ struct DeviceState @0xa4d8b5af2aa492eb {
nvmeTempC @35 :List(Float32);
modemTempC @36 :List(Float32);
pmicTempC @39 :List(Float32);
intakeTempC @46 :Float32;
exhaustTempC @47 :Float32;
caseTempC @48 :Float32;
maxTempC @44 :Float32; # max of other temps, used to control fan
thermalZones @38 :List(ThermalZone);
thermalStatus @14 :ThermalStatus;
@@ -2440,14 +2437,6 @@ struct Microphone {
filteredSoundPressureWeightedDb @2 :Float32;
}
struct Touch {
sec @0 :Int64;
usec @1 :Int64;
type @2 :UInt8;
code @3 :Int32;
value @4 :Int32;
}
struct Event {
logMonoTime @0 :UInt64; # nanoseconds
valid @67 :Bool = true;
@@ -2528,9 +2517,6 @@ struct Event {
logMessage @18 :Text;
errorLogMessage @85 :Text;
# touch frame
touch @135 :List(Touch);
# navigation
navInstruction @82 :NavInstruction;
navRoute @83 :NavRoute;
+1 -1
View File
@@ -17,5 +17,5 @@ class TestServices:
def test_generated_header(self):
with tempfile.NamedTemporaryFile(suffix=".h") as f:
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
assert ret == 0, "generated services header is not valid C"
-1
View File
@@ -22,7 +22,6 @@ _services: dict[str, tuple] = {
"temperatureSensor2": (True, 2., 200),
"gpsNMEA": (True, 9.),
"deviceState": (True, 2., 1),
"touch": (True, 20., 1),
"can": (True, 100., 2053), # decimation gives ~3 msgs in a full segment
"controlsState": (True, 100., 10),
"selfdriveState": (True, 100., 10),
+1
View File
@@ -112,6 +112,7 @@ std::unordered_map<std::string, uint32_t> keys = {
{"DisablePowerDown", PERSISTENT},
{"DisableUpdates", PERSISTENT},
{"DisengageOnAccelerator", PERSISTENT},
{"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
{"DongleId", PERSISTENT},
{"DoReboot", CLEAR_ON_MANAGER_START},
{"DoShutdown", CLEAR_ON_MANAGER_START},
+1 -1
View File
@@ -13,7 +13,7 @@ public:
if (prefix.empty()) {
prefix = util::random_string(15);
}
msgq_path = Path::shm_path() + "/" + prefix;
msgq_path = "/dev/shm/" + prefix;
bool ret = util::create_directories(msgq_path, 0777);
assert(ret);
setenv("OPENPILOT_PREFIX", prefix.c_str(), 1);
+1 -1
View File
@@ -11,7 +11,7 @@ from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
class OpenpilotPrefix:
def __init__(self, prefix: str = None, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15])
self.msgq_path = os.path.join(Paths.shm_path(), self.prefix)
self.msgq_path = os.path.join('/dev/shm', self.prefix)
self.clean_dirs_on_exit = clean_dirs_on_exit
self.shared_download_cache = shared_download_cache
+2 -6
View File
@@ -48,13 +48,13 @@ class Ratekeeper:
def __init__(self, rate: float, print_delay_threshold: float | None = 0.0) -> None:
"""Rate in Hz for ratekeeping. print_delay_threshold must be nonnegative."""
self._interval = 1. / rate
self._next_frame_time = time.monotonic() + self._interval
self._print_delay_threshold = print_delay_threshold
self._frame = 0
self._remaining = 0.0
self._process_name = getproctitle()
self._dts = deque([self._interval], maxlen=100)
self._last_monitor_time = -1.
self._next_frame_time = -1.
self._last_monitor_time = time.monotonic()
@property
def frame(self) -> int:
@@ -79,10 +79,6 @@ class Ratekeeper:
# Monitors the cumulative lag, but does not enforce a rate
def monitor_time(self) -> bool:
if self._last_monitor_time < 0:
self._next_frame_time = time.monotonic() + self._interval
self._last_monitor_time = time.monotonic()
prev = self._last_monitor_time
self._last_monitor_time = time.monotonic()
self._dts.append(self._last_monitor_time - prev)
+1 -8
View File
@@ -1,7 +1,7 @@
import numpy as np
from openpilot.common.transformations.orientation import rot_from_euler
from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame, _ar_ox_fisheye
from openpilot.common.transformations.camera import get_view_frame_from_calib_frame, view_frame_from_device_frame
# segnet
SEGNET_SIZE = (512, 384)
@@ -39,13 +39,6 @@ sbigmodel_intrinsics = np.array([
[0.0, sbigmodel_fl, 0.5 * (256 + MEDMODEL_CY)],
[0.0, 0.0, 1.0]])
DM_INPUT_SIZE = (1440, 960)
dmonitoringmodel_fl = _ar_ox_fisheye.focal_length
dmonitoringmodel_intrinsics = np.array([
[dmonitoringmodel_fl, 0.0, DM_INPUT_SIZE[0]/2],
[0.0, dmonitoringmodel_fl, DM_INPUT_SIZE[1]/2 - (_ar_ox_fisheye.height - DM_INPUT_SIZE[1])/2],
[0.0, 0.0, 1.0]])
bigmodel_frame_from_calib_frame = np.dot(bigmodel_intrinsics,
get_view_frame_from_calib_frame(0, 0, 0, 0))
+1 -2
View File
@@ -2,9 +2,8 @@
#include "common/watchdog.h"
#include "common/util.h"
#include "system/hardware/hw.h"
const std::string watchdog_fn_prefix = Path::shm_path() + "/wd_"; // + <pid>
const std::string watchdog_fn_prefix = "/dev/shm/wd_"; // + <pid>
bool watchdog_kick(uint64_t ts) {
static std::string fn = watchdog_fn_prefix + std::to_string(getpid());
+3 -3
View File
@@ -103,7 +103,7 @@ A supported vehicle is one that just works when you install a comma device. All
|Hyundai|Ioniq Plug-in Hybrid 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Ioniq Plug-in Hybrid 2020-22">Buy Here</a></sub></details>||
|Hyundai|Kona 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|6 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona 2020">Buy Here</a></sub></details>||
|Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai G connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2018-21">Buy Here</a></sub></details>||
|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2022-23">Buy Here</a></sub></details>||
|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai O connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric 2022-23">Buy Here</a></sub></details>||
|Hyundai|Kona Electric (with HDA II, Korea only) 2023[<sup>5</sup>](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai R connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Electric (with HDA II, Korea only) 2023">Buy Here</a></sub></details>|<a href="https://www.youtube.com/watch?v=U2fOCmcQ8hw" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai I connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Kona Hybrid 2020">Buy Here</a></sub></details>||
|Hyundai|Palisade 2020-22|All|openpilot available[<sup>1</sup>](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Hyundai H connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Hyundai&model=Palisade 2020-22">Buy Here</a></sub></details>|<a href="https://youtu.be/TAnDqjF4fDY?t=456" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
@@ -167,7 +167,7 @@ A supported vehicle is one that just works when you install a comma device. All
|Lexus|ES 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES 2017-18">Buy Here</a></sub></details>||
|Lexus|ES 2019-24|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES 2019-24">Buy Here</a></sub></details>||
|Lexus|ES Hybrid 2017-18|All|openpilot available[<sup>2</sup>](#footnotes)|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES Hybrid 2017-18">Buy Here</a></sub></details>||
|Lexus|ES Hybrid 2019-25|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES Hybrid 2019-25">Buy Here</a></sub></details>|<a href="https://youtu.be/BZ29osRVJeg?t=12" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|Lexus|ES Hybrid 2019-24|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=ES Hybrid 2019-24">Buy Here</a></sub></details>|<a href="https://youtu.be/BZ29osRVJeg?t=12" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|Lexus|GS F 2016|All|Stock|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=GS F 2016">Buy Here</a></sub></details>||
|Lexus|IS 2017-19|All|Stock|19 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=IS 2017-19">Buy Here</a></sub></details>||
|Lexus|IS 2022-23|All|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 RJ45 cable (7 ft)<br>- 1 Toyota A connector<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lexus&model=IS 2022-23">Buy Here</a></sub></details>||
@@ -188,7 +188,7 @@ A supported vehicle is one that just works when you install a comma device. All
|Lincoln|Aviator Plug-in Hybrid 2020-24|Co-Pilot360 Plus|openpilot|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Ford Q3 connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Lincoln&model=Aviator Plug-in Hybrid 2020-24">Buy Here</a></sub></details>||
|MAN|eTGE 2020-24|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=MAN&model=eTGE 2020-24">Buy Here</a></sub></details>|<a href="https://youtu.be/4100gLeabmo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|MAN|TGE 2017-24|Adaptive Cruise Control (ACC) & Lane Assist|openpilot available[<sup>1,12</sup>](#footnotes)|0 mph|31 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 USB-C coupler<br>- 1 VW J533 connector<br>- 1 angled mount (8 degrees)<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=MAN&model=TGE 2017-24">Buy Here</a></sub></details>|<a href="https://youtu.be/4100gLeabmo" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|Mazda|CX-5 2022-25|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Mazda connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Mazda&model=CX-5 2022-25">Buy Here</a></sub></details>||
|Mazda|CX-5 2022-24|All|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Mazda connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Mazda&model=CX-5 2022-24">Buy Here</a></sub></details>||
|Mazda|CX-9 2021-23|All|Stock|0 mph|28 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|<details><summary>Parts</summary><sub>- 1 Mazda connector<br>- 1 RJ45 cable (7 ft)<br>- 1 comma 3X<br>- 1 comma power v2<br>- 1 harness box<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Mazda&model=CX-9 2021-23">Buy Here</a></sub></details>|<a href="https://youtu.be/dA3duO4a0O4" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
|Nissan|Altima 2019-20|ProPILOT Assist|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Nissan B connector<br>- 1 RJ45 cable (7 ft)<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Nissan&model=Altima 2019-20">Buy Here</a></sub></details>||
|Nissan|Leaf 2018-23|ProPILOT Assist|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|<details><summary>Parts</summary><sub>- 1 Nissan A connector<br>- 1 RJ45 cable (7 ft)<br>- 1 USB-C coupler<br>- 1 comma 3X<br>- 1 harness box<br>- 1 long OBD-C cable<br>- 1 mount<br>- 1 right angle OBD-C cable (1.5 ft)<br><a href="https://comma.ai/shop/comma-3x.html?make=Nissan&model=Leaf 2018-23">Buy Here</a></sub></details>|<a href="https://youtu.be/vaMbtAh_0cY" target="_blank"><img height="18px" src="assets/icon-youtube.svg"></img></a>|
-44
View File
@@ -1,44 +0,0 @@
[data-tooltip] {
position: relative;
display: inline-block;
border-bottom: 1px dotted black;
}
[data-tooltip] .tooltip-content {
width: max-content;
max-width: 25em;
position: absolute;
top: 100%;
left: 50%;
transform: translateX(-50%);
background-color: white;
color: #404040;
box-shadow: 0 4px 14px 0 rgba(0,0,0,.2), 0 0 0 1px rgba(0,0,0,.05);
padding: 10px;
font: 14px/1.5 Lato, proxima-nova, Helvetica Neue, Arial, sans-serif;
text-decoration: none;
opacity: 0;
visibility: hidden;
transition: opacity 0.1s, visibility 0s;
z-index: 1000;
pointer-events: none; /* Prevent accidental interaction */
}
[data-tooltip]:hover .tooltip-content {
opacity: 1;
visibility: visible;
pointer-events: auto; /* Allow interaction when visible */
}
.tooltip-content .tooltip-glossary-link {
display: inline-block;
margin-top: 8px;
font-size: 12px;
color: #007bff;
text-decoration: none;
}
.tooltip-content .tooltip-glossary-link:hover {
color: #0056b3;
text-decoration: underline;
}
View File
-68
View File
@@ -1,68 +0,0 @@
import re
import tomllib
def load_glossary(file_path="docs/glossary.toml"):
with open(file_path, "rb") as f:
glossary_data = tomllib.load(f)
return glossary_data.get("glossary", {})
def generate_anchor_id(name):
return name.replace(" ", "-").replace("_", "-").lower()
def format_markdown_term(name, definition):
anchor_id = generate_anchor_id(name)
markdown = f"* [**{name.replace('_', ' ').title()}**](#{anchor_id})"
if definition.get("abbreviation"):
markdown += f" *({definition['abbreviation']})*"
if definition.get("description"):
markdown += f": {definition['description']}\n"
return markdown
def glossary_markdown(vocabulary):
markdown = ""
for category, terms in vocabulary.items():
markdown += f"## {category.replace('_', ' ').title()}\n\n"
for name, definition in terms.items():
markdown += format_markdown_term(name, definition)
return markdown
def format_tooltip_html(term_key, definition, html):
display_term = term_key.replace("_", " ").title()
clean_description = re.sub(r"\[(.+)]\(.+\)", r"\1", definition["description"])
glossary_link = (
f"<a href='/concepts/glossary#{term_key}' class='tooltip-glossary-link' title='View in glossary'>Glossary🔗</a>"
)
return re.sub(
re.escape(display_term),
lambda
match: f"<span data-tooltip>{match.group(0)}<span class='tooltip-content'>{clean_description} {glossary_link}</span></span>",
html,
flags=re.IGNORECASE,
)
def apply_tooltip(_term_key, _definition, pattern, html):
return re.sub(
pattern,
lambda match: format_tooltip_html(_term_key, _definition, match.group(0)),
html,
flags=re.IGNORECASE,
)
def tooltip_html(vocabulary, html):
for _category, terms in vocabulary.items():
for term_key, definition in terms.items():
if definition.get("description"):
pattern = rf"(?<!\w){re.escape(term_key.replace('_', ' ').title())}(?![^<]*<\/a>)(?!\([^)]*\))"
html = apply_tooltip(term_key, definition, pattern, html)
return html
# Page Hooks
def on_page_markdown(markdown, **kwargs):
glossary = load_glossary()
return markdown.replace("{{GLOSSARY_DEFINITIONS}}", glossary_markdown(glossary))
def on_page_content(html, **kwargs):
if kwargs.get("page").title == "Glossary":
return html
glossary = load_glossary()
return tooltip_html(glossary, html)
+1 -1
View File
@@ -7,7 +7,7 @@ export OPENBLAS_NUM_THREADS=1
export VECLIB_MAXIMUM_THREADS=1
if [ -z "$AGNOS_VERSION" ]; then
export AGNOS_VERSION="11.4"
export AGNOS_VERSION="11.2"
fi
export STAGING_ROOT="/data/safe_staging"
-4
View File
@@ -8,10 +8,6 @@ strict: true
docs_dir: docs
site_dir: docs_site/
hooks:
- docs/hooks/glossary.py
extra_css:
- css/tooltip.css
theme:
name: readthedocs
navigation_depth: 3
+1 -1
Submodule panda updated: dd76e663d2...bc28d3cc51
+1 -4
View File
@@ -31,9 +31,6 @@ dependencies = [
# body / webrtcd
"aiohttp",
"aiortc",
# aiortc does not put an upper bound on pyopenssl and is now incompatible
# with the latest release
"pyopenssl < 24.3.0",
"pyaudio",
# panda
@@ -101,6 +98,7 @@ dev = [
"azure-identity",
"azure-storage-blob",
"dictdiffer",
"flaky",
"lru-dict",
"matplotlib",
"parameterized >=0.8, <0.9",
@@ -241,7 +239,6 @@ exclude = [
"cereal",
"panda",
"opendbc",
"opendbc_repo",
"rednose_repo",
"tinygrad_repo",
"teleoprtc",
+1
View File
@@ -32,6 +32,7 @@ blacklist = [
".git/",
".github/",
".devcontainer/",
"Darwin/",
".vscode",
+5 -5
View File
@@ -37,7 +37,7 @@ for f in sorted(pyf):
lns = len(src.split("\n"))
tree = ast.parse(src)
Analyzer().visit(tree)
print(f"{lns:5d} {f} {xbit}")
print("%5d %s %s" % (lns, f, xbit))
if 'test' in f:
testlns += lns
elif f.startswith(('tools/', 'scripts/', 'selfdrive/debug')):
@@ -47,8 +47,8 @@ for f in sorted(pyf):
else:
tlns += lns
print(f"{tlns} lines of openpilot python")
print(f"{carlns} lines of car ports")
print(f"{scriptlns} lines of tools/scripts/debug")
print(f"{testlns} lines of tests")
print("%d lines of openpilot python" % tlns)
print("%d lines of car ports" % carlns)
print("%d lines of tools/scripts/debug" % scriptlns)
print("%d lines of tests" % testlns)
#print(sorted(list(imps)))
-10
View File
@@ -1,10 +0,0 @@
#!/usr/bin/env bash
FAIL=0
if grep -n '#include "third_party/raylib/include/raylib\.h"' $@ | grep -v '^system/ui/raylib/raylib\.h'; then
echo -e "Bad raylib include found! Use '#include \"system/ui/raylib/raylib.h\"' instead\n"
FAIL=1
fi
exit $FAIL
+1 -2
View File
@@ -13,7 +13,7 @@ cd $ROOT
FAILED=0
IGNORED_FILES="uv\.lock|docs\/CARS.md"
IGNORED_FILES="uv\.lock|docs\/CARS.md|LICENSE\.md"
IGNORED_DIRS="^third_party.*|^msgq.*|^msgq_repo.*|^opendbc.*|^opendbc_repo.*|^cereal.*|^panda.*|^rednose.*|^rednose_repo.*|^tinygrad.*|^tinygrad_repo.*|^teleoprtc.*|^teleoprtc_repo.*"
function run() {
@@ -53,7 +53,6 @@ function run_tests() {
run "check_shebang_scripts_are_executable" python3 -m pre_commit_hooks.check_shebang_scripts_are_executable $ALL_FILES
run "check_shebang_format" $DIR/check_shebang_format.sh $ALL_FILES
run "check_nomerge_comments" $DIR/check_nomerge_comments.sh $ALL_FILES
run "check_raylib_includes" $DIR/check_raylib_includes.sh $ALL_FILES
if [[ -z "$FAST" ]]; then
run "mypy" mypy $PYTHON_FILES
+2 -2
View File
@@ -16,9 +16,9 @@ def waste(core):
j = 0
while 1:
if (i % 100) == 0:
setproctitle(f"{core:3d}: {i:8d}")
setproctitle("%3d: %8d" % (core, i))
lt = time.monotonic()
print(f"{core:3d}: {i:8d} {lt-st:f} {j:.2f}")
print("%3d: %8d %f %.2f" % (core, i, lt-st, j))
st = lt
i += 1
j = np.sum(np.matmul(m1, m2))
@@ -0,0 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:f7565541b4e6213221174839b9b2b67397ced0b9807ea56413989fd37325b3b6
size 4908
+1 -2
View File
@@ -148,8 +148,7 @@ class CarSpecificEvents:
# To avoid re-engaging when openpilot cancels, check user engagement intention via buttons
# Main button also can trigger an engagement on these cars
self.cruise_buttons.append(any(ev.type in HYUNDAI_ENABLE_BUTTONS for ev in CS.buttonEvents))
events = self.create_common_events(CS, CS_prev, extra_gears=(GearShifter.sport, GearShifter.manumatic),
pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons))
events = self.create_common_events(CS, CS_prev, pcm_enable=self.CP.pcmCruise, allow_enable=any(self.cruise_buttons))
# low speed steer alert hysteresis logic (only for cars with steer cut off above 10 m/s)
if CS.vEgo < (self.CP.minSteerSpeed + 2.) and self.CP.minSteerSpeed > 10.:
+1 -1
View File
@@ -17,7 +17,7 @@ class TestCarDocs:
with open(CARS_MD_OUT) as f:
current_cars_md = f.read()
assert generated_cars_md == current_cars_md, "Run selfdrive/car/docs.py to update the compatibility documentation"
assert generated_cars_md == current_cars_md, "Run selfdrive/opcar/docs.py to update the compatibility documentation"
def test_docs_diff(self):
dump_path = os.path.join(BASEDIR, "selfdrive", "car", "tests", "cars_dump")
+1 -1
View File
@@ -394,7 +394,7 @@ class TestCarModelBase(unittest.TestCase):
for msg in filter(lambda m: m.src in range(64), can.can):
to_send = libpanda_py.make_CANPacket(msg.address, msg.src % 4, msg.dat)
ret = self.safety.safety_rx_hook(to_send)
self.assertEqual(1, ret, f"safety rx failed ({ret=}): {(msg.address, msg.src % 4)}")
self.assertEqual(1, ret, f"safety rx failed ({ret=}): {to_send}")
# Skip first frame so CS_prev is properly initialized
if idx == 0:
-3
View File
@@ -5,15 +5,12 @@ from openpilot.common.realtime import DT_CTRL
MIN_SPEED = 1.0
CONTROL_N = 17
CAR_ROTATION_RADIUS = 0.0
# This is a turn radius smaller than most cars can achieve
MAX_CURVATURE = 0.2
# EU guidelines
MAX_LATERAL_JERK = 5.0
MAX_VEL_ERR = 5.0
def clip_curvature(v_ego, prev_curvature, new_curvature):
new_curvature = clip(new_curvature, -MAX_CURVATURE, MAX_CURVATURE)
v_ego = max(MIN_SPEED, v_ego)
max_curvature_rate = MAX_LATERAL_JERK / (v_ego**2) # inexact calculation, check https://github.com/commaai/openpilot/pull/24755
safe_desired_curvature = clip(new_curvature,
+1 -1
View File
@@ -28,7 +28,7 @@ def can_printer(bus, max_msg, addr, ascii_decode):
x = binascii.hexlify(msgs[_addr][-1]).decode('ascii')
freq = len(msgs[_addr]) / (time.monotonic() - start)
if max_msg is None or _addr < max_msg:
dd += f"{_addr:04X}({_addr:4d})({len(msgs[_addr]):6d})({freq:3}dHz) {x.ljust(20)} {a}\n"
dd += "%04X(%4d)(%6d)(%3dHz) %s %s\n" % (_addr, _addr, len(msgs[_addr]), freq, x.ljust(20), a)
print(dd)
lp = time.monotonic()
@@ -29,7 +29,7 @@ if __name__ == '__main__':
start_t = time.process_time_ns()
for msg in msgs:
can_list = can_capnp_to_list([msg])
for cp in tm.CI.can_parsers.values():
for cp in tm.CI.can_parsers:
if cp is not None:
cp.update_strings(can_list)
ets.append((time.process_time_ns() - start_t) * 1e-6)
+1 -2
View File
@@ -8,7 +8,6 @@ from typing import cast
from cereal.services import SERVICE_LIST
from openpilot.tools.lib.logreader import LogReader, ReadMode
from openpilot.selfdrive.test.process_replay.migration import migrate_all
if __name__ == "__main__":
cnt_events: Counter = Counter()
@@ -21,7 +20,7 @@ if __name__ == "__main__":
start_time = math.inf
end_time = -math.inf
ignition_off = None
for msg in migrate_all(LogReader(sys.argv[1], ReadMode.QLOG)):
for msg in LogReader(sys.argv[1], ReadMode.QLOG):
t = (msg.logMonoTime - start_time) / 1e9
end_time = max(end_time, msg.logMonoTime)
start_time = min(start_time, msg.logMonoTime)
+1 -1
View File
@@ -22,7 +22,7 @@ def get_fingerprint(lr):
msgs[c.address] = len(c.dat)
# show CAN fingerprint
fingerprint = ', '.join(f"{v[0]}: {v[1]}" for v in sorted(msgs.items()))
fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items()))
print(f"\nfound {len(msgs)} messages. CAN fingerprint:\n")
print(fingerprint)
+1 -1
View File
@@ -25,7 +25,7 @@ while True:
if c.src % 0x80 == 0 and c.address < 0x800 and c.address not in (0x7df, 0x7e0, 0x7e8):
msgs[c.address] = len(c.dat)
fingerprint = ', '.join(f"{v[0]}: {v[1]}" for v in sorted(msgs.items()))
fingerprint = ', '.join("%d: %d" % v for v in sorted(msgs.items()))
print(f"number of messages {len(msgs)}:")
print(f"fingerprint {fingerprint}")
-54
View File
@@ -1,54 +0,0 @@
#!/usr/bin/env python3
import argparse
import numpy as np
import matplotlib.pyplot as plt
from openpilot.tools.lib.logreader import LogReader
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('--width', default=2160, type=int)
parser.add_argument('--height', default=1080, type=int)
parser.add_argument('--route', default='rlog', type=str)
args = parser.parse_args()
w = args.width
h = args.height
route = args.route
fingers = [[-1, -1]] * 5
touch_points = []
current_slot = 0
lr = list(LogReader(route))
for msg in lr:
if msg.which() == 'touch':
for event in msg.touch:
if event.type == 3 and event.code == 47:
current_slot = event.value
elif event.type == 3 and event.code == 57 and event.value == -1:
fingers[current_slot] = [-1, -1]
elif event.type == 3 and event.code == 53:
fingers[current_slot][1] = h - (h - event.value)
if fingers[current_slot][0] != -1:
touch_points.append(fingers[current_slot].copy())
elif event.type == 3 and event.code == 54:
fingers[current_slot][0] = w - event.value
if fingers[current_slot][1] != -1:
touch_points.append(fingers[current_slot].copy())
if not touch_points:
print(f'No touch events found for {route}')
quit()
unique_points, counts = np.unique(touch_points, axis=0, return_counts=True)
plt.figure(figsize=(10, 3))
plt.scatter(unique_points[:, 0], unique_points[:, 1], c=counts, s=counts * 20, edgecolors='red')
plt.colorbar()
plt.title(f'Touches for {route}')
plt.xlim(0, w)
plt.ylim(0, h)
plt.grid(True)
plt.show()
+9 -18
View File
@@ -8,7 +8,6 @@ from enum import Enum
from collections import defaultdict
from cereal import log, messaging
from cereal.services import SERVICE_LIST
from openpilot.common.transformations.orientation import rot_from_euler
from openpilot.common.realtime import config_realtime_process
from openpilot.common.params import Params
@@ -24,10 +23,8 @@ MIN_STD_SANITY_CHECK = 1e-5 # m or rad
MAX_FILTER_REWIND_TIME = 0.8 # s
MAX_SENSOR_TIME_DIFF = 0.1 # s
YAWRATE_CROSS_ERR_CHECK_FACTOR = 30
INPUT_INVALID_THRESHOLD = 0.5 # 0 bad inputs ignored
TIMING_INVALID_THRESHOLD = 2.5 # 2 bad timings ignored
INPUT_INVALID_DECAY = 0.9993 # ~10 secs to resume after exceeding allowed bad inputs by one (at 100hz)
TIMING_INVALID_DECAY = 0.9990 # ~2 secs to resume after exceeding allowed bad timings by one (at 100hz)
INPUT_INVALID_THRESHOLD = 0.5
INPUT_INVALID_DECAY = 0.9993 # ~10 secs to resume after a bad input
POSENET_STD_INITIAL_VALUE = 10.0
POSENET_STD_HIST_HALF = 20
@@ -268,13 +265,10 @@ def main():
estimator = LocationEstimator(DEBUG)
filter_initialized = False
critcal_services = ["accelerometer", "gyroscope", "cameraOdometry"]
observation_timing_invalid = defaultdict(int)
critcal_services = ["accelerometer", "gyroscope", "liveCalibration", "cameraOdometry"]
observation_timing_invalid = False
observation_input_invalid = defaultdict(int)
input_invalid_decay = {s: INPUT_INVALID_DECAY ** (100. / SERVICE_LIST[s].frequency) for s in critcal_services}
timing_invalid_decay = {s: TIMING_INVALID_DECAY ** (100. / SERVICE_LIST[s].frequency) for s in critcal_services}
initial_pose = params.get("LocationFilterInitialState")
if initial_pose is not None:
initial_pose = json.loads(initial_pose)
@@ -288,6 +282,8 @@ def main():
acc_msgs, gyro_msgs = (messaging.drain_sock(sock) for sock in sensor_sockets)
if filter_initialized:
observation_timing_invalid = False
msgs = []
for msg in acc_msgs + gyro_msgs:
t, valid, which, data = msg.logMonoTime, msg.valid, msg.which(), getattr(msg, msg.which())
@@ -302,23 +298,18 @@ def main():
if valid:
t = log_mono_time * 1e-9
res = estimator.handle_log(t, which, msg)
if which not in critcal_services:
continue
if res == HandleLogResult.TIMING_INVALID:
observation_timing_invalid[which] += 1
observation_timing_invalid = True
elif res == HandleLogResult.INPUT_INVALID:
observation_input_invalid[which] += 1
else:
observation_input_invalid[which] *= input_invalid_decay[which]
observation_timing_invalid[which] *= timing_invalid_decay[which]
observation_input_invalid[which] *= INPUT_INVALID_DECAY
else:
filter_initialized = sm.all_checks() and sensor_all_checks(acc_msgs, gyro_msgs, sensor_valid, sensor_recv_time, sensor_alive, SIMULATION)
if sm.updated["cameraOdometry"]:
critical_service_inputs_valid = all(observation_input_invalid[s] < INPUT_INVALID_THRESHOLD for s in critcal_services)
critical_service_timing_valid = all(observation_timing_invalid[s] < TIMING_INVALID_THRESHOLD for s in critcal_services)
inputs_valid = sm.all_valid() and critical_service_inputs_valid and critical_service_timing_valid
inputs_valid = sm.all_valid() and critical_service_inputs_valid and not observation_timing_invalid
sensors_valid = sensor_all_checks(acc_msgs, gyro_msgs, sensor_valid, sensor_recv_time, sensor_alive, SIMULATION)
msg = estimator.get_msg(sensors_valid, inputs_valid, filter_initialized)
@@ -1,3 +1,4 @@
import pytest
import numpy as np
from collections import defaultdict
from enum import Enum
@@ -16,7 +17,6 @@ SELECT_COMPARE_FIELDS = {
'sensors_flag': ['sensorsOK'],
}
JUNK_IDX = 100
CONSISTENT_SPIKES_COUNT = 10
class Scenario(Enum):
@@ -25,8 +25,6 @@ class Scenario(Enum):
GYRO_SPIKE_MIDWAY = 'gyro_spike_midway'
ACCEL_OFF = 'accel_off'
ACCEL_SPIKE_MIDWAY = 'accel_spike_midway'
SENSOR_TIMING_SPIKE_MIDWAY = 'timing_spikes'
SENSOR_TIMING_CONSISTENT_SPIKES = 'timing_consistent_spikes'
def get_select_fields_data(logs):
@@ -45,17 +43,6 @@ def get_select_fields_data(logs):
return data
def modify_logs_midway(logs, which, count, fn):
non_which = [x for x in logs if x.which() != which]
which = [x for x in logs if x.which() == which]
temps = which[len(which) // 2:len(which) // 2 + count]
for i, temp in enumerate(temps):
temp = temp.as_builder()
fn(temp)
which[len(which) // 2 + i] = temp.as_reader()
return sorted(non_which + which, key=lambda x: x.logMonoTime)
def run_scenarios(scenario, logs):
if scenario == Scenario.BASE:
pass
@@ -64,28 +51,30 @@ def run_scenarios(scenario, logs):
logs = sorted([x for x in logs if x.which() != 'gyroscope'], key=lambda x: x.logMonoTime)
elif scenario == Scenario.GYRO_SPIKE_MIDWAY:
def gyro_spike(msg):
msg.gyroscope.gyroUncalibrated.v[0] += 3.0
logs = modify_logs_midway(logs, 'gyroscope', 1, gyro_spike)
non_gyro = [x for x in logs if x.which() not in 'gyroscope']
gyro = [x for x in logs if x.which() in 'gyroscope']
temp = gyro[len(gyro) // 2].as_builder()
temp.gyroscope.gyroUncalibrated.v[0] += 3.0
gyro[len(gyro) // 2] = temp.as_reader()
logs = sorted(non_gyro + gyro, key=lambda x: x.logMonoTime)
elif scenario == Scenario.ACCEL_OFF:
logs = sorted([x for x in logs if x.which() != 'accelerometer'], key=lambda x: x.logMonoTime)
elif scenario == Scenario.ACCEL_SPIKE_MIDWAY:
def acc_spike(msg):
msg.accelerometer.acceleration.v[0] += 10.0
logs = modify_logs_midway(logs, 'accelerometer', 1, acc_spike)
elif scenario == Scenario.SENSOR_TIMING_SPIKE_MIDWAY or scenario == Scenario.SENSOR_TIMING_CONSISTENT_SPIKES:
def timing_spike(msg):
msg.accelerometer.timestamp -= int(0.150 * 1e9)
count = 1 if scenario == Scenario.SENSOR_TIMING_SPIKE_MIDWAY else CONSISTENT_SPIKES_COUNT
logs = modify_logs_midway(logs, 'accelerometer', count, timing_spike)
non_accel = [x for x in logs if x.which() not in 'accelerometer']
accel = [x for x in logs if x.which() in 'accelerometer']
temp = accel[len(accel) // 2].as_builder()
temp.accelerometer.acceleration.v[0] += 10.0
accel[len(accel) // 2] = temp.as_reader()
logs = sorted(non_accel + accel, key=lambda x: x.logMonoTime)
replayed_logs = replay_process_with_name(name='locationd', lr=logs)
return get_select_fields_data(logs), get_select_fields_data(replayed_logs)
@pytest.mark.xdist_group("test_locationd_scenarios")
@pytest.mark.shared_download_cache
class TestLocationdScenarios:
"""
Test locationd with different scenarios. In all these scenarios, we expect the following:
@@ -133,7 +122,7 @@ class TestLocationdScenarios:
assert np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.35))
assert np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.55))
assert np.diff(replayed_data['inputs_flag'])[499] == -1.0
assert np.diff(replayed_data['inputs_flag'])[704] == 1.0
assert np.diff(replayed_data['inputs_flag'])[696] == 1.0
def test_accel_off(self):
"""
@@ -157,21 +146,3 @@ class TestLocationdScenarios:
orig_data, replayed_data = run_scenarios(Scenario.ACCEL_SPIKE_MIDWAY, self.logs)
assert np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.35))
assert np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.55))
def test_single_timing_spike(self):
"""
Test: timing of 150ms off for the single accelerometer message in the middle of the segment
Expected Result: the message is ignored, and inputsOK is False for that time
"""
orig_data, replayed_data = run_scenarios(Scenario.SENSOR_TIMING_SPIKE_MIDWAY, self.logs)
assert np.all(replayed_data['inputs_flag'] == orig_data['inputs_flag'])
assert np.all(replayed_data['sensors_flag'] == orig_data['sensors_flag'])
def test_consistent_timing_spikes(self):
"""
Test: consistent timing spikes for N accelerometer messages in the middle of the segment
Expected Result: inputsOK becomes False after N of bad measurements
"""
orig_data, replayed_data = run_scenarios(Scenario.SENSOR_TIMING_CONSISTENT_SPIKES, self.logs)
assert np.diff(replayed_data['inputs_flag'])[500] == -1.0
assert np.diff(replayed_data['inputs_flag'])[787] == 1.0
+14 -13
View File
@@ -12,14 +12,15 @@ from cereal import messaging
from cereal.messaging import PubMaster, SubMaster
from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from openpilot.common.swaglog import cloudlog
from openpilot.common.params import Params
from openpilot.common.realtime import set_realtime_priority
from openpilot.common.transformations.model import dmonitoringmodel_intrinsics
from openpilot.common.transformations.camera import _ar_ox_fisheye, _os_fisheye
from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext, MonitoringModelFrame
from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
from openpilot.selfdrive.modeld.models.commonmodel_pyx import CLContext
from openpilot.selfdrive.modeld.parse_model_outputs import sigmoid
CALIB_LEN = 3
MODEL_WIDTH = 1440
MODEL_HEIGHT = 960
FEATURE_LEN = 512
OUTPUT_SIZE = 84 + FEATURE_LEN
@@ -62,21 +63,25 @@ class ModelState:
def __init__(self, cl_ctx):
assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float)
self.frame = MonitoringModelFrame(cl_ctx)
self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32)
self.inputs = {
'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8),
'calib': np.zeros(CALIB_LEN, dtype=np.float32)}
self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, cl_ctx)
self.model.addInput("input_img", None)
self.model.addInput("calib", self.inputs['calib'])
def run(self, buf:VisionBuf, calib:np.ndarray, transform:np.ndarray) -> tuple[np.ndarray, float]:
def run(self, buf:VisionBuf, calib:np.ndarray) -> tuple[np.ndarray, float]:
self.inputs['calib'][:] = calib
self.model.setInputBuffer("input_img", self.frame.prepare(buf, transform.flatten(), None).view(np.float32))
v_offset = buf.height - MODEL_HEIGHT
h_offset = (buf.width - MODEL_WIDTH) // 2
buf_data = buf.data.reshape(-1, buf.stride)
input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH)
input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH]
self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32))
t1 = time.perf_counter()
self.model.execute()
t2 = time.perf_counter()
@@ -121,6 +126,7 @@ def main():
cl_context = CLContext()
model = ModelState(cl_context)
cloudlog.warning("models loaded, dmonitoringmodeld starting")
Params().put_bool("DmModelInitialized", True)
cloudlog.warning("connecting to driver stream")
vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True, cl_context)
@@ -133,23 +139,18 @@ def main():
pm = PubMaster(["driverStateV2"])
calib = np.zeros(CALIB_LEN, dtype=np.float32)
model_transform = None
while True:
buf = vipc_client.recv()
if buf is None:
continue
if model_transform is None:
cam = _os_fisheye if buf.width == _os_fisheye.width else _ar_ox_fisheye
model_transform = np.linalg.inv(np.dot(dmonitoringmodel_intrinsics, np.linalg.inv(cam.intrinsics))).astype(np.float32)
sm.update(0)
if sm.updated["liveCalibration"]:
calib[:] = np.array(sm["liveCalibration"].rpyCalib)
t1 = time.perf_counter()
model_output, gpu_execution_time = model.run(buf, calib, model_transform)
model_output, gpu_execution_time = model.run(buf, calib)
t2 = time.perf_counter()
pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, gpu_execution_time))
+5 -19
View File
@@ -3,22 +3,11 @@ import capnp
import numpy as np
from cereal import log
from openpilot.selfdrive.modeld.constants import ModelConstants, Plan, Meta
from openpilot.selfdrive.controls.lib.drive_helpers import MIN_SPEED
SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
ConfidenceClass = log.ModelDataV2.ConfidenceClass
def curv_from_psis(psi_target, psi_rate, vego, delay):
vego = np.clip(vego, MIN_SPEED, np.inf)
curv_from_psi = psi_target / (vego * delay) # epsilon to prevent divide-by-zero
return 2*curv_from_psi - psi_rate / vego
def get_curvature_from_plan(plan, vego, delay):
psi_target = np.interp(delay, ModelConstants.T_IDXS, plan[:, Plan.T_FROM_CURRENT_EULER][:, 2])
psi_rate = plan[:, Plan.ORIENTATION_RATE][0, 2]
return curv_from_psis(psi_target, psi_rate, vego, delay)
class PublishState:
def __init__(self):
self.disengage_buffer = np.zeros(ModelConstants.CONFIDENCE_BUFFER_LEN*ModelConstants.DISENGAGE_WIDTH, dtype=np.float32)
@@ -66,17 +55,14 @@ def fill_lane_line_meta(builder, lane_lines, lane_line_probs):
builder.rightProb = lane_line_probs[2]
def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._DynamicStructBuilder,
net_output_data: dict[str, np.ndarray], v_ego: float, delay: float,
publish_state: PublishState, vipc_frame_id: int, vipc_frame_id_extra: int,
frame_id: int, frame_drop: float, timestamp_eof: int, model_execution_time: float,
valid: bool) -> None:
net_output_data: dict[str, np.ndarray], publish_state: PublishState,
vipc_frame_id: int, vipc_frame_id_extra: int, frame_id: int, frame_drop: float,
timestamp_eof: int, model_execution_time: float, valid: bool) -> None:
frame_age = frame_id - vipc_frame_id if frame_id > vipc_frame_id else 0
frame_drop_perc = frame_drop * 100
extended_msg.valid = valid
base_msg.valid = valid
desired_curv = float(get_curvature_from_plan(net_output_data['plan'][0], v_ego, delay))
driving_model_data = base_msg.drivingModelData
driving_model_data.frameId = vipc_frame_id
@@ -85,7 +71,7 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
driving_model_data.modelExecutionTime = model_execution_time
action = driving_model_data.action
action.desiredCurvature = desired_curv
action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
modelV2 = extended_msg.modelV2
modelV2.frameId = vipc_frame_id
@@ -120,7 +106,7 @@ def fill_model_msg(base_msg: capnp._DynamicStructBuilder, extended_msg: capnp._D
# lateral planning
action = modelV2.action
action.desiredCurvature = desired_curv
action.desiredCurvature = float(net_output_data['desired_curvature'][0,0])
# times at X_IDXS according to model plan
PLAN_T_IDXS = [np.nan] * ModelConstants.IDX_N
+24 -53
View File
@@ -12,8 +12,6 @@ from msgq.visionipc import VisionIpcClient, VisionStreamType, VisionBuf
from opendbc.car.car_helpers import get_demo_car_params
from openpilot.common.swaglog import cloudlog
from openpilot.common.params import Params
from openpilot.common.realtime import DT_MDL
from openpilot.common.numpy_fast import interp
from openpilot.common.filter_simple import FirstOrderFilter
from openpilot.common.realtime import config_realtime_process
from openpilot.common.transformations.camera import DEVICE_CAMERAS
@@ -24,7 +22,7 @@ from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime
from openpilot.selfdrive.modeld.parse_model_outputs import Parser
from openpilot.selfdrive.modeld.fill_model_msg import fill_model_msg, fill_pose_msg, PublishState
from openpilot.selfdrive.modeld.constants import ModelConstants
from openpilot.selfdrive.modeld.models.commonmodel_pyx import DrivingModelFrame, CLContext
from openpilot.selfdrive.modeld.models.commonmodel_pyx import ModelFrame, CLContext
PROCESS_NAME = "selfdrive.modeld.modeld"
SEND_RAW_PRED = os.getenv('SEND_RAW_PRED')
@@ -46,30 +44,33 @@ class FrameMeta:
self.frame_id, self.timestamp_sof, self.timestamp_eof = vipc.frame_id, vipc.timestamp_sof, vipc.timestamp_eof
class ModelState:
frame: DrivingModelFrame
wide_frame: DrivingModelFrame
frame: ModelFrame
wide_frame: ModelFrame
inputs: dict[str, np.ndarray]
output: np.ndarray
prev_desire: np.ndarray # for tracking the rising edge of the pulse
model: ModelRunner
def __init__(self, context: CLContext):
self.frame = DrivingModelFrame(context)
self.wide_frame = DrivingModelFrame(context)
self.frame = ModelFrame(context)
self.wide_frame = ModelFrame(context)
self.prev_desire = np.zeros(ModelConstants.DESIRE_LEN, dtype=np.float32)
self.full_features_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN, ModelConstants.FEATURE_LEN), dtype=np.float32)
self.desire_20Hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.DESIRE_LEN), dtype=np.float32)
self.prev_desired_curv_20hz = np.zeros((ModelConstants.FULL_HISTORY_BUFFER_LEN + 1, ModelConstants.PREV_DESIRED_CURV_LEN), dtype=np.float32)
# img buffers are managed in openCL transform code
self.inputs = {}
self.inputs = {
'desire': np.zeros(ModelConstants.DESIRE_LEN * (ModelConstants.HISTORY_BUFFER_LEN+1), dtype=np.float32),
'traffic_convention': np.zeros(ModelConstants.TRAFFIC_CONVENTION_LEN, dtype=np.float32),
'lateral_control_params': np.zeros(ModelConstants.LATERAL_CONTROL_PARAMS_LEN, dtype=np.float32),
'prev_desired_curv': np.zeros(ModelConstants.PREV_DESIRED_CURV_LEN * (ModelConstants.HISTORY_BUFFER_LEN+1), dtype=np.float32),
'features_buffer': np.zeros(ModelConstants.HISTORY_BUFFER_LEN * ModelConstants.FEATURE_LEN, dtype=np.float32),
}
with open(METADATA_PATH, 'rb') as f:
model_metadata = pickle.load(f)
for key, shape in model_metadata['input_shapes'].items():
if key not in ["input_imgs", "big_input_imgs"]:
self.inputs[key] = np.zeros(shape, dtype=np.float32).flatten()
self.output_slices = model_metadata['output_slices']
net_output_size = model_metadata['output_shapes']['outputs'][1]
self.output = np.zeros(net_output_size, dtype=np.float32)
@@ -81,14 +82,6 @@ class ModelState:
for k,v in self.inputs.items():
self.model.addInput(k, v)
num_elements = model_metadata['input_shapes']['features_buffer'][1]
step_size = int(-100 / num_elements)
self.full_features_20Hz_idxs = np.arange(step_size, step_size * (num_elements + 1), step_size)[::-1]
desired_shape = int(self.inputs['desire'].shape[0] / self.desire_20Hz.shape[1])
middle_dim = int(self.desire_20Hz.shape[0] / desired_shape)
self.desire_reshape_dims = (desired_shape, middle_dim, -1)
def slice_outputs(self, model_outputs: np.ndarray) -> dict[str, np.ndarray]:
parsed_model_outputs = {k: model_outputs[np.newaxis, v] for k,v in self.output_slices.items()}
if SEND_RAW_PRED:
@@ -104,9 +97,10 @@ class ModelState:
self.desire_20Hz[:-1] = self.desire_20Hz[1:]
self.desire_20Hz[-1] = new_desire
self.inputs['desire'][:] = self.desire_20Hz.reshape(self.desire_reshape_dims).max(axis=1).flatten()
self.inputs['desire'][:] = self.desire_20Hz.reshape((25,4,-1)).max(axis=1).flatten()
self.inputs['traffic_convention'][:] = inputs['traffic_convention']
self.inputs['lateral_control_params'][:] = inputs['lateral_control_params']
self.model.setInputBuffer("input_imgs", self.frame.prepare(buf, transform.flatten(), self.model.getCLBuffer("input_imgs")))
self.model.setInputBuffer("big_input_imgs", self.wide_frame.prepare(wbuf, transform_wide.flatten(), self.model.getCLBuffer("big_input_imgs")))
@@ -120,25 +114,13 @@ class ModelState:
self.full_features_20Hz[:-1] = self.full_features_20Hz[1:]
self.full_features_20Hz[-1] = outputs['hidden_state'][0, :]
# idxs = np.arange(-4,-100,-4)[::-1]
self.inputs['features_buffer'][:] = self.full_features_20Hz[self.full_features_20Hz_idxs].flatten()
self.prev_desired_curv_20hz[:-1] = self.prev_desired_curv_20hz[1:]
self.prev_desired_curv_20hz[-1] = outputs['desired_curvature'][0, :]
if "lat_planner_solution" in outputs:
if "lat_planner_state" in self.inputs.keys():
self.inputs['lat_planner_state'][2] = interp(DT_MDL, ModelConstants.T_IDXS, outputs['lat_planner_solution'][0, :, 2])
self.inputs['lat_planner_state'][3] = interp(DT_MDL, ModelConstants.T_IDXS, outputs['lat_planner_solution'][0, :, 3])
if "desired_curvature" in outputs:
input_name_prev = None
if "prev_desired_curvs" in self.inputs.keys():
input_name_prev = 'prev_desired_curvs'
elif "prev_desired_curv" in self.inputs.keys():
input_name_prev = 'prev_desired_curv'
if input_name_prev is not None:
len = outputs['desired_curvature'][0].size
self.inputs[input_name_prev][:-len] = self.inputs[input_name_prev][len:]
self.inputs[input_name_prev][-len:] = outputs['desired_curvature'][0, :len]
idxs = np.arange(-4,-100,-4)[::-1]
self.inputs['features_buffer'][:] = self.full_features_20Hz[idxs].flatten()
# TODO model only uses last value now, once that changes we need to input strided action history buffer
self.inputs['prev_desired_curv'][-ModelConstants.PREV_DESIRED_CURV_LEN:] = 0. * self.prev_desired_curv_20hz[-4, :]
return outputs
@@ -249,6 +231,7 @@ def main(demo=False):
is_rhd = sm["driverMonitoringState"].isRHD
frame_id = sm["roadCameraState"].frameId
v_ego = max(sm["carState"].vEgo, 0.)
lateral_control_params = np.array([v_ego, steer_delay], dtype=np.float32)
if sm.updated["liveCalibration"] and sm.seen['roadCameraState'] and sm.seen['deviceState']:
device_from_calib_euler = np.array(sm["liveCalibration"].rpyCalib, dtype=np.float32)
dc = DEVICE_CAMERAS[(str(sm['deviceState'].deviceType), str(sm['roadCameraState'].sensor))]
@@ -279,20 +262,9 @@ def main(demo=False):
inputs:dict[str, np.ndarray] = {
'desire': vec_desire,
'traffic_convention': traffic_convention,
'lateral_control_params': lateral_control_params,
}
if "lateral_control_params" in model.inputs.keys():
inputs['lateral_control_params'] = np.array([v_ego, steer_delay], dtype=np.float32)
if "driving_style" in model.inputs.keys():
inputs['driving_style'] = np.array([1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0], dtype=np.float32)
if "nav_features" in model.inputs.keys():
inputs['nav_features'] = np.zeros(ModelConstants.NAV_FEATURE_LEN, dtype=np.float32) # Get size from shape
if "nav_instructions" in model.inputs.keys():
inputs['nav_instructions'] = np.zeros(ModelConstants.NAV_INSTRUCTION_LEN, dtype=np.float32) # Get size from shape
mt1 = time.perf_counter()
model_output = model.run(buf_main, buf_extra, model_transform_main, model_transform_extra, inputs, prepare_only)
mt2 = time.perf_counter()
@@ -302,8 +274,7 @@ def main(demo=False):
modelv2_send = messaging.new_message('modelV2')
drivingdata_send = messaging.new_message('drivingModelData')
posenet_send = messaging.new_message('cameraOdometry')
fill_model_msg(drivingdata_send, modelv2_send, model_output, v_ego, steer_delay,
publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
fill_model_msg(drivingdata_send, modelv2_send, model_output, publish_state, meta_main.frame_id, meta_extra.frame_id, frame_id,
frame_drop_ratio, meta_main.timestamp_eof, model_execution_time, live_calib_seen)
desire_state = modelv2_send.modelV2.meta.desireState
+19 -30
View File
@@ -1,30 +1,36 @@
#include "selfdrive/modeld/models/commonmodel.h"
#include <cassert>
#include <cmath>
#include <cstring>
#include "common/clutil.h"
DrivingModelFrame::DrivingModelFrame(cl_device_id device_id, cl_context context) : ModelFrame(device_id, context) {
input_frames = std::make_unique<float[]>(buf_size);
//input_frames_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, buf_size, NULL, &err));
ModelFrame::ModelFrame(cl_device_id device_id, cl_context context) {
input_frames = std::make_unique<uint8_t[]>(buf_size);
q = CL_CHECK_ERR(clCreateCommandQueue(context, device_id, 0, &err));
y_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, MODEL_WIDTH * MODEL_HEIGHT, NULL, &err));
u_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, (MODEL_WIDTH / 2) * (MODEL_HEIGHT / 2), NULL, &err));
v_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, (MODEL_WIDTH / 2) * (MODEL_HEIGHT / 2), NULL, &err));
img_buffer_20hz_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, 5*frame_size_bytes, NULL, &err));
region.origin = 4 * frame_size_bytes;
region.size = frame_size_bytes;
last_img_cl = CL_CHECK_ERR(clCreateSubBuffer(img_buffer_20hz_cl, CL_MEM_READ_WRITE, CL_BUFFER_CREATE_TYPE_REGION, &region, &err));
transform_init(&transform, context, device_id);
loadyuv_init(&loadyuv, context, device_id, MODEL_WIDTH, MODEL_HEIGHT);
init_transform(device_id, context, MODEL_WIDTH, MODEL_HEIGHT);
}
float* DrivingModelFrame::prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection, cl_mem* output) {
run_transform(yuv_cl, MODEL_WIDTH, MODEL_HEIGHT, frame_width, frame_height, frame_stride, frame_uv_offset, projection);
uint8_t* ModelFrame::prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3 &projection, cl_mem *output) {
transform_queue(&this->transform, q,
yuv_cl, frame_width, frame_height, frame_stride, frame_uv_offset,
y_cl, u_cl, v_cl, MODEL_WIDTH, MODEL_HEIGHT, projection);
for (int i = 0; i < 4; i++) {
CL_CHECK(clEnqueueCopyBuffer(q, img_buffer_20hz_cl, img_buffer_20hz_cl, (i+1)*frame_size_bytes, i*frame_size_bytes, frame_size_bytes, 0, nullptr, nullptr));
}
loadyuv_queue(&loadyuv, q, y_cl, u_cl, v_cl, last_img_cl);
if (output == NULL) {
CL_CHECK(clEnqueueReadBuffer(q, img_buffer_20hz_cl, CL_TRUE, 0, frame_size_bytes, &input_frames[0], 0, nullptr, nullptr));
CL_CHECK(clEnqueueReadBuffer(q, last_img_cl, CL_TRUE, 0, frame_size_bytes, &input_frames[MODEL_FRAME_SIZE], 0, nullptr, nullptr));
@@ -40,30 +46,13 @@ float* DrivingModelFrame::prepare(cl_mem yuv_cl, int frame_width, int frame_heig
}
}
DrivingModelFrame::~DrivingModelFrame() {
deinit_transform();
ModelFrame::~ModelFrame() {
transform_destroy(&transform);
loadyuv_destroy(&loadyuv);
CL_CHECK(clReleaseMemObject(img_buffer_20hz_cl));
CL_CHECK(clReleaseMemObject(last_img_cl));
CL_CHECK(clReleaseMemObject(v_cl));
CL_CHECK(clReleaseMemObject(u_cl));
CL_CHECK(clReleaseMemObject(y_cl));
CL_CHECK(clReleaseCommandQueue(q));
}
MonitoringModelFrame::MonitoringModelFrame(cl_device_id device_id, cl_context context) : ModelFrame(device_id, context) {
input_frames = std::make_unique<float[]>(buf_size);
//input_frame_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, buf_size, NULL, &err));
init_transform(device_id, context, MODEL_WIDTH, MODEL_HEIGHT);
}
float* MonitoringModelFrame::prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection, cl_mem* output) {
run_transform(yuv_cl, MODEL_WIDTH, MODEL_HEIGHT, frame_width, frame_height, frame_stride, frame_uv_offset, projection);
CL_CHECK(clEnqueueReadBuffer(q, y_cl, CL_TRUE, 0, MODEL_FRAME_SIZE * sizeof(float), input_frames.get(), 0, nullptr, nullptr));
clFinish(q);
//return &y_cl;
return input_frames.get();
}
MonitoringModelFrame::~MonitoringModelFrame() {
deinit_transform();
CL_CHECK(clReleaseCommandQueue(q));
}
}
+9 -69
View File
@@ -2,7 +2,6 @@
#include <cfloat>
#include <cstdlib>
#include <cassert>
#include <memory>
@@ -19,80 +18,21 @@
class ModelFrame {
public:
ModelFrame(cl_device_id device_id, cl_context context) {
q = CL_CHECK_ERR(clCreateCommandQueue(context, device_id, 0, &err));
}
virtual ~ModelFrame() {}
virtual float* prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection, cl_mem* output) { return NULL; }
/*
uint8_t* buffer_from_cl(cl_mem *in_frames, int buffer_size) {
CL_CHECK(clEnqueueReadBuffer(q, *in_frames, CL_TRUE, 0, buffer_size, input_frames.get(), 0, nullptr, nullptr));
clFinish(q);
return &input_frames[0];
}
*/
int MODEL_WIDTH;
int MODEL_HEIGHT;
int MODEL_FRAME_SIZE;
int buf_size;
protected:
cl_mem y_cl, u_cl, v_cl;
Transform transform;
cl_command_queue q;
std::unique_ptr<float[]> input_frames;
void init_transform(cl_device_id device_id, cl_context context, int model_width, int model_height) {
y_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, model_width * model_height, NULL, &err));
u_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, (model_width / 2) * (model_height / 2), NULL, &err));
v_cl = CL_CHECK_ERR(clCreateBuffer(context, CL_MEM_READ_WRITE, (model_width / 2) * (model_height / 2), NULL, &err));
transform_init(&transform, context, device_id);
}
void deinit_transform() {
transform_destroy(&transform);
CL_CHECK(clReleaseMemObject(v_cl));
CL_CHECK(clReleaseMemObject(u_cl));
CL_CHECK(clReleaseMemObject(y_cl));
}
void run_transform(cl_mem yuv_cl, int model_width, int model_height, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection) {
transform_queue(&transform, q,
yuv_cl, frame_width, frame_height, frame_stride, frame_uv_offset,
y_cl, u_cl, v_cl, model_width, model_height, projection);
}
};
class DrivingModelFrame : public ModelFrame {
public:
DrivingModelFrame(cl_device_id device_id, cl_context context);
~DrivingModelFrame();
float* prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection, cl_mem* output);
ModelFrame(cl_device_id device_id, cl_context context);
~ModelFrame();
uint8_t* prepare(cl_mem yuv_cl, int width, int height, int frame_stride, int frame_uv_offset, const mat3& transform, cl_mem *output);
const int MODEL_WIDTH = 512;
const int MODEL_HEIGHT = 256;
const int MODEL_FRAME_SIZE = MODEL_WIDTH * MODEL_HEIGHT * 3 / 2;
const int buf_size = MODEL_FRAME_SIZE * 2;
const size_t frame_size_bytes = MODEL_FRAME_SIZE * sizeof(float);
const size_t frame_size_bytes = MODEL_FRAME_SIZE * sizeof(uint8_t);
private:
Transform transform;
LoadYUVState loadyuv;
cl_mem img_buffer_20hz_cl, last_img_cl;//, input_frames_cl;
cl_command_queue q;
cl_mem y_cl, u_cl, v_cl, img_buffer_20hz_cl, last_img_cl;
cl_buffer_region region;
};
class MonitoringModelFrame : public ModelFrame {
public:
MonitoringModelFrame(cl_device_id device_id, cl_context context);
~MonitoringModelFrame();
float* prepare(cl_mem yuv_cl, int frame_width, int frame_height, int frame_stride, int frame_uv_offset, const mat3& projection, cl_mem* output);
const int MODEL_WIDTH = 1440;
const int MODEL_HEIGHT = 960;
const int MODEL_FRAME_SIZE = MODEL_WIDTH * MODEL_HEIGHT;
const int buf_size = MODEL_FRAME_SIZE;
private:
// cl_mem input_frame_cl;
};
std::unique_ptr<uint8_t[]> input_frames;
};
+2 -10
View File
@@ -14,13 +14,5 @@ cdef extern from "common/clutil.h":
cdef extern from "selfdrive/modeld/models/commonmodel.h":
cppclass ModelFrame:
int buf_size
# unsigned char * buffer_from_cl(cl_mem*, int);
float * prepare(cl_mem, int, int, int, int, mat3, cl_mem*)
cppclass DrivingModelFrame:
int buf_size
DrivingModelFrame(cl_device_id, cl_context)
cppclass MonitoringModelFrame:
int buf_size
MonitoringModelFrame(cl_device_id, cl_context)
ModelFrame(cl_device_id, cl_context)
unsigned char * prepare(cl_mem, int, int, int, int, mat3, cl_mem*)
+6 -37
View File
@@ -4,12 +4,11 @@
import numpy as np
cimport numpy as cnp
from libc.string cimport memcpy
from libc.stdint cimport uintptr_t
from msgq.visionipc.visionipc cimport cl_mem
from msgq.visionipc.visionipc_pyx cimport VisionBuf, CLContext as BaseCLContext
from .commonmodel cimport CL_DEVICE_TYPE_DEFAULT, cl_get_device_id, cl_create_context
from .commonmodel cimport mat3, ModelFrame as cppModelFrame, DrivingModelFrame as cppDrivingModelFrame, MonitoringModelFrame as cppMonitoringModelFrame
from .commonmodel cimport mat3, ModelFrame as cppModelFrame
cdef class CLContext(BaseCLContext):
@@ -24,17 +23,11 @@ cdef class CLMem:
mem.mem = <cl_mem*> cmem
return mem
@property
def mem_address(self):
return <uintptr_t>(self.mem)
def cl_from_visionbuf(VisionBuf buf):
return CLMem.create(<void*>&buf.buf.buf_cl)
cdef class ModelFrame:
cdef cppModelFrame * frame
cdef int buf_size
def __cinit__(self, CLContext context):
self.frame = new cppModelFrame(context.device_id, context.context)
def __dealloc__(self):
del self.frame
@@ -42,35 +35,11 @@ cdef class ModelFrame:
def prepare(self, VisionBuf buf, float[:] projection, CLMem output):
cdef mat3 cprojection
memcpy(cprojection.v, &projection[0], 9*sizeof(float))
cdef float * data
cdef unsigned char * data
if output is None:
data = self.frame.prepare(buf.buf.buf_cl, buf.width, buf.height, buf.stride, buf.uv_offset, cprojection, NULL)
else:
data = self.frame.prepare(buf.buf.buf_cl, buf.width, buf.height, buf.stride, buf.uv_offset, cprojection, output.mem)
if not data:
return None
return np.asarray(<cnp.float32_t[:self.buf_size]> data)
# return CLMem.create(data)
# def buffer_from_cl(self, CLMem in_frames):
# cdef unsigned char * data2
# data2 = self.frame.buffer_from_cl(in_frames.mem, self.buf_size)
# return np.asarray(<cnp.uint8_t[:self.buf_size]> data2)
cdef class DrivingModelFrame(ModelFrame):
cdef cppDrivingModelFrame * _frame
def __cinit__(self, CLContext context):
self._frame = new cppDrivingModelFrame(context.device_id, context.context)
self.frame = <cppModelFrame*>(self._frame)
self.buf_size = self._frame.buf_size
cdef class MonitoringModelFrame(ModelFrame):
cdef cppMonitoringModelFrame * _frame
def __cinit__(self, CLContext context):
self._frame = new cppMonitoringModelFrame(context.device_id, context.context)
self.frame = <cppModelFrame*>(self._frame)
self.buf_size = self._frame.buf_size
return np.asarray(<cnp.uint8_t[:self.frame.buf_size]> data)
+2 -2
View File
@@ -1,3 +1,3 @@
version https://git-lfs.github.com/spec/v1
oid sha256:b31b504bc0b440d3bc72967507a00eb4f112285626fbfb3135011500325ee6d6
size 51452435
oid sha256:c829d824ebc73d15da82516592c07d9784369ccbf710698e919e06a702e70924
size 50320138
+2
View File
@@ -96,6 +96,8 @@ class Parser:
out_shape=(ModelConstants.LEAD_TRAJ_LEN,ModelConstants.LEAD_WIDTH))
if 'lat_planner_solution' in outs:
self.parse_mdn('lat_planner_solution', outs, in_N=0, out_N=0, out_shape=(ModelConstants.IDX_N,ModelConstants.LAT_PLANNER_SOLUTION_WIDTH))
if 'desired_curvature' in outs:
self.parse_mdn('desired_curvature', outs, in_N=0, out_N=0, out_shape=(ModelConstants.DESIRED_CURV_WIDTH,))
for k in ['lead_prob', 'lane_lines_prob', 'meta']:
self.parse_binary_crossentropy(k, outs)
self.parse_categorical_crossentropy('desire_state', outs, out_shape=(ModelConstants.DESIRE_PRED_WIDTH,))
+31 -4
View File
@@ -1,12 +1,39 @@
import os
import onnx
import itertools
import os
import sys
import numpy as np
from typing import Any
from openpilot.selfdrive.modeld.runners.runmodel_pyx import RunModel
from openpilot.selfdrive.modeld.runners.ort_helpers import convert_fp16_to_fp32, ORT_TYPES_TO_NP_TYPES
ORT_TYPES_TO_NP_TYPES = {'tensor(float16)': np.float16, 'tensor(float)': np.float32, 'tensor(uint8)': np.uint8}
def attributeproto_fp16_to_fp32(attr):
float32_list = np.frombuffer(attr.raw_data, dtype=np.float16)
attr.data_type = 1
attr.raw_data = float32_list.astype(np.float32).tobytes()
def convert_fp16_to_fp32(onnx_path_or_bytes):
if isinstance(onnx_path_or_bytes, bytes):
model = onnx.load_from_string(onnx_path_or_bytes)
elif isinstance(onnx_path_or_bytes, str):
model = onnx.load(onnx_path_or_bytes)
for i in model.graph.initializer:
if i.data_type == 10:
attributeproto_fp16_to_fp32(i)
for i in itertools.chain(model.graph.input, model.graph.output):
if i.type.tensor_type.elem_type == 10:
i.type.tensor_type.elem_type = 1
for i in model.graph.node:
if i.op_type == 'Cast' and i.attribute[0].i == 10:
i.attribute[0].i = 1
for a in i.attribute:
if hasattr(a, 't'):
if a.t.data_type == 10:
attributeproto_fp16_to_fp32(a.t)
return model.SerializeToString()
def create_ort_session(path, fp16_to_fp32):
os.environ["OMP_NUM_THREADS"] = "4"
@@ -22,14 +49,14 @@ def create_ort_session(path, fp16_to_fp32):
provider = 'OpenVINOExecutionProvider'
elif 'CUDAExecutionProvider' in ort.get_available_providers() and 'ONNXCPU' not in os.environ:
options.intra_op_num_threads = 2
provider = ('CUDAExecutionProvider', {'cudnn_conv_algo_search': 'EXHAUSTIVE'})
provider = ('CUDAExecutionProvider', {'cudnn_conv_algo_search': 'DEFAULT'})
else:
options.intra_op_num_threads = 2
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
provider = 'CPUExecutionProvider'
model_data = convert_fp16_to_fp32(onnx.load(path)) if fp16_to_fp32 else path
model_data = convert_fp16_to_fp32(path) if fp16_to_fp32 else path
print("Onnx selected provider: ", [provider], file=sys.stderr)
ort_session = ort.InferenceSession(model_data, options, providers=[provider])
print("Onnx using ", ort_session.get_providers(), file=sys.stderr)
-36
View File
@@ -1,36 +0,0 @@
import onnx
import onnxruntime as ort
import numpy as np
import itertools
ORT_TYPES_TO_NP_TYPES = {'tensor(float16)': np.float16, 'tensor(float)': np.float32, 'tensor(uint8)': np.uint8}
def attributeproto_fp16_to_fp32(attr):
float32_list = np.frombuffer(attr.raw_data, dtype=np.float16)
attr.data_type = 1
attr.raw_data = float32_list.astype(np.float32).tobytes()
def convert_fp16_to_fp32(model):
for i in model.graph.initializer:
if i.data_type == 10:
attributeproto_fp16_to_fp32(i)
for i in itertools.chain(model.graph.input, model.graph.output):
if i.type.tensor_type.elem_type == 10:
i.type.tensor_type.elem_type = 1
for i in model.graph.node:
if i.op_type == 'Cast' and i.attribute[0].i == 10:
i.attribute[0].i = 1
for a in i.attribute:
if hasattr(a, 't'):
if a.t.data_type == 10:
attributeproto_fp16_to_fp32(a.t)
return model.SerializeToString()
def make_onnx_cpu_runner(model_path):
options = ort.SessionOptions()
options.intra_op_num_threads = 4
options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL
options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
model_data = convert_fp16_to_fp32(onnx.load(model_path))
return ort.InferenceSession(model_data, options, providers=['CPUExecutionProvider'])
+13 -13
View File
@@ -1,7 +1,7 @@
#define UV_SIZE ((TRANSFORMED_WIDTH/2)*(TRANSFORMED_HEIGHT/2))
__kernel void loadys(__global uchar8 const * const Y,
__global float * out,
__global uchar * out,
int out_offset)
{
const int gid = get_global_id(0);
@@ -10,13 +10,12 @@ __kernel void loadys(__global uchar8 const * const Y,
const int ox = ois % TRANSFORMED_WIDTH;
const uchar8 ys = Y[gid];
const float8 ysf = convert_float8(ys);
// 02
// 13
__global float* outy0;
__global float* outy1;
__global uchar* outy0;
__global uchar* outy1;
if ((oy & 1) == 0) {
outy0 = out + out_offset; //y0
outy1 = out + out_offset + UV_SIZE*2; //y2
@@ -25,23 +24,24 @@ __kernel void loadys(__global uchar8 const * const Y,
outy1 = out + out_offset + UV_SIZE*3; //y3
}
vstore4(ysf.s0246, 0, outy0 + (oy/2) * (TRANSFORMED_WIDTH/2) + ox/2);
vstore4(ysf.s1357, 0, outy1 + (oy/2) * (TRANSFORMED_WIDTH/2) + ox/2);
vstore4(ys.s0246, 0, outy0 + (oy/2) * (TRANSFORMED_WIDTH/2) + ox/2);
vstore4(ys.s1357, 0, outy1 + (oy/2) * (TRANSFORMED_WIDTH/2) + ox/2);
}
__kernel void loaduv(__global uchar8 const * const in,
__global float8 * out,
__global uchar8 * out,
int out_offset)
{
const int gid = get_global_id(0);
const uchar8 inv = in[gid];
const float8 outv = convert_float8(inv);
out[gid + out_offset / 8] = outv;
out[gid + out_offset / 8] = inv;
}
__kernel void copy(__global float8 * inout,
int in_offset)
__kernel void copy(__global uchar8 * in,
__global uchar8 * out,
int in_offset,
int out_offset)
{
const int gid = get_global_id(0);
inout[gid] = inout[gid + in_offset / 8];
}
out[gid + out_offset / 8] = in[gid + in_offset / 8];
}
-4
View File
@@ -145,10 +145,6 @@ void Panda::set_can_speed_kbps(uint16_t bus, uint16_t speed) {
handle->control_write(0xde, bus, (speed * 10));
}
void Panda::set_can_fd_auto(uint16_t bus, bool enabled) {
handle->control_write(0xe8, bus, enabled);
}
void Panda::set_data_speed_kbps(uint16_t bus, uint16_t speed) {
handle->control_write(0xf9, bus, (speed * 10));
}
-1
View File
@@ -77,7 +77,6 @@ public:
void enable_deepsleep();
void send_heartbeat(bool engaged);
void set_can_speed_kbps(uint16_t bus, uint16_t speed);
void set_can_fd_auto(uint16_t bus, bool enabled);
void set_data_speed_kbps(uint16_t bus, uint16_t speed);
void set_canfd_non_iso(uint16_t bus, bool non_iso);
void can_send(const capnp::List<cereal::CanData>::Reader &can_data_list);
+2 -2
View File
@@ -36,7 +36,7 @@ PandaUsbHandle::PandaUsbHandle(std::string serial) : PandaCommsHandle(serial) {
for (size_t i = 0; i < num_devices; ++i) {
libusb_device_descriptor desc;
libusb_get_device_descriptor(dev_list[i], &desc);
if (desc.idVendor == 0x3801 && desc.idProduct == 0xddcc) {
if (desc.idVendor == 0xbbaa && desc.idProduct == 0xddcc) {
int ret = libusb_open(dev_list[i], &dev_handle);
if (dev_handle == NULL || ret < 0) { goto fail; }
@@ -110,7 +110,7 @@ std::vector<std::string> PandaUsbHandle::list() {
libusb_device *device = dev_list[i];
libusb_device_descriptor desc;
libusb_get_device_descriptor(device, &desc);
if (desc.idVendor == 0x3801 && desc.idProduct == 0xddcc) {
if (desc.idVendor == 0xbbaa && desc.idProduct == 0xddcc) {
libusb_device_handle *handle = NULL;
int ret = libusb_open(device, &handle);
if (ret < 0) { goto finish; }
-5
View File
@@ -67,10 +67,6 @@ Panda *connect(std::string serial="", uint32_t index=0) {
}
//panda->enable_deepsleep();
for (int i = 0; i < PANDA_BUS_CNT; i++) {
panda->set_can_fd_auto(i, true);
}
if (!panda->up_to_date() && !getenv("BOARDD_SKIP_FW_CHECK")) {
throw std::runtime_error("Panda firmware out of date. Run pandad.py to update.");
}
@@ -416,7 +412,6 @@ void process_peripheral_state(Panda *panda, PubMaster *pm, bool no_fan_control)
if (ir_pwr != prev_ir_pwr || sm.frame % 100 == 0 || ir_pwr >= 50.0) {
panda->set_ir_pwr(ir_pwr);
Hardware::set_ir_power(ir_pwr);
prev_ir_pwr = ir_pwr;
}
}
+2 -2
View File
@@ -239,7 +239,7 @@ int PandaSpiHandle::spi_transfer_retry(uint8_t endpoint, uint8_t *tx_data, uint1
// due to full TX buffers
nack_count += 1;
if (nack_count > 3) {
SPILOG(LOGD, "NACK sleep %d", nack_count);
SPILOG(LOGE, "NACK sleep %d", nack_count);
usleep(std::clamp(nack_count*10, 200, 2000));
}
}
@@ -418,7 +418,7 @@ fail:
}
}
if (ret >= 0) ret = -1;
if (ret > 0) ret = -1;
return ret;
}
#endif
-2
View File
@@ -6,7 +6,6 @@ import cereal.messaging as messaging
from cereal import log
from openpilot.common.gpio import gpio_set, gpio_init
from panda import Panda, PandaDFU, PandaProtocolMismatch
from openpilot.common.retry import retry
from openpilot.system.manager.process_config import managed_processes
from openpilot.system.hardware import HARDWARE
from openpilot.system.hardware.tici.pins import GPIO
@@ -52,7 +51,6 @@ class TestPandad:
assert not Panda.wait_for_dfu(None, 3)
assert not Panda.wait_for_panda(None, 3)
@retry(attempts=3)
def _flash_bootstub_and_test(self, fn, expect_mismatch=False):
self._go_to_dfu()
pd = PandaDFU(None)
@@ -14,7 +14,7 @@ from openpilot.common.params import Params
from openpilot.common.timeout import Timeout
from openpilot.selfdrive.pandad import can_list_to_can_capnp
from openpilot.system.hardware import TICI
from openpilot.selfdrive.test.helpers import with_processes
from openpilot.selfdrive.test.helpers import phone_only, with_processes
@retry(attempts=3)
@@ -72,6 +72,7 @@ class TestBoarddLoopback:
os.environ['STARTED'] = '1'
os.environ['BOARDD_LOOPBACK'] = '1'
@phone_only
@with_processes(['pandad'])
def test_loopback(self):
num_pandas = 2 if TICI and "SINGLE_PANDA" not in os.environ else 1
+2 -1
View File
@@ -7,7 +7,7 @@ import random
import cereal.messaging as messaging
from cereal.services import SERVICE_LIST
from openpilot.system.hardware import HARDWARE
from openpilot.selfdrive.test.helpers import with_processes
from openpilot.selfdrive.test.helpers import phone_only, with_processes
from openpilot.selfdrive.pandad.tests.test_pandad_loopback import setup_pandad, send_random_can_messages
JUNGLE_SPAM = "JUNGLE_SPAM" in os.environ
@@ -23,6 +23,7 @@ class TestBoarddSpi:
if not JUNGLE_SPAM:
os.environ['BOARDD_LOOPBACK'] = '1'
@phone_only
@with_processes(['pandad'])
def test_spi_corruption(self, subtests):
setup_pandad(1)
+1 -1
View File
@@ -42,7 +42,7 @@ class TestAlerts:
for name, e in events.items():
if not name.endswith("DEPRECATED"):
fail_msg = f"{name} @{e} not in EVENTS"
fail_msg = "%s @%d not in EVENTS" % (name, e)
assert e in EVENTS.keys(), fail_msg
# ensure alert text doesn't exceed allowed width
+9
View File
@@ -10,6 +10,7 @@ from functools import wraps
import cereal.messaging as messaging
from openpilot.common.params import Params
from openpilot.system.manager.process_config import managed_processes
from openpilot.system.hardware import PC
from openpilot.system.version import training_version, terms_version
@@ -28,6 +29,14 @@ def set_params_enabled():
msg.liveCalibration.rpyCalib = [0.0, 0.0, 0.0]
params.put("CalibrationParams", msg.to_bytes())
def phone_only(f):
@wraps(f)
def wrap(self, *args, **kwargs):
if PC:
pytest.skip("This test is not meant to run on PC")
return f(self, *args, **kwargs)
return wrap
def release_only(f):
@wraps(f)
def wrap(self, *args, **kwargs):
+10 -34
View File
@@ -7,20 +7,19 @@ import tempfile
from itertools import zip_longest
import matplotlib.pyplot as plt
import numpy as np
from openpilot.common.git import get_commit
from openpilot.system.hardware import PC
from openpilot.tools.lib.openpilotci import get_url
from openpilot.selfdrive.test.process_replay.compare_logs import compare_logs, format_diff
from openpilot.selfdrive.test.process_replay.process_replay import get_process_config, replay_process
from openpilot.tools.lib.framereader import FrameReader, NumpyFrameReader
from openpilot.tools.lib.framereader import FrameReader
from openpilot.tools.lib.logreader import LogReader, save_log
from openpilot.tools.lib.github_utils import GithubUtils
TEST_ROUTE = "2f4452b03ccb98f0|2022-12-03--13-45-30"
SEGMENT = 6
MAX_FRAMES = 100 if PC else 400
MAX_FRAMES = 100 if PC else 600
NO_MODEL = "NO_MODEL" in os.environ
SEND_EXTRA_INPUTS = bool(int(os.getenv("SEND_EXTRA_INPUTS", "0")))
@@ -32,14 +31,14 @@ GITHUB = GithubUtils(API_TOKEN, DATA_TOKEN)
def get_log_fn(test_route, ref="master"):
return f"{test_route}_model_tici_{ref}.zst"
return f"{test_route}_model_tici_{ref}.bz2"
def plot(proposed, master, title, tmp):
proposed = list(proposed)
master = list(master)
fig, ax = plt.subplots()
ax.plot(master, label='MASTER')
ax.plot(proposed, label='PROPOSED')
ax.plot(master, label='MASTER')
plt.legend(loc='best')
plt.title(title)
plt.savefig(f'{tmp}/{title}.png')
@@ -152,44 +151,21 @@ def model_replay(lr, frs):
dmonitoringmodeld = get_process_config("dmonitoringmodeld")
modeld_msgs = replay_process(modeld, modeld_logs, frs)
if isinstance(frs['roadCameraState'], NumpyFrameReader):
del frs['roadCameraState'].frames
del frs['wideRoadCameraState'].frames
dmonitoringmodeld_msgs = replay_process(dmonitoringmodeld, dmodeld_logs, frs)
return modeld_msgs + dmonitoringmodeld_msgs
def get_frames():
regen_cache = "--regen-cache" in sys.argv
cache = "--cache" in sys.argv or not PC or regen_cache
videos = ('fcamera.hevc', 'dcamera.hevc', 'ecamera.hevc')
cams = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
if cache:
frames_cache = '/tmp/model_replay_cache' if PC else '/data/model_replay_cache'
os.makedirs(frames_cache, exist_ok=True)
cache_size = 200
for v in videos:
if not all(os.path.isfile(f'{frames_cache}/{TEST_ROUTE}_{v}_{i}.npy') for i in range(MAX_FRAMES//cache_size)) or regen_cache:
f = FrameReader(get_url(TEST_ROUTE, SEGMENT, v)).get(0, MAX_FRAMES + 1, pix_fmt="nv12")
print(f'Caching {v}...')
for i in range(MAX_FRAMES//cache_size):
np.save(f'{frames_cache}/{TEST_ROUTE}_{v}_{i}', f[(i * cache_size) + 1:((i + 1) * cache_size) + 1])
del f
return {c : NumpyFrameReader(f"{frames_cache}/{TEST_ROUTE}_{v}", 1928, 1208, cache_size) for c,v in zip(cams, videos, strict=True)}
else:
return {c : FrameReader(get_url(TEST_ROUTE, SEGMENT, v), readahead=True) for c,v in zip(cams, videos, strict=True)}
if __name__ == "__main__":
update = "--update" in sys.argv or (os.getenv("GIT_BRANCH", "") == 'master')
replay_dir = os.path.dirname(os.path.abspath(__file__))
# load logs
lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.zst")))
frs = get_frames()
lr = list(LogReader(get_url(TEST_ROUTE, SEGMENT, "rlog.bz2")))
frs = {
'roadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "fcamera.hevc"), readahead=True),
'driverCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "dcamera.hevc"), readahead=True),
'wideRoadCameraState': FrameReader(get_url(TEST_ROUTE, SEGMENT, "ecamera.hevc"), readahead=True)
}
log_msgs = []
# run replays
@@ -5,13 +5,13 @@ import copy
import json
import heapq
import signal
import platform
from collections import Counter, OrderedDict
from dataclasses import dataclass, field
from typing import Any
from collections.abc import Callable, Iterable
from tqdm import tqdm
import capnp
from openpilot.system.hardware.hw import Paths
import cereal.messaging as messaging
from cereal import car
@@ -780,7 +780,8 @@ def generate_params_config(lr=None, CP=None, fingerprint=None, custom_params=Non
def generate_environ_config(CP=None, fingerprint=None, log_dir=None) -> dict[str, Any]:
environ_dict = {}
environ_dict["PARAMS_ROOT"] = f"{Paths.shm_path()}/params"
if platform.system() != "Darwin":
environ_dict["PARAMS_ROOT"] = "/dev/shm/params"
if log_dir is not None:
environ_dict["LOG_ROOT"] = log_dir
+1 -1
View File
@@ -1 +1 @@
cae12bc0a2960de17104a9e22fafe33d797fbcee
dd0a40182707975c94a40d19198cd60c88735199
+2 -14
View File
@@ -72,7 +72,7 @@ PROCS = {
PROCS.update({
"tici": {
"./pandad": 5.0,
"./pandad": 4.0,
"./ubloxd": 1.0,
"system.ubloxd.pigeond": 6.0,
},
@@ -138,7 +138,6 @@ class TestOnroad:
proc = None
try:
manager_path = os.path.join(BASEDIR, "system/manager/manager.py")
cls.manager_st = time.monotonic()
proc = subprocess.Popen(["python", manager_path])
sm = messaging.SubMaster(['carState'])
@@ -203,10 +202,6 @@ class TestOnroad:
with subtests.test(service=s):
assert len(msgs) >= math.floor(SERVICE_LIST[s].frequency*int(TEST_DURATION*0.8))
def test_manager_starting_time(self):
st = self.msgs['managerState'][0].logMonoTime / 1e9
assert (st - self.manager_st) < 10, f"manager.py took {st - self.manager_st}s to publish the first 'managerState' msg"
def test_cloudlog_size(self):
msgs = self.msgs['logMessage']
@@ -300,18 +295,13 @@ class TestOnroad:
assert cpu_ok
def test_memory_usage(self):
print("\n------------------------------------------------")
print("--------------- Memory Usage -------------------")
print("------------------------------------------------")
offset = int(SERVICE_LIST['deviceState'].frequency * LOG_OFFSET)
mems = [m.deviceState.memoryUsagePercent for m in self.msgs['deviceState'][offset:]]
print("Memory usage: ", mems)
# check for big leaks. note that memory usage is
# expected to go up while the MSGQ buffers fill up
assert np.average(mems) <= 65, "Average memory usage above 65%"
assert np.max(np.diff(mems)) <= 4, "Max memory increase too high"
assert np.average(np.diff(mems)) <= 1, "Average memory increase too high"
assert max(mems) - min(mems) <= 3.0
def test_gpu_usage(self):
assert self.gpu_procs == {"weston", "ui", "camerad", "selfdrive.modeld.modeld", "selfdrive.modeld.dmonitoringmodeld"}
@@ -378,8 +368,6 @@ class TestOnroad:
]
for (s, instant_max, avg_max) in cfgs:
ts = [getattr(m, s).modelExecutionTime for m in self.msgs[s]]
# TODO some init can happen in first iteration
ts = ts[1:]
assert max(ts) < instant_max, f"high '{s}' execution time: {max(ts)}"
assert np.mean(ts) < avg_max, f"high avg '{s}' execution time: {np.mean(ts)}"
result += f"'{s}' execution time: min {min(ts):.5f}s\n"
+24 -22
View File
@@ -67,13 +67,14 @@ if GetOption('extras'):
qt_src.remove("main.cc") # replaced by test_runner
qt_env.Program('tests/test_translations', [asset_obj, 'tests/test_runner.cc', 'tests/test_translations.cc'] + qt_src, LIBS=qt_libs)
if GetOption('extras'):
if GetOption('extras') and arch != "Darwin":
qt_env.SharedLibrary("qt/python_helpers", ["qt/qt_window.cc"], LIBS=qt_libs)
# spinner and text window
qt_env.Program("_text", ["qt/text.cc"], LIBS=qt_libs)
qt_env.Program("_spinner", ["qt/spinner.cc"], LIBS=qt_libs)
# setup and factory resetter
qt_env.Program("qt/setup/reset", ["qt/setup/reset.cc"], LIBS=qt_libs)
qt_env.Program("qt/setup/setup", ["qt/setup/setup.cc", asset_obj],
@@ -82,31 +83,32 @@ if GetOption('extras'):
# build updater UI
qt_env.Program("qt/setup/updater", ["qt/setup/updater.cc", asset_obj], LIBS=qt_libs)
if arch != "Darwin":
# build installers
senv = qt_env.Clone()
senv['LINKFLAGS'].append('-Wl,-strip-debug')
# build installers
senv = qt_env.Clone()
senv['LINKFLAGS'].append('-Wl,-strip-debug')
release = "release3"
installers = [
("openpilot", release),
("openpilot_test", f"{release}-staging"),
("openpilot_nightly", "nightly"),
("openpilot_internal", "nightly-dev"),
]
release = "release3"
installers = [
("openpilot", release),
("openpilot_test", f"{release}-staging"),
("openpilot_nightly", "nightly"),
("openpilot_internal", "nightly-dev"),
]
cont = senv.Command(f"installer/continue_openpilot.o", f"installer/continue_openpilot.sh",
"ld -r -b binary -o $TARGET $SOURCE")
for name, branch in installers:
d = {'BRANCH': f"'\"{branch}\"'"}
if "internal" in name:
d['INTERNAL'] = "1"
cont = senv.Command(f"installer/continue_openpilot.o", f"installer/continue_openpilot.sh",
"ld -r -b binary -o $TARGET $SOURCE")
for name, branch in installers:
d = {'BRANCH': f"'\"{branch}\"'"}
if "internal" in name:
d['INTERNAL'] = "1"
obj = senv.Object(f"installer/installers/installer_{name}.o", ["installer/installer.cc"], CPPDEFINES=d)
f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=qt_libs)
# keep installers small
assert f[0].get_size() < 370*1e3
obj = senv.Object(f"installer/installers/installer_{name}.o", ["installer/installer.cc"], CPPDEFINES=d)
f = senv.Program(f"installer/installers/installer_{name}", [obj, cont], LIBS=qt_libs)
# keep installers small
assert f[0].get_size() < 370*1e3
# build watch3
if arch in ['x86_64', 'aarch64', 'Darwin'] or GetOption('extras'):
qt_env.Program("watch3", ["watch3.cc"], LIBS=qt_libs + ['common', 'msgq', 'visionipc'])
SConscript(['raylib/SConscript'])
-1
View File
@@ -124,7 +124,6 @@ OffroadHome::OffroadHome(QWidget* parent) : QFrame(parent) {
// left: PrimeAdWidget
QStackedWidget *left_widget = new QStackedWidget(this);
QVBoxLayout *left_prime_layout = new QVBoxLayout();
left_prime_layout->setContentsMargins(0, 0, 0, 0);
QWidget *prime_user = new PrimeUserWidget();
prime_user->setStyleSheet(R"(
border-radius: 10px;
+15 -13
View File
@@ -203,7 +203,7 @@ void WifiManager::connect(const Network &n, const bool is_hidden, const QString
connection["ipv4"]["dns-priority"] = 600;
connection["ipv6"]["method"] = "ignore";
asyncCall(NM_DBUS_PATH_SETTINGS, NM_DBUS_INTERFACE_SETTINGS, "AddConnection", QVariant::fromValue(connection));
call(NM_DBUS_PATH_SETTINGS, NM_DBUS_INTERFACE_SETTINGS, "AddConnection", QVariant::fromValue(connection));
}
void WifiManager::deactivateConnectionBySsid(const QString &ssid) {
@@ -330,10 +330,6 @@ void WifiManager::initConnections() {
lteConnectionPath = path;
}
}
if (!isKnownConnection(tethering_ssid)) {
addTetheringConnection();
}
}
std::optional<QDBusPendingCall> WifiManager::activateWifiConnection(const QString &ssid) {
@@ -403,13 +399,9 @@ void WifiManager::updateGsmSettings(bool roaming, QString apn, bool metered) {
}
if (changes) {
QDBusPendingCall pending_call = asyncCall(lteConnectionPath.path(), NM_DBUS_INTERFACE_SETTINGS_CONNECTION, "UpdateUnsaved", QVariant::fromValue(settings)); // update is temporary
QDBusPendingCallWatcher *watcher = new QDBusPendingCallWatcher(pending_call);
QObject::connect(watcher, &QDBusPendingCallWatcher::finished, this, [this, watcher]() {
deactivateConnection(lteConnectionPath);
activateModemConnection(lteConnectionPath);
watcher->deleteLater();
});
call(lteConnectionPath.path(), NM_DBUS_INTERFACE_SETTINGS_CONNECTION, "UpdateUnsaved", QVariant::fromValue(settings)); // update is temporary
deactivateConnection(lteConnectionPath);
activateModemConnection(lteConnectionPath);
}
}
}
@@ -442,7 +434,10 @@ void WifiManager::addTetheringConnection() {
connection["ipv4"]["route-metric"] = 1100;
connection["ipv6"]["method"] = "ignore";
asyncCall(NM_DBUS_PATH_SETTINGS, NM_DBUS_INTERFACE_SETTINGS, "AddConnection", QVariant::fromValue(connection));
auto path = call<QDBusObjectPath>(NM_DBUS_PATH_SETTINGS, NM_DBUS_INTERFACE_SETTINGS, "AddConnection", QVariant::fromValue(connection));
if (!path.path().isEmpty()) {
knownConnections[path] = tethering_ssid;
}
}
void WifiManager::tetheringActivated(QDBusPendingCallWatcher *call) {
@@ -458,6 +453,10 @@ void WifiManager::tetheringActivated(QDBusPendingCallWatcher *call) {
void WifiManager::setTetheringEnabled(bool enabled) {
if (enabled) {
if (!isKnownConnection(tethering_ssid)) {
addTetheringConnection();
}
auto pending_call = activateWifiConnection(tethering_ssid);
if (pending_call) {
@@ -479,6 +478,9 @@ bool WifiManager::isTetheringEnabled() {
}
QString WifiManager::getTetheringPassword() {
if (!isKnownConnection(tethering_ssid)) {
addTetheringConnection();
}
const QDBusObjectPath &path = getConnectionPath(tethering_ssid);
if (!path.path().isEmpty()) {
QDBusReply<QMap<QString, QVariantMap>> response = call(path.path(), NM_DBUS_INTERFACE_SETTINGS_CONNECTION, "GetSecrets", "802-11-wireless-security");
+1 -1
View File
@@ -408,7 +408,7 @@ Setup::Setup(QWidget *parent) : QStackedWidget(parent) {
std::stringstream buffer;
buffer << std::ifstream("/sys/class/hwmon/hwmon1/in1_input").rdbuf();
float voltage = (float)std::atoi(buffer.str().c_str()) / 1000.;
if (voltage < 7) {
if (voltage > 0 && voltage < 7) {
addWidget(low_voltage());
}
+1
View File
@@ -0,0 +1 @@
_spinner
+17
View File
@@ -0,0 +1,17 @@
Import('env', 'arch', 'common')
raylib_env = env.Clone()
raylib_util_lib = env.Library("raylib_util_lib", ['util.cc'], LIBS='raylib')
linked_libs = ['raylib', raylib_util_lib, common]
raylib_env['LIBPATH'] += [f'#third_party/raylib/{arch}/']
mac_frameworks = []
if arch == "Darwin":
mac_frameworks += ['OpenCL', 'CoreVideo', 'Cocoa', 'GLUT', 'CoreFoundation', 'OpenGL', 'IOKit']
elif arch == 'larch64':
linked_libs += []
else:
linked_libs += ['OpenCL', 'dl', 'pthread']
if arch != 'aarch64':
raylib_env.Program("_spinner", ["spinner.cc"], LIBS=linked_libs, FRAMEWORKS=mac_frameworks)

Some files were not shown because too many files have changed in this diff Show More