Compare commits

..

2 Commits

Author SHA1 Message Date
DevTekVE dff2c1987e Hotfix to avoid infinite loop waiting for comma registration after downgrade 2025-03-18 18:11:14 +01:00
Jason Wen f1cb76c2b4 sunnypilot v0.9.8.0 2024-08-15 10:01:52 +00:00
1539 changed files with 595390 additions and 85833 deletions
-19
View File
@@ -1,19 +0,0 @@
---
Checks: '
bugprone-*,
-bugprone-integer-division,
-bugprone-narrowing-conversions,
performance-*,
clang-analyzer-*,
misc-*,
-misc-unused-parameters,
modernize-*,
-modernize-avoid-c-arrays,
-modernize-deprecated-headers,
-modernize-use-auto,
-modernize-use-using,
-modernize-use-nullptr,
-modernize-use-trailing-return-type,
'
CheckOptions:
...
-3
View File
@@ -1,3 +0,0 @@
.Xauthority
.env
.host/
-18
View File
@@ -1,18 +0,0 @@
FROM ghcr.io/commaai/openpilot-base:latest
RUN apt update && apt install -y vim net-tools usbutils htop ripgrep tmux wget mesa-utils xvfb libxtst6 libxv1 libglu1-mesa gdb bash-completion
RUN python3 -m ensurepip --upgrade
RUN pip3 install ipython jupyter jupyterlab
RUN cd /tmp && \
ARCH=$(arch | sed s/aarch64/arm64/ | sed s/x86_64/amd64/) && \
curl -L -o virtualgl.deb "https://github.com/VirtualGL/virtualgl/releases/download/3.1.1/virtualgl_3.1.1_$ARCH.deb" && \
dpkg -i virtualgl.deb
RUN usermod -aG video batman
USER batman
RUN cd $HOME && \
curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.tmux.conf && \
curl -O https://raw.githubusercontent.com/commaai/agnos-builder/master/userspace/home/.vimrc
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
TARGET_USER=batman
source .devcontainer/.host/.env
# override display flag for mac hosts
if [[ $HOST_OS == darwin ]]; then
echo "Setting up DISPLAY override for macOS..."
cat <<EOF >> /home/$TARGET_USER/.bashrc
source .devcontainer/.host/.env
if [ -n "\$HOST_DISPLAY" ]; then
DISPLAY_NUM=\$(echo "\$HOST_DISPLAY" | awk -F: '{print \$NF}')
export DISPLAY=host.docker.internal:\$DISPLAY_NUM
fi
EOF
fi
# setup virtualgl for mac hosts
if [[ $HOST_OS == darwin ]]; then
echo "Setting up virtualgl for macOS..."
cat <<EOF >> /home/$TARGET_USER/.bashrc
if [ -n "\$HOST_DISPLAY" ]; then
export VGL_PORT=10000
export VGL_CLIENT=host.docker.internal
export VGL_COMPRESS=rgb
export VGL_DISPLAY=:99
export VGL_FPS=60
# prevent vglrun from running exec
alias exec=:; source vglrun :; unalias exec
fi
EOF
fi
# These lines are temporary, to remain backwards compatible with old devcontainers
# that were running as root and therefore had their caches written as root
sudo chown -R $TARGET_USER: /tmp/scons_cache
sudo chown -R $TARGET_USER: /tmp/comma_download_cache
sudo chown -R $TARGET_USER: /home/batman/.comma
-15
View File
@@ -1,15 +0,0 @@
#!/usr/bin/env bash
source .devcontainer/.host/.env
# setup safe directories for submodules
SUBMODULE_DIRS=$(git config --file .gitmodules --get-regexp path | awk '{ print $2 }')
for DIR in $SUBMODULE_DIRS; do
git config --global --add safe.directory "$PWD/$DIR"
done
# virtual display for virtualgl
if [[ "$HOST_OS" == "darwin" ]] && [[ -n "$HOST_DISPLAY" ]]; then
echo "Starting virtual display at :99 ..."
tmux new-session -d -s fakedisplay Xvfb :99 -screen 0 1920x1080x24
fi
-53
View File
@@ -1,53 +0,0 @@
{
"name": "openpilot devcontainer",
"build": {
"dockerfile": "Dockerfile"
},
"postCreateCommand": ".devcontainer/container_post_create.sh",
"postStartCommand": ".devcontainer/container_post_start.sh",
"initializeCommand": [".devcontainer/host_setup"],
"privileged": true,
"containerEnv": {
"DISPLAY": "${localEnv:DISPLAY}",
"PYTHONPATH": "${containerWorkspaceFolder}",
"TERM": "xterm-256color",
"force_color_prompt": "1"
},
"runArgs": [
"--volume=/dev:/dev",
"--volume=/tmp/.X11-unix:/tmp/.X11-unix",
"--volume=${localWorkspaceFolder}/.devcontainer/.host/.Xauthority:/home/batman/.Xauthority",
"--volume=${localEnv:HOME}/.comma:/home/batman/.comma",
"--volume=${localEnv:HOME}/.azure:/home/batman/.azure",
"--volume=/tmp/comma_download_cache:/tmp/comma_download_cache",
"--shm-size=1G",
"--add-host=host.docker.internal:host-gateway", // required to use host.docker.internal on linux
"--publish=0.0.0.0:8070-8079:8070-8079" // body ZMQ services
],
"features": {
"ghcr.io/devcontainers/features/common-utils:2": {
"installZsh": false,
"installOhMyZsh": false,
"upgradePackages": false,
"username": "batman"
},
"ghcr.io/devcontainers-contrib/features/gh-cli:1": {},
"ghcr.io/devcontainers/features/azure-cli:1": {}
},
"containerUser": "batman",
"remoteUser": "batman",
"customizations": {
"vscode": {
"extensions": [
"ms-python.python",
"ms-vscode.cpptools",
"ms-toolsai.jupyter",
"guyskk.language-cython",
"lharri73.dbc"
]
}
},
"mounts": [
"type=volume,source=scons_cache,target=/tmp/scons_cache"
]
}
-47
View File
@@ -1,47 +0,0 @@
#!/usr/bin/env bash
# pull base image
if [[ -z $USE_LOCAL_IMAGE ]]; then
echo "Updating openpilot_base image if needed..."
docker pull ghcr.io/commaai/openpilot-base:latest
fi
# setup .host dir
mkdir -p .devcontainer/.host
# setup links to Xauthority
XAUTHORITY_LINK=".devcontainer/.host/.Xauthority"
rm -f $XAUTHORITY_LINK
if [[ -z $XAUTHORITY ]]; then
echo "XAUTHORITY not set. Fallback to ~/.Xauthority ..."
if ! [[ -f $HOME/.Xauthority ]]; then
echo "~/.XAuthority file does not exist. GUI tools may not work properly."
touch $XAUTHORITY_LINK # dummy file to satisfy container volume mount
else
ln -sf $HOME/.Xauthority $XAUTHORITY_LINK
fi
else
ln -sf $XAUTHORITY $XAUTHORITY_LINK
fi
# setup host env file
HOST_INFO_FILE=".devcontainer/.host/.env"
SYSTEM=$(uname -s | tr '[:upper:]' '[:lower:]')
echo "HOST_OS=\"$SYSTEM\"" > $HOST_INFO_FILE
echo "HOST_DISPLAY=\"$DISPLAY\"" >> $HOST_INFO_FILE
# run virtualgl if macos
if [[ $SYSTEM == "darwin" ]]; then
echo
if [[ -f /opt/VirtualGL/bin/vglclient ]]; then
echo "Starting VirtualGL client at port 10000..."
VGL_LOG_FILE=".devcontainer/.host/.vgl/vglclient.log"
mkdir -p "$(dirname $VGL_LOG_FILE)"
/opt/VirtualGL/bin/vglclient -l "$VGL_LOG_FILE" -display "$DISPLAY" -port 10000 -detach
else
echo "VirtualGL not found. GUI tools may not work properly. Some GUI tools require OpenGL to work properly. To use them with XQuartz on mac, VirtualGL needs to be installed. To install it run:"
echo
echo " brew install --cask virtualgl"
echo
fi
fi
-10
View File
@@ -1,10 +0,0 @@
:: pull base image
IF NOT DEFINED USE_LOCAL_IMAGE ^
echo "Updating openpilot_base image if needed..." && ^
docker pull ghcr.io/commaai/openpilot-base:latest
:: setup .host dir
mkdir .devcontainer\.host
:: setup host env file
echo "" > .devcontainer\.host\.env
-39
View File
@@ -1,39 +0,0 @@
**/.git
.DS_Store
*.dylib
*.DSYM
*.d
*.pyc
*.pyo
.*.swp
.*.swo
.*.un~
*.tmp
*.o
*.o-*
*.os
*.os-*
*.so
*.a
venv/
.venv/
notebooks
phone
massivemap
neos
installer
chffr/app2
chffr/backend/env
selfdrive/nav
selfdrive/baseui
selfdrive/test/simulator2
**/cache_data
xx/plus
xx/community
xx/projects
!xx/projects/eon_testing_master
!xx/projects/map3d
xx/ops
xx/junk
-11
View File
@@ -1,11 +0,0 @@
root = true
[*]
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
[{*.py, *.pyx, *.pxd}]
charset = utf-8
indent_style = space
indent_size = 2
-24
View File
@@ -1,24 +0,0 @@
* text=auto
# to move existing files into LFS:
# git add --renormalize .
*.dlc filter=lfs diff=lfs merge=lfs -text
*.onnx filter=lfs diff=lfs merge=lfs -text
*.svg filter=lfs diff=lfs merge=lfs -text
*.png filter=lfs diff=lfs merge=lfs -text
*.gif filter=lfs diff=lfs merge=lfs -text
*.ttf filter=lfs diff=lfs merge=lfs -text
*.wav filter=lfs diff=lfs merge=lfs -text
selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
system/hardware/tici/updater filter=lfs diff=lfs merge=lfs -text
selfdrive/ui/qt/spinner_larch64 filter=lfs diff=lfs merge=lfs -text
selfdrive/ui/qt/text_larch64 filter=lfs diff=lfs merge=lfs -text
third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text
-12
View File
@@ -1,12 +0,0 @@
# These are supported funding model platforms
github: [sunnyhaibin] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
patreon: sunnyhaibin # Replace with a single Patreon username
open_collective: # Replace with a single Open Collective username
ko_fi: # Replace with a single Ko-fi username
tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
liberapay: # Replace with a single Liberapay username
issuehunt: # Replace with a single IssueHunt username
otechie: # Replace with a single Otechie username
custom: ['https://paypal.me/sunnyhaibin0850'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
-47
View File
@@ -1,47 +0,0 @@
name: Bug report
description: For issues with running openpilot on your comma device
labels: ["bug"]
body:
- type: markdown
attributes:
value: >
Before creating a **bug report**, please check the following:
* If the issue likely only affects your car model or make, go back and open a **car bug report** instead.
* If the issue is related to the driving or driver monitoring models, you should open a [discussion](https://github.com/commaai/openpilot/discussions/categories/model-feedback) instead.
* Ensure you're running the latest openpilot release.
* Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
* Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
* Ensure you're running stock openpilot. We cannot look into bug reports from forks.
If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
- type: textarea
attributes:
label: Describe the bug
description: Also include a description of how to reproduce the bug
validations:
required: true
- type: input
id: route
attributes:
label: Provide a route where the issue occurs
description: Ensure the route is fully uploaded at https://useradmin.comma.ai. We cannot look into issues without routes, or at least a Dongle ID.
placeholder: 77611a1fac303767|2020-05-11--16-37-07
validations:
required: true
- type: input
id: version
attributes:
label: openpilot version
description: If you're not on release, provide the commit hash
placeholder: 0.8.10
validations:
required: true
- type: textarea
attributes:
label: Additional info
-53
View File
@@ -1,53 +0,0 @@
name: Car bug report
description: For issues with a particular car make or model
labels: ["car", "bug"]
body:
- type: markdown
attributes:
value: >
Before creating a **bug report**, please check the following:
* Ensure you're running the latest openpilot release.
* Ensure you're using officially supported hardware. Issues running on PCs have a different issue template.
* Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
* Ensure you're running stock openpilot. We cannot look into bug reports from forks.
If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
- type: textarea
attributes:
label: Describe the bug
description: Also include a description of how to reproduce the bug
validations:
required: true
- type: input
id: car
attributes:
label: Which car does this affect?
placeholder: Toyota Prius 2017
validations:
required: true
- type: input
id: route
attributes:
label: Provide a route where the issue occurs
description: Ensure the route is fully uploaded at https://useradmin.comma.ai
placeholder: 77611a1fac303767|2020-05-11--16-37-07
validations:
required: true
- type: input
id: version
attributes:
label: openpilot version
description: If you're not on release, provide the commit hash
placeholder: 0.8.10
validations:
required: true
- type: textarea
attributes:
label: Additional info
-14
View File
@@ -1,14 +0,0 @@
blank_issues_enabled: false
contact_links:
- name: Report model bugs
url: https://github.com/commaai/openpilot/discussions/categories/model-feedback
about: Provide feedback for the driving or driver monitoring models
- name: Discussions
url: https://github.com/commaai/openpilot/discussions
about: For questions and general discussion about openpilot
- name: Community Wiki
url: https://github.com/commaai/openpilot/wiki
about: Check out our community wiki
- name: Community Discord
url: https://discord.comma.ai
about: Check out our community discord
-8
View File
@@ -1,8 +0,0 @@
---
name: Enhancement
about: For openpilot enhancement suggestions
title: ''
labels: 'enhancement'
assignees: ''
---
-42
View File
@@ -1,42 +0,0 @@
name: PC bug report
description: For issues with running openpilot on PC
labels: ["PC"]
body:
- type: markdown
attributes:
value: >
Before creating a **bug report**, please check the following:
* Ensure you're running the latest openpilot release.
* Ensure there isn't an existing issue for your bug. If there is, leave a comment on the existing issue.
* Ensure you're running stock openpilot. We cannot look into bug reports from forks.
If you're unsure whether you've hit a bug, check out the #installation-help channel in the [community Discord server](https://discord.comma.ai).
- type: textarea
attributes:
label: Describe the bug
description: Also include a description of how to reproduce the bug
validations:
required: true
- type: input
id: os-version
attributes:
label: OS Version
placeholder: Ubuntu 24.04
validations:
required: true
- type: input
id: version
attributes:
label: openpilot version or commit
placeholder: bd36f2ec8d3559909678eff2690c10a520938367
validations:
required: false
- type: textarea
attributes:
label: Additional info
-15
View File
@@ -1,15 +0,0 @@
---
name: Bug fix
about: For openpilot bug fixes
title: ''
labels: 'bugfix'
assignees: ''
---
**Description**
<!-- A description of the bug and the fix. Also link the issue if it exists. -->
**Verification**
<!-- Explain how you tested this bug fix. -->
@@ -1,19 +0,0 @@
---
name: Car Bug fix
about: For vehicle/brand specific bug fixes
title: ''
labels: 'car bug fix'
assignees: ''
---
**Description**
<!-- A description of the bug and the fix. Also link the issue if it exists. -->
**Verification**
<!-- Explain how you tested this bug fix. -->
**Route**
Route: [a route with the bug fix]
-15
View File
@@ -1,15 +0,0 @@
---
name: Car port
about: For new car ports
title: ''
labels: 'car port'
assignees: ''
---
**Checklist**
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
- [ ] route with openpilot:
- [ ] route with stock system:
- [ ] car harness used (if comma doesn't sell it, put N/A):
@@ -1,13 +0,0 @@
---
name: Fingerprint
about: For adding fingerprints to existing cars
title: ''
labels: 'fingerprint'
assignees: ''
---
**Car**
Which car (make, model, year) this fingerprint is for
**Route**
A route with the fingerprint
-15
View File
@@ -1,15 +0,0 @@
---
name: Refactor
about: For code refactors
title: ''
labels: 'refactor'
assignees: ''
---
**Description**
<!-- A description of the refactor, including the goals it accomplishes. -->
**Verification**
<!-- Explain how you tested the refactor for regressions. -->
-31
View File
@@ -1,31 +0,0 @@
---
name: Tuning
about: For openpilot tuning changes
title: ''
labels: 'tuning'
assignees: ''
---
**Description**
<!-- A description of what is wrong with the current tuning and how the PR addresses this. -->
**Verification**
<!-- To verify tuning, capture the following scenarios (broadly, not exactly), with current tune and this tune.
Use the PlotJuggler tuning layout to compare planned versus actual behavior.
Run ./juggle.py <route> --layout layouts/tuning.xml , screenshot the full tab of interest, and paste into this PR.
Longitudinal:
* Maintaining speed at 25, 40, 65mph
* Driving up and down hills
* Accelerating from a stop
* Decelerating to a stop
* Following large changes in set speed
* Coming to a stop behind a lead car
Lateral:
* Straight driving at ~25, ~45 and ~65mph
* Turns driving at ~25, ~45 and ~65mph
-->
-30
View File
@@ -1,30 +0,0 @@
import pathlib
GITHUB_FOLDER = pathlib.Path(__file__).parent
PULL_REQUEST_TEMPLATES = (GITHUB_FOLDER / "PULL_REQUEST_TEMPLATE")
order = ["fingerprint", "car_bugfix", "bugfix", "car_port", "refactor"]
def create_pull_request_template():
with open(GITHUB_FOLDER / "pull_request_template.md", "w") as f:
f.write("<!-- Please copy and paste the relevant template -->\n\n")
for t in order:
template = PULL_REQUEST_TEMPLATES / f"{t}.md"
text = template.read_text()
# Remove metadata for GitHub
start = text.find("---")
end = text.find("---", start+1)
text = text[end + 4:]
# Remove comments
text = text.replace("<!-- ", "").replace("-->", "")
f.write(f"<!--- ***** Template: {template.stem.replace('_', ' ').title()} *****\n")
f.write(text)
f.write("\n\n")
f.write("-->\n\n")
create_pull_request_template()
-79
View File
@@ -1,79 +0,0 @@
CI / testing:
- changed-files:
- any-glob-to-all-files: "{.github/**,**/test_*,Jenkinsfile}"
car:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/**'
body:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/body/*'
chrysler:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/chrysler/*'
ford:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/ford/*'
gm:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/gm/*'
honda:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/honda/*'
hyundai:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/hyundai/*'
mazda:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/mazda/*'
nissan:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/nissan/*'
subaru:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/subaru/*'
tesla:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/tesla/*'
toyota:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/toyota/*'
volkswagen:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/volkswagen/*'
fingerprint:
- changed-files:
- any-glob-to-all-files: 'selfdrive/car/*/fingerprints.py'
simulation:
- changed-files:
- any-glob-to-all-files: 'tools/sim/**'
ui:
- changed-files:
- any-glob-to-all-files: 'selfdrive/ui/**'
tools:
- changed-files:
- any-glob-to-all-files: 'tools/**'
multilanguage:
- changed-files:
- any-glob-to-all-files: 'selfdrive/ui/translations/**'
research:
- changed-files:
- any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit}"
-68
View File
@@ -1,68 +0,0 @@
<!-- Please copy and paste the relevant template -->
<!--- ***** Template: Fingerprint *****
**Car**
Which car (make, model, year) this fingerprint is for
**Route**
A route with the fingerprint
-->
<!--- ***** Template: Car Bugfix *****
**Description**
A description of the bug and the fix. Also link the issue if it exists.
**Verification**
Explain how you tested this bug fix.
**Route**
Route: [a route with the bug fix]
-->
<!--- ***** Template: Bugfix *****
**Description**
A description of the bug and the fix. Also link the issue if it exists.
**Verification**
Explain how you tested this bug fix.
-->
<!--- ***** Template: Car Port *****
**Checklist**
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
- [ ] route with openpilot:
- [ ] route with stock system:
- [ ] car harness used (if comma doesn't sell it, put N/A):
-->
<!--- ***** Template: Refactor *****
**Description**
A description of the refactor, including the goals it accomplishes.
**Verification**
Explain how you tested the refactor for regressions.
-->
-49
View File
@@ -1,49 +0,0 @@
name: 'automatically cache based on current runner'
inputs:
path:
description: 'path to cache'
required: true
key:
description: 'key'
required: true
restore-keys:
description: 'restore-keys'
required: true
save:
description: 'whether to save the cache'
default: 'false'
required: false
runs:
using: "composite"
steps:
- name: setup namespace cache
if: ${{ contains(runner.name, 'nsc') }}
uses: namespacelabs/nscloud-cache-action@v1
with:
path: ${{ inputs.path }}
- name: setup github cache
if: ${{ !contains(runner.name, 'nsc') && inputs.save != 'false' }}
uses: 'actions/cache@v4'
with:
path: ${{ inputs.path }}
key: ${{ inputs.key }}
restore-keys: ${{ inputs.restore-keys }}
- name: setup github cache
if: ${{ !contains(runner.name, 'nsc') && inputs.save == 'false' }}
uses: 'actions/cache/restore@v4'
with:
path: ${{ inputs.path }}
key: ${{ inputs.key }}
restore-keys: ${{ inputs.restore-keys }}
# make the directory manually in case we didn't get a hit, so it doesn't fail on future steps
- id: scons-cache-setup
shell: bash
run: |
mkdir -p ${{ inputs.path }}
sudo chmod -R 777 ${{ inputs.path }}
sudo chown -R $USER ${{ inputs.path }}
-53
View File
@@ -1,53 +0,0 @@
name: "PR review"
on:
pull_request_target:
types: [opened, reopened, synchronize, edited, edited]
jobs:
labeler:
name: review
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: false
# Label PRs
- uses: actions/labeler@v5.0.0
with:
dot: true
configuration-path: .github/labeler.yaml
# Check PR target branch
- name: check branch
uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
if: github.repository == 'commaai/openpilot'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
target: /^(?!master$).*/
exclude: /commaai:.*/
change-to: ${{ github.base_ref }}
already-exists-action: close_this
already-exists-comment: "Your PR should be made against the `master` branch"
# Welcome comment
- name: comment
uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
with:
message: |
<!-- _(run_id **${{ github.run_id }}**)_ -->
Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
* Convert your PR to a draft unless it's ready to review
* Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
* Before marking as "ready for review", ensure:
* the goal is clearly stated in the description
* all the tests are passing
* the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
* include a route or your device' dongle ID if relevant
comment_tag: run_id
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-37
View File
@@ -1,37 +0,0 @@
name: badges
on:
schedule:
- cron: '0 * * * *'
workflow_dispatch:
env:
BASE_IMAGE: openpilot-base
DOCKER_REGISTRY: ghcr.io/commaai
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c
jobs:
badges:
name: create badges
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Push badges
run: |
${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"
rm .gitattributes
git checkout --orphan badges
git rm -rf --cached .
git config user.email "badge-researcher@comma.ai"
git config user.name "Badge Researcher"
git add translation_badge.svg
git commit -m "Add/Update badges"
git push -f origin HEAD
-101
View File
@@ -1,101 +0,0 @@
name: weekly CI test report
on:
schedule:
- cron: '37 9 * * 1' # 9:37AM UTC -> 2:37AM PST every monday
workflow_dispatch:
inputs:
ci_runs:
description: 'The amount of runs to trigger in CI test report'
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
env:
CI_RUNS: ${{ github.event.inputs.ci_runs || '50' }}
jobs:
setup:
if: github.repository == 'commaai/openpilot'
runs-on: ubuntu-latest
outputs:
ci_runs: ${{ steps.ci_runs_setup.outputs.matrix }}
steps:
- id: ci_runs_setup
name: CI_RUNS=${{ env.CI_RUNS }}
run: |
matrix=$(python3 -c "import json; print(json.dumps({ 'run_number' : list(range(${{ env.CI_RUNS }})) }))")
echo "matrix=$matrix" >> $GITHUB_OUTPUT
ci_matrix_run:
needs: [ setup ]
strategy:
fail-fast: false
matrix: ${{fromJSON(needs.setup.outputs.ci_runs)}}
uses: commaai/openpilot/.github/workflows/ci_weekly_run.yaml@master
with:
run_number: ${{ matrix.run_number }}
report:
needs: [ci_matrix_run]
runs-on: ubuntu-latest
if: always()
steps:
- name: Get job results
uses: actions/github-script@v7
id: get-job-results
with:
script: |
const jobs = await github
.paginate("GET /repos/{owner}/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs", {
owner: "commaai",
repo: "${{ github.event.repository.name }}",
run_id: "${{ github.run_id }}",
attempt: "${{ github.run_attempt }}",
})
var report = {}
jobs.slice(1, jobs.length-1).forEach(job => {
if (job.conclusion === "skipped") return;
const jobName = job.name.split(" / ")[2];
const runRegex = /\((.*?)\)/;
const run = job.name.match(runRegex)[1];
report[jobName] = report[jobName] || { successes: [], failures: [], cancelled: [] };
switch (job.conclusion) {
case "success":
report[jobName].successes.push({ "run_number": run, "link": job.html_url}); break;
case "failure":
report[jobName].failures.push({ "run_number": run, "link": job.html_url }); break;
case "cancelled":
report[jobName].cancelled.push({ "run_number": run, "link": job.html_url }); break;
}
});
return JSON.stringify({"jobs": report});
- name: Add job results to summary
env:
JOB_RESULTS: ${{ fromJSON(steps.get-job-results.outputs.result) }}
run: |
cat <<EOF >> template.html
<table>
<thead>
<tr>
<th></th>
<th>Job</th>
<th>✅ Passing</th>
<th>❌ Failure Details</th>
</tr>
</thead>
<tbody>
{% for key in jobs.keys() %}<tr>
<td>{% for i in range(5) %}{% if i+1 <= (5 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }}) %}🟩{% else %}🟥{% endif %}{% endfor%}</td>
<td>{{ key }}</td>
<td>{{ 100 * jobs[key]["successes"]|length // ${{ env.CI_RUNS }} }}%</td>
<td>{% if jobs[key]["failures"]|length > 0 %}<details>{% for failure in jobs[key]["failures"] %}<a href="{{ failure['link'] }}">Log for run #{{ failure['run_number'] }}</a><br>{% endfor %}</details>{% else %}{% endif %}</td>
</td>
</tr>{% endfor %}
</table>
EOF
pip install jinja2-cli
echo $JOB_RESULTS | jinja2 template.html > report.html
echo "# CI Test Report - ${{ env.CI_RUNS }} Runs" >> $GITHUB_STEP_SUMMARY
cat report.html >> $GITHUB_STEP_SUMMARY
-21
View File
@@ -1,21 +0,0 @@
name: weekly CI test run
on:
workflow_call:
inputs:
run_number:
required: true
type: string
concurrency:
group: ci-run-${{ inputs.run_number }}-${{ github.ref }}
cancel-in-progress: true
jobs:
selfdrive_tests:
uses: commaai/openpilot/.github/workflows/selfdrive_tests.yaml@master
with:
run_number: ${{ inputs.run_number }}
tools_tests:
uses: commaai/openpilot/.github/workflows/tools_tests.yaml@master
with:
run_number: ${{ inputs.run_number }}
-97
View File
@@ -1,97 +0,0 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"
on:
push:
branches: [ "master", "*-c3", "master*" ]
pull_request:
branches: [ "master", "*-c3", "master*" ]
schedule:
- cron: '39 7 * * 2'
jobs:
analyze:
name: Analyze (${{ matrix.language }})
# Runner size impacts CodeQL analysis time. To learn more, please see:
# - https://gh.io/recommended-hardware-resources-for-running-codeql
# - https://gh.io/supported-runners-and-hardware-resources
# - https://gh.io/using-larger-runners (GitHub.com only)
# Consider using larger runners or machines with greater resources for possible analysis time improvements.
runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
permissions:
# required for all workflows
security-events: write
# required to fetch internal or private CodeQL packs
packages: read
# only required for workflows in private repositories
actions: read
contents: read
strategy:
fail-fast: false
matrix:
include:
- language: c-cpp
build-mode: autobuild
- language: javascript-typescript
build-mode: none
- language: python
build-mode: none
# CodeQL supports the following values keywords for 'language': 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift'
# Use `c-cpp` to analyze code written in C, C++ or both
# Use 'java-kotlin' to analyze code written in Java, Kotlin or both
# Use 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
# To learn more about changing the languages that are analyzed or customizing the build mode for your analysis,
# see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/customizing-your-advanced-setup-for-code-scanning.
# If you are analyzing a compiled language, you can modify the 'build-mode' for that language to customize how
# your codebase is analyzed, see https://docs.github.com/en/code-security/code-scanning/creating-an-advanced-setup-for-code-scanning/codeql-code-scanning-for-compiled-languages
steps:
- name: Checkout repository
uses: actions/checkout@v4
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
build-mode: ${{ matrix.build-mode }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
# queries: security-extended,security-and-quality
# If the analyze step fails for one of the languages you are analyzing with
# "We were unable to automatically build your code", modify the matrix above
# to set the build mode to "manual" for that language. Then modify this step
# to build your code.
# ️ Command-line programs to run using the OS shell.
# 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
- if: matrix.build-mode == 'manual'
shell: bash
run: |
echo 'If you are using a "manual" build mode for one or more of the' \
'languages you are analyzing, replace this with the commands to build' \
'your code, for example:'
echo ' make bootstrap'
echo ' make release'
exit 1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"
@@ -1,21 +0,0 @@
name: 'compile openpilot'
runs:
using: "composite"
steps:
- shell: bash
name: Build openpilot with all flags
run: |
${{ env.RUN }} "scons -j$(nproc)"
${{ env.RUN }} "release/check-dirty.sh"
- shell: bash
name: Cleanup scons cache and rebuild
run: |
${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \
scons -j$(nproc) --cache-populate"
- name: Save scons cache
uses: actions/cache/save@v4
if: github.ref == 'refs/heads/master'
with:
path: .ci_cache/scons_cache
key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
-64
View File
@@ -1,64 +0,0 @@
name: docs
on:
push:
branches:
- master
pull_request:
workflow_call:
inputs:
run_number:
default: '1'
required: true
type: string
concurrency:
group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
jobs:
docs:
name: build docs
runs-on: ubuntu-latest
timeout-minutes: 1
steps:
- uses: actions/checkout@v4
with:
submodules: true
# Build
- name: Build docs
run: |
# TODO: can we install just the "docs" dependency group without the normal deps?
pip install mkdocs
mkdocs build
# Push to docs.comma.ai
- uses: actions/checkout@v4
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
with:
path: openpilot-docs
ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
repository: commaai/openpilot-docs
- name: Push
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
run: |
set -x
source release/identity.sh
cd openpilot-docs
git checkout --orphan tmp
git rm -rf .
# copy over docs
cp -r ../docs_site/ docs/
# GitHub pages config
touch docs/.nojekyll
echo -n docs.comma.ai > docs/CNAME
git add -f .
git commit -m "build docs"
# docs live in different repo to not bloat openpilot's full clone size
git push -f origin tmp:gh-pages
-45
View File
@@ -1,45 +0,0 @@
name: jenkins scan
on:
issue_comment:
types: [created, edited]
jobs:
# TODO: gc old branches in a separate job in this workflow
scan-comments:
runs-on: ubuntu-latest
if: ${{ github.event.issue.pull_request }}
steps:
- name: Check for trigger phrase
id: check_comment
uses: actions/github-script@v7
with:
script: |
const triggerPhrase = "trigger-jenkins";
const comment = context.payload.comment.body;
const commenter = context.payload.comment.user.login;
const { data: permissions } = await github.rest.repos.getCollaboratorPermissionLevel({
owner: context.repo.owner,
repo: context.repo.repo,
username: commenter
});
const hasWriteAccess = permissions.permission === 'write' || permissions.permission === 'admin';
return (hasWriteAccess && comment.includes(triggerPhrase));
result-encoding: json
- name: Checkout repository
if: steps.check_comment.outputs.result == 'true'
uses: actions/checkout@v4
with:
ref: refs/pull/${{ github.event.issue.number }}/head
- name: Push to tmp-jenkins branch
if: steps.check_comment.outputs.result == 'true'
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b tmp-jenkins-${{ github.event.issue.number }}
GIT_LFS_SKIP_PUSH=1 git push -f origin tmp-jenkins-${{ github.event.issue.number }}
-46
View File
@@ -1,46 +0,0 @@
name: Mirror to GitLab
on:
push:
delete:
workflow_dispatch: # This enables manual triggering
jobs:
sync:
runs-on: ubuntu-latest
steps:
- name: Wait for other instances of this workflow to conclude
uses: softprops/turnstyle@8db075d65b19bf94e6e8687b504db69938dc3c65
with:
same-branch-only: 'true'
abort-after-seconds: 300
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- name: Checkout Repository
uses: actions/checkout@v3
with:
ref: ${{ github.ref }}
fetch-depth: 0 # Fetch full history
- name: Set up Git
run: |
git config --global user.name 'GitHub Action'
git config --global user.email 'action@github.com'
- name: Set up SSH
uses: webfactory/ssh-agent@v0.9.0
with:
ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }}
- name: Add GitLab public keys
run: |
ssh-keyscan -H gitlab.com >> ~/.ssh/known_hosts
# Note: If you have issues with "push rejected missing LFS" or something make sure you disabled LFS on the GITLAB repo if you intend to use a different LFS repo other than the target repo.
- name: Sync and commit changes
id: sync-and-commit
run: |
# Add GitLab remote
git remote add gitlab git@gitlab.com:sunnypilot/sunnyhaibin/sunnypilot-github-mirror.git
git push -u --force gitlab ${{ github.ref }}
-39
View File
@@ -1,39 +0,0 @@
name: prebuilt
on:
schedule:
- cron: '0 * * * *'
workflow_dispatch:
env:
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh prebuilt
jobs:
build_prebuilt:
name: build prebuilt
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
env:
PUSH_IMAGE: true
permissions:
checks: read
contents: read
packages: write
steps:
- name: Wait for green check mark
if: ${{ github.event_name != 'workflow_dispatch' }}
uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
with:
ref: master
wait-interval: 30
running-workflow-name: 'build prebuilt'
repo-token: ${{ secrets.GITHUB_TOKEN }}
check-regexp: ^((?!.*(build master-ci).*).)*$
- uses: actions/checkout@v4
with:
submodules: true
- run: git lfs pull
- name: Build and Push docker image
run: |
$DOCKER_LOGIN
eval "$BUILD"
-54
View File
@@ -1,54 +0,0 @@
name: release
on:
schedule:
- cron: '0 10 * * *'
workflow_dispatch:
jobs:
build_masterci:
name: build master-ci
env:
TARGET_DIR: /tmp/openpilot
ImageOS: ubuntu20
container:
image: ghcr.io/commaai/openpilot-base:latest
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
permissions:
checks: read
contents: write
steps:
- name: Install wait-on-check-action dependencies
run: |
sudo apt-get update
sudo apt-get install -y libyaml-dev
- name: Wait for green check mark
if: ${{ github.event_name != 'workflow_dispatch' }}
uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
with:
ref: master
wait-interval: 30
running-workflow-name: 'build master-ci'
repo-token: ${{ secrets.GITHUB_TOKEN }}
check-regexp: ^((?!.*(build prebuilt).*).)*$
- uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- name: Pull LFS
run: |
git config --global --add safe.directory '*'
git lfs pull
- name: Build master-ci
run: |
release/build_devel.sh
- name: Run tests
run: |
export PYTHONPATH=$TARGET_DIR
cd $TARGET_DIR
scons -j$(nproc)
pytest -n logical selfdrive/car/tests/test_car_interfaces.py
- name: Push master-ci
run: |
unset TARGET_DIR
BRANCH=master-ci release/build_devel.sh
-60
View File
@@ -1,60 +0,0 @@
name: repo maintenance
on:
schedule:
- cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
workflow_dispatch:
jobs:
bump_submodules:
name: bump_submodules
runs-on: ubuntu-latest
container:
image: ghcr.io/commaai/openpilot-base:latest
if: github.repository == 'commaai/openpilot'
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: bump submodules
run: |
git config --global --add safe.directory '*'
git -c submodule."tinygrad".update=none submodule update --remote
git add .
- name: Create Pull Request
uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
with:
author: Vehicle Researcher <user@comma.ai>
token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
commit-message: bump submodules
title: '[bot] Bump submodules'
branch: auto-bump-submodules
base: master
delete-branch: true
body: 'Automatic PR from repo-maintenance -> bump_submodules'
labels: bot
package_updates:
name: package_updates
runs-on: ubuntu-latest
container:
image: ghcr.io/commaai/openpilot-base:latest
if: github.repository == 'commaai/openpilot'
steps:
- uses: actions/checkout@v4
- name: uv lock
run: |
python3 -m ensurepip --upgrade
pip3 install uv
uv lock --upgrade
- name: Create Pull Request
uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
with:
author: Vehicle Researcher <user@comma.ai>
token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
commit-message: Update Python packages
title: '[bot] Update Python packages'
branch: auto-package-updates
base: master
delete-branch: true
body: 'Automatic PR from repo-maintenance -> package_updates'
labels: bot
-359
View File
@@ -1,359 +0,0 @@
name: selfdrive
on:
push:
branches:
- master
pull_request:
workflow_dispatch:
workflow_call:
inputs:
run_number:
default: '1'
required: true
type: string
concurrency:
group: selfdrive-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
PYTHONWARNINGS: error
BASE_IMAGE: openpilot-base
AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh base
RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical
jobs:
build_release:
name: build release
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
env:
STRIPPED_DIR: /tmp/releasepilot
steps:
- uses: actions/checkout@v4
with:
submodules: true
- run: git lfs pull
- name: Build devel
timeout-minutes: 1
run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
- uses: ./.github/workflows/setup-with-retry
- name: Check submodules
if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
timeout-minutes: 1
run: release/check-submodules.sh
- name: Build openpilot and run checks
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "python system/manager/build.py"
- name: Run tests
timeout-minutes: 3
run: |
cd $STRIPPED_DIR
${{ env.RUN }} "release/check-dirty.sh && \
MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
- name: static analysis
timeout-minutes: 1
run: |
cd $GITHUB_WORKSPACE
cp pyproject.toml $STRIPPED_DIR
cd $STRIPPED_DIR
${{ env.RUN }} "scripts/lint.sh"
build:
strategy:
matrix:
arch: ${{ fromJson(
((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && '["x86_64", "aarch64"]' || '["x86_64"]' ) }}
runs-on: ${{ (matrix.arch == 'aarch64') && 'namespace-profile-arm64-2x8' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Setup docker push
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
run: |
echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
$DOCKER_LOGIN
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- uses: ./.github/workflows/compile-openpilot
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache
build_mac:
name: build macOS
runs-on: macos-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- run: git lfs pull
- name: Install dependencies
run: ./tools/mac_setup.sh
env:
# package install has DeprecationWarnings
PYTHONWARNINGS: default
- run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
- name: Getting scons cache
uses: 'actions/cache@v4'
with:
path: /tmp/scons_cache
key: scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
restore-keys: |
scons-${{ runner.arch }}-macos-${{ env.CACHE_COMMIT_DATE }}
scons-${{ runner.arch }}-macos
- name: Building openpilot
run: . .venv/bin/activate && scons -j$(nproc)
docker_push_multiarch:
name: docker push multiarch tag
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
needs: [build]
steps:
- uses: actions/checkout@v4
with:
submodules: false
- name: Setup docker
run: |
$DOCKER_LOGIN
- name: Merge x64 and arm64 tags
run: |
export PUSH_IMAGE=true
scripts/retry.sh selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64
static_analysis:
name: static analysis
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: static analysis
timeout-minutes: 1
run: ${{ env.RUN }} "scripts/lint.sh"
unit_tests:
name: unit tests
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- name: Build openpilot
timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Run unit tests
timeout-minutes: 15
run: |
${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
$PYTEST --timeout 60 -m 'not slow' && \
./selfdrive/ui/tests/create_test_translations.sh && \
QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
pytest ./selfdrive/ui/tests/test_translations.py"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
process_replay:
name: process replay
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
with:
docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
- name: Cache test routes
id: dependency-cache
uses: actions/cache@v4
with:
path: .ci_cache/comma_download_cache
key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Run replay
timeout-minutes: 30
run: |
${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
chmod -R 777 /tmp/comma_download_cache && \
coverage combine && \
coverage xml"
- name: Print diff
id: print-diff
if: always()
run: cat selfdrive/test/process_replay/diff.txt
- uses: actions/upload-artifact@v4
if: always()
continue-on-error: true
with:
name: process_replay_diff.txt
path: selfdrive/test/process_replay/diff.txt
- name: Upload reference logs
if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
run: |
${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
# PYTHONWARNINGS triggers a SyntaxError in onnxruntime
- name: Run model replay with ONNX
timeout-minutes: 4
run: |
${{ env.RUN }} "unset PYTHONWARNINGS && \
ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
coverage combine && coverage xml"
- name: Run regen
timeout-minutes: 4
run: |
${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
chmod -R 777 /tmp/comma_download_cache"
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
test_cars:
name: cars
runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
(github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
strategy:
fail-fast: false
matrix:
job: [0, 1]
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Cache test routes
id: dependency-cache
uses: ./.github/workflows/auto-cache
with:
path: .ci_cache/comma_download_cache
key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Test car models
timeout-minutes: 20
run: |
${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
chmod -R 777 /tmp/comma_download_cache"
env:
NUM_JOBS: 2
JOB_ID: ${{ matrix.job }}
- name: "Upload coverage to Codecov"
uses: codecov/codecov-action@v4
with:
name: ${{ github.job }}-${{ matrix.job }}
env:
CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
car_docs_diff:
name: PR comments
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v4
with:
submodules: true
ref: ${{ github.event.pull_request.base.ref }}
- run: git lfs pull
- uses: ./.github/workflows/setup-with-retry
- name: Get base car info
run: |
${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
sudo chown -R $USER:$USER ${{ github.workspace }}
- uses: actions/checkout@v4
with:
submodules: true
path: current
- run: cd current && git lfs pull
- name: Save car docs diff
id: save_diff
run: |
cd current
${{ env.RUN }} "scons -j$(nproc)"
output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
output="${output//$'\n'/'%0A'}"
echo "::set-output name=diff::$output"
- name: Find comment
if: ${{ env.AZURE_TOKEN != '' }}
uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
body-includes: This PR makes changes to
- name: Update comment
if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
issue-number: ${{ github.event.pull_request.number }}
body: "${{ steps.save_diff.outputs.diff }}"
edit-mode: replace
- name: Delete comment
if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
uses: actions/github-script@v7
with:
script: |
github.rest.issues.deleteComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: ${{ steps.fc.outputs.comment-id }}
})
create_ui_report:
# This job name needs to be the same as UI_JOB_NAME in ui_preview.yaml
name: Create UI Report
runs-on: ubuntu-latest
if: github.event_name == 'pull_request'
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Build openpilot
run: ${{ env.RUN }} "scons -j$(nproc)"
- name: Create Test Report
run: >
${{ env.RUN }} "PYTHONWARNINGS=ignore &&
source selfdrive/test/setup_xvfb.sh &&
export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' &&
python selfdrive/ui/tests/test_ui/run.py"
- name: Upload Test Report
uses: actions/upload-artifact@v4
with:
name: report-${{ github.event.number }}
path: selfdrive/ui/tests/test_ui/report_1/screenshots
@@ -1,40 +0,0 @@
name: 'openpilot env setup, with retry on failure'
inputs:
docker_hub_pat:
description: 'Auth token for Docker Hub, required for BuildJet jobs'
required: false
default: ''
sleep_time:
description: 'Time to sleep between retries'
required: false
default: 30
runs:
using: "composite"
steps:
- id: setup1
uses: ./.github/workflows/setup
continue-on-error: true
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
- if: steps.setup1.outcome == 'failure'
shell: bash
run: sleep ${{ inputs.sleep_time }}
- id: setup2
if: steps.setup1.outcome == 'failure'
uses: ./.github/workflows/setup
continue-on-error: true
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
- if: steps.setup2.outcome == 'failure'
shell: bash
run: sleep ${{ inputs.sleep_time }}
- id: setup3
if: steps.setup2.outcome == 'failure'
uses: ./.github/workflows/setup
with:
docker_hub_pat: ${{ inputs.docker_hub_pat }}
is_retried: true
-63
View File
@@ -1,63 +0,0 @@
name: 'openpilot env setup'
inputs:
docker_hub_pat:
description: 'Auth token for Docker Hub, required for BuildJet jobs'
required: true
default: ''
is_retried:
description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly'
required: false
default: 'false'
runs:
using: "composite"
steps:
# assert that this action is retried using the setup-with-retry
- shell: bash
if: ${{ inputs.is_retried == 'false' }}
run: |
echo "You should not run this action directly. Use setup-with-retry instead"
exit 1
# do this after checkout to ensure our custom LFS config is used to pull from GitLab
- shell: bash
run: git lfs pull
# on BuildJet runners, must be logged into DockerHub to avoid rate limiting
# https://buildjet.com/for-github-actions/docs/guides/docker
- shell: bash
if: ${{ contains(runner.name, 'buildjet') && inputs.docker_hub_pat == '' }}
run: |
echo "Need to set the Docker Hub PAT secret as an input to this action"
exit 1
- name: Login to Docker Hub
if: contains(runner.name, 'buildjet')
shell: bash
run: |
docker login -u adeebshihadeh -p ${{ inputs.docker_hub_pat }}
# build cache
- id: date
shell: bash
run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
- shell: bash
run: echo "$CACHE_COMMIT_DATE"
- id: scons-cache
uses: ./.github/workflows/auto-cache
with:
path: .ci_cache/scons_cache
key: scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
restore-keys: |
scons-${{ runner.arch }}-${{ env.CACHE_COMMIT_DATE }}
scons-${{ runner.arch }}
# as suggested here: https://github.com/moby/moby/issues/32816#issuecomment-910030001
- id: normalize-file-permissions
shell: bash
name: Normalize file permissions to ensure a consistent docker build cache
run: |
find . -type f -executable -not -perm 755 -exec chmod 755 {} \;
find . -type f -not -executable -not -perm 644 -exec chmod 644 {} \;
# build our docker image
- shell: bash
run: eval ${{ env.BUILD }}
-29
View File
@@ -1,29 +0,0 @@
name: stale
on:
schedule:
- cron: '30 1 * * *'
workflow_dispatch:
env:
DAYS_BEFORE_PR_CLOSE: 2
DAYS_BEFORE_PR_STALE: 9
jobs:
stale:
runs-on: ubuntu-latest
steps:
- uses: actions/stale@v9
with:
exempt-all-milestones: true
# pull request config
stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
stale-pr-label: stale
delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
exempt-pr-labels: "ignore stale,needs testing,car port,car" # if wip or it needs testing from the community, don't mark as stale
days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}
# issue config
days-before-issue-stale: -1 # ignore issues for now
-103
View File
@@ -1,103 +0,0 @@
name: tools
on:
push:
branches:
- master
pull_request:
workflow_call:
inputs:
run_number:
default: '1'
required: true
type: string
concurrency:
group: tools-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
BASE_IMAGE: openpilot-base
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: selfdrive/test/docker_build.sh base
RUN: docker run --shm-size 2G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c
jobs:
simulator_driving:
name: simulator driving
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Build openpilot
run: |
${{ env.RUN }} "scons -j$(nproc)"
- name: Run bridge test
run: |
${{ env.RUN }} "export MAPBOX_TOKEN='pk.eyJ1Ijoiam5ld2IiLCJhIjoiY2xxNW8zZXprMGw1ZzJwbzZneHd2NHljbSJ9.gV7VPRfbXFetD-1OVF0XZg' && \
source selfdrive/test/setup_xvfb.sh && \
source selfdrive/test/setup_vsound.sh && \
CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"
test_compatibility:
name: test 20.04 + 3.11
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v4
with:
submodules: true
- name: Installing ubuntu dependencies
run: INSTALL_EXTRA_PACKAGES=no tools/install_ubuntu_dependencies.sh
- name: Installing python
uses: actions/setup-python@v5
with:
python-version: '3.11.4'
- name: Installing pip
run: pip install pip==24.0
- name: Installing uv
run: pip install uv
- name: git LFS
run: git lfs pull
- run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
- name: Getting scons cache
uses: 'actions/cache@v4'
with:
path: /tmp/scons_cache
key: scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
restore-keys: |
scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
scons-${{ runner.arch }}-ubuntu2004
- name: Building openpilot
run: uv run scons -u -j$(nproc)
devcontainer:
name: devcontainer
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
submodules: true
- uses: ./.github/workflows/setup-with-retry
- name: Use local image for testing devcontainer with latest base image
run: |
echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
- name: Setup Dev Container CLI
run: npm install -g @devcontainers/cli
- name: Build dev container image
run: ./scripts/retry.sh devcontainer build --workspace-folder .
- name: Run dev container
run: |
mkdir -p /tmp/devcontainer_scons_cache/
cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
devcontainer up --workspace-folder .
- name: Test environment
run: |
devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
devcontainer exec --workspace-folder . pip3 install pip-install-test
devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
devcontainer exec --workspace-folder . sudo touch /root/test.txt
-91
View File
@@ -1,91 +0,0 @@
name: "ui preview"
on:
pull_request_target:
types: [assigned, opened, synchronize, reopened, edited]
branches:
- 'master'
paths:
- 'selfdrive/ui/**'
env:
UI_JOB_NAME: "Create UI Report"
jobs:
preview:
if: github.repository == 'commaai/openpilot'
name: preview
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
actions: read
steps:
- name: Waiting for ui test to start
run: sleep 30
- name: Wait for ui report
uses: lewagon/wait-on-check-action@v1.3.4
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: ${{ env.UI_JOB_NAME }}
repo-token: ${{ secrets.GITHUB_TOKEN }}
allowed-conclusions: success
wait-interval: 20
- name: Get workflow run ID
id: get_run_id
run: |
echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ github.event.pull_request.head.sha }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
- name: Checkout ci-artifacts
uses: actions/checkout@v4
with:
repository: commaai/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
path: ${{ github.workspace }}/ci-artifacts
ref: master
- name: Download artifact
id: download-artifact
uses: dawidd6/action-download-artifact@v6
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ steps.get_run_id.outputs.run_id }}
search_artifacts: true
name: report-${{ github.event.number }}
path: ${{ github.workspace }}/ci-artifacts
- name: Push Screenshots
working-directory: ${{ github.workspace }}/ci-artifacts
run: |
git checkout -b openpilot/pr-${{ github.event.number }}
git config user.name "GitHub Actions Bot"
git config user.email "<>"
git add ${{ github.workspace }}/ci-artifacts/*
git commit -m "screenshots for PR #${{ github.event.number }}"
git push origin openpilot/pr-${{ github.event.number }} --force
- name: Comment Screenshots on PR
uses: thollander/actions-comment-pull-request@v2
with:
message: |
<!-- _(run_id_screenshots **${{ github.run_id }}**)_ -->
## UI Screenshots
<table>
<tr>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/homescreen.png"></td>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad.png"></td>
</tr>
<tr>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_map.png"></td>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/onroad_sidebar.png"></td>
</tr>
<tr>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_network.png"></td>
<td><img src="https://raw.githubusercontent.com/commaai/ci-artifacts/openpilot/pr-${{ github.event.number }}/settings_device.png"></td>
</tr>
</table>
comment_tag: run_id_screenshots
pr_number: ${{ github.event.number }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-240
View File
@@ -1,240 +0,0 @@
variables:
BUILD_DIR: "/data/openpilot"
OUTPUT_DIR: "${CI_PROJECT_DIR}/output"
CI_DIR: "${CI_PROJECT_DIR}/release/ci"
VERSION: 'echo $(date "+%Y.%m.%d")'
GIT_SUBMODULE_STRATEGY: recursive
GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
DEV_C3_SOURCE_BRANCH: "master-dev-c3"
GIT_CONFIG_USER_EMAIL: "gitlab@pipeline.com"
GIT_CONFIG_USER_NAME: "Gitlab Pipeline"
PUBLIC_REPO_URL: "https://github.com/sunnyhaibin/sunnypilot"
BASE_BUILD_NUMER: 3000
EXTRA_VERSION_IDENTIFIER: "${CI_PIPELINE_IID}"
NEW_BRANCH: ${CI_COMMIT_REF_NAME}-prebuilt
stages:
- build
- sanity
- publish
- notify
default:
retry: 2
tags:
- sunnypilot
- x86
.default_before_script: &default_before_script
- 'if [ "$EXTRA_VERSION_IDENTIFIER" = "$CI_PIPELINE_IID" ]; then export EXTRA_VERSION_IDENTIFIER=$((CI_PIPELINE_IID + BASE_BUILD_NUMER)); fi'
- 'export VERSION=$(eval $VERSION)-${EXTRA_VERSION_IDENTIFIER}'
- 'mkdir -p "${BUILD_DIR}/"'
- 'git config --global user.email "${GIT_CONFIG_USER_EMAIL}"'
- 'git config --global user.name "${GIT_CONFIG_USER_NAME}"'
workflow: # If running on any branch other than main.
rules:
# We are an MR, but it's a draft, we won't proceed with anything.
- if: '$CI_MERGE_REQUEST_TITLE =~ /^wip:/i || $CI_MERGE_REQUEST_TITLE =~ /^draft:/i'
when: never
# Below are the rules when a commit is done (code has been added to the branch)
# Commit to master-dev-c3
- if: $CI_COMMIT_REF_NAME == $DEV_C3_SOURCE_BRANCH
variables:
EXTRA_VERSION_IDENTIFIER: "${CI_PIPELINE_IID}"
NEW_BRANCH: "dev-c3"
AUTO_BUILD: true
when: always
#commit made to main (master)
- if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
variables:
NEW_BRANCH: "staging-c3"
VERSION: 'cat common/version.h | grep COMMA_VERSION | sed -e "s/[^0-9|.]//g"'
EXTRA_VERSION_IDENTIFIER: "staging"
AUTO_PUBLISH: true
AUTO_BUILD: true
when: always
# if tag
- if: $CI_COMMIT_TAG
variables:
NEW_BRANCH: "release-c3"
VERSION: 'cat common/version.h | grep COMMA_VERSION | sed -e "s/[^0-9|.]//g"'
EXTRA_VERSION_IDENTIFIER: "release"
AUTO_BUILD: true
- when: always
build:
stage: build
cache:
key: "${CI_COMMIT_REF_SLUG}"
paths:
- "${CI_DIR}/scons_cache"
before_script:
- find $BUILD_DIR/ -mindepth 1 -delete
- *default_before_script
- "echo Starting build stage..."
- "echo BUILD_DIR: ${BUILD_DIR}"
- "echo CI_DIR: ${CI_DIR}"
- "echo VERSION: ${VERSION}"
- "echo CI_COMMIT_REF_NAME: ${CI_COMMIT_REF_NAME}"
- git config --global --add safe.directory ${CI_PROJECT_DIR}
script:
- export PYTHONPATH="$BUILD_DIR"
- "echo Building Panda..."
- scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache ${CI_PROJECT_DIR}/panda
- mkdir -p ${BUILD_DIR}
- ls -la ${BUILD_DIR}
- "echo Building Rest..."
- ./release/release_files.py | sort | uniq | rsync -vrRl --files-from=- . $BUILD_DIR/
# - cp -pR --parents $(cat release/files_common release/files_tici | sort | uniq) $BUILD_DIR/ 2> >(grep -v 'warning:' >&2)
- cd $BUILD_DIR
- sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py # comment panda jungle when prebuilt
- scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache --minimal
- touch ${BUILD_DIR}/prebuilt
- sudo rm -rf ${OUTPUT_DIR}
- mkdir -p ${OUTPUT_DIR}
# We first include the paths we want to keep, even if we later will be excluding the other things on those paths
- rsync -avm
--include='**/panda/board/'
--include='**/panda/board/obj'
--include='**/panda/board/obj/panda.bin.signed'
--include='**/panda/board/obj/panda_h7.bin.signed'
--include='**/panda/board/obj/bootstub.panda.bin'
--include='**/panda/board/obj/bootstub.panda_h7.bin'
--exclude='.sconsign.dblite'
--exclude='*.a'
--exclude='*.A'
--exclude='*.o'
--exclude='*.O'
--exclude='*.os'
--exclude='*.OS'
--exclude='*.pyc'
--exclude='*.PYC'
--exclude='moc_*'
--exclude='MOC_*'
--exclude='*.cc'
--exclude='*.CC'
--exclude='Jenkinsfile'
--exclude='supercombo.onnx'
--exclude='**/panda/board/*'
--exclude='**/panda/board/obj/**'
--exclude='**/panda/certs/'
--exclude='**/panda/crypto/'
--exclude='**/release/'
--exclude='**/.github/'
--exclude='**/selfdrive/ui/replay/'
--exclude='**/__pycache__/'
--exclude='**/selfdrive/ui/*.h'
--exclude='**/selfdrive/ui/**/*.h'
--exclude='**/selfdrive/ui/qt/offroad/sunnypilot/'
--exclude='**/.git/'
--exclude='**/SConstruct'
--exclude='**/SConscript'
--delete-excluded
--chown=comma:comma
${BUILD_DIR}/ ${OUTPUT_DIR}/
after_script:
# cleanup build dir after doing work
- find $BUILD_DIR/ -mindepth 1 -delete
artifacts:
paths:
- ${OUTPUT_DIR}/
tags: [ 'sunnypilot', 'tici' ]
rules:
- if: $AUTO_BUILD
when: always
- when: manual
.publish_base: &publish_base
image: alpine
variables:
GIT_SUBMODULE_STRATEGY: normal
stage: publish
needs:
- job: build
artifacts: true
before_script:
- 'apk update && apk upgrade'
- 'apk add git bash openssh'
- 'eval $(ssh-agent -s)'
- echo "$SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
- 'mkdir -p ~/.ssh/'
- 'chmod 700 ~/.ssh'
- export HOST=$(echo ${GIT_ORIGIN} | cut -d'@' -f2 | cut -d':' -f1)
- echo fetching ssh pub keys for ${HOST}
- 'ssh-keyscan -H ${HOST} >> ~/.ssh/known_hosts' # Adding gitlab to trusted
- 'chmod 644 ~/.ssh/known_hosts'
- *default_before_script
script:
- echo "${GIT_ORIGIN}"
- echo "Calling to publish [${CI_DIR}/publish.sh ${CI_PROJECT_DIR} ${OUTPUT_DIR} ${NEW_BRANCH} ${VERSION} ${GIT_ORIGIN}]"
- git config --global --add safe.directory ${OUTPUT_DIR}
- $CI_DIR/publish.sh "${CI_PROJECT_DIR}" "${OUTPUT_DIR}" "${NEW_BRANCH}" "${VERSION}" "${GIT_ORIGIN}" "-${EXTRA_VERSION_IDENTIFIER}"
allow_failure: false
publish to private gitlab prebuilt:
extends: ".publish_base"
variables:
GIT_ORIGIN: git@gitlab.com:sunnypilot/public/sunnypilot-prebuilts.git
rules:
- if: $AUTO_BUILD
when: on_success
- if: $CI_MERGE_REQUEST_IID
when: on_success
- when: manual
publish to public github prebuilt:
extends: ".publish_base"
variables:
GIT_ORIGIN: git@github.com:sunnyhaibin/sunnypilot.git
GIT_CONFIG_USER_EMAIL: "jason.wen@sunnypilot.ai"
GIT_CONFIG_USER_NAME: "Jason Wen"
rules:
- if: $AUTO_PUBLISH
when: on_success
- when: manual
.notify_discord: &notify_discord
image: alpine
stage: notify
needs: ["build"]
variables:
DISCORD_HOOK: "${DISCORD_MANUAL_BUILD_WEBHOOK_URL}" # Default hook if not overriden by children
before_script:
- 'apk add curl jq envsubst'
script:
- echo using [${TEMPLATE}]
- cat release/ci/${TEMPLATE} | envsubst | tee payload.json
- 'curl -X POST -H "Content-Type: application/json" -d "$(cat payload.json)" ${DISCORD_HOOK} | jq .'
rules:
- if: $NEW_BRANCH
when: on_success
- when: never
allow_failure: true
notify pending action:
extends: ".notify_discord"
variables:
TEMPLATE: "discord_template_notify_dev_private.json"
before_script:
- !reference [".notify_discord", "before_script"]
- export AVATAR_URL=$(curl -s -X GET "https://gitlab.com/api/v4/avatar?email=${GITLAB_USER_EMAIL}" | jq -r '.avatar_url')
notify new dev build:
extends: ".notify_discord"
stage: notify
needs: ["publish to public github prebuilt"] # This notify shall only happen after a publish to github public
variables:
TEMPLATE: "discord_template_notify_dev_public.json"
before_script:
- !reference [".notify_discord", "before_script"]
- export EXTRA_VERSION_IDENTIFIER=$((CI_PIPELINE_IID + BASE_BUILD_NUMER))
rules:
- if: $NEW_BRANCH == "dev-c3"
variables:
DISCORD_HOOK: "${DISCORD_NEW_BUILD_WEBHOOK_URL}" # Overriding hook because we know we are dev-c3
- !reference [".notify_discord", "rules"]
-18
View File
@@ -1,18 +0,0 @@
[submodule "panda"]
path = panda
url = https://github.com/sunnyhaibin/panda.git
[submodule "opendbc"]
path = opendbc_repo
url = https://github.com/sunnypilot/opendbc.git
[submodule "msgq"]
path = msgq_repo
url = https://github.com/sunnypilot/msgq.git
[submodule "rednose_repo"]
path = rednose_repo
url = https://github.com/commaai/rednose.git
[submodule "teleoprtc_repo"]
path = teleoprtc_repo
url = https://github.com/commaai/teleoprtc
[submodule "tinygrad"]
path = tinygrad_repo
url = https://github.com/tinygrad/tinygrad.git
-25
View File
@@ -1,25 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CLionExternalBuildManager">
<target id="a62f99e8-5ec4-434c-8122-49efed5af108" name="Poetry SCons Build Debug" defaultType="TOOL">
<configuration id="b93ec964-16e5-4962-a12e-3ed360ce8f02" name="Poetry SCons Build Debug">
<build type="TOOL">
<tool actionId="Tool_External Tools_Poetry SCons Build Debug" />
</build>
<clean type="TOOL">
<tool actionId="Tool_External Tools_Poetry SCons Clean" />
</clean>
</configuration>
</target>
<target id="edd8ad9d-183b-467c-a355-0d9a0ecab026" name="Poetry SCons Build Release" defaultType="TOOL">
<configuration id="09523339-5ce3-4223-ab9e-904f38ad7752" name="Poetry SCons Build Release">
<build type="TOOL">
<tool actionId="Tool_External Tools_Poetry SCons Build Release" />
</build>
<clean type="TOOL">
<tool actionId="Tool_External Tools_Poetry SCons Clean" />
</clean>
</configuration>
</target>
</component>
</project>
-23
View File
@@ -1,23 +0,0 @@
<toolSet name="External Tools">
<tool name="Poetry SCons Build Debug" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
<exec>
<option name="COMMAND" value="bash" />
<option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db --ccflags=\&quot;-fno-inline\&quot;&quot;" />
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
</exec>
</tool>
<tool name="Poetry SCons Clean" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
<exec>
<option name="COMMAND" value="bash" />
<option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -c&quot; " />
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
</exec>
</tool>
<tool name="Poetry SCons Build Release" showInMainMenu="false" showInEditor="false" showInProject="false" showInSearchPopup="false" disabled="false" useConsole="true" showConsoleOnStdOut="false" showConsoleOnStdErr="false" synchronizeAfterRun="true">
<exec>
<option name="COMMAND" value="bash" />
<option name="PARAMETERS" value="-c &quot;source .venv/bin/activate &amp;&amp; scons -u -j$(nproc) --compile_db&quot; " />
<option name="WORKING_DIRECTORY" value="$ProjectFileDir$" />
</exec>
</tool>
</toolSet>
-4
View File
@@ -1,4 +0,0 @@
[lfs]
url = https://gitlab.com/sunnypilot/public/sunnypilot-lfs.git/info/lfs
pushurl = ssh://git@gitlab.com/sunnypilot/public/sunnypilot-lfs.git
locksverify = false
-4
View File
@@ -1,4 +0,0 @@
[lfs]
url = https://gitlab.com/commaai/openpilot-lfs.git/info/lfs
pushurl = ssh://git@gitlab.com/commaai/openpilot-lfs.git
locksverify = false
-10
View File
@@ -1,10 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Build Debug" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="openpilot-special" TARGET_NAME="Poetry SCons Build Debug" CONFIG_NAME="Poetry SCons Build Debug" RUN_PATH="ui">
<envs>
<env name="QT_DBL_CLICK_DIST" value="150" />
</envs>
<method v="2">
<option name="CLION.EXTERNAL.BUILD" enabled="true" />
</method>
</configuration>
</component>
-10
View File
@@ -1,10 +0,0 @@
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Build Release" type="CLionExternalRunConfiguration" factoryName="Application" REDIRECT_INPUT="false" ELEVATE="false" USE_EXTERNAL_CONSOLE="false" EMULATE_TERMINAL="false" WORKING_DIR="file://$ProjectFileDir$/selfdrive/ui" PASS_PARENT_ENVS_2="true" PROJECT_NAME="openpilot-special" TARGET_NAME="Poetry SCons Build Release" CONFIG_NAME="Poetry SCons Build Release" RUN_PATH="ui">
<envs>
<env name="QT_DBL_CLICK_DIST" value="150" />
</envs>
<method v="2">
<option name="CLION.EXTERNAL.BUILD" enabled="true" />
</method>
</configuration>
</component>
-8
View File
@@ -1,8 +0,0 @@
{
"recommendations": [
"ms-python.python",
"ms-vscode.cpptools",
"elagil.pre-commit-helper",
"charliermarsh.ruff",
]
}
-47
View File
@@ -1,47 +0,0 @@
{
"version": "0.2.0",
"inputs": [
{
"id": "python_process",
"type": "pickString",
"description": "Select the process to debug",
"options": [
"selfdrive/controls/controlsd.py",
"selfdrive/navd/navd.py",
"system/timed/timed.py",
"tools/sim/run_bridge.py"
]
},
{
"id": "cpp_process",
"type": "pickString",
"description": "Select the process to debug",
"options": [
"selfdrive/ui/ui"
]
},
{
"id": "args",
"description": "Arguments to pass to the process",
"type": "promptString"
}
],
"configurations": [
{
"name": "Python: openpilot Process",
"type": "debugpy",
"request": "launch",
"program": "${input:python_process}",
"console": "integratedTerminal",
"justMyCode": true,
"args": "${input:args}"
},
{
"name": "C++: openpilot Process",
"type": "cppdbg",
"request": "launch",
"program": "${workspaceFolder}/${input:cpp_process}",
"cwd": "${workspaceFolder}",
}
]
}
-27
View File
@@ -1,27 +0,0 @@
{
"editor.tabSize": 2,
"editor.insertSpaces": true,
"editor.renderWhitespace": "trailing",
"files.trimTrailingWhitespace": true,
"search.exclude": {
"**/.git": true,
"**/.venv": true,
"**/__pycache__": true
},
"files.exclude": {
"**/.git": true,
"**/.venv": true,
"**/__pycache__": true
},
"python.analysis.exclude": [
"**/.git",
"**/.venv",
"**/__pycache__",
// exclude directories that should be using the symlinked version
"common/**",
"selfdrive/**",
"system/**",
"third_party/**",
"tools/**",
]
}
-11
View File
@@ -31,9 +31,6 @@ sunnypilot - 0.9.8.0 (2024-xx-xx)
* UPDATED: Driving Model Selector v5
* NEW❗: Driving Model additions
* Notre Dame (July 01, 2024) - NDv3
* UPDATED: Neural Network Lateral Control (NNLC)
* NEW❗: Remove Lateral Jerk Response (Alpha)
* FIXED: Hotfix for "lazy" steering performance in tighter curves thanks to twilsonco!
* UPDATED: Toyota: Continued support for Smart DSU (SDSU) and Radar CAN Filter
* In response to the official deprecation of support for Smart DSU (SDSU) and Radar CAN Filter in the upstream ([commaai/openpilot#32777](https://github.com/commaai/openpilot/pull/32777)), sunnypilot will continue maintaining software support for Smart DSU (SDSU) and Radar CAN Filter
* UPDATED: Continued support for Mapbox navigation
@@ -51,10 +48,6 @@ sunnypilot - 0.9.8.0 (2024-xx-xx)
* NEW❗: Time to Lead Car
* Displays the time to reach the position previously occupied by the lead car
* NEW❗: Display Distance, Speed, and Time to Lead Car simultaneously
* Ford F-150 2022-23 support
* Ford F-150 Lightning 2021-23 support
* Ford Mustang Mach-E 2021-23 support
* Hyundai Kona Electric Non-SCC 2019 support thanks to NikitaNekrasov!
* Kia Ceed Plug-in Hybrid Non-SCC 2022 support thanks to TerminatorNL!
sunnypilot - 0.9.7.1 (2024-06-13)
@@ -92,8 +85,6 @@ sunnypilot - 0.9.7.1 (2024-06-13)
* Force sunnypilot in the offroad state even when the car is on
* When Forced Offroad mode is on, allows changing offroad-only settings even when the car is turned on
* To engage/disengage Force Offroad, go to Settings -> Device panel
* NEW❗: Ford CAN-FD longitudinal
* NEW❗: Parse speed limit sign recognition from camera for certain supported platforms
* UPDATED: Auto Lane Change Timer -> Auto Lane Change by Blinker
* NEW❗: New "Off" option to disable lane change by blinker
* UPDATED: Pause Lateral Below Speed with Blinker
@@ -101,8 +92,6 @@ sunnypilot - 0.9.7.1 (2024-06-13)
* Pause lateral actuation with blinker when traveling below the desired speed selected. Default is 20 MPH or 32 km/h.
* UPDATED: Hyundai CAN Longitudinal
* Auto-enable radar tracks on platforms with applicable Mando radar
* UPDATED: Hyundai CAN-FD Radar-based SCC
* Longitudinal support for CAN-FD Radar-based SCC cars
* UPDATED: Hyundai CAN-FD Camera-based SCC
* NEW❗: Parse lead info for camera-based SCC platforms with longitudinal support
* Improve lead tracking when using openpilot longitudinal
-13
View File
@@ -1,13 +0,0 @@
FROM ghcr.io/commaai/openpilot-base:latest
ENV PYTHONUNBUFFERED 1
ENV OPENPILOT_PATH /home/batman/openpilot
ENV PYTHONPATH ${OPENPILOT_PATH}:${PYTHONPATH}
RUN mkdir -p ${OPENPILOT_PATH}
WORKDIR ${OPENPILOT_PATH}
COPY . ${OPENPILOT_PATH}/
RUN scons --cache-readonly -j$(nproc)
-84
View File
@@ -1,84 +0,0 @@
FROM ubuntu:24.04
ENV PYTHONUNBUFFERED 1
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y --no-install-recommends sudo tzdata locales ssh pulseaudio xvfb x11-xserver-utils gnome-screenshot && \
rm -rf /var/lib/apt/lists/*
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
COPY tools/install_ubuntu_dependencies.sh /tmp/tools/
RUN INSTALL_EXTRA_PACKAGES=no /tmp/tools/install_ubuntu_dependencies.sh && \
rm -rf /var/lib/apt/lists/* /tmp/* && \
cd /usr/lib/gcc/arm-none-eabi/* && \
rm -rf arm/ thumb/nofp thumb/v6* thumb/v8* thumb/v7+fp thumb/v7-r+fp.sp
# Add OpenCL
RUN apt-get update && apt-get install -y --no-install-recommends \
apt-utils \
alien \
unzip \
tar \
curl \
xz-utils \
dbus \
gcc-arm-none-eabi \
tmux \
vim \
libx11-6 \
wget \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir -p /tmp/opencl-driver-intel && \
cd /tmp/opencl-driver-intel && \
wget https://github.com/intel/llvm/releases/download/2024-WW14/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
wget https://github.com/oneapi-src/oneTBB/releases/download/v2021.12.0/oneapi-tbb-2021.12.0-lin.tgz && \
mkdir -p /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
cd /opt/intel/oclcpuexp_2024.17.3.0.09_rel && \
tar -zxvf /tmp/opencl-driver-intel/oclcpuexp-2024.17.3.0.09_rel.tar.gz && \
mkdir -p /etc/OpenCL/vendors && \
echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64/libintelocl.so > /etc/OpenCL/vendors/intel_expcpu.icd && \
cd /opt/intel && \
tar -zxvf /tmp/opencl-driver-intel/oneapi-tbb-2021.12.0-lin.tgz && \
ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbb.so.12 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
ln -s /opt/intel/oneapi-tbb-2021.12.0/lib/intel64/gcc4.8/libtbbmalloc.so.2 /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 && \
mkdir -p /etc/ld.so.conf.d && \
echo /opt/intel/oclcpuexp_2024.17.3.0.09_rel/x64 > /etc/ld.so.conf.d/libintelopenclexp.conf && \
ldconfig -f /etc/ld.so.conf.d/libintelopenclexp.conf && \
cd / && \
rm -rf /tmp/opencl-driver-intel
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES graphics,utility,compute
ENV QTWEBENGINE_DISABLE_SANDBOX 1
RUN dbus-uuidgen > /etc/machine-id
ARG USER=batman
ARG USER_UID=1001
RUN useradd -m -s /bin/bash -u $USER_UID $USER
RUN usermod -aG sudo $USER
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER $USER
COPY --chown=$USER pyproject.toml uv.lock /tmp/
COPY --chown=$USER tools/install_python_dependencies.sh /tmp/tools/
ENV VIRTUAL_ENV=/home/$USER/.venv
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
RUN cd /tmp && \
tools/install_python_dependencies.sh && \
mkdir -p $VIRTUAL_ENV && \
cp -r /tmp/.venv/* $VIRTUAL_ENV && \
rm -rf /tmp/* && \
rm -rf /home/$USER/.cache
USER root
RUN sudo git config --global --add safe.directory /tmp/openpilot
Vendored
-254
View File
@@ -1,254 +0,0 @@
def retryWithDelay(int maxRetries, int delay, Closure body) {
for (int i = 0; i < maxRetries; i++) {
try {
return body()
} catch (Exception e) {
sleep(delay)
}
}
throw Exception("Failed after ${maxRetries} retries")
}
def device(String ip, String step_label, String cmd) {
withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
def ssh_cmd = """
ssh -tt -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END'
set -e
shopt -s huponexit # kill all child processes when the shell exits
export CI=1
export PYTHONWARNINGS=error
export LOGPRINT=debug
export TEST_DIR=${env.TEST_DIR}
export SOURCE_DIR=${env.SOURCE_DIR}
export GIT_BRANCH=${env.GIT_BRANCH}
export GIT_COMMIT=${env.GIT_COMMIT}
export AZURE_TOKEN='${env.AZURE_TOKEN}'
export MAPBOX_TOKEN='${env.MAPBOX_TOKEN}'
# only use 1 thread for tici tests since most require HIL
export PYTEST_ADDOPTS="-n 0"
export GIT_SSH_COMMAND="ssh -i /data/gitkey"
source ~/.bash_profile
if [ -f /TICI ]; then
source /etc/profile
rm -rf /tmp/tmp*
rm -rf ~/.commacache
rm -rf /dev/shm/*
rm -rf /dev/tmp/tmp*
if ! systemctl is-active --quiet systemd-resolved; then
echo "restarting resolved"
sudo systemctl start systemd-resolved
sleep 3
fi
# restart aux USB
if [ -e /sys/bus/usb/drivers/hub/3-0:1.0 ]; then
echo "restarting aux usb"
echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/unbind
sleep 0.5
echo "3-0:1.0" | sudo tee /sys/bus/usb/drivers/hub/bind
fi
fi
if [ -f /data/openpilot/launch_env.sh ]; then
source /data/openpilot/launch_env.sh
fi
ln -snf ${env.TEST_DIR} /data/pythonpath
cd ${env.TEST_DIR} || true
${cmd}
exit 0
END"""
sh script: ssh_cmd, label: step_label
}
}
def deviceStage(String stageName, String deviceType, List extra_env, def steps) {
stage(stageName) {
if (currentBuild.result != null) {
return
}
def extra = extra_env.collect { "export ${it}" }.join('\n');
def branch = env.BRANCH_NAME ?: 'master';
lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1, resourceSelectStrategy: 'random') {
docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') {
timeout(time: 35, unit: 'MINUTES') {
retry (3) {
device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
}
steps.each { item ->
device(device_ip, item[0], item[1])
}
}
}
}
}
}
def pcStage(String stageName, Closure body) {
node {
stage(stageName) {
if (currentBuild.result != null) {
return
}
checkout scm
def dockerArgs = "--user=batman -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache -e PYTHONPATH=${env.WORKSPACE} --cpus=8 --memory 16g -e PYTEST_ADDOPTS='-n8'";
def openpilot_base = retryWithDelay (3, 15) {
return docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .")
}
lock(resource: "", label: 'pc', inversePrecedence: true, quantity: 1) {
openpilot_base.inside(dockerArgs) {
timeout(time: 20, unit: 'MINUTES') {
try {
retryWithDelay (3, 15) {
sh "git config --global --add safe.directory '*'"
sh "git submodule update --init --recursive"
sh "git lfs pull"
}
body()
} finally {
sh "rm -rf ${env.WORKSPACE}/* || true"
sh "rm -rf .* || true"
}
}
}
}
}
}
}
def setupCredentials() {
withCredentials([
string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'),
string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN')
]) {
env.AZURE_TOKEN = "${AZURE_TOKEN}"
env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}"
}
}
node {
env.CI = "1"
env.PYTHONWARNINGS = "error"
env.TEST_DIR = "/data/openpilot"
env.SOURCE_DIR = "/data/openpilot_source/"
setupCredentials()
env.GIT_BRANCH = checkout(scm).GIT_BRANCH
env.GIT_COMMIT = checkout(scm).GIT_COMMIT
def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging',
'testing-closet*', 'hotfix-*']
def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
if (env.BRANCH_NAME != 'master') {
properties([
disableConcurrentBuilds(abortPrevious: true)
])
}
try {
if (env.BRANCH_NAME == 'devel-staging') {
deviceStage("build release3-staging", "tici-needs-can", [], [
["build release3-staging", "RELEASE_BRANCH=release3-staging $SOURCE_DIR/release/build_release.sh"],
])
}
if (env.BRANCH_NAME == 'master-ci') {
deviceStage("build nightly", "tici-needs-can", [], [
["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"],
])
}
if (!env.BRANCH_NAME.matches(excludeRegex)) {
parallel (
// tici tests
'onroad tests': {
deviceStage("onroad", "tici-needs-can", [], [
// TODO: ideally, this test runs in master-ci, but it takes 5+m to build it
//["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR $SOURCE_DIR/scripts/retry.sh ./build_devel.sh"],
["build openpilot", "cd system/manager && ./build.py"],
["check dirty", "release/check-dirty.sh"],
["onroad tests", "pytest selfdrive/test/test_onroad.py -s"],
["time to onroad", "pytest selfdrive/test/test_time_to_onroad.py"],
])
},
'HW + Unit Tests': {
deviceStage("tici-hardware", "tici-common", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"],
["test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"],
["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"],
["test pigeond", "pytest system/ubloxd/tests/test_pigeond.py"],
["test manager", "pytest system/manager/test/test_manager.py"],
])
},
'loopback': {
deviceStage("loopback", "tici-loopback", ["UNSAFE=1"], [
["build openpilot", "cd system/manager && ./build.py"],
["test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"],
])
},
'camerad': {
deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["test camerad", "pytest system/camerad/test/test_camerad.py"],
["test exposure", "pytest system/camerad/test/test_exposure.py"],
])
deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["test camerad", "pytest system/camerad/test/test_camerad.py"],
["test exposure", "pytest system/camerad/test/test_exposure.py"],
])
},
'sensord': {
deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
])
deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["test sensord", "pytest system/sensord/tests/test_sensord.py"],
])
},
'replay': {
deviceStage("model-replay", "tici-replay", ["UNSAFE=1"], [
["build", "cd system/manager && ./build.py"],
["model replay", "selfdrive/test/process_replay/model_replay.py"],
])
},
'tizi': {
deviceStage("tizi", "tizi", ["UNSAFE=1"], [
["build openpilot", "cd system/manager && ./build.py"],
["test pandad loopback", "SINGLE_PANDA=1 pytest selfdrive/pandad/tests/test_pandad_loopback.py"],
["test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"],
["test pandad", "pytest selfdrive/pandad/tests/test_pandad.py"],
["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"],
["test hw", "pytest system/hardware/tici/tests/test_hardware.py"],
["test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py"],
])
},
)
}
} catch (Exception e) {
currentBuild.result = 'FAILED'
throw e
}
}
-7
View File
@@ -1,7 +0,0 @@
Copyright (c) 2018, Comma.ai, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-21
View File
@@ -1,21 +0,0 @@
# Custom MIT License
Copyright (c) 2024, Haibin Wen, SUNNYPILOT LLC
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to view and modify the Software, subject to the following conditions:
1. **Permission Required**: Permission Required for Commercial, For-Profit, or Closed Source Use: Use of the Software, in whole or in part, for any commercial purposes, for-profit projects, or in closed source projects requires explicit written permission from the original author(s).
2. **Redistribution**: Any redistribution of the Software, modified or unmodified, must retain this license notice and the following acknowledgment:
"This software is licensed under a custom license requiring permission for use."
3. **Visibility**: Any project that uses the Software must visibly mention the following acknowledgment:
"This project uses software from Haibin Wen and SUNNYPILOT LLC and is licensed under a custom license requiring permission for use."
4. **No Warranty**: THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
Contact sunnypilot Support <support@sunnypilot.ai> for permission requests.
---
Haibin Wen, SUNNYPILOT LLC
+3 -32
View File
@@ -48,7 +48,6 @@ Join the official sunnypilot Discord server to stay up to date with all the late
To use sunnypilot in a car, you need the following:
* A supported device to run this software
* a [comma three](https://comma.ai/shop/products/three), or
* a comma two (only with older versions below 0.8.13)
* This software
* One of [the 250+ supported cars](https://github.com/commaai/openpilot/blob/master/docs/CARS.md). We support Honda, Toyota, Hyundai, Nissan, Kia, Chrysler, Lexus, Acura, Audi, VW, Ford and more. If your car is not supported but has adaptive cruise control and lane-keeping assist, it's likely able to run sunnypilot.
* A [car harness](https://comma.ai/shop/products/car-harness) to connect to your car
@@ -115,40 +114,12 @@ Please refer to [Recommended Branches](#-recommended-branches) to find your pref
Requires further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
comma two
------
1. [Factory reset/uninstall](https://github.com/commaai/openpilot/wiki/FAQ#how-can-i-reset-the-device) the previous software if you have another software/fork installed.
2. After factory reset/uninstall and upon reboot, select `Custom Software` when given the option.
3. Input the installation URL per [Recommended Branches](#-recommended-branches). Example: ```https://smiskol.com/fork/sunnyhaibin/0.8.12-4-prod```
4. Complete the rest of the installation following the onscreen instructions.
Requires further assistance with software installation? Join the [sunnypilot Discord server](https://discord.sunnypilot.com) and message us in the `#installation-help` channel.
</details>
<details>
<summary>SSH (More Versatile)</summary>
<br>
Prerequisites: [How to SSH](https://github.com/commaai/openpilot/wiki/SSH)
If you are looking to install sunnypilot via SSH, run the following command in an SSH terminal after connecting to your device:
comma three:
------
* [`release-c3`](https://github.com/sunnyhaibin/openpilot/tree/release-c3):
```
cd /data; rm -rf ./openpilot; git clone -b release-c3 --recurse-submodules https://github.com/sunnyhaibin/sunnypilot.git openpilot; cd openpilot; sudo reboot
```
comma two:
------
* [`0.8.12-prod-personal-hkg`](https://github.com/sunnyhaibin/openpilot/tree/0.8.12-prod-personal-hkg):
```
cd /data; rm -rf ./openpilot; git clone -b 0.8.12-prod-personal-hkg --recurse-submodules https://github.com/sunnyhaibin/sunnypilot.git openpilot; cd openpilot; sudo reboot
cd /data && rm -rf ./openpilot && git clone -b release-c3 --recurse-submodules https://github.com/sunnyhaibin/sunnypilot.git openpilot && cd openpilot && sudo reboot
```
After running the command to install the desired branch, your comma device should reboot.
@@ -223,7 +194,7 @@ The goal of Modified Assistive Driving Safety (MADS) is to enhance the user driv
* `SET-` button enables ACC/SCC
* `CANCEL` button only disables ACC/SCC
* `CRUISE (MAIN)` must be `ON` to use ACC/SCC
* `CRUISE (MAIN)` button disables ACC/SCC completely when `OFF` **(strictly enforced in panda safety code)**
* `CRUISE (MAIN)` button disables sunnypilot completely when `OFF` **(strictly enforced in panda safety code)**
### Disengage Lateral ALC on Brake Press Mode toggle
Dedicated toggle to handle Lateral state on brake pedal press and release:
@@ -355,7 +326,7 @@ Example:
---
How-To instructions can be found in [HOW-TOS.md](https://github.com/sunnyhaibin/openpilot/blob/(!)README/HOW-TOS.md).
How-To instructions can be found in [HOW-TOS.md](HOW-TOS.md).
</details>
-414
View File
@@ -1,414 +0,0 @@
import os
import subprocess
import sys
import sysconfig
import platform
import numpy as np
import SCons.Errors
SCons.Warnings.warningAsException(True)
# pending upstream fix - https://github.com/SCons/scons/issues/4461
#SetOption('warn', 'all')
TICI = os.path.isfile('/TICI')
AGNOS = TICI
UBUNTU_FOCAL = int(subprocess.check_output('[ -f /etc/os-release ] && . /etc/os-release && [ "$ID" = "ubuntu" ] && [ "$VERSION_ID" = "20.04" ] && echo 1 || echo 0', shell=True, encoding='utf-8').rstrip())
Export('UBUNTU_FOCAL')
Decider('MD5-timestamp')
SetOption('num_jobs', int(os.cpu_count()/2))
AddOption('--kaitai',
action='store_true',
help='Regenerate kaitai struct parsers')
AddOption('--asan',
action='store_true',
help='turn on ASAN')
AddOption('--ubsan',
action='store_true',
help='turn on UBSan')
AddOption('--coverage',
action='store_true',
help='build with test coverage options')
AddOption('--clazy',
action='store_true',
help='build with clazy')
AddOption('--compile_db',
action='store_true',
help='build clang compilation database')
AddOption('--ccflags',
action='store',
type='string',
default='',
help='pass arbitrary flags over the command line')
AddOption('--snpe',
action='store_true',
help='use SNPE on PC')
AddOption('--external-sconscript',
action='store',
metavar='FILE',
dest='external_sconscript',
help='add an external SConscript to the build')
AddOption('--pc-thneed',
action='store_true',
dest='pc_thneed',
help='use thneed on pc')
AddOption('--minimal',
action='store_false',
dest='extras',
default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS)
help='the minimum build to run openpilot. no tests, tools, etc.')
AddOption('--stock-ui',
action='store_true',
dest='stock_ui',
default=False,
help='Build stock UI instead of sunnypilot UI')
## Architecture name breakdown (arch)
## - larch64: linux tici aarch64
## - aarch64: linux pc aarch64
## - x86_64: linux pc x64
## - Darwin: mac x64 or arm64
real_arch = arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
if platform.system() == "Darwin":
arch = "Darwin"
brew_prefix = subprocess.check_output(['brew', '--prefix'], encoding='utf8').strip()
elif arch == "aarch64" and AGNOS:
arch = "larch64"
assert arch in ["larch64", "aarch64", "x86_64", "Darwin"]
lenv = {
"PATH": os.environ['PATH'],
"LD_LIBRARY_PATH": [Dir(f"#third_party/acados/{arch}/lib").abspath],
"PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
"ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
"ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
"TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
}
rpath = lenv["LD_LIBRARY_PATH"].copy()
if arch == "larch64":
cpppath = [
"#third_party/opencl/include",
]
libpath = [
"/usr/local/lib",
"/system/vendor/lib64",
f"#third_party/acados/{arch}/lib",
]
libpath += [
"#third_party/snpe/larch64",
"#third_party/libyuv/larch64/lib",
"/usr/lib/aarch64-linux-gnu"
]
cflags = ["-DQCOM2", "-mcpu=cortex-a57"]
cxxflags = ["-DQCOM2", "-mcpu=cortex-a57"]
rpath += ["/usr/local/lib"]
else:
cflags = []
cxxflags = []
cpppath = []
rpath += []
# MacOS
if arch == "Darwin":
libpath = [
f"#third_party/libyuv/{arch}/lib",
f"#third_party/acados/{arch}/lib",
f"{brew_prefix}/lib",
f"{brew_prefix}/opt/openssl@3.0/lib",
"/System/Library/Frameworks/OpenGL.framework/Libraries",
]
cflags += ["-DGL_SILENCE_DEPRECATION"]
cxxflags += ["-DGL_SILENCE_DEPRECATION"]
cpppath += [
f"{brew_prefix}/include",
f"{brew_prefix}/opt/openssl@3.0/include",
]
lenv["DYLD_LIBRARY_PATH"] = lenv["LD_LIBRARY_PATH"]
# Linux
else:
libpath = [
f"#third_party/acados/{arch}/lib",
f"#third_party/libyuv/{arch}/lib",
"/usr/lib",
"/usr/local/lib",
]
if arch == "x86_64":
libpath += [
f"#third_party/snpe/{arch}"
]
rpath += [
Dir(f"#third_party/snpe/{arch}").abspath,
]
if GetOption('asan'):
ccflags = ["-fsanitize=address", "-fno-omit-frame-pointer"]
ldflags = ["-fsanitize=address"]
elif GetOption('ubsan'):
ccflags = ["-fsanitize=undefined"]
ldflags = ["-fsanitize=undefined"]
else:
ccflags = []
ldflags = []
# no --as-needed on mac linker
if arch != "Darwin":
ldflags += ["-Wl,--as-needed", "-Wl,--no-undefined"]
# Enable swaglog include in submodules
cxxflags += ['-DSWAGLOG="\\"common/swaglog.h\\""']
if not GetOption('stock_ui'):
cflags += ["-DSUNNYPILOT"]
cxxflags += ["-DSUNNYPILOT"]
ccflags_option = GetOption('ccflags')
if ccflags_option:
ccflags += ccflags_option.split(' ')
env = Environment(
ENV=lenv,
CCFLAGS=[
"-g",
"-fPIC",
"-O2",
"-Wunused",
"-Werror",
"-Wshadow",
"-Wno-unknown-warning-option",
"-Wno-deprecated-register",
"-Wno-register",
"-Wno-inconsistent-missing-override",
"-Wno-c99-designator",
"-Wno-reorder-init-list",
"-Wno-error=unused-but-set-variable",
"-Wno-vla-cxx-extension",
] + cflags + ccflags,
CPPPATH=cpppath + [
"#",
"#third_party/acados/include",
"#third_party/acados/include/blasfeo/include",
"#third_party/acados/include/hpipm/include",
"#third_party/catch2/include",
"#third_party/libyuv/include",
"#third_party/json11",
"#third_party/linux/include",
"#third_party/snpe/include",
"#third_party",
"#cereal",
"#msgq",
"#opendbc/can",
"#third_party/maplibre-native-qt/include",
f"#third_party/maplibre-native-qt/{arch}/include"
],
CC='clang',
CXX='clang++',
LINKFLAGS=ldflags,
RPATH=rpath,
CFLAGS=["-std=gnu11"] + cflags,
CXXFLAGS=["-std=c++1z"] + cxxflags,
LIBPATH=libpath + [
"#msgq_repo",
"#third_party",
"#selfdrive/pandad",
"#common",
"#rednose/helpers",
],
CYTHONCFILESUFFIX=".cpp",
COMPILATIONDB_USE_ABSPATH=True,
REDNOSE_ROOT="#",
tools=["default", "cython", "compilation_db", "rednose_filter"],
toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
)
if arch == "Darwin":
# RPATH is not supported on macOS, instead use the linker flags
darwin_rpath_link_flags = [f"-Wl,-rpath,{path}" for path in env["RPATH"]]
env["LINKFLAGS"] += darwin_rpath_link_flags
if GetOption('compile_db'):
env.CompilationDatabase('compile_commands.json')
# Setup cache dir
default_cache_dir = '/data/scons_cache' if AGNOS else '/tmp/scons_cache'
cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
CacheDir(cache_dir)
Clean(["."], cache_dir)
node_interval = 5
node_count = 0
def progress_function(node):
global node_count
node_count += node_interval
sys.stderr.write("progress: %d\n" % node_count)
if os.environ.get('SCONS_PROGRESS'):
Progress(progress_function, interval=node_interval)
# Cython build environment
py_include = sysconfig.get_paths()['include']
envCython = env.Clone()
envCython["CPPPATH"] += [py_include, np.get_include()]
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-shadow", "-Wno-deprecated-declarations"]
envCython["CCFLAGS"].remove("-Werror")
envCython["LIBS"] = []
if arch == "Darwin":
envCython["LINKFLAGS"] = ["-bundle", "-undefined", "dynamic_lookup"] + darwin_rpath_link_flags
else:
envCython["LINKFLAGS"] = ["-pthread", "-shared"]
np_version = SCons.Script.Value(np.__version__)
Export('envCython', 'np_version')
# Qt build environment
qt_env = env.Clone()
qt_modules = ["Widgets", "Gui", "Core", "Network", "Concurrent", "Multimedia", "Quick", "Qml", "QuickWidgets", "Location", "Positioning", "DBus", "Xml"]
qt_libs = []
if arch == "Darwin":
qt_env['QTDIR'] = f"{brew_prefix}/opt/qt@5"
qt_dirs = [
os.path.join(qt_env['QTDIR'], "include"),
]
qt_dirs += [f"{qt_env['QTDIR']}/include/Qt{m}" for m in qt_modules]
qt_env["LINKFLAGS"] += ["-F" + os.path.join(qt_env['QTDIR'], "lib")]
qt_env["FRAMEWORKS"] += [f"Qt{m}" for m in qt_modules] + ["OpenGL"]
qt_env.AppendENVPath('PATH', os.path.join(qt_env['QTDIR'], "bin"))
else:
qt_install_prefix = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_PREFIX'], encoding='utf8').strip()
qt_install_headers = subprocess.check_output(['qmake', '-query', 'QT_INSTALL_HEADERS'], encoding='utf8').strip()
qt_env['QTDIR'] = qt_install_prefix
qt_dirs = [
f"{qt_install_headers}",
]
qt_gui_path = os.path.join(qt_install_headers, "QtGui")
qt_gui_dirs = [d for d in os.listdir(qt_gui_path) if os.path.isdir(os.path.join(qt_gui_path, d))]
qt_dirs += [f"{qt_install_headers}/QtGui/{qt_gui_dirs[0]}/QtGui", ] if qt_gui_dirs else []
qt_dirs += [f"{qt_install_headers}/Qt{m}" for m in qt_modules]
qt_libs = [f"Qt5{m}" for m in qt_modules]
if arch == "larch64":
qt_libs += ["GLESv2", "wayland-client"]
qt_env.PrependENVPath('PATH', Dir("#third_party/qt5/larch64/bin/").abspath)
elif arch != "Darwin":
qt_libs += ["GL"]
qt_env['QT3DIR'] = qt_env['QTDIR']
# compatibility for older SCons versions
try:
qt_env.Tool('qt3')
except SCons.Errors.UserError:
qt_env.Tool('qt')
qt_env['CPPPATH'] += qt_dirs + ["#third_party/qrcode"]
qt_flags = [
"-D_REENTRANT",
"-DQT_NO_DEBUG",
"-DQT_WIDGETS_LIB",
"-DQT_GUI_LIB",
"-DQT_QUICK_LIB",
"-DQT_QUICKWIDGETS_LIB",
"-DQT_QML_LIB",
"-DQT_CORE_LIB",
"-DQT_MESSAGELOGCONTEXT",
]
qt_env['CXXFLAGS'] += qt_flags
qt_env['LIBPATH'] += ['#selfdrive/ui', f"#third_party/maplibre-native-qt/{arch}/lib"]
qt_env['RPATH'] += [Dir(f"#third_party/maplibre-native-qt/{arch}/lib").srcnode().abspath]
qt_env['LIBS'] = qt_libs
if GetOption("clazy"):
checks = [
"level0",
"level1",
"no-range-loop",
"no-non-pod-global-static",
]
qt_env['CXX'] = 'clazy'
qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0]
qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks)
Export('env', 'qt_env', 'arch', 'real_arch')
# Build common module
SConscript(['common/SConscript'])
Import('_common', '_gpucommon')
common = [_common, 'json11']
gpucommon = [_gpucommon]
Export('common', 'gpucommon')
# Build messaging (cereal + msgq + socketmaster + their dependencies)
SConscript(['msgq_repo/SConscript'])
SConscript(['cereal/SConscript'])
Import('socketmaster', 'msgq')
messaging = [socketmaster, msgq, 'zmq', 'capnp', 'kj',]
Export('messaging')
# Build other submodules
SConscript([
'opendbc/can/SConscript',
'panda/SConscript',
])
# Build rednose library
SConscript(['rednose/SConscript'])
# Build system services
SConscript([
'system/proclogd/SConscript',
'system/ubloxd/SConscript',
'system/loggerd/SConscript',
])
if arch != "Darwin":
SConscript([
'system/sensord/SConscript',
'system/logcatd/SConscript',
])
if arch == "larch64":
SConscript(['system/camerad/SConscript'])
# Build openpilot
SConscript(['third_party/SConscript'])
SConscript(['selfdrive/SConscript'])
if Dir('#tools/cabana/').exists() and GetOption('extras'):
SConscript(['tools/replay/SConscript'])
if arch != "larch64":
SConscript(['tools/cabana/SConscript'])
external_sconscript = GetOption('external_sconscript')
if external_sconscript:
SConscript([external_sconscript])
-5
View File
@@ -1,5 +0,0 @@
# Security Policy
## Reporting a Vulnerability
Suspected vulnerabilities can be reported to both `adeeb@comma.ai` and `security@comma.ai`.
-51
View File
@@ -1,51 +0,0 @@
# What is cereal?
cereal is the messaging system for openpilot. It uses [msgq](https://github.com/commaai/msgq) as a pub/sub backend, and [Cap'n proto](https://capnproto.org/capnp-tool.html) for serialization of the structs.
## Messaging Spec
You'll find the message types in [log.capnp](log.capnp). It uses [Cap'n proto](https://capnproto.org/capnp-tool.html) and defines one struct called `Event`.
All `Events` have a `logMonoTime` and a `valid`. Then a big union defines the packet type.
### Best Practices
- **All fields must describe quantities in SI units**, unless otherwise specified in the field name.
- In the context of the message they are in, field names should be completely unambiguous.
- All values should be easy to plot and be human-readable with minimal parsing.
### Maintaining backwards-compatibility
When making changes to the messaging spec you want to maintain backwards-compatibility, such that old logs can
be parsed with a new version of cereal. Adding structs and adding members to structs is generally safe, most other
things are not. Read more details [here](https://capnproto.org/language.html).
### Custom forks
Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
Example
---
```python
import cereal.messaging as messaging
# in subscriber
sm = messaging.SubMaster(['sensorEvents'])
while 1:
sm.update()
print(sm['sensorEvents'])
```
```python
# in publisher
pm = messaging.PubMaster(['sensorEvents'])
dat = messaging.new_message('sensorEvents', size=1)
dat.sensorEvents[0] = {"gyro": {"v": [0.1, -0.1, 0.1]}}
pm.send('sensorEvents', dat)
```
-31
View File
@@ -1,31 +0,0 @@
Import('env', 'envCython', 'arch', 'common', 'msgq')
import shutil
cereal_dir = Dir('.')
gen_dir = Dir('gen')
other_dir = Dir('#msgq')
# Build cereal
schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
env.Command(["gen/c/include/c++.capnp.h"], [], "mkdir -p " + gen_dir.path + "/c/include && touch $TARGETS")
env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
schema_files,
f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
# TODO: remove non shared cereal and messaging
cereal_objects = env.SharedObject([f'gen/cpp/{s}.c++' for s in schema_files])
cereal = env.Library('cereal', cereal_objects)
env.SharedLibrary('cereal_shared', cereal_objects)
# Build messaging
services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
env.Program('messaging/bridge', ['messaging/bridge.cc'], LIBS=[msgq, 'zmq', common])
socketmaster = env.SharedObject(['messaging/socketmaster.cc'])
socketmaster = env.Library('socketmaster', socketmaster)
Export('cereal', 'socketmaster')
+2 -28
View File
@@ -16,7 +16,6 @@ enum LongitudinalPersonalitySP {
moderate @1;
standard @2;
relaxed @3;
overtake @4;
}
enum AccelerationPersonality {
@@ -35,17 +34,11 @@ enum ModelGeneration {
five @5;
}
enum MpcSource {
acc @0;
blended @1;
}
struct ControlsStateSP @0x81c2f05a394cf4af {
lateralState @0 :Text;
personality @8 :LongitudinalPersonalitySP;
dynamicPersonality @9 :Bool;
accelPersonality @10 :AccelerationPersonality;
overtakingAccelerationAssist @11 :Bool;
lateralControlState :union {
indiState @1 :LateralINDIState;
@@ -97,10 +90,8 @@ struct LongitudinalPlanSP @0xaedffd8f31e7b55d {
desiredTF @13 :Float32;
notSpeedLimit @14 :Int16;
e2eX @15 :List(Float32);
e2eBlendedDEPRECATED @18 :Text;
e2eBlended @18 :Text;
e2eStatus @22 :Bool;
mpcSource @23 :MpcSource;
dynamicExperimentalControl @24 :Bool;
distToTurn @7 :Float32;
turnSpeed @8 :Float32;
@@ -212,24 +203,7 @@ struct ModelDataV2SP @0xf98d843bfd7004a3 {
modelCapabilities @4 :UInt32;
}
struct CarControlSP @0xb86e6369214c01c8 {
customStockLongitudinalControl @0 :CustomStockLongitudinalControl;
struct CustomStockLongitudinalControl {
state @0 :ButtonControlState;
cruiseButton @1 :Int16;
finalSpeedKphDEPRECATED @2 :Float32;
vTarget @3 :Float32;
vCruiseCluster @4 :Float32;
enum ButtonControlState {
inactive @0; # No button press or default state
loading @1; # Loading state before transitioning to accelerating or decelerating
accelerating @2; # Increasing speed
decelerating @3; # Decreasing speed
holding @4; # Holding steady speed
}
}
struct CustomReserved7 @0xb86e6369214c01c8 {
}
struct CustomReserved8 @0xf416ec09499d9d19 {
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
BIN
View File
Binary file not shown.
+1 -1
View File
@@ -2388,7 +2388,7 @@ struct Event {
liveMapDataSP @111 :Custom.LiveMapDataSP;
e2eLongStateSP @112 :Custom.E2eLongStateSP;
modelV2SP @113 :Custom.ModelDataV2SP;
carControlSP @114 :Custom.CarControlSP;
customReserved7 @114 :Custom.CustomReserved7;
customReserved8 @115 :Custom.CustomReserved8;
customReserved9 @116 :Custom.CustomReserved9;
BIN
View File
Binary file not shown.
-92
View File
@@ -1,92 +0,0 @@
#include <algorithm>
#include <cassert>
#include <csignal>
#include <iostream>
#include <map>
#include <string>
typedef void (*sighandler_t)(int sig);
#include "cereal/services.h"
#include "msgq/impl_msgq.h"
#include "msgq/impl_zmq.h"
std::atomic<bool> do_exit = false;
static void set_do_exit(int sig) {
do_exit = true;
}
void sigpipe_handler(int sig) {
assert(sig == SIGPIPE);
std::cout << "SIGPIPE received" << std::endl;
}
static std::vector<std::string> get_services(std::string whitelist_str, bool zmq_to_msgq) {
std::vector<std::string> service_list;
for (const auto& it : services) {
std::string name = it.second.name;
bool in_whitelist = whitelist_str.find(name) != std::string::npos;
if (name == "plusFrame" || name == "uiLayoutState" || (zmq_to_msgq && !in_whitelist)) {
continue;
}
service_list.push_back(name);
}
return service_list;
}
int main(int argc, char** argv) {
signal(SIGPIPE, (sighandler_t)sigpipe_handler);
signal(SIGINT, (sighandler_t)set_do_exit);
signal(SIGTERM, (sighandler_t)set_do_exit);
bool zmq_to_msgq = argc > 2;
std::string ip = zmq_to_msgq ? argv[1] : "127.0.0.1";
std::string whitelist_str = zmq_to_msgq ? std::string(argv[2]) : "";
Poller *poller;
Context *pub_context;
Context *sub_context;
if (zmq_to_msgq) { // republishes zmq debugging messages as msgq
poller = new ZMQPoller();
pub_context = new MSGQContext();
sub_context = new ZMQContext();
} else {
poller = new MSGQPoller();
pub_context = new ZMQContext();
sub_context = new MSGQContext();
}
std::map<SubSocket*, PubSocket*> sub2pub;
for (auto endpoint : get_services(whitelist_str, zmq_to_msgq)) {
PubSocket * pub_sock;
SubSocket * sub_sock;
if (zmq_to_msgq) {
pub_sock = new MSGQPubSocket();
sub_sock = new ZMQSubSocket();
} else {
pub_sock = new ZMQPubSocket();
sub_sock = new MSGQSubSocket();
}
pub_sock->connect(pub_context, endpoint);
sub_sock->connect(sub_context, endpoint, ip, false);
poller->registerSocket(sub_sock);
sub2pub[sub_sock] = pub_sock;
}
while (!do_exit) {
for (auto sub_sock : poller->poll(100)) {
Message * msg = sub_sock->receive();
if (msg == NULL) continue;
int ret;
do {
ret = sub2pub[sub_sock]->sendMessage(msg);
} while (ret == -1 && errno == EINTR && !do_exit);
assert(ret >= 0 || do_exit);
delete msg;
if (do_exit) break;
}
}
return 0;
}
-203
View File
@@ -1,203 +0,0 @@
#include <assert.h>
#include <stdlib.h>
#include <string>
#include <mutex>
#include "cereal/services.h"
#include "cereal/messaging/messaging.h"
const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
static inline bool inList(const std::vector<const char *> &list, const char *value) {
for (auto &v : list) {
if (strcmp(value, v) == 0) return true;
}
return false;
}
class MessageContext {
public:
MessageContext() : ctx_(nullptr) {}
~MessageContext() { delete ctx_; }
inline Context *context() {
std::call_once(init_flag, [=]() { ctx_ = Context::create(); });
return ctx_;
}
private:
Context *ctx_;
std::once_flag init_flag;
};
MessageContext message_context;
struct SubMaster::SubMessage {
std::string name;
SubSocket *socket = nullptr;
int freq = 0;
bool updated = false, alive = false, valid = true, ignore_alive;
uint64_t rcv_time = 0, rcv_frame = 0;
void *allocated_msg_reader = nullptr;
bool is_polled = false;
capnp::FlatArrayMessageReader *msg_reader = nullptr;
AlignedBuffer aligned_buf;
cereal::Event::Reader event;
};
SubMaster::SubMaster(const std::vector<const char *> &service_list, const std::vector<const char *> &poll,
const char *address, const std::vector<const char *> &ignore_alive) {
poller_ = Poller::create();
for (auto name : service_list) {
assert(services.count(std::string(name)) > 0);
service serv = services.at(std::string(name));
SubSocket *socket = SubSocket::create(message_context.context(), name, address ? address : "127.0.0.1", true);
assert(socket != 0);
bool is_polled = inList(poll, name) || poll.empty();
if (is_polled) poller_->registerSocket(socket);
SubMessage *m = new SubMessage{
.name = name,
.socket = socket,
.freq = serv.frequency,
.ignore_alive = inList(ignore_alive, name),
.allocated_msg_reader = malloc(sizeof(capnp::FlatArrayMessageReader)),
.is_polled = is_polled};
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader({});
messages_[socket] = m;
services_[name] = m;
}
}
void SubMaster::update(int timeout) {
for (auto &kv : messages_) kv.second->updated = false;
auto sockets = poller_->poll(timeout);
// add non-polled sockets for non-blocking receive
for (auto &kv : messages_) {
SubMessage *m = kv.second;
SubSocket *s = kv.first;
if (!m->is_polled) sockets.push_back(s);
}
uint64_t current_time = nanos_since_boot();
std::vector<std::pair<std::string, cereal::Event::Reader>> messages;
for (auto s : sockets) {
Message *msg = s->receive(true);
if (msg == nullptr) continue;
SubMessage *m = messages_.at(s);
m->msg_reader->~FlatArrayMessageReader();
capnp::ReaderOptions options;
options.traversalLimitInWords = kj::maxValue; // Don't limit
m->msg_reader = new (m->allocated_msg_reader) capnp::FlatArrayMessageReader(m->aligned_buf.align(msg), options);
delete msg;
messages.push_back({m->name, m->msg_reader->getRoot<cereal::Event>()});
}
update_msgs(current_time, messages);
}
void SubMaster::update_msgs(uint64_t current_time, const std::vector<std::pair<std::string, cereal::Event::Reader>> &messages){
if (++frame == UINT64_MAX) frame = 1;
for (auto &kv : messages) {
auto m_find = services_.find(kv.first);
if (m_find == services_.end()){
continue;
}
SubMessage *m = m_find->second;
m->event = kv.second;
m->updated = true;
m->rcv_time = current_time;
m->rcv_frame = frame;
m->valid = m->event.getValid();
if (SIMULATION) m->alive = true;
}
if (!SIMULATION) {
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->alive = (m->freq <= (1e-5) || ((current_time - m->rcv_time) * (1e-9)) < (10.0 / m->freq));
}
}
}
bool SubMaster::all_(const std::vector<const char *> &service_list, bool valid, bool alive) {
int found = 0;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
if (service_list.size() == 0 || inList(service_list, m->name.c_str())) {
found += (!valid || m->valid) && (!alive || (m->alive || m->ignore_alive));
}
}
return service_list.size() == 0 ? found == messages_.size() : found == service_list.size();
}
void SubMaster::drain() {
while (true) {
auto polls = poller_->poll(0);
if (polls.size() == 0)
break;
for (auto sock : polls) {
Message *msg = sock->receive(true);
delete msg;
}
}
}
bool SubMaster::updated(const char *name) const {
return services_.at(name)->updated;
}
bool SubMaster::alive(const char *name) const {
return services_.at(name)->alive;
}
bool SubMaster::valid(const char *name) const {
return services_.at(name)->valid;
}
uint64_t SubMaster::rcv_frame(const char *name) const {
return services_.at(name)->rcv_frame;
}
uint64_t SubMaster::rcv_time(const char *name) const {
return services_.at(name)->rcv_time;
}
cereal::Event::Reader &SubMaster::operator[](const char *name) const {
return services_.at(name)->event;
}
SubMaster::~SubMaster() {
delete poller_;
for (auto &kv : messages_) {
SubMessage *m = kv.second;
m->msg_reader->~FlatArrayMessageReader();
free(m->allocated_msg_reader);
delete m->socket;
delete m;
}
}
PubMaster::PubMaster(const std::vector<const char *> &service_list) {
for (auto name : service_list) {
assert(services.count(name) > 0);
PubSocket *socket = PubSocket::create(message_context.context(), name);
assert(socket);
sockets_[name] = socket;
}
}
int PubMaster::send(const char *name, MessageBuilder &msg) {
auto bytes = msg.toBytes();
return send(name, bytes.begin(), bytes.size());
}
PubMaster::~PubMaster() {
for (auto s : sockets_) delete s.second;
}
-185
View File
@@ -1,185 +0,0 @@
import os
import capnp
import multiprocessing
import numbers
import random
import threading
import time
from parameterized import parameterized
import pytest
from cereal import log, car
import cereal.messaging as messaging
from cereal.services import SERVICE_LIST
events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
def random_sock():
return random.choice(events)
def random_socks(num_socks=10):
return list({random_sock() for _ in range(num_socks)})
def random_bytes(length=1000):
return bytes([random.randrange(0xFF) for _ in range(length)])
def zmq_sleep(t=1):
if "ZMQ" in os.environ:
time.sleep(t)
# TODO: this should take any capnp struct and returrn a msg with random populated data
def random_carstate():
fields = ["vEgo", "aEgo", "gas", "steeringAngleDeg"]
msg = messaging.new_message("carState")
cs = msg.carState
for f in fields:
setattr(cs, f, random.random() * 10)
return msg
# TODO: this should compare any capnp structs
def assert_carstate(cs1, cs2):
for f in car.CarState.schema.non_union_fields:
# TODO: check all types
val1, val2 = getattr(cs1, f), getattr(cs2, f)
if isinstance(val1, numbers.Number):
assert val1 == val2, f"{f}: sent '{val1}' vs recvd '{val2}'"
def delayed_send(delay, sock, dat):
def send_func():
sock.send(dat)
threading.Timer(delay, send_func).start()
class TestMessaging:
def setUp(self):
# TODO: ZMQ tests are too slow; all sleeps will need to be
# replaced with logic to block on the necessary condition
if "ZMQ" in os.environ:
pytest.skip()
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep()
@parameterized.expand(events)
def test_new_message(self, evt):
try:
msg = messaging.new_message(evt)
except capnp.lib.capnp.KjException:
msg = messaging.new_message(evt, random.randrange(200))
assert (time.monotonic() - msg.logMonoTime) < 0.1
assert not msg.valid
assert evt == msg.which()
@parameterized.expand(events)
def test_pub_sock(self, evt):
messaging.pub_sock(evt)
@parameterized.expand(events)
def test_sub_sock(self, evt):
messaging.sub_sock(evt)
@parameterized.expand([
(messaging.drain_sock, capnp._DynamicStructReader),
(messaging.drain_sock_raw, bytes),
])
def test_drain_sock(self, func, expected_type):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no wait and no msgs in queue
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert len(msgs) == 0
# no wait but msgs are queued up
num_msgs = random.randrange(3, 10)
for _ in range(num_msgs):
pub_sock.send(messaging.new_message(sock).to_bytes())
time.sleep(0.1)
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert all(isinstance(msg, expected_type) for msg in msgs)
assert len(msgs) == num_msgs
def test_recv_sock(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=100)
zmq_sleep()
# no wait and no msg in queue, socket should timeout
recvd = messaging.recv_sock(sub_sock)
assert recvd is None
# no wait and one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
recvd = messaging.recv_sock(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
# https://github.com/python/mypy/issues/13038
assert_carstate(msg.carState, recvd.carState)
def test_recv_one(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no msg in queue, socket should timeout
recvd = messaging.recv_one(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
@pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
def test_recv_one_or_none(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock)
zmq_sleep()
# no msg in queue, socket shouldn't block
recvd = messaging.recv_one_or_none(sub_sock)
assert recvd is None
# one msg in queue
msg = random_carstate()
pub_sock.send(msg.to_bytes())
recvd = messaging.recv_one_or_none(sub_sock)
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
def test_recv_one_retry(self):
sock = "carState"
sock_timeout = 0.1
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
zmq_sleep()
# this test doesn't work with ZMQ since multiprocessing interrupts it
if "ZMQ" not in os.environ:
# wait 15 socket timeouts and make sure it's still retrying
p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
p.start()
time.sleep(sock_timeout*15)
assert p.is_alive()
p.terminate()
# wait 15 socket timeouts before sending
msg = random_carstate()
delayed_send(sock_timeout*15, pub_sock, msg.to_bytes())
start_time = time.monotonic()
recvd = messaging.recv_one_retry(sub_sock)
assert (time.monotonic() - start_time) >= sock_timeout*15
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
@@ -1,157 +0,0 @@
import random
import time
from typing import Sized, cast
import cereal.messaging as messaging
from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
random_bytes, random_carstate, assert_carstate, \
zmq_sleep
class TestSubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
sm = messaging.SubMaster(events)
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(events)
def test_init_state(self):
socks = random_socks()
sm = messaging.SubMaster(socks)
assert sm.frame == -1
assert not any(sm.updated.values())
assert not any(sm.alive.values())
assert all(t == 0. for t in sm.recv_time.values())
assert all(f == 0 for f in sm.recv_frame.values())
assert all(t == 0 for t in sm.logMonoTime.values())
for p in [sm.updated, sm.recv_time, sm.recv_frame, sm.alive,
sm.sock, sm.data, sm.logMonoTime, sm.valid]:
assert len(cast(Sized, p)) == len(socks)
def test_getitem(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
msg = random_carstate()
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert_carstate(msg.carState, sm[sock])
# TODO: break this test up to individually test SubMaster.update and SubMaster.update_msgs
def test_update(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
for i in range(10):
msg = messaging.new_message(sock)
pub_sock.send(msg.to_bytes())
sm.update(1000)
assert sm.frame == i
assert all(sm.updated.values())
def test_update_timeout(self):
sock = random_sock()
sm = messaging.SubMaster([sock,])
for _ in range(5):
timeout = random.randrange(1000, 5000)
start_time = time.monotonic()
sm.update(timeout)
t = time.monotonic() - start_time
assert t >= timeout/1000.
assert t < 5
assert not any(sm.updated.values())
def test_avg_frequency_checks(self):
for poll in (True, False):
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
poll=("modelV2" if poll else None),
frequency=(20. if not poll else None))
checks = {
"carState": (20, 20),
"modelV2": (20, 20 if poll else 10),
"cameraOdometry": (20, 10),
"liveCalibration": (4, 4),
"carParams": (None, None),
}
for service, (max_freq, min_freq) in checks.items():
if max_freq is not None:
assert sm._check_avg_freq(service)
assert sm.max_freq[service] == max_freq*1.2
assert sm.min_freq[service] == min_freq*0.8
else:
assert not sm._check_avg_freq(service)
def test_alive(self):
pass
def test_ignore_alive(self):
pass
def test_valid(self):
pass
# SubMaster should always conflate
def test_conflate(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
n = 10
for i in range(n+1):
msg = messaging.new_message(sock)
msg.carState.vEgo = i
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
sm.update(1000)
assert sm[sock].vEgo == n
class TestPubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
messaging.PubMaster(events)
def test_send(self):
socks = random_socks()
pm = messaging.PubMaster(socks)
sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
zmq_sleep()
# PubMaster accepts either a capnp msg builder or bytes
for capnp in [True, False]:
for i in range(100):
sock = socks[i % len(socks)]
if capnp:
try:
msg = messaging.new_message(sock)
except Exception:
msg = messaging.new_message(sock, random.randrange(50))
else:
msg = random_bytes()
pm.send(sock, msg)
recvd = sub_socks[sock].receive()
if capnp:
msg.clear_write_flag()
msg = msg.to_bytes()
assert msg == recvd, i
-21
View File
@@ -1,21 +0,0 @@
import os
import tempfile
from typing import Dict
from parameterized import parameterized
import cereal.services as services
from cereal.services import SERVICE_LIST
class TestServices:
@parameterized.expand(SERVICE_LIST.keys())
def test_services(self, s):
service = SERVICE_LIST[s]
assert service.frequency <= 104
assert service.decimation != 0
def test_generated_header(self):
with tempfile.NamedTemporaryFile(suffix=".h") as f:
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name}")
assert ret == 0, "generated services header is not valid C"
+95
View File
@@ -0,0 +1,95 @@
/* THIS IS AN AUTOGENERATED FILE, PLEASE EDIT services.py */
#ifndef __SERVICES_H
#define __SERVICES_H
#include <map>
#include <string>
struct service { std::string name; bool should_log; int frequency; int decimation; };
static std::map<std::string, service> services = {
{ "gyroscope", {"gyroscope", true, 104, 104}},
{ "gyroscope2", {"gyroscope2", true, 100, 100}},
{ "accelerometer", {"accelerometer", true, 104, 104}},
{ "accelerometer2", {"accelerometer2", true, 100, 100}},
{ "magnetometer", {"magnetometer", true, 25, -1}},
{ "lightSensor", {"lightSensor", true, 100, 100}},
{ "temperatureSensor", {"temperatureSensor", true, 2, 200}},
{ "temperatureSensor2", {"temperatureSensor2", true, 2, 200}},
{ "gpsNMEA", {"gpsNMEA", true, 9, -1}},
{ "deviceState", {"deviceState", true, 2, 1}},
{ "can", {"can", true, 100, 2053}},
{ "controlsState", {"controlsState", true, 100, 10}},
{ "pandaStates", {"pandaStates", true, 10, 1}},
{ "peripheralState", {"peripheralState", true, 2, 1}},
{ "radarState", {"radarState", true, 20, 5}},
{ "roadEncodeIdx", {"roadEncodeIdx", false, 20, 1}},
{ "liveTracks", {"liveTracks", true, 20, -1}},
{ "sendcan", {"sendcan", true, 100, 139}},
{ "logMessage", {"logMessage", true, 0, -1}},
{ "errorLogMessage", {"errorLogMessage", true, 0, 1}},
{ "liveCalibration", {"liveCalibration", true, 4, 4}},
{ "liveTorqueParameters", {"liveTorqueParameters", true, 4, 1}},
{ "androidLog", {"androidLog", true, 0, -1}},
{ "carState", {"carState", true, 100, 10}},
{ "carControl", {"carControl", true, 100, 10}},
{ "carOutput", {"carOutput", true, 100, 10}},
{ "longitudinalPlan", {"longitudinalPlan", true, 20, 10}},
{ "procLog", {"procLog", true, 0, 15}},
{ "gpsLocationExternal", {"gpsLocationExternal", true, 10, 10}},
{ "gpsLocation", {"gpsLocation", true, 1, 1}},
{ "ubloxGnss", {"ubloxGnss", true, 10, -1}},
{ "qcomGnss", {"qcomGnss", true, 2, -1}},
{ "gnssMeasurements", {"gnssMeasurements", true, 10, 10}},
{ "clocks", {"clocks", true, 0, 1}},
{ "ubloxRaw", {"ubloxRaw", true, 20, -1}},
{ "livePose", {"livePose", true, 20, 4}},
{ "liveLocationKalman", {"liveLocationKalman", true, 20, -1}},
{ "liveParameters", {"liveParameters", true, 20, 5}},
{ "cameraOdometry", {"cameraOdometry", true, 20, 10}},
{ "lateralPlanDEPRECATED", {"lateralPlanDEPRECATED", true, 20, 5}},
{ "thumbnail", {"thumbnail", true, 0, 1}},
{ "onroadEvents", {"onroadEvents", true, 1, 1}},
{ "carParams", {"carParams", true, 0, 1}},
{ "roadCameraState", {"roadCameraState", true, 20, 20}},
{ "driverCameraState", {"driverCameraState", true, 20, 20}},
{ "driverEncodeIdx", {"driverEncodeIdx", false, 20, 1}},
{ "driverStateV2", {"driverStateV2", true, 20, 10}},
{ "driverMonitoringState", {"driverMonitoringState", true, 20, 10}},
{ "wideRoadEncodeIdx", {"wideRoadEncodeIdx", false, 20, 1}},
{ "wideRoadCameraState", {"wideRoadCameraState", true, 20, 20}},
{ "drivingModelData", {"drivingModelData", true, 20, 10}},
{ "modelV2", {"modelV2", true, 20, -1}},
{ "managerState", {"managerState", true, 2, 1}},
{ "uploaderState", {"uploaderState", true, 0, 1}},
{ "navInstruction", {"navInstruction", true, 1, 10}},
{ "navRoute", {"navRoute", true, 0, -1}},
{ "navThumbnail", {"navThumbnail", true, 0, -1}},
{ "navModelDEPRECATED", {"navModelDEPRECATED", true, 2, 4}},
{ "mapRenderState", {"mapRenderState", true, 2, 1}},
{ "uiPlanDEPRECATED", {"uiPlanDEPRECATED", true, 20, 40}},
{ "qRoadEncodeIdx", {"qRoadEncodeIdx", false, 20, -1}},
{ "userFlag", {"userFlag", true, 0, 1}},
{ "microphone", {"microphone", true, 10, 10}},
{ "controlsStateSP", {"controlsStateSP", true, 100, 10}},
{ "longitudinalPlanSP", {"longitudinalPlanSP", true, 20, 5}},
{ "lateralPlanSPDEPRECATED", {"lateralPlanSPDEPRECATED", true, 20, 5}},
{ "driverMonitoringStateSP", {"driverMonitoringStateSP", true, 20, 10}},
{ "liveMapDataSP", {"liveMapDataSP", true, 0, -1}},
{ "e2eLongStateSP", {"e2eLongStateSP", true, 0, -1}},
{ "modelV2SP", {"modelV2SP", true, 20, 40}},
{ "uiDebug", {"uiDebug", true, 0, 1}},
{ "testJoystick", {"testJoystick", true, 0, -1}},
{ "roadEncodeData", {"roadEncodeData", false, 20, -1}},
{ "driverEncodeData", {"driverEncodeData", false, 20, -1}},
{ "wideRoadEncodeData", {"wideRoadEncodeData", false, 20, -1}},
{ "qRoadEncodeData", {"qRoadEncodeData", false, 20, -1}},
{ "livestreamWideRoadEncodeIdx", {"livestreamWideRoadEncodeIdx", false, 20, -1}},
{ "livestreamRoadEncodeIdx", {"livestreamRoadEncodeIdx", false, 20, -1}},
{ "livestreamDriverEncodeIdx", {"livestreamDriverEncodeIdx", false, 20, -1}},
{ "livestreamWideRoadEncodeData", {"livestreamWideRoadEncodeData", false, 20, -1}},
{ "livestreamRoadEncodeData", {"livestreamRoadEncodeData", false, 20, -1}},
{ "livestreamDriverEncodeData", {"livestreamDriverEncodeData", false, 20, -1}},
{ "customReservedRawData0", {"customReservedRawData0", true, 0, -1}},
{ "customReservedRawData1", {"customReservedRawData1", true, 0, -1}},
{ "customReservedRawData2", {"customReservedRawData2", true, 0, -1}},
};
#endif
-1
View File
@@ -83,7 +83,6 @@ _services: dict[str, tuple] = {
"liveMapDataSP": (True, 0.),
"e2eLongStateSP": (True, 0.),
"modelV2SP": (True, 20., 40),
"carControlSP": (True, 100., 10),
# debug
"uiDebug": (True, 0., 1),
-13
View File
@@ -1,13 +0,0 @@
comment: false
coverage:
status:
project:
default:
informational: true
patch: off
ignore:
- "**/test_*.py"
- "selfdrive/test/**"
- "system/version.py" # codecov changes depending on if we are in a branch or not
- "tools"
-39
View File
@@ -1,39 +0,0 @@
Import('env', 'envCython', 'arch')
common_libs = [
'params.cc',
'swaglog.cc',
'util.cc',
'i2c.cc',
'watchdog.cc',
'ratekeeper.cc'
]
if arch != "Darwin":
common_libs.append('gpio.cc')
_common = env.Library('common', common_libs, LIBS="json11")
files = [
'clutil.cc',
]
_gpucommon = env.Library('gpucommon', files)
Export('_common', '_gpucommon')
if GetOption('extras'):
env.Program('tests/test_common',
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
LIBS=[_common, 'json11', 'zmq', 'pthread'])
# Cython bindings
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
SConscript([
'transformations/SConscript',
])
Import('transformations_python')
common_python = [params_python, transformations_python]
Export('common_python')
-200
View File
@@ -1,200 +0,0 @@
#include "common/clutil.h"
#include <cassert>
#include <iostream>
#include <memory>
#include "common/util.h"
#include "common/swaglog.h"
namespace { // helper functions
template <typename Func, typename Id, typename Name>
std::string get_info(Func get_info_func, Id id, Name param_name) {
size_t size = 0;
CL_CHECK(get_info_func(id, param_name, 0, NULL, &size));
std::string info(size, '\0');
CL_CHECK(get_info_func(id, param_name, size, info.data(), NULL));
return info;
}
inline std::string get_platform_info(cl_platform_id id, cl_platform_info name) { return get_info(&clGetPlatformInfo, id, name); }
inline std::string get_device_info(cl_device_id id, cl_device_info name) { return get_info(&clGetDeviceInfo, id, name); }
void cl_print_info(cl_platform_id platform, cl_device_id device) {
size_t work_group_size = 0;
cl_device_type device_type = 0;
clGetDeviceInfo(device, CL_DEVICE_MAX_WORK_GROUP_SIZE, sizeof(work_group_size), &work_group_size, NULL);
clGetDeviceInfo(device, CL_DEVICE_TYPE, sizeof(device_type), &device_type, NULL);
const char *type_str = "Other...";
switch (device_type) {
case CL_DEVICE_TYPE_CPU: type_str ="CL_DEVICE_TYPE_CPU"; break;
case CL_DEVICE_TYPE_GPU: type_str = "CL_DEVICE_TYPE_GPU"; break;
case CL_DEVICE_TYPE_ACCELERATOR: type_str = "CL_DEVICE_TYPE_ACCELERATOR"; break;
}
LOGD("vendor: %s", get_platform_info(platform, CL_PLATFORM_VENDOR).c_str());
LOGD("platform version: %s", get_platform_info(platform, CL_PLATFORM_VERSION).c_str());
LOGD("profile: %s", get_platform_info(platform, CL_PLATFORM_PROFILE).c_str());
LOGD("extensions: %s", get_platform_info(platform, CL_PLATFORM_EXTENSIONS).c_str());
LOGD("name: %s", get_device_info(device, CL_DEVICE_NAME).c_str());
LOGD("device version: %s", get_device_info(device, CL_DEVICE_VERSION).c_str());
LOGD("max work group size: %zu", work_group_size);
LOGD("type = %d, %s", (int)device_type, type_str);
}
void cl_print_build_errors(cl_program program, cl_device_id device) {
cl_build_status status;
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_STATUS, sizeof(status), &status, NULL);
size_t log_size;
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, 0, NULL, &log_size);
std::string log(log_size, '\0');
clGetProgramBuildInfo(program, device, CL_PROGRAM_BUILD_LOG, log_size, &log[0], NULL);
LOGE("build failed; status=%d, log: %s", status, log.c_str());
}
} // namespace
cl_device_id cl_get_device_id(cl_device_type device_type) {
cl_uint num_platforms = 0;
CL_CHECK(clGetPlatformIDs(0, NULL, &num_platforms));
std::unique_ptr<cl_platform_id[]> platform_ids = std::make_unique<cl_platform_id[]>(num_platforms);
CL_CHECK(clGetPlatformIDs(num_platforms, &platform_ids[0], NULL));
for (size_t i = 0; i < num_platforms; ++i) {
LOGD("platform[%zu] CL_PLATFORM_NAME: %s", i, get_platform_info(platform_ids[i], CL_PLATFORM_NAME).c_str());
// Get first device
if (cl_device_id device_id = NULL; clGetDeviceIDs(platform_ids[i], device_type, 1, &device_id, NULL) == 0 && device_id) {
cl_print_info(platform_ids[i], device_id);
return device_id;
}
}
LOGE("No valid openCL platform found");
assert(0);
return nullptr;
}
cl_context cl_create_context(cl_device_id device_id) {
return CL_CHECK_ERR(clCreateContext(NULL, 1, &device_id, NULL, NULL, &err));
}
cl_program cl_program_from_file(cl_context ctx, cl_device_id device_id, const char* path, const char* args) {
return cl_program_from_source(ctx, device_id, util::read_file(path), args);
}
cl_program cl_program_from_source(cl_context ctx, cl_device_id device_id, const std::string& src, const char* args) {
const char *csrc = src.c_str();
cl_program prg = CL_CHECK_ERR(clCreateProgramWithSource(ctx, 1, &csrc, NULL, &err));
if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
cl_print_build_errors(prg, device_id);
assert(0);
}
return prg;
}
cl_program cl_program_from_binary(cl_context ctx, cl_device_id device_id, const uint8_t* binary, size_t length, const char* args) {
cl_program prg = CL_CHECK_ERR(clCreateProgramWithBinary(ctx, 1, &device_id, &length, &binary, NULL, &err));
if (int err = clBuildProgram(prg, 1, &device_id, args, NULL, NULL); err != 0) {
cl_print_build_errors(prg, device_id);
assert(0);
}
return prg;
}
// Given a cl code and return a string representation
#define CL_ERR_TO_STR(err) case err: return #err
const char* cl_get_error_string(int err) {
switch (err) {
CL_ERR_TO_STR(CL_SUCCESS);
CL_ERR_TO_STR(CL_DEVICE_NOT_FOUND);
CL_ERR_TO_STR(CL_DEVICE_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_COMPILER_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_MEM_OBJECT_ALLOCATION_FAILURE);
CL_ERR_TO_STR(CL_OUT_OF_RESOURCES);
CL_ERR_TO_STR(CL_OUT_OF_HOST_MEMORY);
CL_ERR_TO_STR(CL_PROFILING_INFO_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_MEM_COPY_OVERLAP);
CL_ERR_TO_STR(CL_IMAGE_FORMAT_MISMATCH);
CL_ERR_TO_STR(CL_IMAGE_FORMAT_NOT_SUPPORTED);
CL_ERR_TO_STR(CL_MAP_FAILURE);
CL_ERR_TO_STR(CL_MISALIGNED_SUB_BUFFER_OFFSET);
CL_ERR_TO_STR(CL_EXEC_STATUS_ERROR_FOR_EVENTS_IN_WAIT_LIST);
CL_ERR_TO_STR(CL_COMPILE_PROGRAM_FAILURE);
CL_ERR_TO_STR(CL_LINKER_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_LINK_PROGRAM_FAILURE);
CL_ERR_TO_STR(CL_DEVICE_PARTITION_FAILED);
CL_ERR_TO_STR(CL_KERNEL_ARG_INFO_NOT_AVAILABLE);
CL_ERR_TO_STR(CL_INVALID_VALUE);
CL_ERR_TO_STR(CL_INVALID_DEVICE_TYPE);
CL_ERR_TO_STR(CL_INVALID_PLATFORM);
CL_ERR_TO_STR(CL_INVALID_DEVICE);
CL_ERR_TO_STR(CL_INVALID_CONTEXT);
CL_ERR_TO_STR(CL_INVALID_QUEUE_PROPERTIES);
CL_ERR_TO_STR(CL_INVALID_COMMAND_QUEUE);
CL_ERR_TO_STR(CL_INVALID_HOST_PTR);
CL_ERR_TO_STR(CL_INVALID_MEM_OBJECT);
CL_ERR_TO_STR(CL_INVALID_IMAGE_FORMAT_DESCRIPTOR);
CL_ERR_TO_STR(CL_INVALID_IMAGE_SIZE);
CL_ERR_TO_STR(CL_INVALID_SAMPLER);
CL_ERR_TO_STR(CL_INVALID_BINARY);
CL_ERR_TO_STR(CL_INVALID_BUILD_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_PROGRAM);
CL_ERR_TO_STR(CL_INVALID_PROGRAM_EXECUTABLE);
CL_ERR_TO_STR(CL_INVALID_KERNEL_NAME);
CL_ERR_TO_STR(CL_INVALID_KERNEL_DEFINITION);
CL_ERR_TO_STR(CL_INVALID_KERNEL);
CL_ERR_TO_STR(CL_INVALID_ARG_INDEX);
CL_ERR_TO_STR(CL_INVALID_ARG_VALUE);
CL_ERR_TO_STR(CL_INVALID_ARG_SIZE);
CL_ERR_TO_STR(CL_INVALID_KERNEL_ARGS);
CL_ERR_TO_STR(CL_INVALID_WORK_DIMENSION);
CL_ERR_TO_STR(CL_INVALID_WORK_GROUP_SIZE);
CL_ERR_TO_STR(CL_INVALID_WORK_ITEM_SIZE);
CL_ERR_TO_STR(CL_INVALID_GLOBAL_OFFSET);
CL_ERR_TO_STR(CL_INVALID_EVENT_WAIT_LIST);
CL_ERR_TO_STR(CL_INVALID_EVENT);
CL_ERR_TO_STR(CL_INVALID_OPERATION);
CL_ERR_TO_STR(CL_INVALID_GL_OBJECT);
CL_ERR_TO_STR(CL_INVALID_BUFFER_SIZE);
CL_ERR_TO_STR(CL_INVALID_MIP_LEVEL);
CL_ERR_TO_STR(CL_INVALID_GLOBAL_WORK_SIZE);
CL_ERR_TO_STR(CL_INVALID_PROPERTY);
CL_ERR_TO_STR(CL_INVALID_IMAGE_DESCRIPTOR);
CL_ERR_TO_STR(CL_INVALID_COMPILER_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_LINKER_OPTIONS);
CL_ERR_TO_STR(CL_INVALID_DEVICE_PARTITION_COUNT);
case -69: return "CL_INVALID_PIPE_SIZE";
case -70: return "CL_INVALID_DEVICE_QUEUE";
case -71: return "CL_INVALID_SPEC_ID";
case -72: return "CL_MAX_SIZE_RESTRICTION_EXCEEDED";
case -1002: return "CL_INVALID_D3D10_DEVICE_KHR";
case -1003: return "CL_INVALID_D3D10_RESOURCE_KHR";
case -1004: return "CL_D3D10_RESOURCE_ALREADY_ACQUIRED_KHR";
case -1005: return "CL_D3D10_RESOURCE_NOT_ACQUIRED_KHR";
case -1006: return "CL_INVALID_D3D11_DEVICE_KHR";
case -1007: return "CL_INVALID_D3D11_RESOURCE_KHR";
case -1008: return "CL_D3D11_RESOURCE_ALREADY_ACQUIRED_KHR";
case -1009: return "CL_D3D11_RESOURCE_NOT_ACQUIRED_KHR";
case -1010: return "CL_INVALID_DX9_MEDIA_ADAPTER_KHR";
case -1011: return "CL_INVALID_DX9_MEDIA_SURFACE_KHR";
case -1012: return "CL_DX9_MEDIA_SURFACE_ALREADY_ACQUIRED_KHR";
case -1013: return "CL_DX9_MEDIA_SURFACE_NOT_ACQUIRED_KHR";
case -1093: return "CL_INVALID_EGL_OBJECT_KHR";
case -1092: return "CL_EGL_RESOURCE_NOT_ACQUIRED_KHR";
case -1001: return "CL_PLATFORM_NOT_FOUND_KHR";
case -1057: return "CL_DEVICE_PARTITION_FAILED_EXT";
case -1058: return "CL_INVALID_PARTITION_COUNT_EXT";
case -1059: return "CL_INVALID_PARTITION_NAME_EXT";
case -1094: return "CL_INVALID_ACCELERATOR_INTEL";
case -1095: return "CL_INVALID_ACCELERATOR_TYPE_INTEL";
case -1096: return "CL_INVALID_ACCELERATOR_DESCRIPTOR_INTEL";
case -1097: return "CL_ACCELERATOR_TYPE_NOT_SUPPORTED_INTEL";
case -1000: return "CL_INVALID_GL_SHAREGROUP_REFERENCE_KHR";
case -1098: return "CL_INVALID_VA_API_MEDIA_ADAPTER_INTEL";
case -1099: return "CL_INVALID_VA_API_MEDIA_SURFACE_INTEL";
case -1100: return "CL_VA_API_MEDIA_SURFACE_ALREADY_ACQUIRED_INTEL";
case -1101: return "CL_VA_API_MEDIA_SURFACE_NOT_ACQUIRED_INTEL";
default: return "CL_UNKNOWN_ERROR";
}
}
-84
View File
@@ -1,84 +0,0 @@
#include "common/gpio.h"
#include <string>
#ifdef __APPLE__
int gpio_init(int pin_nr, bool output) {
return 0;
}
int gpio_set(int pin_nr, bool high) {
return 0;
}
int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
return 0;
}
#else
#include <fcntl.h>
#include <unistd.h>
#include <cstring>
#include <linux/gpio.h>
#include <sys/ioctl.h>
#include "common/util.h"
#include "common/swaglog.h"
int gpio_init(int pin_nr, bool output) {
char pin_dir_path[50];
int pin_dir_path_len = snprintf(pin_dir_path, sizeof(pin_dir_path),
"/sys/class/gpio/gpio%d/direction", pin_nr);
if (pin_dir_path_len <= 0) {
return -1;
}
const char *value = output ? "out" : "in";
return util::write_file(pin_dir_path, (void*)value, strlen(value));
}
int gpio_set(int pin_nr, bool high) {
char pin_val_path[50];
int pin_val_path_len = snprintf(pin_val_path, sizeof(pin_val_path),
"/sys/class/gpio/gpio%d/value", pin_nr);
if (pin_val_path_len <= 0) {
return -1;
}
return util::write_file(pin_val_path, (void*)(high ? "1" : "0"), 1);
}
int gpiochip_get_ro_value_fd(const char* consumer_label, int gpiochiop_id, int pin_nr) {
// Assumed that all interrupt pins are unexported and rights are given to
// read from gpiochip0.
std::string gpiochip_path = "/dev/gpiochip" + std::to_string(gpiochiop_id);
int fd = open(gpiochip_path.c_str(), O_RDONLY);
if (fd < 0) {
LOGE("Error opening gpiochip0 fd");
return -1;
}
// Setup event
struct gpioevent_request rq;
rq.lineoffset = pin_nr;
rq.handleflags = GPIOHANDLE_REQUEST_INPUT;
/* Requesting both edges as the data ready pulse from the lsm6ds sensor is
very short(75us) and is mostly detected as falling edge instead of rising.
So if it is detected as rising the following falling edge is skipped. */
rq.eventflags = GPIOEVENT_REQUEST_BOTH_EDGES;
strncpy(rq.consumer_label, consumer_label, std::size(rq.consumer_label) - 1);
int ret = util::safe_ioctl(fd, GPIO_GET_LINEEVENT_IOCTL, &rq);
if (ret == -1) {
LOGE("Unable to get line event from ioctl : %s", strerror(errno));
close(fd);
return -1;
}
close(fd);
return rq.fd;
}
#endif
-92
View File
@@ -1,92 +0,0 @@
#include "common/i2c.h"
#include <fcntl.h>
#include <sys/ioctl.h>
#include <unistd.h>
#include <cassert>
#include <cstdio>
#include <stdexcept>
#include "common/swaglog.h"
#include "common/util.h"
#define UNUSED(x) (void)(x)
#ifdef QCOM2
// TODO: decide if we want to install libi2c-dev everywhere
extern "C" {
#include <linux/i2c-dev.h>
#include <i2c/smbus.h>
}
I2CBus::I2CBus(uint8_t bus_id) {
char bus_name[20];
snprintf(bus_name, 20, "/dev/i2c-%d", bus_id);
i2c_fd = HANDLE_EINTR(open(bus_name, O_RDWR));
if (i2c_fd < 0) {
throw std::runtime_error("Failed to open I2C bus");
}
}
I2CBus::~I2CBus() {
if (i2c_fd >= 0) {
close(i2c_fd);
}
}
int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
std::lock_guard lk(m);
int ret = 0;
ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
if (ret < 0) { goto fail; }
ret = i2c_smbus_read_i2c_block_data(i2c_fd, register_address, len, buffer);
if ((ret < 0) || (ret != len)) { goto fail; }
fail:
return ret;
}
int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
std::lock_guard lk(m);
int ret = 0;
ret = HANDLE_EINTR(ioctl(i2c_fd, I2C_SLAVE, device_address));
if (ret < 0) { goto fail; }
ret = i2c_smbus_write_byte_data(i2c_fd, register_address, data);
if (ret < 0) { goto fail; }
fail:
return ret;
}
#else
I2CBus::I2CBus(uint8_t bus_id) {
UNUSED(bus_id);
i2c_fd = -1;
}
I2CBus::~I2CBus() {}
int I2CBus::read_register(uint8_t device_address, uint register_address, uint8_t *buffer, uint8_t len) {
UNUSED(device_address);
UNUSED(register_address);
UNUSED(buffer);
UNUSED(len);
return -1;
}
int I2CBus::set_register(uint8_t device_address, uint register_address, uint8_t data) {
UNUSED(device_address);
UNUSED(register_address);
UNUSED(data);
return -1;
}
#endif
-512
View File
@@ -1,512 +0,0 @@
#include "common/params.h"
#include <dirent.h>
#include <sys/file.h>
#include <algorithm>
#include <cassert>
#include <csignal>
#include <unordered_map>
#include "common/queue.h"
#include "common/swaglog.h"
#include "common/util.h"
#include "system/hardware/hw.h"
namespace {
volatile sig_atomic_t params_do_exit = 0;
void params_sig_handler(int signal) {
params_do_exit = 1;
}
int fsync_dir(const std::string &path) {
int result = -1;
int fd = HANDLE_EINTR(open(path.c_str(), O_RDONLY, 0755));
if (fd >= 0) {
result = HANDLE_EINTR(fsync(fd));
HANDLE_EINTR(close(fd));
}
return result;
}
bool create_params_path(const std::string &param_path, const std::string &key_path) {
// Make sure params path exists
if (!util::file_exists(param_path) && !util::create_directories(param_path, 0775)) {
return false;
}
// See if the symlink exists, otherwise create it
if (!util::file_exists(key_path)) {
// 1) Create temp folder
// 2) Symlink it to temp link
// 3) Move symlink to <params>/d
std::string tmp_path = param_path + "/.tmp_XXXXXX";
// this should be OK since mkdtemp just replaces characters in place
char *tmp_dir = mkdtemp((char *)tmp_path.c_str());
if (tmp_dir == NULL) {
return false;
}
std::string link_path = std::string(tmp_dir) + ".link";
if (symlink(tmp_dir, link_path.c_str()) != 0) {
return false;
}
// don't return false if it has been created by other
if (rename(link_path.c_str(), key_path.c_str()) != 0 && errno != EEXIST) {
return false;
}
}
return true;
}
std::string ensure_params_path(const std::string &prefix, const std::string &path = {}) {
std::string params_path = path.empty() ? Path::params() : path;
if (!create_params_path(params_path, params_path + prefix)) {
throw std::runtime_error(util::string_format(
"Failed to ensure params path, errno=%d, path=%s, param_prefix=%s",
errno, params_path.c_str(), prefix.c_str()));
}
return params_path;
}
class FileLock {
public:
FileLock(const std::string &fn) {
fd_ = HANDLE_EINTR(open(fn.c_str(), O_CREAT, 0775));
if (fd_ < 0 || HANDLE_EINTR(flock(fd_, LOCK_EX)) < 0) {
LOGE("Failed to lock file %s, errno=%d", fn.c_str(), errno);
}
}
~FileLock() { close(fd_); }
private:
int fd_ = -1;
};
std::unordered_map<std::string, uint32_t> keys = {
{"AccessToken", CLEAR_ON_MANAGER_START | DONT_LOG},
{"AlwaysOnDM", PERSISTENT},
{"ApiCache_Device", PERSISTENT},
{"ApiCache_NavDestinations", PERSISTENT},
{"AssistNowToken", PERSISTENT},
{"AthenadPid", PERSISTENT},
{"AthenadUploadQueue", PERSISTENT},
{"AthenadRecentlyViewedRoutes", PERSISTENT},
{"BootCount", PERSISTENT},
{"CalibrationParams", PERSISTENT},
{"CameraDebugExpGain", CLEAR_ON_MANAGER_START},
{"CameraDebugExpTime", CLEAR_ON_MANAGER_START},
{"CarBatteryCapacity", PERSISTENT},
{"CarParams", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CarParamsCache", CLEAR_ON_MANAGER_START},
{"CarParamsPersistent", PERSISTENT},
{"CarParamsPrevRoute", PERSISTENT},
{"CarVin", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CompletedTrainingVersion", PERSISTENT},
{"ControlsReady", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"CurrentBootlog", PERSISTENT},
{"CurrentRoute", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"DisableLogging", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"DisablePowerDown", PERSISTENT | BACKUP},
{"DisableUpdates", PERSISTENT | BACKUP},
{"DisengageOnAccelerator", PERSISTENT | BACKUP},
{"DmModelInitialized", CLEAR_ON_ONROAD_TRANSITION},
{"DongleId", PERSISTENT},
{"DoReboot", CLEAR_ON_MANAGER_START},
{"DoShutdown", CLEAR_ON_MANAGER_START},
{"DoUninstall", CLEAR_ON_MANAGER_START},
{"ExperimentalLongitudinalEnabled", PERSISTENT | DEVELOPMENT_ONLY | BACKUP},
{"ExperimentalMode", PERSISTENT | BACKUP},
{"ExperimentalModeConfirmed", PERSISTENT | BACKUP},
{"FirmwareQueryDone", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"ForcePowerDown", PERSISTENT},
{"GitBranch", PERSISTENT},
{"GitCommit", PERSISTENT},
{"GitCommitDate", PERSISTENT},
{"GitDiff", PERSISTENT},
{"GithubSshKeys", PERSISTENT | BACKUP},
{"GithubUsername", PERSISTENT | BACKUP},
{"GitRemote", PERSISTENT},
{"GsmApn", PERSISTENT | BACKUP},
{"GsmMetered", PERSISTENT | BACKUP},
{"GsmRoaming", PERSISTENT | BACKUP},
{"HardwareSerial", PERSISTENT},
{"HasAcceptedTerms", PERSISTENT},
{"IMEI", PERSISTENT},
{"InstallDate", PERSISTENT},
{"IsDriverViewEnabled", CLEAR_ON_MANAGER_START},
{"IsEngaged", PERSISTENT},
{"IsLdwEnabled", PERSISTENT | BACKUP},
{"IsMetric", PERSISTENT | BACKUP},
{"IsOffroad", CLEAR_ON_MANAGER_START},
{"IsOnroad", PERSISTENT},
{"IsRhdDetected", PERSISTENT},
{"IsReleaseBranch", CLEAR_ON_MANAGER_START},
{"IsReleaseSPBranch", CLEAR_ON_MANAGER_START},
{"IsTakingSnapshot", CLEAR_ON_MANAGER_START},
{"IsTestedBranch", CLEAR_ON_MANAGER_START},
{"JoystickDebugMode", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"LanguageSetting", PERSISTENT | BACKUP},
{"LastAthenaPingTime", CLEAR_ON_MANAGER_START},
{"LastGPSPosition", PERSISTENT},
{"LastManagerExitReason", CLEAR_ON_MANAGER_START},
{"LastOffroadStatusPacket", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"LastPowerDropDetected", CLEAR_ON_MANAGER_START},
{"LastUpdateException", CLEAR_ON_MANAGER_START},
{"LastUpdateTime", PERSISTENT},
{"LiveParameters", PERSISTENT},
{"LiveTorqueParameters", PERSISTENT | DONT_LOG},
{"LongitudinalPersonality", PERSISTENT | BACKUP},
{"NavDestination", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"NavDestinationWaypoints", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"NavPastDestinations", PERSISTENT},
{"NavSettingLeftSide", PERSISTENT | BACKUP},
{"NavSettingTime24h", PERSISTENT | BACKUP},
{"NetworkMetered", PERSISTENT},
{"ObdMultiplexingChanged", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"ObdMultiplexingEnabled", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_BadNvme", CLEAR_ON_MANAGER_START},
{"Offroad_CarUnrecognized", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_ConnectivityNeeded", CLEAR_ON_MANAGER_START},
{"Offroad_ConnectivityNeededPrompt", CLEAR_ON_MANAGER_START},
{"Offroad_IsTakingSnapshot", CLEAR_ON_MANAGER_START},
{"Offroad_NeosUpdate", CLEAR_ON_MANAGER_START},
{"Offroad_NoFirmware", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_Recalibration", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"Offroad_StorageMissing", CLEAR_ON_MANAGER_START},
{"Offroad_TemperatureTooHigh", CLEAR_ON_MANAGER_START},
{"Offroad_UnofficialHardware", CLEAR_ON_MANAGER_START},
{"Offroad_UpdateFailed", CLEAR_ON_MANAGER_START},
{"Offroad_OSMUpdateRequired", CLEAR_ON_MANAGER_START},
{"OpenpilotEnabledToggle", PERSISTENT | BACKUP},
{"PandaHeartbeatLost", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"PandaSomResetTriggered", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"PandaSignatures", CLEAR_ON_MANAGER_START},
{"PrimeType", PERSISTENT},
{"RecordFront", PERSISTENT | BACKUP},
{"RecordFrontLock", PERSISTENT}, // for the internal fleet
{"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"RouteCount", PERSISTENT},
{"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
{"SshEnabled", PERSISTENT | BACKUP},
{"TermsVersion", PERSISTENT},
{"TrainingVersion", PERSISTENT},
{"UbloxAvailable", PERSISTENT},
{"UpdateAvailable", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
{"UpdateFailedCount", CLEAR_ON_MANAGER_START},
{"UpdaterAvailableBranches", PERSISTENT},
{"UpdaterCurrentDescription", CLEAR_ON_MANAGER_START},
{"UpdaterCurrentReleaseNotes", CLEAR_ON_MANAGER_START},
{"UpdaterFetchAvailable", CLEAR_ON_MANAGER_START},
{"UpdaterNewDescription", CLEAR_ON_MANAGER_START},
{"UpdaterNewReleaseNotes", CLEAR_ON_MANAGER_START},
{"UpdaterState", CLEAR_ON_MANAGER_START},
{"UpdaterTargetBranch", CLEAR_ON_MANAGER_START},
{"UpdaterLastFetchTime", PERSISTENT},
{"Version", PERSISTENT},
{"AccelPersonality", PERSISTENT | BACKUP},
{"AccMadsCombo", PERSISTENT | BACKUP},
{"AmapKey1", PERSISTENT | BACKUP},
{"AmapKey2", PERSISTENT | BACKUP},
{"ApiCache_DriveStats", PERSISTENT | BACKUP},
{"AutoLaneChangeTimer", PERSISTENT | BACKUP},
{"AutoLaneChangeBsmDelay", PERSISTENT | BACKUP},
{"BelowSpeedPause", PERSISTENT | BACKUP},
{"BrakeLights", PERSISTENT | BACKUP},
{"BrightnessControl", PERSISTENT | BACKUP},
{"ButtonAutoHide", PERSISTENT | BACKUP},
{"CameraControl", PERSISTENT | BACKUP},
{"CameraControlToggle", PERSISTENT | BACKUP},
{"CameraOffset", PERSISTENT | BACKUP},
{"CarModel", PERSISTENT | BACKUP},
{"CarModelText", PERSISTENT | BACKUP},
{"ChevronInfo", PERSISTENT | BACKUP},
{"CustomBootScreen", PERSISTENT | BACKUP},
{"CustomDrivingModel", PERSISTENT},
{"CustomMapboxTokenPk", PERSISTENT | BACKUP},
{"CustomMapboxTokenSk", PERSISTENT | BACKUP},
{"CustomOffsets", PERSISTENT | BACKUP},
{"CustomStockLong", PERSISTENT | BACKUP},
{"CustomStockLongPlanner", PERSISTENT | BACKUP},
{"CustomTorqueLateral", PERSISTENT | BACKUP},
{"DevUIInfo", PERSISTENT | BACKUP},
{"DisableOnroadUploads", PERSISTENT | BACKUP},
{"DisengageLateralOnBrake", PERSISTENT | BACKUP},
{"DrivingModelGeneration", PERSISTENT},
{"DrivingModelMetadataText", PERSISTENT},
{"DrivingModelName", PERSISTENT},
{"DrivingModelText", PERSISTENT},
{"DrivingModelUrl", PERSISTENT},
{"DynamicExperimentalControl", PERSISTENT | BACKUP},
{"DynamicLaneProfile", PERSISTENT | BACKUP},
{"DynamicPersonality", PERSISTENT | BACKUP},
{"EnableAmap", PERSISTENT | BACKUP},
{"EnableGmap", PERSISTENT | BACKUP},
{"EnableMads", PERSISTENT | BACKUP},
{"EnableSlc", PERSISTENT | BACKUP},
{"EndToEndLongAlertLead", PERSISTENT | BACKUP},
{"EndToEndLongAlertLight", PERSISTENT | BACKUP},
{"EndToEndLongAlertUI", PERSISTENT | BACKUP},
{"EndToEndLongToggle", PERSISTENT | BACKUP},
{"EnforceTorqueLateral", PERSISTENT | BACKUP},
{"EnhancedScc", PERSISTENT | BACKUP},
{"FeatureStatus", PERSISTENT | BACKUP},
{"FleetManagerPin", PERSISTENT},
{"ForceOffroad", CLEAR_ON_MANAGER_START},
{"GmapKey", PERSISTENT | BACKUP},
{"HandsOnWheelMonitoring", PERSISTENT | BACKUP},
{"HasAcceptedTermsSP", PERSISTENT},
{"HideVEgoUi", PERSISTENT | BACKUP},
{"HyundaiCruiseMainDefault", PERSISTENT | BACKUP},
{"HkgCustomLongTuning", PERSISTENT | BACKUP},
{"HkgSmoothStop", PERSISTENT | BACKUP},
{"HyundaiCruiseMainDefault", PERSISTENT | BACKUP},
{"HotspotOnBoot", PERSISTENT},
{"HotspotOnBootConfirmed", PERSISTENT},
{"HyundaiRadarTracksAvailable", PERSISTENT},
{"HyundaiRadarTracksAvailableCache", PERSISTENT},
{"HyundaiRadarTracksAvailablePersistent", PERSISTENT},
{"LastCarModel", PERSISTENT | BACKUP},
{"LastSpeedLimitSignTap", PERSISTENT},
{"LastSunnylinkPingTime", CLEAR_ON_MANAGER_START},
{"LiveTorque", PERSISTENT | BACKUP},
{"LiveTorqueRelaxed", PERSISTENT | BACKUP},
{"LkasToggle", PERSISTENT | BACKUP},
{"MadsCruiseMain", PERSISTENT | BACKUP},
{"MadsIconToggle", PERSISTENT | BACKUP},
{"MapboxFullScreen", PERSISTENT | BACKUP},
{"MapTargetVelocities", PERSISTENT},
{"Map3DBuildings", PERSISTENT | BACKUP},
{"MaxTimeOffroad", PERSISTENT | BACKUP},
{"NavModelText", PERSISTENT | BACKUP},
{"NavModelUrl", PERSISTENT | BACKUP},
{"NNFF", PERSISTENT | BACKUP},
{"NNFFCarModel", PERSISTENT | BACKUP},
{"NNFFNoLateralJerk", PERSISTENT | BACKUP},
{"OnroadScreenOff", PERSISTENT | BACKUP},
{"OnroadScreenOffBrightness", PERSISTENT | BACKUP},
{"OnroadScreenOffEvent", PERSISTENT | BACKUP},
{"OnroadSettings", PERSISTENT | BACKUP},
{"OsmLocal", PERSISTENT},
{"OsmLocationName", PERSISTENT},
{"OsmLocationTitle", PERSISTENT},
{"OsmLocationUrl", PERSISTENT},
{"OsmWayTest", PERSISTENT},
{"OsmDownloadedDate", PERSISTENT},
{"OvertakingAccelerationAssist", PERSISTENT},
{"PathOffset", PERSISTENT | BACKUP},
{"PauseLateralSpeed", PERSISTENT | BACKUP},
{"PCMVCruiseOverride", PERSISTENT | BACKUP},
{"PCMVCruiseOverrideSpeed", PERSISTENT | BACKUP},
{"QuietDrive", PERSISTENT | BACKUP},
{"RoadEdge", PERSISTENT | BACKUP},
{"ReverseAccChange", PERSISTENT | BACKUP},
{"ReverseDmCam", PERSISTENT | BACKUP},
{"ScreenRecorder", PERSISTENT | BACKUP},
{"ShowDebugUI", PERSISTENT | BACKUP},
{"SidebarTemperatureOptions", PERSISTENT | BACKUP},
{"SpeedLimitControlPolicy", PERSISTENT | BACKUP},
{"SpeedLimitEngageType", PERSISTENT | BACKUP},
{"SpeedLimitValueOffset", PERSISTENT | BACKUP},
{"SpeedLimitOffsetType", PERSISTENT | BACKUP},
{"SpeedLimitWarningFlash", PERSISTENT | BACKUP},
{"SpeedLimitWarningType", PERSISTENT | BACKUP},
{"SpeedLimitWarningValueOffset", PERSISTENT | BACKUP},
{"SpeedLimitWarningOffsetType", PERSISTENT | BACKUP},
{"StandStillTimer", PERSISTENT | BACKUP},
{"StockLongToyota", PERSISTENT | BACKUP},
{"SubaruManualParkingBrakeSng", PERSISTENT | BACKUP},
{"SunnylinkDongleId", PERSISTENT},
{"SunnylinkEnabled", PERSISTENT},
{"SunnylinkdPid", PERSISTENT},
{"TermsVersionSunnypilot", PERSISTENT},
{"TorqueDeadzoneDeg", PERSISTENT | BACKUP},
{"TorqueFriction", PERSISTENT | BACKUP},
{"TorqueLateralJerk", PERSISTENT | BACKUP},
{"TorqueMaxLatAccel", PERSISTENT | BACKUP},
{"TorquedOverride", PERSISTENT | BACKUP},
{"ToyotaAutoLockBySpeed", PERSISTENT | BACKUP},
{"ToyotaAutoUnlockByShifter", PERSISTENT | BACKUP},
{"ToyotaEnhancedBsm", PERSISTENT | BACKUP},
{"ToyotaSnG", PERSISTENT | BACKUP},
{"ToyotaTSS2Long", PERSISTENT | BACKUP},
{"TrueVEgoUi", PERSISTENT | BACKUP},
{"TurnSpeedControl", PERSISTENT | BACKUP},
{"TurnVisionControl", PERSISTENT | BACKUP},
{"DriverCameraHardwareMissing", PERSISTENT},
{"VisionCurveLaneless", PERSISTENT | BACKUP},
{"VwAccType", PERSISTENT | BACKUP},
{"VwCCOnly", PERSISTENT | BACKUP},
{"Offroad_ForceStatus", CLEAR_ON_MANAGER_START},
{"Offroad_SupersededUpdate", PERSISTENT},
{"SunnylinkCache_Users", PERSISTENT},
{"SunnylinkCache_Roles", PERSISTENT},
{"EnableGitlabRunner", PERSISTENT | BACKUP},
{"EnableSunnylinkUploader", PERSISTENT | BACKUP},
// PFEIFER - MAPD {{
{"MapdVersion", PERSISTENT},
{"RoadName", CLEAR_ON_ONROAD_TRANSITION},
{"MapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
{"MapAdvisorySpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
{"NextMapSpeedLimit", CLEAR_ON_ONROAD_TRANSITION},
{"OSMDownloadBounds", PERSISTENT},
{"OSMDownloadLocations", PERSISTENT},
{"OsmDownloadedDate", PERSISTENT},
{"OsmStateTitle", PERSISTENT},
{"OsmStateName", PERSISTENT},
{"OSMDownloadProgress", CLEAR_ON_MANAGER_START},
{"OsmDbUpdatesCheck", CLEAR_ON_MANAGER_START}, // mapd database update happens with device ON, reset on boot
// }} PFEIFER - MAPD
};
} // namespace
Params::Params(const std::string &path) {
params_prefix = "/" + util::getenv("OPENPILOT_PREFIX", "d");
params_path = ensure_params_path(params_prefix, path);
}
Params::~Params() {
if (future.valid()) {
future.wait();
}
assert(queue.empty());
}
std::vector<std::string> Params::allKeys() const {
std::vector<std::string> ret;
for (auto &p : keys) {
ret.push_back(p.first);
}
return ret;
}
bool Params::checkKey(const std::string &key) {
return keys.find(key) != keys.end();
}
ParamKeyType Params::getKeyType(const std::string &key) {
return static_cast<ParamKeyType>(keys[key]);
}
int Params::put(const char* key, const char* value, size_t value_size) {
// Information about safely and atomically writing a file: https://lwn.net/Articles/457667/
// 1) Create temp file
// 2) Write data to temp file
// 3) fsync() the temp file
// 4) rename the temp file to the real name
// 5) fsync() the containing directory
std::string tmp_path = params_path + "/.tmp_value_XXXXXX";
int tmp_fd = mkstemp((char*)tmp_path.c_str());
if (tmp_fd < 0) return -1;
int result = -1;
do {
// Write value to temp.
ssize_t bytes_written = HANDLE_EINTR(write(tmp_fd, value, value_size));
if (bytes_written < 0 || (size_t)bytes_written != value_size) {
result = -20;
break;
}
// fsync to force persist the changes.
if ((result = fsync(tmp_fd)) < 0) break;
FileLock file_lock(params_path + "/.lock");
// Move temp into place.
if ((result = rename(tmp_path.c_str(), getParamPath(key).c_str())) < 0) break;
// fsync parent directory
result = fsync_dir(getParamPath());
} while (false);
close(tmp_fd);
if (result != 0) {
::unlink(tmp_path.c_str());
}
return result;
}
int Params::remove(const std::string &key) {
FileLock file_lock(params_path + "/.lock");
int result = unlink(getParamPath(key).c_str());
if (result != 0) {
return result;
}
return fsync_dir(getParamPath());
}
std::string Params::get(const std::string &key, bool block) {
if (!block) {
return util::read_file(getParamPath(key));
} else {
// blocking read until successful
params_do_exit = 0;
void (*prev_handler_sigint)(int) = std::signal(SIGINT, params_sig_handler);
void (*prev_handler_sigterm)(int) = std::signal(SIGTERM, params_sig_handler);
std::string value;
while (!params_do_exit) {
if (value = util::read_file(getParamPath(key)); !value.empty()) {
break;
}
util::sleep_for(100); // 0.1 s
}
std::signal(SIGINT, prev_handler_sigint);
std::signal(SIGTERM, prev_handler_sigterm);
return value;
}
}
std::map<std::string, std::string> Params::readAll() {
FileLock file_lock(params_path + "/.lock");
return util::read_files_in_dir(getParamPath());
}
void Params::clearAll(ParamKeyType key_type) {
FileLock file_lock(params_path + "/.lock");
// 1) delete params of key_type
// 2) delete files that are not defined in the keys.
if (DIR *d = opendir(getParamPath().c_str())) {
struct dirent *de = NULL;
while ((de = readdir(d))) {
if (de->d_type != DT_DIR) {
auto it = keys.find(de->d_name);
if (it == keys.end() || (it->second & key_type)) {
unlink(getParamPath(de->d_name).c_str());
}
}
}
closedir(d);
}
fsync_dir(getParamPath());
}
void Params::putNonBlocking(const std::string &key, const std::string &val) {
queue.push(std::make_pair(key, val));
// start thread on demand
if (!future.valid() || future.wait_for(std::chrono::milliseconds(0)) == std::future_status::ready) {
future = std::async(std::launch::async, &Params::asyncWriteThread, this);
}
}
void Params::asyncWriteThread() {
// TODO: write the latest one if a key has multiple values in the queue.
std::pair<std::string, std::string> p;
while (queue.try_pop(p, 0)) {
// Params::put is Thread-Safe
put(p.first, p.second);
}
}
+18382
View File
File diff suppressed because it is too large Load Diff
BIN
View File
Binary file not shown.
-40
View File
@@ -1,40 +0,0 @@
#include "common/ratekeeper.h"
#include <algorithm>
#include "common/swaglog.h"
#include "common/timing.h"
#include "common/util.h"
RateKeeper::RateKeeper(const std::string &name, float rate, float print_delay_threshold)
: name(name),
print_delay_threshold(std::max(0.f, print_delay_threshold)) {
interval = 1 / rate;
last_monitor_time = seconds_since_boot();
next_frame_time = last_monitor_time + interval;
}
bool RateKeeper::keepTime() {
bool lagged = monitorTime();
if (remaining_ > 0) {
util::sleep_for(remaining_ * 1000);
}
return lagged;
}
bool RateKeeper::monitorTime() {
++frame_;
last_monitor_time = seconds_since_boot();
remaining_ = next_frame_time - last_monitor_time;
bool lagged = remaining_ < 0;
if (lagged) {
if (print_delay_threshold > 0 && remaining_ < -print_delay_threshold) {
LOGW("%s lagging by %.2f ms", name.c_str(), -remaining_ * 1000);
}
next_frame_time = last_monitor_time + interval;
} else {
next_frame_time += interval;
}
return lagged;
}
-152
View File
@@ -1,152 +0,0 @@
#ifndef _GNU_SOURCE
#define _GNU_SOURCE
#endif
#include "common/swaglog.h"
#include <cassert>
#include <limits>
#include <mutex>
#include <string>
#include <zmq.h>
#include <stdarg.h>
#include "third_party/json11/json11.hpp"
#include "common/version.h"
#include "system/hardware/hw.h"
class SwaglogState {
public:
SwaglogState() {
zctx = zmq_ctx_new();
sock = zmq_socket(zctx, ZMQ_PUSH);
// Timeout on shutdown for messages to be received by the logging process
int timeout = 100;
zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout));
zmq_connect(sock, Path::swaglog_ipc().c_str());
print_level = CLOUDLOG_WARNING;
if (const char* print_lvl = getenv("LOGPRINT")) {
if (strcmp(print_lvl, "debug") == 0) {
print_level = CLOUDLOG_DEBUG;
} else if (strcmp(print_lvl, "info") == 0) {
print_level = CLOUDLOG_INFO;
} else if (strcmp(print_lvl, "warning") == 0) {
print_level = CLOUDLOG_WARNING;
}
}
ctx_j = json11::Json::object{};
if (char* dongle_id = getenv("DONGLE_ID")) {
ctx_j["dongle_id"] = dongle_id;
}
if (char* git_origin = getenv("GIT_ORIGIN")) {
ctx_j["origin"] = git_origin;
}
if (char* git_branch = getenv("GIT_BRANCH")) {
ctx_j["branch"] = git_branch;
}
if (char* git_commit = getenv("GIT_COMMIT")) {
ctx_j["commit"] = git_commit;
}
if (char* daemon_name = getenv("MANAGER_DAEMON")) {
ctx_j["daemon"] = daemon_name;
}
ctx_j["version"] = COMMA_VERSION;
ctx_j["dirty"] = !getenv("CLEAN");
ctx_j["device"] = Hardware::get_name();
}
~SwaglogState() {
zmq_close(sock);
zmq_ctx_destroy(zctx);
}
void log(int levelnum, const char* filename, int lineno, const char* func, const char* msg, const std::string& log_s) {
std::lock_guard lk(lock);
if (levelnum >= print_level) {
printf("%s: %s\n", filename, msg);
}
zmq_send(sock, log_s.data(), log_s.length(), ZMQ_NOBLOCK);
}
std::mutex lock;
void* zctx = nullptr;
void* sock = nullptr;
int print_level;
json11::Json::object ctx_j;
};
bool LOG_TIMESTAMPS = getenv("LOG_TIMESTAMPS");
uint32_t NO_FRAME_ID = std::numeric_limits<uint32_t>::max();
static void cloudlog_common(int levelnum, const char* filename, int lineno, const char* func,
char* msg_buf, const json11::Json::object &msg_j={}) {
static SwaglogState s;
json11::Json::object log_j = json11::Json::object {
{"ctx", s.ctx_j},
{"levelnum", levelnum},
{"filename", filename},
{"lineno", lineno},
{"funcname", func},
{"created", seconds_since_epoch()}
};
if (msg_j.empty()) {
log_j["msg"] = msg_buf;
} else {
log_j["msg"] = msg_j;
}
std::string log_s;
log_s += (char)levelnum;
((json11::Json)log_j).dump(log_s);
s.log(levelnum, filename, lineno, func, msg_buf, log_s);
free(msg_buf);
}
void cloudlog_e(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
va_end(args);
if (ret <= 0 || !msg_buf) return;
cloudlog_common(levelnum, filename, lineno, func, msg_buf);
}
void cloudlog_t_common(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, va_list args) {
if (!LOG_TIMESTAMPS) return;
char* msg_buf = nullptr;
int ret = vasprintf(&msg_buf, fmt, args);
if (ret <= 0 || !msg_buf) return;
json11::Json::object tspt_j = json11::Json::object{
{"event", msg_buf},
{"time", std::to_string(nanos_since_boot())}
};
if (frame_id < NO_FRAME_ID) {
tspt_j["frame_id"] = std::to_string(frame_id);
}
tspt_j = json11::Json::object{{"timestamp", tspt_j}};
cloudlog_common(levelnum, filename, lineno, func, msg_buf, tspt_j);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, NO_FRAME_ID, fmt, args);
va_end(args);
}
void cloudlog_te(int levelnum, const char* filename, int lineno, const char* func,
uint32_t frame_id, const char* fmt, ...) {
va_list args;
va_start(args, fmt);
cloudlog_t_common(levelnum, filename, lineno, func, frame_id, fmt, args);
va_end(args);
}
-1
View File
@@ -1 +0,0 @@
test_common
-19
View File
@@ -1,19 +0,0 @@
import os
from uuid import uuid4
from openpilot.common.file_helpers import atomic_write_in_dir
class TestFileHelpers:
def run_atomic_write_func(self, atomic_write_func):
path = f"/tmp/tmp{uuid4()}"
with atomic_write_func(path) as f:
f.write("test")
assert not os.path.exists(path)
with open(path) as f:
assert f.read() == "test"
os.remove(path)
def test_atomic_write_in_dir(self):
self.run_atomic_write_func(atomic_write_in_dir)
-15
View File
@@ -1,15 +0,0 @@
import os
from openpilot.common.basedir import BASEDIR
from openpilot.common.markdown import parse_markdown
class TestMarkdown:
def test_all_release_notes(self):
with open(os.path.join(BASEDIR, "RELEASES.md")) as f:
release_notes = f.read().split("\n\n")
assert len(release_notes) > 10
for rn in release_notes:
md = parse_markdown(rn)
assert len(md) > 0

Some files were not shown because too many files have changed in this diff Show More