Compare commits

..

10 Commits

Author SHA1 Message Date
royjr 13f5ae7a2d Update opendbc_repo 2026-03-05 22:10:29 -05:00
royjr 0f86cff7a3 Merge branch 'master' into henc 2026-03-05 20:53:10 -05:00
royjr 59270d641a Update opendbc_repo 2026-03-05 20:53:03 -05:00
royjr 55f6a8b4f5 Update opendbc_repo 2026-02-28 02:22:34 -05:00
royjr dd1e84ccce Update opendbc_repo 2026-02-28 01:54:13 -05:00
royjr c6575b5870 Update opendbc_repo 2026-02-28 01:43:06 -05:00
royjr 0498bdb532 Update opendbc_repo 2026-02-28 00:35:35 -05:00
royjr ffb2d66e8c Update opendbc_repo 2026-02-28 00:18:36 -05:00
royjr f726c2392a Update opendbc_repo 2026-02-28 00:10:51 -05:00
royjr 3c32b61d7f Update opendbc_repo 2026-02-27 14:15:20 -05:00
2132 changed files with 133539 additions and 66699 deletions
+18
View File
@@ -0,0 +1,18 @@
**/.git
.DS_Store
*.dylib
*.DSYM
*.d
*.pyc
*.pyo
.*.swp
.*.swo
.*.un~
*.tmp
*.o
*.o-*
*.os
*.os-*
venv/
.venv/
+11 -2
View File
@@ -10,5 +10,14 @@
*.otf filter=lfs diff=lfs merge=lfs -text
*.wav filter=lfs diff=lfs merge=lfs -text
openpilot/selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
openpilot/common/hardware/comma/updater filter=lfs diff=lfs merge=lfs -text
selfdrive/car/tests/test_models_segs.txt filter=lfs diff=lfs merge=lfs -text
system/hardware/tici/updater_weston filter=lfs diff=lfs merge=lfs -text
system/hardware/tici/updater_magic filter=lfs diff=lfs merge=lfs -text
third_party/**/*.a filter=lfs diff=lfs merge=lfs -text
third_party/**/*.so filter=lfs diff=lfs merge=lfs -text
third_party/**/*.so.* filter=lfs diff=lfs merge=lfs -text
third_party/**/*.dylib filter=lfs diff=lfs merge=lfs -text
third_party/acados/*/t_renderer filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lrelease filter=lfs diff=lfs merge=lfs -text
third_party/qt5/larch64/bin/lupdate filter=lfs diff=lfs merge=lfs -text
third_party/catch2/include/catch2/catch.hpp filter=lfs diff=lfs merge=lfs -text
+8
View File
@@ -0,0 +1,8 @@
---
name: Enhancement
about: For openpilot enhancement suggestions
title: ''
labels: 'enhancement'
assignees: ''
---
+6 -6
View File
@@ -8,24 +8,24 @@ chore:
car:
- changed-files:
- any-glob-to-all-files: '{openpilot/selfdrive/car/**,opendbc_repo}'
- any-glob-to-all-files: '{selfdrive/car/**,opendbc_repo}'
simulation:
- changed-files:
- any-glob-to-all-files: 'openpilot/tools/sim/**'
- any-glob-to-all-files: 'tools/sim/**'
ui:
- changed-files:
- any-glob-to-all-files: '{openpilot/selfdrive/assets/**,openpilot/selfdrive/ui/**,openpilot/system/ui/**}'
- any-glob-to-all-files: '{selfdrive/assets/**,selfdrive/ui/**,system/ui/**}'
tools:
- changed-files:
- any-glob-to-all-files: 'openpilot/tools/**'
- any-glob-to-all-files: 'tools/**'
multilanguage:
- changed-files:
- any-glob-to-all-files: 'openpilot/selfdrive/ui/translations/**'
- any-glob-to-all-files: 'selfdrive/ui/translations/**'
autonomy:
- changed-files:
- any-glob-to-all-files: "{openpilot/selfdrive/modeld/models/**,openpilot/selfdrive/test/process_replay/model_replay_ref_commit,openpilot/sunnypilot/modeld*/models/**}"
- any-glob-to-all-files: "{selfdrive/modeld/models/**,selfdrive/test/process_replay/model_replay_ref_commit,sunnypilot/modeld*/models/**}"
+68
View File
@@ -0,0 +1,68 @@
<!-- Please copy and paste the relevant template -->
<!--- ***** Template: Fingerprint *****
**Car**
Which car (make, model, year) this fingerprint is for
**Route**
A route with the fingerprint
-->
<!--- ***** Template: Car Bugfix *****
**Description**
A description of the bug and the fix. Also link the issue if it exists.
**Verification**
Explain how you tested this bug fix.
**Route**
Route: [a route with the bug fix]
-->
<!--- ***** Template: Bugfix *****
**Description**
A description of the bug and the fix. Also link the issue if it exists.
**Verification**
Explain how you tested this bug fix.
-->
<!--- ***** Template: Car Port *****
**Checklist**
- [ ] added entry to CAR in selfdrive/car/*/values.py and ran `selfdrive/car/docs.py` to generate new docs
- [ ] test route added to [routes.py](https://github.com/commaai/openpilot/blob/master/selfdrive/car/tests/routes.py)
- [ ] route with openpilot:
- [ ] route with stock system:
- [ ] car harness used (if comma doesn't sell it, put N/A):
-->
<!--- ***** Template: Refactor *****
**Description**
A description of the refactor, including the goals it accomplishes.
**Verification**
Explain how you tested the refactor for regressions.
-->
+2 -2
View File
@@ -12,7 +12,7 @@ jobs:
issues: write
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: false
@@ -24,7 +24,7 @@ jobs:
# Check PR target branch
- name: check branch
uses: Vankka/pr-target-branch-action@5da68a42bcb7b43d39104295a876a6f3f8d7908b
uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
if: github.repository == 'sunnypilot/sunnypilot'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+35
View File
@@ -0,0 +1,35 @@
name: badges
on:
schedule:
- cron: '0 * * * *'
workflow_dispatch:
env:
PYTHONPATH: ${{ github.workspace }}
jobs:
badges:
name: create badges
runs-on: ubuntu-latest
if: github.repository == 'sunnypilot/sunnypilot'
permissions:
contents: write
steps:
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Push badges
run: |
python3 selfdrive/ui/translations/create_badges.py
rm .gitattributes
git checkout --orphan badges
git rm -rf --cached .
git config user.email "badge-researcher@sunnypilot.ai"
git config user.name "Badge Researcher"
git add translation_badge.svg
git commit -m "Add/Update badges"
git push -f origin HEAD
@@ -30,14 +30,14 @@ jobs:
run: |
cd sunnypilot
export PYTHONPATH=$(pwd)
ref=$(python3 openpilot/sunnypilot/models/tinygrad_ref.py)
ref=$(python3 sunnypilot/models/tinygrad_ref.py)
echo "tinygrad_ref=$ref" >> $GITHUB_OUTPUT
echo "tinygrad_ref is $ref"
- name: Checkout docs repo (sunnypilot-models, gh-pages)
- name: Checkout docs repo (sunnypilot-docs, gh-pages)
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot-models
repository: sunnypilot/sunnypilot-docs
ref: gh-pages
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -120,7 +120,6 @@ jobs:
with:
upstream_branch: ${{ matrix.model.ref }}
custom_name: ${{ matrix.model.display_name }}
is_20hz: ${{ matrix.model.is_20hz }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
secrets: inherit
@@ -158,7 +157,6 @@ jobs:
with:
upstream_branch: ${{ matrix.model.ref }}
custom_name: ${{ matrix.model.display_name }}
is_20hz: ${{ matrix.model.is_20hz }}
recompiled_dir: ${{ needs.setup.outputs.recompiled_dir }}
json_version: ${{ needs.setup.outputs.json_version }}
artifact_suffix: -retry
@@ -204,7 +202,7 @@ jobs:
- name: Checkout docs repo
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot-models
repository: sunnypilot/sunnypilot-docs
ref: gh-pages
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
@@ -12,11 +12,11 @@ on:
required: false
type: string
recompiled_dir:
description: 'Existing recompiled directory number (e.g. 1 for recompiled1)'
description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
required: true
type: string
json_version:
description: 'driving_models version number to update (e.g. 18 for driving_models_v18.json)'
description: 'driving_models version number to update (e.g. 5 for driving_models_v5.json)'
required: true
type: string
artifact_suffix:
@@ -24,11 +24,6 @@ on:
required: false
type: string
default: ''
is_20hz:
description: 'Is this a 20Hz model'
required: false
type: boolean
default: true
bypass_push:
description: 'Bypass pushing to GitLab for build-all'
required: false
@@ -44,11 +39,6 @@ on:
description: 'Custom name for the model (no date, only name)'
required: false
type: string
is_20hz:
description: 'Is this a 20Hz model'
required: false
type: boolean
default: true
recompiled_dir:
description: 'Existing recompiled directory number (e.g. 3 for recompiled3)'
required: true
@@ -63,11 +53,12 @@ on:
default: 'None'
options:
- None
- Master Models
- Release Models
- 2025 World Models
- 2026 World Models
- Simple Plan Models
- Space Lab Models
- TR Models
- DTR Models
- Custom Merge Models
- FOF series models
- Other
custom_model_folder:
description: 'Custom model folder name (if "Other" selected)'
@@ -91,7 +82,7 @@ jobs:
with:
upstream_branch: ${{ inputs.upstream_branch }}
custom_name: ${{ inputs.custom_name || inputs.upstream_branch }}
is_20hz: ${{ inputs.is_20hz }}
is_20hz: true
artifact_suffix: ${{ inputs.artifact_suffix }}
secrets: inherit
@@ -128,7 +119,7 @@ jobs:
- name: Checkout docs repo
uses: actions/checkout@v4
with:
repository: sunnypilot/sunnypilot-models
repository: sunnypilot/sunnypilot-docs
ref: gh-pages
path: docs
ssh-key: ${{ secrets.CI_SUNNYPILOT_DOCS_PRIVATE_KEY }}
+43 -62
View File
@@ -6,7 +6,7 @@ on:
- master
pull_request:
paths:
- 'openpilot/cereal/**'
- 'cereal/**'
workflow_dispatch:
workflow_call:
inputs:
@@ -23,75 +23,56 @@ env:
CI: 1
jobs:
generate_cereal_artifact:
name: Generate cereal validation artifacts
runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Build openpilot
run: scons -j$(nproc) cereal
- name: Generate the log file
run: |
export PYTHONPATH=${{ github.workspace }}
python3 cereal/messaging/tests/validate_sp_cereal_upstream.py -g -f schema_instances.bin
- name: 'Prepare artifact'
run: |
mkdir -p "cereal/messaging/tests/cereal_validations"
cp cereal/messaging/tests/validate_sp_cereal_upstream.py "cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py"
cp schema_instances.bin "cereal/messaging/tests/cereal_validations/schema_instances.bin"
- name: 'Upload Artifact'
uses: actions/upload-artifact@v4
with:
name: cereal_validations
path: cereal/messaging/tests/cereal_validations
validate_cereal_with_upstream:
name: Validate cereal with Upstream
runs-on: ubuntu-24.04
needs: generate_cereal_artifact
steps:
- name: Checkout sunnypilot cereal
- name: Checkout sunnypilot
uses: actions/checkout@v6
with:
sparse-checkout: |
openpilot/cereal
uv.lock
submodules: false
- name: Init sunnypilot opendbc submodule
run: git submodule update --init --depth 1 opendbc_repo
- name: Locate sunnypilot capnp import paths
id: locate-sp-capnp
run: |
SP_IMPORT_ARGS=""
SP_CAR_CAPNP=$(find opendbc_repo -maxdepth 4 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
if [ -n "$SP_CAR_CAPNP" ]; then
SP_IMPORT_ARGS="-I $SP_CAR_CAPNP"
echo "Found sunnypilot car.capnp at: $SP_CAR_CAPNP"
fi
echo "import_args=$SP_IMPORT_ARGS" >> "$GITHUB_OUTPUT"
- name: Checkout upstream openpilot
uses: actions/checkout@v6
with:
repository: 'commaai/openpilot'
path: upstream_openpilot
path: openpilot
submodules: true
ref: "refs/heads/master"
- name: Init upstream opendbc submodule
working-directory: upstream_openpilot
run: git submodule update --init --depth 1 opendbc_repo
- name: Locate upstream capnp paths
id: locate-capnp
- run: ./tools/op.sh setup
- name: Build openpilot
working-directory: openpilot
run: scons -j$(nproc) cereal
- name: Download build artifacts
uses: actions/download-artifact@v4
with:
name: cereal_validations
path: openpilot/cereal/messaging/tests/cereal_validations
- name: 'Run the validation'
run: |
CEREAL_DIR=$(find upstream_openpilot -maxdepth 4 -name log.capnp -path '*/cereal/log.capnp' -printf '%h\n' -quit)
if [ -z "$CEREAL_DIR" ]; then
echo "::error::Could not locate cereal/log.capnp in upstream openpilot"
exit 1
fi
echo "cereal_dir=$CEREAL_DIR" >> "$GITHUB_OUTPUT"
echo "Found upstream cereal at: $CEREAL_DIR"
IMPORT_ARGS=""
CAR_CAPNP=$(find upstream_openpilot -maxdepth 5 -name car.capnp -path '*/opendbc/car/car.capnp' -printf '%h\n' -quit)
if [ -n "$CAR_CAPNP" ]; then
IMPORT_ARGS="-I $CAR_CAPNP"
echo "Found car.capnp at: $CAR_CAPNP"
fi
echo "import_args=$IMPORT_ARGS" >> "$GITHUB_OUTPUT"
- name: Install pycapnp
run: |
PYCAPNP_VER=$(python3 -c "import re; m=re.search(r'name = \"pycapnp\"\nversion = \"([^\"]+)\"', open('uv.lock').read()); print(m.group(1))")
pip install "pycapnp==${PYCAPNP_VER}"
- name: Generate sunnypilot schema
run: |
python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-g -f /tmp/sp_schema.json --cereal-dir openpilot/cereal \
${{ steps.locate-sp-capnp.outputs.import_args }}
- name: Validate against upstream
run: |
python3 openpilot/cereal/messaging/tests/validate_sp_cereal_upstream.py \
-r -f /tmp/sp_schema.json --cereal-dir ${{ steps.locate-capnp.outputs.cereal_dir }} \
${{ steps.locate-capnp.outputs.import_args }}
export PYTHONPATH=${{ github.workspace }}/openpilot
chmod +x openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py
python3 openpilot/cereal/messaging/tests/cereal_validations/validate_sp_cereal_upstream.py -r -f openpilot/cereal/messaging/tests/cereal_validations/schema_instances.bin
-58
View File
@@ -1,58 +0,0 @@
name: diff report
on:
pull_request_target:
types: [opened, synchronize, reopened]
jobs:
comment:
name: comment
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
pull-requests: write
actions: read
steps:
- name: Wait for process replay
id: wait
continue-on-error: true
uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
with:
ref: ${{ github.event.pull_request.head.sha }}
check-name: process replay
repo-token: ${{ secrets.GITHUB_TOKEN }}
allowed-conclusions: success,failure
wait-interval: 20
- name: Get tests run ID
if: steps.wait.outcome == 'success'
id: tests-run
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
run_id=$(gh api \
"repos/${{ github.repository }}/actions/workflows/tests.yaml/runs?event=pull_request&head_sha=${{ github.event.pull_request.head.sha }}&per_page=1" \
--jq '.workflow_runs[0].id // empty')
if [ -z "$run_id" ]; then
echo "No tests.yaml run found for ${{ github.event.pull_request.head.sha }}" >&2
exit 1
fi
echo "run-id=$run_id" >> "$GITHUB_OUTPUT"
- name: Download diff
if: steps.wait.outcome == 'success'
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
repository: ${{ github.repository }}
run-id: ${{ steps.tests-run.outputs.run-id }}
name: diff_report_${{ github.event.number }}
path: .
- name: Comment on PR
if: steps.wait.outcome == 'success' && hashFiles('diff_report.txt') != ''
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
with:
file-path: diff_report.txt
comment-tag: diff_report
pr-number: ${{ github.event.number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
+7 -11
View File
@@ -15,11 +15,6 @@ concurrency:
group: docs-tests-ci-run-${{ inputs.run_number }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
cancel-in-progress: true
env:
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
jobs:
docs:
name: build docs
@@ -27,18 +22,19 @@ jobs:
steps:
- uses: commaai/timeout@v1
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
# Build
- name: Build docs
run: |
git lfs pull
python docs/serve.py --build
# TODO: can we install just the "docs" dependency group without the normal deps?
pip install mkdocs
mkdocs build
# Push to docs.comma.ai
- uses: actions/checkout@v7
- uses: actions/checkout@v6
if: github.ref == 'refs/heads/master' && github.repository == 'sunnypilot/sunnypilot'
with:
path: openpilot-docs
@@ -49,14 +45,14 @@ jobs:
run: |
set -x
source tools/release/identity.sh
source release/identity.sh
cd openpilot-docs
git checkout --orphan tmp
git rm -rf .
# copy over docs
cp -r ../docs/_site/ docs/
cp -r ../docs_site/ docs/
# GitHub pages config
touch docs/.nojekyll
+4 -4
View File
@@ -11,7 +11,7 @@ jobs:
contents: write
steps:
- name: Delete stale Jenkins branches
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const cutoff = Date.now() - 24 * 60 * 60 * 1000;
@@ -52,7 +52,7 @@ jobs:
steps:
- name: Check for trigger phrase
id: check_comment
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const triggerPhrase = "trigger-jenkins";
@@ -72,7 +72,7 @@ jobs:
- name: Checkout repository
if: steps.check_comment.outputs.result == 'true'
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
ref: refs/pull/${{ github.event.issue.number }}/head
@@ -86,7 +86,7 @@ jobs:
- name: Delete trigger comment
if: steps.check_comment.outputs.result == 'true' && always()
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
await github.rest.issues.deleteComment({
+1 -14
View File
@@ -65,21 +65,8 @@ jobs:
echo ' pushurl = ${{ env.LFS_PUSH_URL }}' >> .lfsconfig
echo ' locksverify = false' >> .lfsconfig
- name: Configure LFS transfer settings
run: |
git config lfs.activitytimeout 300
git config lfs.transfer.maxretries 5
git config lfs.concurrenttransfers 4
- name: Push LFS
id: sync-and-commit
run: |
git lfs ls-files -l
for attempt in 1 2 3; do
echo "Push attempt $attempt..."
git lfs push --all origin && exit 0
echo "Attempt $attempt failed, retrying in 30s..."
sleep 30
done
echo "All push attempts failed"
exit 1
git lfs push --all origin
+42
View File
@@ -0,0 +1,42 @@
name: "model review"
on:
pull_request:
types: [opened, reopened, synchronize]
paths:
- 'selfdrive/modeld/models/*.onnx'
workflow_dispatch:
jobs:
comment:
permissions:
contents: read
pull-requests: write
runs-on: ubuntu-latest
if: github.repository == 'commaai/openpilot'
steps:
- name: Checkout
uses: actions/checkout@v6
with:
submodules: true
- name: Checkout master
uses: actions/checkout@v6
with:
ref: master
path: base
- run: git lfs pull
- run: cd base && git lfs pull
- name: scripts/reporter.py
id: report
run: |
echo "content<<EOF" >> $GITHUB_OUTPUT
echo "## Model Review" >> $GITHUB_OUTPUT
PYTHONPATH=${{ github.workspace }} MASTER_PATH=${{ github.workspace }}/base python scripts/reporter.py >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
- name: Post model report comment
uses: marocchino/sticky-pull-request-comment@baa7203ed60924babbe5dcd0ac8eae3b66ec5e16
with:
header: model-review
message: ${{ steps.report.outputs.content }}
+39
View File
@@ -0,0 +1,39 @@
name: prebuilt
on:
schedule:
- cron: '0 * * * *'
workflow_dispatch:
env:
DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
BUILD: release/ci/docker_build_sp.sh
jobs:
build_prebuilt:
name: build prebuilt
runs-on: ubuntu-latest
if: github.repository == 'sunnypilot/sunnypilot'
env:
PUSH_IMAGE: true
permissions:
checks: read
contents: read
packages: write
steps:
- name: Wait for green check mark
if: ${{ github.event_name != 'workflow_dispatch' }}
uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
with:
ref: master
wait-interval: 30
running-workflow-name: 'build prebuilt'
repo-token: ${{ secrets.GITHUB_TOKEN }}
check-regexp: ^((?!.*(build master-ci|create badges).*).)*$
- uses: actions/checkout@v6
with:
submodules: true
- run: git lfs pull
- name: Build and Push docker image
run: |
$DOCKER_LOGIN
eval "$BUILD"
+3 -8
View File
@@ -4,11 +4,6 @@ on:
- cron: '0 9 * * *'
workflow_dispatch:
env:
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
jobs:
build___nightly:
name: build __nightly
@@ -20,17 +15,17 @@ jobs:
steps:
- name: Wait for green check mark
if: ${{ github.event_name == 'schedule' }}
uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
with:
ref: master
wait-interval: 30
running-workflow-name: 'build __nightly'
repo-token: ${{ secrets.GITHUB_TOKEN }}
check-regexp: ^((?!.*(build prebuilt|create badges).*).)*$
- uses: actions/checkout@v4 # checkout version > v4 breaks nightly release script
- uses: actions/checkout@v4
with:
submodules: true
fetch-depth: 0
- run: ./tools/op.sh setup
- name: Push __nightly
run: BRANCH=__nightly tools/release/build_stripped.sh
run: BRANCH=__nightly release/build_stripped.sh
+27 -59
View File
@@ -3,25 +3,40 @@ name: repo maintenance
on:
schedule:
- cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
pull_request:
types: [closed]
workflow_dispatch:
env:
PYTHONPATH: ${{ github.workspace }}
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
jobs:
update_translations:
runs-on: ubuntu-latest
if: github.repository == 'sunnypilot/sunnypilot'
steps:
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Update translations
run: python3 selfdrive/ui/update_translations.py --vanish
- name: Create Pull Request
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
with:
author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
commit-message: "Update translations"
title: "[bot] Update translations"
body: "Automatic PR from repo-maintenance -> update_translations"
branch: "update-translations"
base: "master"
delete-branch: true
labels: bot
package_updates:
name: package_updates
runs-on: ubuntu-latest
if: >-
github.repository == 'sunnypilot/sunnypilot' &&
(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch')
if: github.repository == 'sunnypilot/sunnypilot'
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
@@ -57,10 +72,11 @@ jobs:
git add .
- name: update car docs
run: |
python openpilot/selfdrive/car/docs.py
scons -j$(nproc) --minimal opendbc_repo
python selfdrive/car/docs.py
git add docs/CARS.md
- name: Create Pull Request
uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1
uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0
with:
author: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
token: ${{ github.repository == 'commaai/openpilot' && secrets.ACTIONS_CREATE_PR_PAT || secrets.GITHUB_TOKEN }}
@@ -82,51 +98,3 @@ jobs:
${{ steps.pip_tree.outputs.PIP_TREE }}
```
labels: bot
cleanup_closed_branches:
if: github.repository == 'commaai/openpilot' && github.event_name == 'pull_request'
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: read
steps:
- uses: actions/github-script@v9
with:
script: |
const { owner, repo } = context.repo;
const upstream = `${owner}/${repo}`;
const closed = context.payload.pull_request;
if (closed) {
if (closed.head.repo?.full_name === upstream) {
await github.rest.git.deleteRef({ owner, repo, ref: `heads/${closed.head.ref}` }).catch(console.log);
}
return;
}
for await (const response of github.paginate.iterator(github.rest.pulls.list, {
owner,
repo,
state: 'closed',
per_page: 100,
})) {
for (const pr of response.data) {
if (pr.head.repo?.full_name !== upstream) continue;
const branch = pr.head.ref;
try {
await github.rest.git.deleteRef({
owner,
repo,
ref: `heads/${branch}`,
});
console.log(`Deleted branch ${branch} (PR #${pr.number})`);
} catch (error) {
if (error.status === 422 || error.status === 403) {
console.log(`Skipping branch ${branch} (PR #${pr.number}): ${error.message}`);
continue;
}
throw error;
}
}
}
+20 -84
View File
@@ -6,7 +6,7 @@ env:
SCONS_CACHE_DIR: ${{ github.workspace }}/release/ci/scons_cache
UPSTREAM_REPO: "commaai/openpilot"
TINYGRAD_PATH: ${{ github.workspace }}/tinygrad_repo
MODELS_DIR: ${{ github.workspace }}/openpilot/selfdrive/modeld/models
MODELS_DIR: ${{ github.workspace }}/selfdrive/modeld/models
on:
workflow_call:
@@ -30,11 +30,6 @@ on:
required: false
type: string
default: ''
target_hardware:
description: 'Hardware target to compile for (qcom or usbgpu)'
required: false
type: string
default: 'qcom'
workflow_dispatch:
inputs:
upstream_branch:
@@ -51,14 +46,6 @@ on:
required: false
type: boolean
default: true
target_hardware:
description: 'Hardware target to compile for'
required: true
type: choice
options:
- qcom
- usbgpu
default: 'qcom'
run-name: Build model [${{ inputs.custom_name || inputs.upstream_branch }}] from ref [${{ inputs.upstream_branch }}]
@@ -105,7 +92,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ github.workspace }}/openpilot/openpilot/selfdrive/modeld/models/*.onnx
path: ${{ github.workspace }}/openpilot/selfdrive/modeld/models/*.onnx
build_model:
runs-on: [self-hosted, tici]
@@ -141,10 +128,7 @@ jobs:
# Set up common environment
source /etc/profile;
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export UV_PYTHON_PREFERENCE=managed
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
uv sync
printenv >> $GITHUB_ENV
if [[ "${{ runner.debug }}" == "1" ]]; then
cat $GITHUB_OUTPUT
@@ -164,7 +148,6 @@ jobs:
if [[ "${{ runner.debug }}" == "1" ]]; then
printenv
fi
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
rm -rf ${{ env.MODELS_DIR }}/*.onnx
@@ -172,79 +155,35 @@ jobs:
uses: actions/download-artifact@v4
with:
name: models-${{ env.REF }}${{ inputs.artifact_suffix }}
path: ${{ env.MODELS_DIR }}
path: ${{ github.workspace }}/selfdrive/modeld/models
- run: |
rm -f ${{ env.MODELS_DIR }}/{dmonitoring_model,big_driving_policy,big_driving_vision,big_driving_supercombo}.onnx
rm -f ${{ github.workspace }}/selfdrive/modeld/models/{dmonitoring_model,big_driving_policy,big_driving_vision}.onnx
- name: Build Model
run: |
source /etc/profile
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}:${{ github.workspace }}"
export PYTHONPATH="${PYTHONPATH}:${{ env.TINYGRAD_PATH }}"
COMPILE_MODELD="${{ github.workspace }}/openpilot/sunnypilot/modeld_v2/compile_modeld.py"
MODEL_SIZE=$(python3 -c "from openpilot.common.transformations.model import MEDMODEL_INPUT_SIZE as s; print(f'{s[0]}x{s[1]}')")
CAMERA_RES=$(python3 -c "from openpilot.common.transformations.camera import _ar_ox_fisheye as a, _os_fisheye as o; print(f'{a.width}x{a.height} {o.width}x{o.height}')")
if [ "${{ inputs.target_hardware }}" == "usbgpu" ]; then
echo "USBGPU build"
export USBGPU=1
TG_FLAGS="DEV=AMD USBGPU=1 IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
OUTPUT_PKL="${{ env.MODELS_DIR }}/big_driving_tinygrad.pkl"
else
echo "QCOM build"
TG_FLAGS="DEV=QCOM IMAGE=1 FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 OPENPILOT_HACKS=1"
OUTPUT_PKL="${{ env.MODELS_DIR }}/driving_tinygrad.pkl"
fi
# Generate metadata for all ONNX files
# Loop through all .onnx files
find "${{ env.MODELS_DIR }}" -maxdepth 1 -name '*.onnx' | while IFS= read -r onnx_file; do
echo "Generating metadata: $onnx_file"
env ${TG_FLAGS} python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
base_name=$(basename "$onnx_file" .onnx)
output_file="${{ env.MODELS_DIR }}/${base_name}_tinygrad.pkl"
echo "Compiling: $onnx_file -> $output_file"
QCOM=1 python3 "${{ env.TINYGRAD_PATH }}/examples/openpilot/compile3.py" "$onnx_file" "$output_file"
DEV=QCOM FLOAT16=1 NOLOCALS=1 JIT_BATCH_SIZE=0 python3 "${{ env.MODELS_DIR }}/../get_model_metadata.py" "$onnx_file" || true
done
# Detect model type and build compile args
VISION_ONNX="${{ env.MODELS_DIR }}/driving_vision.onnx"
POLICY_ONNX="${{ env.MODELS_DIR }}/driving_policy.onnx"
OFF_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_off_policy.onnx"
ON_POLICY_ONNX="${{ env.MODELS_DIR }}/driving_on_policy.onnx"
SUPERCOMBO_ONNX=""
for f in "${{ env.MODELS_DIR }}/supercombo.onnx" "${{ env.MODELS_DIR }}/driving_supercombo.onnx"; do
if [ -f "$f" ]; then
SUPERCOMBO_ONNX="$f"
break
fi
done
MODEL_TYPE="" ONNX_ARGS="" OUTPUT_NAME=""
if [ -f "$VISION_ONNX" ]; then
ONNX_ARGS="--vision-onnx $VISION_ONNX"
if [ -f "$ON_POLICY_ONNX" ] && [ -f "$OFF_POLICY_ONNX" ]; then
MODEL_TYPE=vision_multi_policy
ONNX_ARGS="$ONNX_ARGS --off-policy-onnx $OFF_POLICY_ONNX --on-policy-onnx $ON_POLICY_ONNX"
elif [ -f "$OFF_POLICY_ONNX" ] && [ -f "$POLICY_ONNX" ]; then
MODEL_TYPE=vision_multi_policy
ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX --off-policy-onnx $OFF_POLICY_ONNX"
elif [ -f "$POLICY_ONNX" ]; then
MODEL_TYPE=vision_policy
ONNX_ARGS="$ONNX_ARGS --policy-onnx $POLICY_ONNX"
fi
elif [ -f "$SUPERCOMBO_ONNX" ]; then
MODEL_TYPE=supercombo
ONNX_ARGS="--supercombo-onnx $SUPERCOMBO_ONNX"
fi
if [ -n "$MODEL_TYPE" ]; then
echo "Detected: $MODEL_TYPE -> $OUTPUT_PKL"
env ${TG_FLAGS} python3 "$COMPILE_MODELD" \
--model-type $MODEL_TYPE \
--model-size $MODEL_SIZE \
--camera-resolutions $CAMERA_RES \
$ONNX_ARGS \
--output "$OUTPUT_PKL"
fi
- name: Validate Model Outputs
run: |
source /etc/profile
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
python3 "${{ github.workspace }}/release/ci/model_generator.py" \
--validate-only \
--model-dir "${{ env.MODELS_DIR }}"
- name: Prepare Output
run: |
@@ -255,8 +194,6 @@ jobs:
rsync -avm \
--include='*.dlc' \
--include='*.pkl' \
--include='*.chunk*' \
--include='*.chunkmanifest' \
--include='*.onnx' \
--exclude='*' \
--delete-excluded \
@@ -289,5 +226,4 @@ jobs:
- name: Re-enable powersave
if: always()
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
@@ -79,7 +79,7 @@ jobs:
is_stable_branch="$(echo "$CONFIG" | jq -r '.stable_branch // false')";
echo "is_stable_branch=$is_stable_branch" >> $GITHUB_OUTPUT
stable_version=$(cat openpilot/sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
stable_version=$(cat sunnypilot/common/version.h | grep SUNNYPILOT_VERSION | sed -e 's/[^0-9|.]//g');
echo "version=$([ "$is_stable_branch" = "true" ] && echo "$stable_version" || echo "$BUILD")" >> $GITHUB_OUTPUT
echo "extra_version_identifier=${environment}" >> $GITHUB_OUTPUT
fi
@@ -152,10 +152,7 @@ jobs:
# Set up common environment
source /etc/profile;
export UV_PROJECT_ENVIRONMENT=${HOME}/venv
export UV_PYTHON_PREFERENCE=managed
export UV_PYTHON_INSTALL_DIR=${HOME}/uv/python
export VIRTUAL_ENV=$UV_PROJECT_ENVIRONMENT
uv sync
printenv >> $GITHUB_ENV
if [[ "${{ runner.debug }}" == "1" ]]; then
cat $GITHUB_OUTPUT
@@ -175,26 +172,20 @@ jobs:
if [[ "${{ runner.debug }}" == "1" ]]; then
printenv
fi
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --disable
- name: Build Main Project
run: |
export PYTHONPATH="$BUILD_DIR"
./tools/release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
./release/release_files.py | sort | uniq | rsync -rRl${RUNNER_DEBUG:+v} --files-from=- . $BUILD_DIR/
cd $BUILD_DIR
ln -sfn msgq_repo/msgq msgq
ln -sfn opendbc_repo/opendbc opendbc
ln -sfn rednose_repo/rednose rednose
ln -sfn teleoprtc_repo/teleoprtc teleoprtc
ln -sfn tinygrad_repo/tinygrad tinygrad
sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py
echo "Building sunnypilot's modeld_v2..."
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/modeld_v2
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/modeld_v2
echo "Building sunnypilot's locationd..."
scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/sunnypilot/selfdrive/locationd
scons -j2 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal sunnypilot/selfdrive/locationd
echo "Building openpilot's locationd..."
scons -j1 cache_dir=${{env.SCONS_CACHE_DIR}} --minimal openpilot/selfdrive/locationd
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal selfdrive/locationd
echo "Building rest of sunnypilot"
scons -j$(nproc) cache_dir=${{env.SCONS_CACHE_DIR}} --minimal
touch ${BUILD_DIR}/prebuilt
@@ -217,17 +208,17 @@ jobs:
--exclude='Jenkinsfile' \
--exclude='**/release/' \
--exclude='**/.github/' \
--exclude='**/openpilot/selfdrive/ui/replay/' \
--exclude='**/selfdrive/ui/replay/' \
--exclude='**/__pycache__/' \
--exclude='${{env.SCONS_CACHE_DIR}}' \
--exclude='**/.git/' \
--exclude='**/SConstruct' \
--exclude='**/SConscript' \
--exclude='**/.venv/' \
--exclude='openpilot/selfdrive/modeld/models/*.onnx*' \
--exclude='openpilot/sunnypilot/modeld*/models/*.onnx*' \
--exclude='openpilot/third_party/*x86*' \
--exclude='openpilot/third_party/*Darwin*' \
--exclude='selfdrive/modeld/models/driving_vision.onnx' \
--exclude='selfdrive/modeld/models/driving_policy.onnx' \
--exclude='third_party/*x86*' \
--exclude='third_party/*Darwin*' \
--delete-excluded \
--chown=comma:comma \
${BUILD_DIR}/ ${OUTPUT_DIR}/
@@ -246,7 +237,6 @@ jobs:
- name: Re-enable powersave
if: always()
run: |
source ${UV_PROJECT_ENVIRONMENT}/bin/activate
PYTHONPATH=$PYTHONPATH:${{ github.workspace }}/ ${{ github.workspace }}/scripts/manage-powersave.py --enable
@@ -373,7 +363,7 @@ jobs:
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
name: process.env.LABELf
name: process.env.LABEL
});
console.log(`Removed '${process.env.LABEL}' label from PR #${prNumber}`);
+61 -66
View File
@@ -20,9 +20,7 @@ concurrency:
env:
CI: 1
PYTHONPATH: ${{ github.workspace }}
GIT_CONFIG_COUNT: 1
GIT_CONFIG_KEY_0: lfs.fetchexclude
GIT_CONFIG_VALUE_0: openpilot/selfdrive/modeld/models/big_driving_supercombo.onnx
PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical
jobs:
build_release:
@@ -37,27 +35,27 @@ jobs:
STRIPPED_DIR: /tmp/releasepilot
PYTHONPATH: /tmp/releasepilot
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- name: Getting LFS files
uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e
with:
timeout_minutes: 2
max_attempts: 3
command: git lfs pull
- name: Build devel
timeout-minutes: 3
run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
timeout-minutes: 1
run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
- run: ./tools/op.sh setup
- name: Build openpilot and run checks
timeout-minutes: 30
working-directory: ${{ env.STRIPPED_DIR }}
run: python3 openpilot/system/manager/build.py
run: python3 system/manager/build.py
- name: Run tests
timeout-minutes: 1
working-directory: ${{ env.STRIPPED_DIR }}
run: tools/release/check-dirty.sh
run: release/check-dirty.sh
- name: Check submodules
if: github.repository == 'sunnypilot/sunnypilot'
timeout-minutes: 3
@@ -77,13 +75,15 @@ jobs:
export CHECK_PR_REFS=true
fi
fi
tools/release/check-submodules.sh
release/check-submodules.sh
build_mac:
name: build macOS
runs-on: ${{ ((github.repository == 'commaai/openpilot') && ((github.event_name != 'pull_request') || (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-macos-8x14' || 'macos-latest' }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- name: Remove Homebrew from environment
run: |
FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
@@ -101,7 +101,9 @@ jobs:
&& fromJSON('["namespace-profile-amd64-8x16"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Static analysis
timeout-minutes: 1
@@ -116,21 +118,23 @@ jobs:
&& fromJSON('["namespace-profile-amd64-8x16"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Install raylib headless dependencies
if: ${{ !contains(runner.name, 'nsc') }}
run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
- name: Build openpilot
run: scons
run: scons -j$(nproc)
- name: Run unit tests
timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 999 }}
env:
RAYLIB_BACKEND: headless
run: tools/op.sh test
run: |
source selfdrive/test/setup_xvfb.sh
# Pre-compile Python bytecode so each pytest worker doesn't need to
$PYTEST --collect-only -m 'not slow' -qq
MAX_EXAMPLES=1 $PYTEST -m 'not slow'
process_replay:
name: process replay
if: false # disable process_replay for forks
runs-on: ${{
(github.repository == 'commaai/openpilot') &&
((github.event_name != 'pull_request') ||
@@ -138,60 +142,52 @@ jobs:
&& fromJSON('["namespace-profile-amd64-8x16"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Build openpilot
run: scons
run: scons -j$(nproc)
- name: Run replay
timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }}
continue-on-error: ${{ github.ref == 'refs/heads/master' }}
run: openpilot/selfdrive/test/process_replay/test_processes.py -j$(nproc)
run: selfdrive/test/process_replay/test_processes.py -j$(nproc)
- name: Print diff
id: print-diff
if: always()
run: cat openpilot/selfdrive/test/process_replay/diff.txt
- name: Print diff report
if: always()
run: cat openpilot/selfdrive/test/process_replay/diff_report.txt
- uses: actions/upload-artifact@v7
run: cat selfdrive/test/process_replay/diff.txt
- uses: actions/upload-artifact@v6
if: always()
continue-on-error: true
with:
name: process_replay_diff.txt
path: openpilot/selfdrive/test/process_replay/diff.txt
- name: Upload diff report
uses: actions/upload-artifact@v7
if: always() && github.event_name == 'pull_request'
continue-on-error: true
with:
name: diff_report_${{ github.event.number }}
path: openpilot/selfdrive/test/process_replay/diff_report.txt
path: selfdrive/test/process_replay/diff.txt
- name: Checkout ci-artifacts
if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
uses: actions/checkout@v7
if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
uses: actions/checkout@v4
with:
repository: sunnypilot/ci-artifacts
repository: commaai/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
path: ${{ github.workspace }}/ci-artifacts
- name: Prepare refs
if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
- name: Push refs
if: github.repository == 'commaai/openpilot' && github.ref == 'refs/heads/master'
working-directory: ${{ github.workspace }}/ci-artifacts
run: |
git config user.name "GitHub Actions Bot"
git config user.email "<>"
git fetch origin process-replay || true
git checkout process-replay 2>/dev/null || git checkout --orphan process-replay
cp ${{ github.workspace }}/openpilot/selfdrive/test/process_replay/fakedata/*.zst .
cp ${{ github.workspace }}/selfdrive/test/process_replay/fakedata/*.zst .
echo "${{ github.sha }}" > ref_commit
git add .
git commit -m "process-replay refs for ${{ github.repository }}@${{ github.sha }}" || echo "No changes to commit"
- name: Push refs
if: github.repository == 'sunnypilot/sunnypilot' && github.ref == 'refs/heads/master'
uses: nick-fields/retry@ad984534de44a9489a53aefd81eb77f87c70dc60
with:
timeout_minutes: 2
max_attempts: 3
command: cd ${{ github.workspace }}/ci-artifacts && git push origin process-replay --force
git push origin process-replay
- name: Run regen
if: false
timeout-minutes: 4
env:
ONNXCPU: 1
run: $PYTEST selfdrive/test/process_replay/test_regen.py
simulator_driving:
name: simulator driving
@@ -203,16 +199,17 @@ jobs:
|| fromJSON('["ubuntu-24.04"]') }}
if: false # FIXME: Started to timeout recently
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Build openpilot
run: scons
run: scons -j$(nproc)
- name: Driving test
timeout-minutes: 2
env:
# MetaDrive renders offscreen through panda3d's EGL pipe on llvmpipe
EGL_PLATFORM: surfaceless
run: python openpilot/tools/sim/tests/test_metadrive_bridge.py
run: |
source selfdrive/test/setup_xvfb.sh
pytest -s tools/sim/tests/test_metadrive_bridge.py
create_ui_report:
name: Create UI Report
@@ -223,21 +220,19 @@ jobs:
&& fromJSON('["namespace-profile-amd64-8x16"]')
|| fromJSON('["ubuntu-24.04"]') }}
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- run: ./tools/op.sh setup
- name: Install raylib headless dependencies
if: ${{ !contains(runner.name, 'nsc') }}
run: sudo apt-get install -y --no-install-recommends libgles2 libegl1
- name: Build openpilot
run: scons
run: scons -j$(nproc)
- name: Create UI Report
env:
RAYLIB_BACKEND: headless
run: |
python3 openpilot/selfdrive/ui/tests/diff/replay.py
python3 openpilot/selfdrive/ui/tests/diff/replay.py --big
source selfdrive/test/setup_xvfb.sh
python3 selfdrive/ui/tests/diff/replay.py
python3 selfdrive/ui/tests/diff/replay.py --big
- name: Upload UI Report
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ui-report-${{ inputs.run_number || '1' }}-${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && 'master' || github.event.number }}
path: openpilot/selfdrive/ui/tests/diff/report
path: selfdrive/ui/tests/diff/report
+17 -17
View File
@@ -8,9 +8,9 @@ on:
branches:
- 'master'
paths:
- 'openpilot/selfdrive/assets/**'
- 'openpilot/selfdrive/ui/**'
- 'openpilot/system/ui/**'
- 'selfdrive/assets/**'
- 'selfdrive/ui/**'
- 'system/ui/**'
workflow_dispatch:
env:
@@ -34,12 +34,12 @@ jobs:
pull-requests: write
actions: read
steps:
- uses: actions/checkout@v7
- uses: actions/checkout@v6
with:
submodules: true
- name: Waiting for ui generation to end
uses: lewagon/wait-on-check-action@1d57e2c51a58d812d2765e036a028b6bdb5a6154
uses: lewagon/wait-on-check-action@v1.3.4
with:
ref: ${{ env.SHA }}
check-name: ${{ env.UI_JOB_NAME }}
@@ -53,7 +53,7 @@ jobs:
echo "run_id=$(curl https://api.github.com/repos/${{ github.repository }}/commits/${{ env.SHA }}/check-runs | jq -r '.check_runs[] | select(.name == "${{ env.UI_JOB_NAME }}") | .html_url | capture("(?<number>[0-9]+)") | .number')" >> $GITHUB_OUTPUT
- name: Getting proposed ui
uses: dawidd6/action-download-artifact@b6e2e70617bc3265edd6dab6c906732b2f1ae151
uses: dawidd6/action-download-artifact@v6
with:
github_token: ${{ secrets.GITHUB_TOKEN }}
run_id: ${{ steps.get_run_id.outputs.run_id }}
@@ -62,7 +62,7 @@ jobs:
path: ${{ github.workspace }}/pr_ui
- name: Getting mici master ui
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
repository: sunnypilot/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -70,7 +70,7 @@ jobs:
ref: openpilot_master_ui_mici_raylib
- name: Getting big master ui
uses: actions/checkout@v7
uses: actions/checkout@v6
with:
repository: sunnypilot/ci-artifacts
ssh-key: ${{ secrets.CI_ARTIFACTS_DEPLOY_KEY }}
@@ -97,7 +97,7 @@ jobs:
done
- name: Setup FFmpeg
uses: AnimMouse/setup-ffmpeg@178e0b4a408f06ce40d896c3cd79f50fa6f8b0c3
uses: AnimMouse/setup-ffmpeg@ae28d57dabbb148eff63170b6bf7f2b60062cbae
- name: Finding diffs
if: github.event_name == 'pull_request_target'
@@ -115,13 +115,13 @@ jobs:
cp "${{ github.workspace }}/master_${name}/${video}.mp4" "${{ github.workspace }}/pr_ui/${video}_master.mp4"
diff_exit_code=0
python3 ${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/diff.py \
python3 ${{ github.workspace }}/selfdrive/ui/tests/diff/diff.py \
"${{ github.workspace }}/pr_ui/${video}_master.mp4" \
"${{ github.workspace }}/pr_ui/${video}_proposed.mp4" \
"${diff_name}.html" --basedir "$baseurl" --no-open || diff_exit_code=$?
cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${{ github.workspace }}/pr_ui/"
cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.mp4" "${{ github.workspace }}/pr_ui/"
REPORT_URL="https://sunnypilot.github.io/ci-artifacts/${diff_name}_pr_${{ github.event.number }}.html"
if [ $diff_exit_code -eq 0 ]; then
@@ -156,7 +156,7 @@ jobs:
for variant in $VARIANTS; do
IFS=':' read -r name _ _ <<< "$variant"
diff_name="${name}_diff"
cp "${{ github.workspace }}/openpilot/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
cp "${{ github.workspace }}/selfdrive/ui/tests/diff/report/${diff_name}.html" "${diff_name}_pr_${{ github.event.number }}.html"
git add "${diff_name}_pr_${{ github.event.number }}.html"
done
git commit -m "ui diff reports for PR #${{ github.event.number }}" || echo "No changes to commit"
@@ -164,12 +164,12 @@ jobs:
- name: Comment on PR
if: github.event_name == 'pull_request_target'
uses: thollander/actions-comment-pull-request@24bffb9b452ba05a4f3f77933840a6a841d1b32b
uses: thollander/actions-comment-pull-request@v2
with:
message: |
<!-- _(run_id_ui_preview **${{ github.run_id }}**)_ -->
## UI Preview
${{ steps.find_diff.outputs.COMMENT }}
comment-tag: run_id_ui_preview
pr-number: ${{ github.event.number }}
github-token: ${{ secrets.GITHUB_TOKEN }}
comment_tag: run_id_ui_preview
pr_number: ${{ github.event.number }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+33 -31
View File
@@ -13,20 +13,13 @@ venv/
a.out
.hypothesis
.cache/
bin/
# created at launch for comma hardware PYTHONPATH (PC uses editable installs via pyproject.toml)
/msgq
/opendbc
/rednose
/teleoprtc
/tinygrad
/docs_site/
*.mp4
*.dylib
*.DSYM
*.d
*.pem
*.pyc
*.pyo
.*.swp
@@ -39,8 +32,6 @@ bin/
*.os-*
*.so
*.a
st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
*.unchunked
*.clb
*.class
*.pyxbldc
@@ -48,57 +39,68 @@ st[0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z][0-9A-Za-z]
*.mo
*_pyx.cpp
*.stats
*.pkl
*.pkl*
config.json
clcache
compile_commands.json
compare_runtime*.html
openpilot/selfdrive/modeld/models/tg_input_devices.json
# build artifacts
docs/_site/
openpilot/selfdrive/pandad/pandad
openpilot/cereal/services.h
openpilot/cereal/gen
openpilot/cereal/messaging/bridge
openpilot/selfdrive/ui/translations/tmp
openpilot/selfdrive/car/tests/cars_dump
openpilot/system/camerad/camerad
openpilot/system/camerad/test/ae_gray_test
selfdrive/pandad/pandad
cereal/services.h
cereal/gen
cereal/messaging/bridge
selfdrive/ui/translations/tmp
selfdrive/car/tests/cars_dump
system/camerad/camerad
system/camerad/test/ae_gray_test
.coverage*
coverage.xml
htmlcov
pandaextra
.mypy_cache/
flycheck_*
cppcheck_report.txt
comma*.sh
selfdrive/modeld/models/*.pkl*
sunnypilot/modeld*/models/*.pkl
# openpilot log files
*.bz2
*.zst
*.rlog
build/
!**/.gitkeep
poetry.toml
Pipfile
### VisualStudioCode ###
*.vsix
.history
.ionide
.vscode/*
.history/
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# agents
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix
### VisualStudioCode Patch ###
# Ignore all local history of files
.history
.ionide
.claude/
.context/
PLAN.md
TASK.md
CLAUDE.md
SKILL.md
### JetBrains ###
!.idea/customTargets.xml
+2 -2
View File
@@ -6,7 +6,7 @@
url = https://github.com/sunnypilot/opendbc.git
[submodule "msgq"]
path = msgq_repo
url = https://github.com/sunnypilot/msgq.git
url = https://github.com/commaai/msgq.git
[submodule "rednose_repo"]
path = rednose_repo
url = https://github.com/commaai/rednose.git
@@ -17,5 +17,5 @@
path = tinygrad_repo
url = https://github.com/sunnypilot/tinygrad.git
[submodule "sunnypilot/neural_network_data"]
path = openpilot/sunnypilot/neural_network_data
path = sunnypilot/neural_network_data
url = https://github.com/sunnypilot/neural-network-data.git
-1
View File
@@ -1 +0,0 @@
3.12.13
+4 -7
View File
@@ -6,9 +6,9 @@
"type": "pickString",
"description": "Select the process to debug",
"options": [
"openpilot/selfdrive/controls/controlsd.py",
"openpilot/system/timed/timed.py",
"openpilot/tools/sim/run_bridge.py"
"selfdrive/controls/controlsd.py",
"system/timed/timed.py",
"tools/sim/run_bridge.py"
]
},
{
@@ -16,7 +16,7 @@
"type": "pickString",
"description": "Select the process to debug",
"options": [
"openpilot/selfdrive/ui/ui"
"selfdrive/ui/ui"
]
},
{
@@ -52,9 +52,6 @@
"type": "lldb",
"request": "attach",
"pid": "${command:pickMyProcess}",
"sourceMap": {
".": "${workspaceFolder}/opendbc/safety"
},
"initCommands": [
"script import time; time.sleep(3)"
]
+6
View File
@@ -17,5 +17,11 @@
"**/.git",
"**/.venv",
"**/__pycache__",
// exclude directories that should be using the symlinked version
"common/**",
"selfdrive/**",
"system/**",
"third_party/**",
"tools/**",
]
}
+1 -118
View File
@@ -1,36 +1,4 @@
sunnypilot Version 2026.002.000 (2026-06-28)
========================
* What's Changed (sunnypilot/sunnypilot)
* ui: update gates for certain toggles by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1830
* release: ignore upstream IsReleaseBranch by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1831
* manager: disable DEVELOPMENT_ONLY reset by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1833
* sunnylink: fix max time offroad values by @nayan8teen in https://github.com/sunnypilot/sunnypilot/pull/1835
* ui: show default model name by @nayan8teen in https://github.com/sunnypilot/sunnypilot/pull/1837
* sunnylink: add CarParams fallback for brand-specific capabilities by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1839
* sunnylink SDUI: tweak DisableUpdate param for clarity by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1842
* Revert "DM: Lancia Delta HF Integrale model" by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1849
* modeld_v2: safe model validation by @Discountchubbs in https://github.com/sunnypilot/sunnypilot/pull/1855
* Revert "deprecate `carState.brake`" for Honda Gas Interceptor by @mvl-boston in https://github.com/sunnypilot/sunnypilot/pull/1860
* sunnylink: deprecate legacy params metadata by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1862
* ui: reset Enforce Torque Control and NNLC if both are enabled by @sunnyhaibin in https://github.com/sunnypilot/sunnypilot/pull/1863
* What's Changed (sunnypilot/opendbc)
* Rivian: suppress ACM hold-the-wheel warning during MADS-only lateral by @lukasloetkolben in https://github.com/sunnypilot/opendbc/pull/465
* Sync: `commaai/opendbc:master``sunnypilot/opendbc:master` by @sunnyhaibin in https://github.com/sunnypilot/opendbc/pull/479
* safety: add option to ignore frequency check for RX checks by @sunnyhaibin in https://github.com/sunnypilot/opendbc/pull/480
* Revert "deprecate carState.brake" for Honda Gas Interceptor by @mvl-boston in https://github.com/sunnypilot/opendbc/pull/481
* New Contributors (sunnypilot/sunnypilot)
* @mvl-boston made their first contribution in https://github.com/sunnypilot/sunnypilot/pull/1860
* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2026.001.007...v2026.002.000
************************
* Synced with commaai's openpilot (v0.11.1)
* master commit 69e2c321e49760e52f7983eaa0a5f77cb95de637 (June 02, 2026)
* New driver monitoring model
* Improved image processing pipeline for driver camera
* Improved thermal policy for comma four
* Acura MDX 2022-24 support thanks to mvl-boston!
* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
sunnypilot Version 2026.001.000 (2026-05-06)
sunnypilot Version 2026.001.000 (2026-03-xx)
========================
* What's Changed (sunnypilot/sunnypilot)
* Complete rewrite of the user interface from Qt C++ to Raylib Python
@@ -98,64 +66,6 @@ sunnypilot Version 2026.001.000 (2026-05-06)
* Pause Lateral Control with Blinker: Post-Blinker Delay by @CHaucke89
* SCC-V: Use p97 for predicted lateral accel by @yasu-oh
* Controls: Support for Torque Lateral Control v0 Tune by @sunnyhaibin
* [TIZI/TICI] ui: ensure null checks for `CarParams` and `CarParamsSP` by @sunnyhaibin
* [TIZI/TICI] ui: use `vCruiseCluster` and `vEgoCluster` for SLA `preActive` by @sunnyhaibin
* Fix display of values when using use_float_scaling by @CHaucke89
* models: fix default & index "0" by @nayan8teen
* [TIZI/TICI] visuals: Improved speed limit by @angaz
* ICBM: ensure button timers update on disable to clear stale presses by @jamesmikesell
* [TIZI/TICI] ui: simplify Smart Cruise Control text rendering by @sunnyhaibin
* controlsd: fix steer_limited_by_safety not updating under MADS by @zephleggett
* soundd: trigger timeout warning during MADS lateral-only by @zephleggett
* pandad: flasher for Rivian long upgrade module by @lukasloetkolben
* modeld_v2: tinygrad transformation warp by @Discountchubbs
* tools: block `manage_sunnylinkd` in sim startup script by @sunnyhaibin
* [MICI] ui: need superclass `_render` in `HudRendererSP` by @sunnyhaibin
* [TIZI/TICI] ui: Speed Limit Assist active status by @sunnyhaibin
* ui: reimplement "Screen Off" option to Onroad Brightness by @sunnyhaibin
* ui: don't hide steering wheel when blindspot disabled by @royjr
* ui: Speed Limit Assist `preActive` improvements by @sunnyhaibin
* ui: consolidate Speed Limit Assist `preActive` status rendering by @sunnyhaibin
* [MICI] ui: Speed Limit Assist `preActive` status by @sunnyhaibin
* sunnypilot modeld: remove thneed modeld by @Discountchubbs
* modeld_v2: decouple planplus scaling from accel by @Discountchubbs
* sunnylink: Handle exceptions in `getParamsAllKeysV1` to log crashes by @devtekve
* [TIZI/TICI] ui: Developer UI cleanup by @sunnyhaibin
* [TIZI/TICI] ui: dynamic alert size by @nayan8teen
* i18n(fr): Add French translations by @didlawowo
* Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
* ui: `AlertFadeAnimator` for longitudinal-related statuses by @sunnyhaibin
* pandad: gate unsupported pandas before flashing by @sunnyhaibin
* Rivian: Flash xnor's Longitudinal Upgrade Kit prior supported panda check by @lukasloetkolben
* [TIZI/TICI] ui: add back gate steering arc behind toggle by @sunnyhaibin
* ui: gate Onroad Brightness Delay on readiness by @sunnyhaibin
* ui: add new timer options for Onroad Brightness Delay by @sunnyhaibin
* [TIZI/TICI] ui: branch switcher is always available by @sunnyhaibin
* pandad: always prioritize internal panda by @sunnyhaibin
* sunnylinkd: fetch compressed params schema by @sunnyhaibin
* sunnypilot locationd: remove unused car_ekf filter by @sunnyhaibin
* modeld_v2: update deprecated temporalPose ref by @sunnyhaibin
* NNLC: restore pre-v1 PID gains in torque extension by @mmmorks
* MADS safety: enable heartbeat and lateral controls mismatch checks by @sunnyhaibin
* [MICI] ui: models panel enhancements by @nayan8teen
* [TIZI/TICI] ui: fix unintended selection while scrolling in TreeOptionDialog by @TheSecurityDev
* tools: script for video concatenation by @Discountchubbs
* tools: profile memory usage by @Discountchubbs
* [TIZI/TICI] ui: remove per-frame param sync by @sunnyhaibin
* [MICI] ui: always offroad by @nayan8teen
* controls: always default Torque Lateral Control to v0 Tune by @sunnyhaibin
* Revert "controls: always default Torque Lateral Control to v0 Tune" by @sunnyhaibin
* Reapply "controls: always default Torque Lateral Control to v0 Tune" (#1806) by @sunnyhaibin
* [MICI] ui: add sunnylink info & connectivity check by @nayan8teen
* sunnylink: Remove unused API endpoint by @devtekve
* DM: wheel touch enforcement in MADS by @sunnyhaibin
* torque: show static override values in Dev UI & gate `useParams` on custom torque tune by @sunnyhaibin
* MADS: suppress espActive event when long is not engaged by @sunnyhaibin
* sunnylink: SDUI by @sunnyhaibin
* [MICI] ui: align upstream changes with sunnypilot settings buttons by @nayan8teen
* ui: fix cellular toggles by @AmyJeanes
* sunnylink: switch athena domain by @devtekve
* Platform List: dynamically migrate CarPlatformBundle by @sunnyhaibin
* What's Changed (sunnypilot/opendbc)
* Honda: DBC for Accord 9th Generation by @mvl-boston
* FCA: update tire stiffness values for `RAM_HD` by @dparring
@@ -174,25 +84,12 @@ sunnypilot Version 2026.001.000 (2026-05-06)
* Honda: add missing `GasInterceptor` messages to Taiwan Odyssey DBC by @mvl-boston
* GM: remove `CHEVROLET_EQUINOX_NON_ACC_3RD_GEN` from `dashcamOnly` by @sunnyhaibin
* GM: remove `CHEVROLET_BOLT_NON_ACC_2ND_GEN` from `dashcamOnly` by @sunnyhaibin
* Hyundai Longitudinal: deprecate ramp update for dynamic tune by @Discountchubbs
* Rivian: long upgrade messages on bus 1 by @lukasloetkolben
* Toyota: Stop and Go Hack (Alpha) by @sunnyhaibin
* Toyota: gate Smart DSU behind Alpha Longitudinal by @sunnyhaibin
* Toyota: Gas Interceptor always set `standstill_req` by @sunnyhaibin
* MADS safety: dedicated `controls_allowed_lateral` by @sunnyhaibin
* Platform List: include community supported platforms by @sunnyhaibin
* New Contributors (sunnypilot/sunnypilot)
* @TheSecurityDev made their first contribution in "ui: fix sidebar scroll in UI screenshots"
* @zikeji made their first contribution in "sunnylink: block remote modification of SSH key parameters"
* @Candy0707 made their first contribution in "[TIZI/TICI] ui: Fix misaligned turn signals and blindspot indicators with sidebar"
* @CHaucke89 made their first contribution in "Pause Lateral Control with Blinker: Post-Blinker Delay"
* @yasu-oh made their first contribution in "SCC-V: Use p97 for predicted lateral accel"
* @angaz made their first contribution in "[TIZI/TICI] visuals: Improved speed limit"
* @jamesmikesell made their first contribution in "ICBM: ensure button timers update on disable to clear stale presses"
* @zephleggett made their first contribution in "controlsd: fix steer_limited_by_safety not updating under MADS"
* @lukasloetkolben made their first contribution in "pandad: flasher for Rivian long upgrade module"
* @didlawowo made their first contribution in "i18n(fr): Add French translations"
* @mmmorks made their first contribution in "NNLC: restore pre-v1 PID gains in torque extension"
* New Contributors (sunnypilot/opendbc)
* @AmyJeanes made their first contribution in "Tesla: Fix stock LKAS being blocked when MADS is enabled"
* @mvl-boston made their first contribution in "Honda: Update Clarity brake to renamed DBC message name"
@@ -202,20 +99,6 @@ sunnypilot Version 2026.001.000 (2026-05-06)
* @royjr made their first contribution in "HKG: add KIA_FORTE_2019_NON_SCC fingerprint"
* @ssysm made their first contribution in "Tesla: remove `TESLA_MODEL_X` from `dashcamOnly`"
* Full Changelog: https://github.com/sunnypilot/sunnypilot/compare/v2025.002.000...v2026.001.000
************************
* Synced with commaai's openpilot (v0.11.1)
* master commit c001f3c9b490a80e69539f0af6022f6e07ceb721 (April 16, 2026)
* New driver monitoring model
* Improved image processing pipeline for driver camera
* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
* New driving model #36798
* Fully trained using a learned simulator
* Improved longitudinal performance in Experimental mode
* Reduce comma four standby power usage by 77% to 52 mW
* Kia K7 2017 support thanks to royjr!
* Lexus LS 2018 support thanks to Hacheoy!
* Improved inter-process communication memory efficiency
* comma four support
sunnypilot Version 2025.002.000 (2025-11-06)
========================
+38
View File
@@ -0,0 +1,38 @@
FROM ubuntu:24.04
ENV PYTHONUNBUFFERED=1
ENV DEBIAN_FRONTEND=noninteractive
RUN apt-get update && \
apt-get install -y --no-install-recommends sudo tzdata locales && \
rm -rf /var/lib/apt/lists/*
RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
ENV LANG=en_US.UTF-8
ENV LANGUAGE=en_US:en
ENV LC_ALL=en_US.UTF-8
ENV NVIDIA_VISIBLE_DEVICES=all
ENV NVIDIA_DRIVER_CAPABILITIES=graphics,utility,compute
ARG USER=batman
ARG USER_UID=1001
RUN useradd -m -s /bin/bash -u $USER_UID $USER
RUN usermod -aG sudo $USER
RUN echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
USER $USER
ENV OPENPILOT_PATH=/home/$USER/openpilot
RUN mkdir -p ${OPENPILOT_PATH}
WORKDIR ${OPENPILOT_PATH}
COPY --chown=$USER . ${OPENPILOT_PATH}/
ENV UV_BIN="/home/$USER/.local/bin/"
ENV VIRTUAL_ENV=${OPENPILOT_PATH}/.venv
ENV PATH="$UV_BIN:$VIRTUAL_ENV/bin:$PATH"
RUN tools/setup_dependencies.sh && \
sudo rm -rf /var/lib/apt/lists/*
USER root
RUN git config --global --add safe.directory '*'
Vendored
+31 -32
View File
@@ -12,7 +12,7 @@ def retryWithDelay(int maxRetries, int delay, Closure body) {
def device(String ip, String step_label, String cmd) {
withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) {
def ssh_cmd = """
ssh -o ControlMaster=auto -o ControlPath=/tmp/ssh_control_%C -o ControlPersist=yes -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
ssh -o ConnectTimeout=5 -o ServerAliveInterval=5 -o ServerAliveCountMax=2 -o BatchMode=yes -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' exec /usr/bin/bash <<'END'
set -e
@@ -30,14 +30,14 @@ export GIT_COMMIT=${env.GIT_COMMIT}
export CI_ARTIFACTS_TOKEN=${env.CI_ARTIFACTS_TOKEN}
export GITHUB_COMMENTS_TOKEN=${env.GITHUB_COMMENTS_TOKEN}
export AZURE_TOKEN='${env.AZURE_TOKEN}'
# only use 1 thread since most require real hardware that can't be shared
# only use 1 thread for tici tests since most require HIL
export PYTEST_ADDOPTS="-n0 -s"
export GIT_SSH_COMMAND="ssh -i /data/gitkey"
source ~/.bash_profile
if [ -f /AGNOS ]; then
if [ -f /TICI ]; then
source /etc/profile
rm -rf /tmp/tmp*
@@ -63,8 +63,6 @@ if [ -f /data/openpilot/launch_env.sh ]; then
source /data/openpilot/launch_env.sh
fi
export LD_LIBRARY_PATH="\$(python -c 'import ffmpeg; print(ffmpeg.LIB_DIR)'):/usr/local/lib:\${LD_LIBRARY_PATH:-}"
ln -snf ${env.TEST_DIR} /data/pythonpath
cd ${env.TEST_DIR} || true
@@ -95,7 +93,7 @@ def deviceStage(String stageName, String deviceType, List extra_env, def steps)
retry (3) {
def date = sh(script: 'date', returnStdout: true).trim();
device(device_ip, "set time", "date -s '" + date + "'")
device(device_ip, "git checkout", extra + "\n" + readFile("openpilot/selfdrive/test/setup_device_ci.sh"))
device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh"))
}
steps.each { item ->
def name = item[0]
@@ -168,8 +166,8 @@ node {
env.GIT_BRANCH = checkout(scm).GIT_BRANCH
env.GIT_COMMIT = checkout(scm).GIT_COMMIT
def excludeBranches = ['__nightly', 'devel', 'devel-staging',
'release-tizi', 'release-tizi-staging', 'release-mici', 'release-mici-staging', 'testing-closet*', 'hotfix-*']
def excludeBranches = ['__nightly', 'devel', 'devel-staging', 'release3', 'release3-staging',
'release-tici', 'release-tizi', 'release-tizi-staging', 'testing-closet*', 'hotfix-*']
def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*')
if (env.BRANCH_NAME != 'master' && !env.BRANCH_NAME.contains('__jenkins_loop_')) {
@@ -181,7 +179,7 @@ node {
try {
if (env.BRANCH_NAME == 'devel-staging') {
deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/tools/release/build_release.sh"),
step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging $SOURCE_DIR/release/build_release.sh"),
])
}
@@ -189,12 +187,12 @@ node {
parallel (
'nightly': {
deviceStage("build nightly", "tizi-needs-can", [], [
step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/tools/release/build_release.sh"),
step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
])
},
'nightly-dev': {
deviceStage("build nightly-dev", "tizi-needs-can", [], [
step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/tools/release/build_release.sh"),
step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
])
},
)
@@ -204,51 +202,52 @@ node {
parallel (
'onroad tests': {
deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
step("build openpilot", "cd openpilot/system/manager && ./build.py"),
step("check dirty", "tools/release/check-dirty.sh"),
step("onroad tests", "./openpilot/selfdrive/test/test_onroad.py", [timeout: 60]),
step("build openpilot", "cd system/manager && ./build.py"),
step("check dirty", "release/check-dirty.sh"),
step("onroad tests", "pytest selfdrive/test/test_onroad.py -s", [timeout: 60]),
])
},
'HW + Unit Tests': {
deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
step("build", "cd openpilot/system/manager && ./build.py"),
step("test power draw", "./openpilot/selfdrive/test/test_power_draw.py"),
step("test encoder", "./openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
step("test manager", "./openpilot/system/manager/test/test_manager.py"),
step("build", "cd system/manager && ./build.py"),
step("test power draw", "pytest -s system/hardware/tici/tests/test_power_draw.py"),
step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py", [diffPaths: ["system/loggerd/"]]),
step("test manager", "pytest system/manager/test/test_manager.py"),
])
},
'camerad OX03C10': {
deviceStage("OX03C10", "tizi-ox03c10", ["UNSAFE=1"], [
step("build", "cd openpilot/system/manager && ./build.py"),
step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
step("build", "cd system/manager && ./build.py"),
step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
])
},
'camerad OS04C10': {
deviceStage("OS04C10", "tici-os04c10", ["UNSAFE=1"], [
step("build", "cd openpilot/system/manager && ./build.py"),
step("test pandad", "./openpilot/selfdrive/pandad/tests/test_pandad.py"),
step("test camerad", "./openpilot/system/camerad/test/test_camerad.py", [timeout: 90]),
step("build", "cd system/manager && ./build.py"),
step("test pandad", "pytest selfdrive/pandad/tests/test_pandad.py", [diffPaths: ["panda", "selfdrive/pandad/"]]),
step("test camerad", "pytest system/camerad/test/test_camerad.py", [timeout: 90]),
])
},
'sensord': {
deviceStage("LSM + MMC", "tizi-lsmc", ["UNSAFE=1"], [
step("build", "cd openpilot/system/manager && ./build.py"),
step("test sensord", "./openpilot/system/sensord/tests/test_sensord.py"),
step("build", "cd system/manager && ./build.py"),
step("test sensord", "pytest system/sensord/tests/test_sensord.py"),
])
},
'replay': {
deviceStage("model-replay", "tizi-replay", ["UNSAFE=1"], [
step("build", "cd openpilot/system/manager && ./build.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
step("model replay", "openpilot/selfdrive/test/process_replay/model_replay.py", [diffPaths: ["openpilot/selfdrive/modeld/", "tinygrad_repo", "openpilot/selfdrive/test/process_replay/model_replay.py"]]),
step("build", "cd system/manager && ./build.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
step("model replay", "selfdrive/test/process_replay/model_replay.py", [diffPaths: ["selfdrive/modeld/", "tinygrad_repo", "selfdrive/test/process_replay/model_replay.py"]]),
])
},
'tizi': {
deviceStage("tizi", "tizi", ["UNSAFE=1"], [
step("build openpilot", "cd openpilot/system/manager && ./build.py"),
step("test pandad loopback", "./openpilot/selfdrive/pandad/tests/test_pandad_loopback.py"),
step("test pandad spi", "./openpilot/selfdrive/pandad/tests/test_pandad_spi.py"),
step("test amp", "./openpilot/common/hardware/comma/tests/test_amplifier.py"),
step("build openpilot", "cd system/manager && ./build.py"),
step("test pandad loopback", "pytest selfdrive/pandad/tests/test_pandad_loopback.py"),
step("test pandad spi", "pytest selfdrive/pandad/tests/test_pandad_spi.py"),
step("test amp", "pytest system/hardware/tici/tests/test_amplifier.py"),
step("test qcomgpsd", "pytest system/qcomgpsd/tests/test_qcomgpsd.py", [diffPaths: ["system/qcomgpsd/"]]),
])
},
+3 -26
View File
@@ -1,31 +1,8 @@
Version 0.11.2 (2026-08-12)
=======================
* New driving model
* Big model with 880M parameters
* Support for big models running on an external GPU
* Live stream cameras from comma connect
* Generate dashcam clips from comma connect
* Remote comma body control from comma connect
* New alert sounds
* CUPRA Born 2021-2023 support thanks to DaHansi!
* Volkswagen ID.4 2021-2025 support thanks to DaHansi!
Version 0.11.1 (2026-05-18)
Version 0.10.4 (2026-02-17)
========================
* New driver monitoring model
* Improved image processing pipeline for driver camera
* Improved thermal policy for comma four
* Acura MDX 2022-24 support thanks to mvl-boston!
* Rivian R1S and R1T 2025 support thanks to lukasloetkolben!
Version 0.11.0 (2026-03-17)
========================
* New driving model #36798
* Fully trained using a learned simulator
* Improved longitudinal performance in Experimental mode
* Reduce comma four standby power usage by 77% to 52 mW
* Kia K7 2017 support thanks to royjr!
* Lexus LS 2018 support thanks to Hacheoy!
* Reduce comma four standby power usage by 77% to 52 mW
Version 0.10.3 (2025-12-17)
========================
@@ -1112,7 +1089,7 @@ Version 0.2.1 (2016-12-14)
Version 0.2 (2016-12-12)
=========================
* Car/Radar abstraction layers have shipped, see openpilot/cereal/car.capnp
* Car/Radar abstraction layers have shipped, see cereal/car.capnp
* controlsd has been refactored
* Shipped plant model and testing maneuvers
* visiond exits more gracefully now
+94 -199
View File
@@ -4,136 +4,73 @@ import sys
import sysconfig
import platform
import shlex
import importlib
import numpy as np
import SCons.Errors
from SCons.Defaults import _stripixes
COMMA_HARDWARE = os.path.isfile('/AGNOS')
SCons.Warnings.warningAsException(True)
Decider('MD5-timestamp')
SetOption('num_jobs', max(1, int(os.cpu_count()/(1 if "CI" in os.environ else 2))))
SetOption('num_jobs', max(1, int(os.cpu_count()/2)))
AddOption('--asan', action='store_true', help='turn on ASAN')
AddOption('--ubsan', action='store_true', help='turn on UBSan')
AddOption('--mutation', action='store_true', help='generate mutation-ready code')
AddOption('--ccflags', action='store', type='string', default='', help='pass arbitrary flags over the command line')
AddOption('--verbose', action='store_true', default=False, help='show full build commands')
release = not os.path.exists(File('#.gitattributes').abspath) # file absent on release branch, see release_files.py
AddOption('--minimal',
action='store_false',
dest='extras',
default=(not COMMA_HARDWARE and not release),
default=os.path.exists(File('#.gitattributes').abspath), # minimal by default on release branch (where there's no LFS)
help='the minimum build to run openpilot. no tests, tools, etc.')
submodule_python_paths = [
Dir("#").abspath,
Dir("#msgq_repo").abspath,
Dir("#opendbc_repo").abspath,
Dir("#rednose_repo").abspath,
Dir("#teleoprtc_repo").abspath,
Dir("#tinygrad_repo").abspath,
]
for p in reversed(submodule_python_paths):
if p not in sys.path:
sys.path.insert(0, p)
if external_pythonpath := os.environ.get("PYTHONPATH"):
submodule_python_paths += [p for p in external_pythonpath.split(os.pathsep) if p and p not in submodule_python_paths]
# Detect platform
arch = subprocess.check_output(["uname", "-m"], encoding='utf8').rstrip()
if platform.system() == "Darwin":
arch = "Darwin"
elif arch == "aarch64" and COMMA_HARDWARE:
arch = "comma_arm64"
elif arch == "aarch64" and os.path.isfile('/TICI'):
arch = "larch64"
assert arch in [
"comma_arm64", # linux comma hardware (AGNOS) arm64
"aarch64", # linux pc arm64
"x86_64", # linux pc x64
"Darwin", # macOS arm64 (x86 not supported)
"larch64", # linux tici arm64
"aarch64", # linux pc arm64
"x86_64", # linux pc x64
"Darwin", # macOS arm64 (x86 not supported)
]
pkg_names = ['acados', 'capnproto', 'eigen', 'ffmpeg', 'json11', 'ncurses', 'zeromq', 'zstd']
pkgs = [importlib.import_module(name) for name in pkg_names]
acados = pkgs[pkg_names.index('acados')]
ffmpeg = pkgs[pkg_names.index('ffmpeg')]
# Shared package ships .so/.dylib; older device venvs still have static .a only.
# Keep static link deps (x264/z/va/drm) when the installed package is static so
# COMMA_HARDWARE CI works without upgrading the device venv yet.
# TODO: drop the static fallback once device venvs have comma-deps-ffmpeg>=7.1.0.post94
_ffmpeg_lib_names = os.listdir(ffmpeg.LIB_DIR) if os.path.isdir(ffmpeg.LIB_DIR) else []
ffmpeg_shared = any(
n.startswith('libavcodec.so') or (n.startswith('libavcodec') and n.endswith('.dylib'))
for n in _ffmpeg_lib_names
)
ffmpeg_libs = ['avformat', 'avcodec', 'swresample', 'avutil']
if not ffmpeg_shared:
ffmpeg_libs += ['x264', 'z']
if arch != "Darwin":
ffmpeg_libs += ['va', 'va-drm', 'drm']
acados_include_dirs = [
acados.INCLUDE_DIR,
os.path.join(acados.INCLUDE_DIR, "blasfeo", "include"),
os.path.join(acados.INCLUDE_DIR, "hpipm", "include"),
]
# ***** enforce a whitelist of system libraries *****
# this prevents silently relying on a 3rd party package,
# e.g. apt-installed libusb. all libraries should either
# be distributed with all Linux distros and macOS, or
# vendored in commaai/dependencies.
allowed_system_libs = {
"EGL", "GLESv2", "GL",
"Qt5Charts", "Qt5Core", "Qt5Gui", "Qt5Widgets",
"dl", "drm", "gbm", "m", "pthread",
}
def _resolve_lib(env, name):
for d in env.Flatten(env.get('LIBPATH', [])):
p = Dir(str(d)).abspath
for ext in ('.a', '.so', '.dylib'):
f = File(os.path.join(p, f'lib{name}{ext}'))
if f.exists() or f.has_builder():
return name
if name in allowed_system_libs:
return name
raise SCons.Errors.UserError(f"Unexpected non-vendored library '{name}'")
def _libflags(target, source, env, for_signature):
libs = []
lp = env.subst('$LIBLITERALPREFIX')
for lib in env.Flatten(env.get('LIBS', [])):
if isinstance(lib, str):
if os.sep in lib or lib.startswith('#'):
libs.append(File(lib))
elif lib.startswith('-') or (lp and lib.startswith(lp)):
libs.append(lib)
else:
libs.append(_resolve_lib(env, lib))
else:
libs.append(lib)
return _stripixes(env['LIBLINKPREFIX'], libs, env['LIBLINKSUFFIX'],
env['LIBPREFIXES'], env['LIBSUFFIXES'], env, env['LIBLITERALPREFIX'])
if arch != "larch64":
import bzip2
import capnproto
import eigen
import ffmpeg as ffmpeg_pkg
import libjpeg
import libyuv
import ncurses
import python3_dev
import zeromq
import zstd
pkgs = [bzip2, capnproto, eigen, ffmpeg_pkg, libjpeg, libyuv, ncurses, zeromq, zstd]
py_include = python3_dev.INCLUDE_DIR
else:
# TODO: remove when AGNOS has our new vendor pkgs
pkgs = []
py_include = sysconfig.get_paths()['include']
env = Environment(
ENV={
"PATH": os.environ['PATH'],
"PYTHONPATH": os.pathsep.join(submodule_python_paths),
"ACADOS_SOURCE_DIR": acados.DIR,
"ACADOS_PYTHON_INTERFACE_PATH": acados.TEMPLATE_DIR,
"TERA_PATH": acados.TERA_PATH
"PYTHONPATH": Dir("#").abspath + ':' + Dir(f"#third_party/acados").abspath,
"ACADOS_SOURCE_DIR": Dir("#third_party/acados").abspath,
"ACADOS_PYTHON_INTERFACE_PATH": Dir("#third_party/acados/acados_template").abspath,
"TERA_PATH": Dir("#").abspath + f"/third_party/acados/{arch}/t_renderer"
},
CCFLAGS=[
"-g",
"-fPIC",
"-pipe",
"-O2",
"-Wunused",
"-Werror",
"-Wshadow" if arch in ("Darwin", "comma_arm64") else "-Wshadow=local",
"-Wshadow" if arch in ("Darwin", "larch64") else "-Wshadow=local",
"-Wno-unknown-warning-option",
"-Wno-inconsistent-missing-override",
"-Wno-c99-designator",
@@ -143,46 +80,44 @@ env = Environment(
CFLAGS=["-std=gnu11"],
CXXFLAGS=["-std=c++1z"],
CPPPATH=[
"#openpilot",
"#msgq_repo", # #include "msgq/..."
"#opendbc_repo", # #include "opendbc/..."
"#rednose_repo", # #include "rednose/..."
"#rednose_repo/rednose", # #include "logger/..." (rednose package root)
"#openpilot/cereal/gen/cpp",
acados_include_dirs,
[x.INCLUDE_DIR for x in pkgs],
"#",
"#msgq",
"#third_party",
"#third_party/json11",
"#third_party/linux/include",
"#third_party/acados/include",
"#third_party/acados/include/blasfeo/include",
"#third_party/acados/include/hpipm/include",
"#third_party/catch2/include",
[x.INCLUDE_DIR for x in pkgs],
],
LIBPATH=[
"#openpilot/common",
"#common",
"#msgq_repo",
"#openpilot/selfdrive/pandad",
"#rednose_repo/rednose/helpers",
"#third_party",
"#selfdrive/pandad",
"#rednose/helpers",
f"#third_party/acados/{arch}/lib",
[x.LIB_DIR for x in pkgs],
],
RPATH=[ffmpeg.LIB_DIR] if ffmpeg_shared else [],
RPATH=[],
CYTHONCFILESUFFIX=".cpp",
COMPILATIONDB_USE_ABSPATH=True,
REDNOSE_ROOT="#rednose_repo",
REDNOSE_ROOT="#",
tools=["default", "cython", "compilation_db", "rednose_filter"],
toolpath=["#msgq_repo/site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
toolpath=["#site_scons/site_tools", "#rednose_repo/site_scons/site_tools"],
)
# SCons' Darwin linker tool doesn't define the variables used to expand RPATH.
if arch == "Darwin":
env["RPATHPREFIX"] = "-Wl,-rpath,"
env["RPATHSUFFIX"] = ""
env["_RPATH"] = "${_concat(RPATHPREFIX, RPATH, RPATHSUFFIX, __env__)}"
if arch != "comma_arm64":
env['_LIBFLAGS'] = _libflags
# Arch-specific flags and paths
if arch == "comma_arm64":
if arch == "larch64":
env["CC"] = "clang"
env["CXX"] = "clang++"
env.Append(LIBPATH=[
"/usr/local/lib",
"/system/vendor/lib64",
"/usr/lib/aarch64-linux-gnu",
])
arch_flags = ["-D__COMMA_HARDWARE__", "-mcpu=cortex-a57"]
arch_flags = ["-D__TICI__", "-mcpu=cortex-a57", "-DQCOM2"]
env.Append(CCFLAGS=arch_flags)
env.Append(CXXFLAGS=arch_flags)
elif arch == "Darwin":
@@ -191,6 +126,19 @@ elif arch == "Darwin":
])
env.Append(CCFLAGS=["-DGL_SILENCE_DEPRECATION"])
env.Append(CXXFLAGS=["-DGL_SILENCE_DEPRECATION"])
else:
env.Append(LIBPATH=[
"/usr/lib",
"/usr/local/lib",
])
# Sanitizers and extra CCFLAGS from CLI
if GetOption('asan'):
env.Append(CCFLAGS=["-fsanitize=address", "-fno-omit-frame-pointer"])
env.Append(LINKFLAGS=["-fsanitize=address"])
elif GetOption('ubsan'):
env.Append(CCFLAGS=["-fsanitize=undefined"])
env.Append(LINKFLAGS=["-fsanitize=undefined"])
_extra_cc = shlex.split(GetOption('ccflags') or '')
if _extra_cc:
@@ -216,9 +164,19 @@ if not GetOption('verbose'):
):
env[f"{action}COMSTR"] = f" [{short}] $TARGET"
# progress output
node_interval = 5
node_count = 0
def progress_function(node):
global node_count
node_count += node_interval
sys.stderr.write("progress: %d\n" % node_count)
if os.environ.get('SCONS_PROGRESS'):
Progress(progress_function, interval=node_interval)
# ********** Cython build environment **********
envCython = env.Clone()
envCython["CPPPATH"] += [sysconfig.get_paths()['include'], np.get_include()]
envCython["CPPPATH"] += [py_include, np.get_include()]
envCython["CCFLAGS"] += ["-Wno-#warnings", "-Wno-cpp", "-Wno-shadow", "-Wno-deprecated-declarations"]
envCython["CCFLAGS"].remove("-Werror")
@@ -231,28 +189,18 @@ else:
np_version = SCons.Script.Value(np.__version__)
Export('envCython', 'np_version')
Export('env', 'arch', 'acados', 'release', 'ffmpeg_libs')
Export('env', 'arch')
# Setup cache dir
default_cache_dir = '/data/scons_cache' if arch == "comma_arm64" else '/tmp/scons_cache'
default_cache_dir = '/data/scons_cache' if arch == "larch64" else '/tmp/scons_cache'
cache_dir = ARGUMENTS.get('cache_dir', default_cache_dir)
cache_size_limit = 4e9 if "CI" in os.environ else 2e9
CacheDir(cache_dir)
Clean(["."], cache_dir)
def prune_cache_dir(target=None, source=None, env=None):
cache_files = sorted((os.path.join(root, f) for root, _, files in os.walk(cache_dir) for f in files), key=os.path.getmtime)
cache_size = sum(os.path.getsize(f) for f in cache_files)
for f in cache_files:
if cache_size < cache_size_limit:
break
cache_size -= os.path.getsize(f)
os.unlink(f)
# ********** start building stuff **********
# Build common module
SConscript(['openpilot/common/SConscript'])
SConscript(['common/SConscript'])
Import('_common')
common = [_common, 'json11', 'zmq']
Export('common')
@@ -262,8 +210,9 @@ Export('common')
env_swaglog = env.Clone()
env_swaglog['CXXFLAGS'].append('-DSWAGLOG="\\"common/swaglog.h\\""')
SConscript(['msgq_repo/SConscript'], exports={'env': env_swaglog})
SConscript(['opendbc_repo/SConscript'], exports={'env': env_swaglog})
SConscript(['openpilot/cereal/SConscript'])
SConscript(['cereal/SConscript'])
Import('socketmaster', 'msgq')
messaging = [socketmaster, msgq, 'capnp', 'kj',]
@@ -274,79 +223,25 @@ Export('messaging')
SConscript(['panda/SConscript'])
# Build rednose library
SConscript(['rednose_repo/rednose/SConscript'])
SConscript(['rednose/SConscript'])
# Build system services
SConscript([
'openpilot/system/loggerd/SConscript',
'system/loggerd/SConscript',
])
if arch == "comma_arm64":
SConscript(['openpilot/system/camerad/SConscript'])
if arch == "larch64":
SConscript(['system/camerad/SConscript'])
# Build selfdrive
SConscript([
'openpilot/selfdrive/pandad/SConscript',
'openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript',
'openpilot/selfdrive/locationd/SConscript',
'openpilot/selfdrive/modeld/SConscript',
'openpilot/selfdrive/ui/SConscript',
])
# Build openpilot
SConscript(['third_party/SConscript'])
SConscript(['openpilot/sunnypilot/SConscript'])
SConscript(['selfdrive/SConscript'])
# Build desktop-only tools
if GetOption('extras') and arch != "comma_arm64":
SConscript([
'openpilot/tools/replay/SConscript',
'openpilot/tools/cabana/SConscript',
'openpilot/tools/jotpluggler/SConscript',
])
SConscript(['sunnypilot/SConscript'])
if Dir('#tools/cabana/').exists() and arch != "larch64":
SConscript(['tools/cabana/SConscript'])
env.CompilationDatabase('compile_commands.json')
# progress output
def count_scons_nodes(nodes):
seen = set()
stack = list(nodes)
while stack:
node = stack.pop().disambiguate()
if node in seen:
continue
seen.add(node)
if hasattr(node, 'has_builder') and node.has_builder():
build_product_nodes.add(node)
executor = node.get_executor()
if executor is not None:
stack += executor.get_all_prerequisites() + executor.get_all_children()
return len(seen)
progress_interval = 5
progress_count = 0
build_product_nodes = set()
progress_total = max(1, count_scons_nodes(env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)))
def progress_function(node):
global progress_count
if progress_count >= progress_total:
return
progress_count = min(progress_count + progress_interval, progress_total)
progress = round(100. * progress_count / progress_total, 1)
sys.stderr.write("\rBuilding: %5.1f%%" % progress if sys.stderr.isatty() else "progress: %.1f\n" % progress)
if progress == 100. and sys.stderr.isatty():
sys.stderr.write("\n")
sys.stderr.flush()
Progress(progress_function, interval=progress_interval)
AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
def check_build_product_size(target, source, env):
limit = 50 * 1024 * 1024 # GitHub max size
for t in target:
if hasattr(t, 'isfile') and t.isfile() and (size := os.path.getsize(t.abspath)) > limit:
raise SCons.Errors.UserError(f"{t} is {size / (1024 * 1024):.1f} MiB, exceeding the {limit / (1024 * 1024):.1f} MiB limit")
if not GetOption('extras'):
AddPostAction(list(build_product_nodes), Action(check_build_product_size, None))
@@ -24,17 +24,17 @@ things are not. Read more details [here](https://capnproto.org/language.html).
### Custom forks
Forks of [openpilot](https://github.com/commaai/openpilot) might want to add things to the messaging
spec, however this could conflict with future changes made in mainline openpilot's cereal spec. Rebasing against mainline openpilot
spec, however this could conflict with future changes made in mainline cereal/openpilot. Rebasing against mainline openpilot
then means breaking backwards-compatibility with all old logs of your fork. So we added reserved events in
[custom.capnp](custom.capnp) that we will leave empty in mainline openpilot's cereal spec. **If you only modify those, you can ensure your
[custom.capnp](custom.capnp) that we will leave empty in mainline cereal/openpilot. **If you only modify those, you can ensure your
fork will remain backwards-compatible with all versions of mainline openpilot and your fork.**
An example of compatible changes:
```diff
diff --git a/openpilot/cereal/custom.capnp b/openpilot/cereal/custom.capnp
diff --git a/cereal/custom.capnp b/cereal/custom.capnp
index 3348e859e..3365c7b98 100644
--- a/openpilot/cereal/custom.capnp
+++ b/openpilot/cereal/custom.capnp
--- a/cereal/custom.capnp
+++ b/cereal/custom.capnp
@@ -10,7 +10,11 @@ $Cxx.namespace("cereal");
# DO rename the structs
# DON'T change the identifier (e.g. @0x81c2f05a394cf4af)
@@ -48,10 +48,10 @@ index 3348e859e..3365c7b98 100644
}
struct CustomReserved1 @0xaedffd8f31e7b55d {
diff --git a/openpilot/cereal/log.capnp b/openpilot/cereal/log.capnp
diff --git a/cereal/log.capnp b/cereal/log.capnp
index 1209f3fd9..b189f58b6 100644
--- a/openpilot/cereal/log.capnp
+++ b/openpilot/cereal/log.capnp
--- a/cereal/log.capnp
+++ b/cereal/log.capnp
@@ -2558,14 +2558,14 @@ struct Event {
# DO change the name of the field
@@ -76,7 +76,7 @@ index 1209f3fd9..b189f58b6 100644
Example
---
```python
import openpilot.cereal.messaging as messaging
import cereal.messaging as messaging
# in subscriber
sm = messaging.SubMaster(['sensorEvents'])
+20
View File
@@ -0,0 +1,20 @@
Import('env', 'common', 'msgq')
cereal_dir = Dir('.')
gen_dir = Dir('gen')
# Build cereal
schema_files = ['log.capnp', 'car.capnp', 'legacy.capnp', 'custom.capnp']
env.Command([f'gen/cpp/{s}.c++' for s in schema_files] + [f'gen/cpp/{s}.h' for s in schema_files],
schema_files,
f"capnpc --src-prefix={cereal_dir.path} $SOURCES -o c++:{gen_dir.path}/cpp/")
cereal = env.Library('cereal', [f'gen/cpp/{s}.c++' for s in schema_files])
# Build messaging
services_h = env.Command(['services.h'], ['services.py'], 'python3 ' + cereal_dir.path + '/services.py > $TARGET')
env.Program('messaging/bridge', ['messaging/bridge.cc', 'messaging/msgq_to_zmq.cc', 'messaging/bridge_zmq.cc'], LIBS=[msgq, common, 'pthread'])
socketmaster = env.Library('socketmaster', ['messaging/socketmaster.cc'])
Export('cereal', 'socketmaster')
+11
View File
@@ -0,0 +1,11 @@
import os
import capnp
from importlib.resources import as_file, files
capnp.remove_import_hook()
with as_file(files("cereal")) as fspath:
CEREAL_PATH = fspath.as_posix()
log = capnp.load(os.path.join(CEREAL_PATH, "log.capnp"))
car = capnp.load(os.path.join(CEREAL_PATH, "car.capnp"))
custom = capnp.load(os.path.join(CEREAL_PATH, "custom.capnp"))
+1
View File
@@ -0,0 +1 @@
../opendbc_repo/opendbc/car/car.capnp
@@ -1,4 +1,4 @@
using Cxx = import "/include/c++.capnp";
using Cxx = import "./include/c++.capnp";
$Cxx.namespace("cereal");
@0xb526ba661d550a59;
@@ -69,8 +69,6 @@ struct LeadData {
struct SelfdriveStateSP @0x81c2f05a394cf4af {
mads @0 :ModularAssistiveDrivingSystem;
intelligentCruiseButtonManagement @1 :IntelligentCruiseButtonManagement;
buttonsPressed @2 :UInt16;
buttonsReleaseToggle @3 :UInt16;
enum AudibleAlert {
none @0;
@@ -139,16 +137,10 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
eta @2 :UInt32;
}
struct Chunk {
fileName @0 :Text;
sha256 @1 :Text;
}
struct Artifact {
fileName @0 :Text;
downloadUri @1 :DownloadUri;
downloadProgress @2 :DownloadProgress;
chunks @3 :List(Chunk);
}
struct Model {
@@ -162,8 +154,6 @@ struct ModelManagerSP @0xaedffd8f31e7b55d {
vision @2;
policy @3;
offPolicy @4;
onPolicy @5;
chunked @6;
}
}
@@ -351,7 +341,6 @@ struct OnroadEventSP @0xda96579883444c35 {
speedLimitChanged @21;
speedLimitPending @22;
e2eChime @23;
laneChangeRoadEdge @24;
}
}
@@ -458,8 +447,6 @@ struct LiveMapDataSP @0xf416ec09499d9d19 {
struct ModelDataV2SP @0xa1680744031fdb2d {
laneTurnDirection @0 :TurnDirection;
leftLaneChangeEdgeBlock @1 :Bool;
rightLaneChangeEdgeBlock @2 :Bool;
enum TurnDirection {
none @0;
@@ -1,9 +1,9 @@
using Cxx = import "/include/c++.capnp";
using Cxx = import "./include/c++.capnp";
$Cxx.namespace("cereal");
@0x80ef1ec4889c2a63;
# deprecated.capnp: a home for deprecated structs
# legacy.capnp: a home for deprecated structs
struct LogRotate @0x9811e1f38f62f2d1 {
segmentNum @0 :Int32;
@@ -571,217 +571,4 @@ struct LidarPts @0xe3d6685d4e9d8f7a {
pkt @4 :Data;
}
struct LiveTracksDEPRECATED @0xb16f60103159415a {
trackId @0 :Int32;
dRel @1 :Float32;
yRel @2 :Float32;
vRel @3 :Float32;
aRel @4 :Float32;
timeStamp @5 :Float32;
status @6 :Float32;
currentTime @7 :Float32;
stationary @8 :Bool;
oncoming @9 :Bool;
}
struct LiveMpcData @0x92a5e332a85f32a0 {
x @0 :List(Float32);
y @1 :List(Float32);
psi @2 :List(Float32);
curvature @3 :List(Float32);
qpIterations @4 :UInt32;
calculationTime @5 :UInt64;
cost @6 :Float64;
}
struct LiveLongitudinalMpcData @0xe7e17c434f865ae2 {
xEgo @0 :List(Float32);
vEgo @1 :List(Float32);
aEgo @2 :List(Float32);
xLead @3 :List(Float32);
vLead @4 :List(Float32);
aLead @5 :List(Float32);
aLeadTau @6 :Float32; # lead accel time constant
qpIterations @7 :UInt32;
mpcId @8 :UInt32;
calculationTime @9 :UInt64;
cost @10 :Float64;
}
struct DriverStateDEPRECATED @0xb83c6cc593ed0a00 {
frameId @0 :UInt32;
modelExecutionTime @14 :Float32;
dspExecutionTime @16 :Float32;
rawPredictions @15 :Data;
faceOrientation @3 :List(Float32);
facePosition @4 :List(Float32);
faceProb @5 :Float32;
leftEyeProb @6 :Float32;
rightEyeProb @7 :Float32;
leftBlinkProb @8 :Float32;
rightBlinkProb @9 :Float32;
faceOrientationStd @11 :List(Float32);
facePositionStd @12 :List(Float32);
sunglassesProb @13 :Float32;
poorVision @17 :Float32;
partialFace @18 :Float32;
distractedPose @19 :Float32;
distractedEyes @20 :Float32;
eyesOnRoad @21 :Float32;
phoneUse @22 :Float32;
occludedProb @23 :Float32;
readyProb @24 :List(Float32);
notReadyProb @25 :List(Float32);
irPwrDEPRECATED @10 :Float32;
descriptorDEPRECATED @1 :List(Float32);
stdDEPRECATED @2 :Float32;
}
struct NavModelData @0xac3de5c437be057a {
frameId @0 :UInt32;
locationMonoTime @6 :UInt64;
modelExecutionTime @1 :Float32;
dspExecutionTime @2 :Float32;
features @3 :List(Float32);
# predicted future position
position @4 :XYData;
desirePrediction @5 :List(Float32);
# All SI units and in device frame
struct XYData @0xbe09e615b2507e26 {
x @0 :List(Float32);
y @1 :List(Float32);
xStd @2 :List(Float32);
yStd @3 :List(Float32);
}
}
struct AndroidBuildInfo @0xfe2919d5c21f426c {
board @0 :Text;
bootloader @1 :Text;
brand @2 :Text;
device @3 :Text;
display @4 :Text;
fingerprint @5 :Text;
hardware @6 :Text;
host @7 :Text;
id @8 :Text;
manufacturer @9 :Text;
model @10 :Text;
product @11 :Text;
radioVersion @12 :Text;
serial @13 :Text;
supportedAbis @14 :List(Text);
tags @15 :Text;
time @16 :Int64;
type @17 :Text;
user @18 :Text;
versionCodename @19 :Text;
versionRelease @20 :Text;
versionSdk @21 :Int32;
versionSecurityPatch @22 :Text;
}
struct AndroidSensor @0x9b513b93a887dbcd {
id @0 :Int32;
name @1 :Text;
vendor @2 :Text;
version @3 :Int32;
handle @4 :Int32;
type @5 :Int32;
maxRange @6 :Float32;
resolution @7 :Float32;
power @8 :Float32;
minDelay @9 :Int32;
fifoReservedEventCount @10 :UInt32;
fifoMaxEventCount @11 :UInt32;
stringType @12 :Text;
maxDelay @13 :Int32;
}
struct IosBuildInfo @0xd97e3b28239f5580 {
appVersion @0 :Text;
appBuild @1 :UInt32;
osVersion @2 :Text;
deviceModel @3 :Text;
}
enum FrameTypeDEPRECATED @0xa37f0d8558e193fd {
unknown @0;
neo @1;
chffrAndroid @2;
front @3;
}
struct AndroidCaptureResult @0xbcc3efbac41d2048 {
sensitivity @0 :Int32;
frameDuration @1 :Int64;
exposureTime @2 :Int64;
rollingShutterSkew @3 :UInt64;
colorCorrectionTransform @4 :List(Int32);
colorCorrectionGains @5 :List(Float32);
displayRotation @6 :Int8;
}
enum UsbPowerModeDEPRECATED @0xa8883583b32c9877 {
none @0;
client @1;
cdp @2;
dcp @3;
}
struct LateralINDIState @0x939463348632375e {
active @0 :Bool;
steeringAngleDeg @1 :Float32;
steeringRateDeg @2 :Float32;
steeringAccelDeg @3 :Float32;
rateSetPoint @4 :Float32;
accelSetPoint @5 :Float32;
accelError @6 :Float32;
delayedOutput @7 :Float32;
delta @8 :Float32;
output @9 :Float32;
saturated @10 :Bool;
steeringAngleDesiredDeg @11 :Float32;
steeringRateDesiredDeg @12 :Float32;
}
struct LateralLQRState @0x9024e2d790c82ade {
active @0 :Bool;
steeringAngleDeg @1 :Float32;
i @2 :Float32;
output @3 :Float32;
lqrOutput @4 :Float32;
saturated @5 :Bool;
steeringAngleDesiredDeg @6 :Float32;
}
struct LateralPlannerSolution @0x84caeca5a6b4acfe {
x @0 :List(Float32);
y @1 :List(Float32);
yaw @2 :List(Float32);
yawRate @3 :List(Float32);
xStd @4 :List(Float32);
yStd @5 :List(Float32);
yawStd @6 :List(Float32);
yawRateStd @7 :List(Float32);
}
struct GpsTrajectory @0x8cfeb072f5301000 {
x @0 :List(Float32);
y @1 :List(Float32);
}
struct AudioFeedbackDEPRECATED @0xed47e3c075be372a {
audio @0 :AudioData;
blockNum @1 :UInt16;
struct AudioData {
data @0 :Data;
sampleRate @1 :UInt32;
}
}
File diff suppressed because it is too large Load Diff
@@ -7,43 +7,12 @@ import os
import capnp
import time
from typing import Union
from typing import Optional, List, Union, Dict
from openpilot.cereal import log
from openpilot.cereal.services import SERVICE_LIST
from cereal import log
from cereal.services import SERVICE_LIST
from openpilot.common.utils import MovingAverage
__all__ = (
"NO_TRAVERSAL_LIMIT",
"Context",
"FrequencyTracker",
"IpcError",
"MultiplePublishersError",
"Poller",
"PubMaster",
"PubSocket",
"SocketEventHandle",
"SubMaster",
"SubSocket",
"delete_fake_prefix",
"drain_sock",
"drain_sock_raw",
"fake_event_handle",
"get_fake_prefix",
"log_from_bytes",
"new_message",
"pub_sock",
"recv_one",
"recv_one_or_none",
"recv_one_retry",
"recv_sock",
"reset_context",
"set_fake_prefix",
"sub_sock",
"toggle_fake_events",
"wait_for_one_event",
)
NO_TRAVERSAL_LIMIT = 2**64-1
@@ -53,8 +22,8 @@ def pub_sock(endpoint: str) -> PubSocket:
return msgq.pub_sock(endpoint, segment_size)
def sub_sock(endpoint: str, poller: Poller | None = None, addr: str = "127.0.0.1",
conflate: bool = False, timeout: int | None = None) -> SubSocket:
def sub_sock(endpoint: str, poller: Optional[Poller] = None, addr: str = "127.0.0.1",
conflate: bool = False, timeout: Optional[int] = None) -> SubSocket:
service = SERVICE_LIST.get(endpoint)
segment_size = service.queue_size if service else 0
return msgq.sub_sock(endpoint, poller=poller, addr=addr, conflate=conflate,
@@ -70,7 +39,7 @@ def log_from_bytes(dat: bytes, struct: capnp.lib.capnp._StructModule = log.Event
return msg
def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
def new_message(service: Optional[str], size: Optional[int] = None, **kwargs) -> capnp.lib.capnp._DynamicStructBuilder:
args = {
'valid': False,
'logMonoTime': int(time.monotonic() * 1e9),
@@ -85,14 +54,14 @@ def new_message(service: str | None, size: int | None = None, **kwargs) -> capnp
return dat
def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> list[capnp.lib.capnp._DynamicStructReader]:
def drain_sock(sock: SubSocket, wait_for_one: bool = False) -> List[capnp.lib.capnp._DynamicStructReader]:
"""Receive all message currently available on the queue"""
msgs = drain_sock_raw(sock, wait_for_one=wait_for_one)
return [log_from_bytes(m) for m in msgs]
# TODO: print when we drop packets?
def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicStructReader | None:
def recv_sock(sock: SubSocket, wait: bool = False) -> Optional[capnp.lib.capnp._DynamicStructReader]:
"""Same as drain sock, but only returns latest message. Consider using conflate instead."""
dat = None
@@ -113,14 +82,14 @@ def recv_sock(sock: SubSocket, wait: bool = False) -> capnp.lib.capnp._DynamicSt
return dat
def recv_one(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
def recv_one(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
dat = sock.receive()
if dat is not None:
dat = log_from_bytes(dat)
return dat
def recv_one_or_none(sock: SubSocket) -> capnp.lib.capnp._DynamicStructReader | None:
def recv_one_or_none(sock: SubSocket) -> Optional[capnp.lib.capnp._DynamicStructReader]:
dat = sock.receive(non_blocking=True)
if dat is not None:
dat = log_from_bytes(dat)
@@ -179,27 +148,27 @@ class FrequencyTracker:
class SubMaster:
def __init__(self, services: list[str], poll: str | None = None,
ignore_alive: list[str] | None = None, ignore_avg_freq: list[str] | None = None,
ignore_valid: list[str] | None = None, addr: str = "127.0.0.1", frequency: float | None = None):
def __init__(self, services: List[str], poll: Optional[str] = None,
ignore_alive: Optional[List[str]] = None, ignore_avg_freq: Optional[List[str]] = None,
ignore_valid: Optional[List[str]] = None, addr: str = "127.0.0.1", frequency: Optional[float] = None):
self.frame = -1
self.services = services
self.seen = dict.fromkeys(services, False)
self.updated = dict.fromkeys(services, False)
self.recv_time = dict.fromkeys(services, 0.0)
self.recv_frame = dict.fromkeys(services, 0)
self.seen = {s: False for s in services}
self.updated = {s: False for s in services}
self.recv_time = {s: 0. for s in services}
self.recv_frame = {s: 0 for s in services}
self.sock = {}
self.data = {}
self.logMonoTime = dict.fromkeys(services, 0)
self.logMonoTime = {s: 0 for s in services}
# zero-frequency / on-demand services are always alive and presumed valid; all others must pass checks
on_demand = {s: SERVICE_LIST[s].frequency <= 1e-5 for s in services}
self.static_freq_services = {s for s in services if not on_demand[s]}
self.static_freq_services = set(s for s in services if not on_demand[s])
self.alive = {s: on_demand[s] for s in services}
self.freq_ok = {s: on_demand[s] for s in services}
self.valid = {s: on_demand[s] for s in services}
self.freq_tracker: dict[str, FrequencyTracker] = {}
self.freq_tracker: Dict[str, FrequencyTracker] = {}
self.poller = Poller()
polled_services = set([poll, ] if poll is not None else services)
self.non_polled_services = set(services) - polled_services
@@ -242,7 +211,7 @@ class SubMaster:
msgs.append(recv_one_or_none(self.sock[s]))
self.update_msgs(time.monotonic(), msgs)
def update_msgs(self, cur_time: float, msgs: list[capnp.lib.capnp._DynamicStructReader]) -> None:
def update_msgs(self, cur_time: float, msgs: List[capnp.lib.capnp._DynamicStructReader]) -> None:
self.frame += 1
self.updated = dict.fromkeys(self.services, False)
for msg in msgs:
@@ -265,21 +234,21 @@ class SubMaster:
self.alive[s] = (cur_time - self.recv_time[s]) < (10. / SERVICE_LIST[s].frequency) or (self.seen[s] and self.simulation)
self.freq_ok[s] = self.freq_tracker[s].valid or self.simulation
def all_alive(self, service_list: list[str] | None = None) -> bool:
def all_alive(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.alive[s] for s in (service_list or self.services) if s not in self.ignore_alive)
def all_freq_ok(self, service_list: list[str] | None = None) -> bool:
def all_freq_ok(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.freq_ok[s] for s in (service_list or self.services) if self._check_avg_freq(s))
def all_valid(self, service_list: list[str] | None = None) -> bool:
def all_valid(self, service_list: Optional[List[str]] = None) -> bool:
return all(self.valid[s] for s in (service_list or self.services) if s not in self.ignore_valid)
def all_checks(self, service_list: list[str] | None = None) -> bool:
def all_checks(self, service_list: Optional[List[str]] = None) -> bool:
return self.all_alive(service_list) and self.all_freq_ok(service_list) and self.all_valid(service_list)
class PubMaster:
def __init__(self, services: list[str]):
def __init__(self, services: List[str]):
self.sock = {}
for s in services:
self.sock[s] = pub_sock(s)
@@ -290,11 +259,11 @@ class PubMaster:
self.sock[s].send(dat)
def wait_for_readers_to_update(self, s: str, timeout: int, dt: float = 0.05) -> bool:
try:
self.sock[s].wait_for_readers(timeout=timeout, interval=dt)
return True
except TimeoutError:
return False
for _ in range(int(timeout*(1./dt))):
if self.sock[s].all_readers_updated():
return True
time.sleep(dt)
return False
def all_readers_updated(self, s: str) -> bool:
return self.sock[s].all_readers_updated()
return self.sock[s].all_readers_updated() # type: ignore
@@ -1,7 +1,7 @@
#include <cassert>
#include "openpilot/cereal/messaging/msgq_to_zmq.h"
#include "openpilot/cereal/services.h"
#include "cereal/messaging/msgq_to_zmq.h"
#include "cereal/services.h"
#include "common/util.h"
ExitHandler do_exit;
@@ -1,4 +1,4 @@
#include "openpilot/cereal/messaging/bridge_zmq.h"
#include "cereal/messaging/bridge_zmq.h"
#include <cassert>
#include <cstring>
@@ -8,7 +8,7 @@
#include <capnp/serialize.h>
#include "openpilot/cereal/gen/cpp/log.capnp.h"
#include "cereal/gen/cpp/log.capnp.h"
#include "common/timing.h"
#include "msgq/ipc.h"
@@ -1,8 +1,8 @@
#include "openpilot/cereal/messaging/msgq_to_zmq.h"
#include "cereal/messaging/msgq_to_zmq.h"
#include <cassert>
#include "openpilot/cereal/services.h"
#include "cereal/services.h"
#include "common/util.h"
extern ExitHandler do_exit;
@@ -8,7 +8,7 @@
#include <vector>
#include "msgq/impl_msgq.h"
#include "openpilot/cereal/messaging/bridge_zmq.h"
#include "cereal/messaging/bridge_zmq.h"
class MsgqToZmq {
public:
@@ -3,8 +3,8 @@
#include <string>
#include <mutex>
#include "openpilot/cereal/services.h"
#include "openpilot/cereal/messaging/messaging.h"
#include "cereal/services.h"
#include "cereal/messaging/messaging.h"
const bool SIMULATION = (getenv("SIMULATION") != nullptr) && (std::string(getenv("SIMULATION")) == "1");
@@ -1,16 +1,16 @@
import os
import capnp
import multiprocessing
import numbers
import random
import threading
import time
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.parameterized import parameterized
import pytest
from openpilot.cereal import log
from opendbc.car.structs import car
import openpilot.cereal.messaging as messaging
from openpilot.cereal.services import SERVICE_LIST
from cereal import log, car
import cereal.messaging as messaging
from cereal.services import SERVICE_LIST
events = [evt for evt in log.Event.schema.union_fields if evt in SERVICE_LIST.keys()]
@@ -23,10 +23,14 @@ def random_socks(num_socks=10):
def random_bytes(length=1000):
return bytes([random.randrange(0xFF) for _ in range(length)])
def zmq_sleep(t=1):
if "ZMQ" in os.environ:
time.sleep(t)
# TODO: this should take any capnp struct and returrn a msg with random populated data
def random_carstate():
fields = ["vEgo", "aEgo", "steeringTorque", "steeringAngleDeg"]
fields = ["vEgo", "aEgo", "brake", "steeringAngleDeg"]
msg = messaging.new_message("carState")
cs = msg.carState
for f in fields:
@@ -47,7 +51,17 @@ def delayed_send(delay, sock, dat):
threading.Timer(delay, send_func).start()
class TestMessaging(OpenpilotTestCase):
class TestMessaging:
def setUp(self):
# TODO: ZMQ tests are too slow; all sleeps will need to be
# replaced with logic to block on the necessary condition
if "ZMQ" in os.environ:
pytest.skip()
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep()
@parameterized.expand(events)
def test_new_message(self, evt):
try:
@@ -74,6 +88,7 @@ class TestMessaging(OpenpilotTestCase):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no wait and no msgs in queue
msgs = func(sub_sock)
@@ -81,11 +96,10 @@ class TestMessaging(OpenpilotTestCase):
assert len(msgs) == 0
# no wait but msgs are queued up
pub_sock.send(messaging.new_message(sock).to_bytes())
assert sub_sock.receive() is not None # synchronize the PUB/SUB connection
num_msgs = random.randrange(3, 10)
for _ in range(num_msgs):
pub_sock.send(messaging.new_message(sock).to_bytes())
time.sleep(0.1)
msgs = func(sub_sock)
assert isinstance(msgs, list)
assert all(isinstance(msg, expected_type) for msg in msgs)
@@ -95,6 +109,7 @@ class TestMessaging(OpenpilotTestCase):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=100)
zmq_sleep()
# no wait and no msg in queue, socket should timeout
recvd = messaging.recv_sock(sub_sock)
@@ -112,7 +127,8 @@ class TestMessaging(OpenpilotTestCase):
def test_recv_one(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=10)
sub_sock = messaging.sub_sock(sock, timeout=1000)
zmq_sleep()
# no msg in queue, socket should timeout
recvd = messaging.recv_one(sub_sock)
@@ -125,10 +141,12 @@ class TestMessaging(OpenpilotTestCase):
assert isinstance(recvd, capnp._DynamicStructReader)
assert_carstate(msg.carState, recvd.carState)
@pytest.mark.xfail(condition="ZMQ" in os.environ, reason='ZMQ detected')
def test_recv_one_or_none(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock)
zmq_sleep()
# no msg in queue, socket shouldn't block
recvd = messaging.recv_one_or_none(sub_sock)
@@ -143,16 +161,19 @@ class TestMessaging(OpenpilotTestCase):
def test_recv_one_retry(self):
sock = "carState"
sock_timeout = 0.005
sock_timeout = 0.1
pub_sock = messaging.pub_sock(sock)
sub_sock = messaging.sub_sock(sock, timeout=round(sock_timeout*1000))
zmq_sleep()
# wait 5 socket timeouts and make sure it's still retrying
p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
p.start()
time.sleep(sock_timeout*5)
assert p.is_alive()
p.terminate()
# this test doesn't work with ZMQ since multiprocessing interrupts it
if "ZMQ" not in os.environ:
# wait 5 socket timeouts and make sure it's still retrying
p = multiprocessing.Process(target=messaging.recv_one_retry, args=(sub_sock,))
p.start()
time.sleep(sock_timeout*5)
assert p.is_alive()
p.terminate()
# wait 5 socket timeouts before sending
msg = random_carstate()
@@ -1,16 +1,20 @@
import random
import time
from typing import cast
from collections.abc import Sized
from typing import Sized, cast
from openpilot.common.test import OpenpilotTestCase
import openpilot.cereal.messaging as messaging
from openpilot.cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
random_bytes, random_carstate, assert_carstate
from openpilot.cereal.services import SERVICE_LIST
import cereal.messaging as messaging
from cereal.messaging.tests.test_messaging import events, random_sock, random_socks, \
random_bytes, random_carstate, assert_carstate, \
zmq_sleep
from cereal.services import SERVICE_LIST
class TestSubMaster(OpenpilotTestCase):
class TestSubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
sm = messaging.SubMaster(events)
@@ -38,6 +42,7 @@ class TestSubMaster(OpenpilotTestCase):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
msg = random_carstate()
pub_sock.send(msg.to_bytes())
@@ -49,6 +54,7 @@ class TestSubMaster(OpenpilotTestCase):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
zmq_sleep()
for i in range(10):
msg = messaging.new_message(sock)
@@ -60,17 +66,17 @@ class TestSubMaster(OpenpilotTestCase):
def test_update_timeout(self):
sock = random_sock()
sm = messaging.SubMaster([sock,])
timeout = random.randrange(10, 30)
timeout = random.randrange(1000, 3000)
start_time = time.monotonic()
sm.update(timeout)
t = time.monotonic() - start_time
assert t >= timeout/1000.
assert t < 0.1
assert t < 3
assert not any(sm.updated.values())
def test_avg_frequency_checks(self):
for poll in (True, False):
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "extrinsicsCalibration"],
sm = messaging.SubMaster(["modelV2", "carParams", "carState", "cameraOdometry", "liveCalibration"],
poll=("modelV2" if poll else None),
frequency=(20. if not poll else None))
@@ -78,39 +84,50 @@ class TestSubMaster(OpenpilotTestCase):
"carState": (20, 20),
"modelV2": (20, 20 if poll else 10),
"cameraOdometry": (20, 10),
"extrinsicsCalibration": (4, 4),
"liveCalibration": (4, 4),
"carParams": (None, None),
"userBookmark": (None, None),
}
for service, (max_freq, min_freq) in checks.items():
if max_freq is not None:
assert min_freq is not None
assert sm._check_avg_freq(service)
assert sm.freq_tracker[service].max_freq == max_freq*1.2
assert sm.freq_tracker[service].min_freq == min_freq*0.8
else:
assert not sm._check_avg_freq(service)
def test_alive(self):
pass
def test_ignore_alive(self):
pass
def test_valid(self):
pass
# SubMaster should always conflate
def test_conflate(self):
sock = "carState"
pub_sock = messaging.pub_sock(sock)
sm = messaging.SubMaster([sock,])
pub_sock.send(messaging.new_message(sock).to_bytes())
sm.update(1000) # synchronize the PUB/SUB connection
n = 10
for i in range(n+1):
msg = messaging.new_message(sock)
msg.carState.vEgo = i
pub_sock.send(msg.to_bytes())
time.sleep(0.01)
sm.update(1000)
assert sm[sock].vEgo == n
class TestPubMaster(OpenpilotTestCase):
class TestPubMaster:
def setup_method(self):
# ZMQ pub socket takes too long to die
# sleep to prevent multiple publishers error between tests
zmq_sleep(3)
def test_init(self):
messaging.PubMaster(events)
@@ -119,6 +136,7 @@ class TestPubMaster(OpenpilotTestCase):
socks = random_socks()
pm = messaging.PubMaster(socks)
sub_socks = {s: messaging.sub_sock(s, conflate=True, timeout=1000) for s in socks}
zmq_sleep()
# PubMaster accepts either a capnp msg builder or bytes
for capnp in [True, False]:
@@ -1,14 +1,13 @@
import subprocess
import os
import tempfile
from openpilot.common.test import OpenpilotTestCase
from typing import Dict
from openpilot.common.parameterized import parameterized
import openpilot.cereal.services as services
from openpilot.cereal.services import SERVICE_LIST
import cereal.services as services
from cereal.services import SERVICE_LIST
class TestServices(OpenpilotTestCase):
class TestServices:
@parameterized.expand(SERVICE_LIST.keys())
def test_services(self, s):
@@ -18,5 +17,5 @@ class TestServices(OpenpilotTestCase):
def test_generated_header(self):
with tempfile.NamedTemporaryFile(suffix=".h") as f:
ret = subprocess.run(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11", shell=True).returncode
ret = os.system(f"python3 {services.__file__} > {f.name} && clang++ {f.name} -std=c++11")
assert ret == 0, "generated services header is not valid C"
+222
View File
@@ -0,0 +1,222 @@
#!/usr/bin/env python3
import argparse
import sys
from typing import Any, List, Tuple
DEBUG = False
def print_debug(string: str) -> None:
if DEBUG:
print(string)
def create_schema_instance(struct: Any, prop: Tuple[str, Any]) -> Any:
"""
Create a new instance of a schema type, handling different field types.
Args:
struct: The Cap'n Proto schema structure
prop: A tuple containing the field name and field metadata
Returns:
A new initialized schema instance
"""
struct_instance = struct.new_message()
field_name, field_metadata = prop
try:
field_type = field_metadata.proto.slot.type.which()
# Initialize different types of fields
if field_type in ('list', 'text', 'data'):
struct_instance.init(field_name, 1)
print_debug(f"Initialized list/text/data field: {field_name}")
elif field_type in ('struct', 'object'):
struct_instance.init(field_name)
print_debug(f"Initialized struct/object field: {field_name}")
return struct_instance
except Exception as e:
print(f"Error creating instance for {field_name}: {e}")
return None
def get_schema_fields(schema_struct: Any) -> List[Tuple[str, Any]]:
"""
Retrieve all fields from a given schema structure.
Args:
schema_struct: The Cap'n Proto schema structure
Returns:
A list of field names and their metadata
"""
try:
# Get all fields from the schema
schema_fields = list(schema_struct.schema.fields.items())
print_debug("Discovered schema fields:")
for field_name, field_metadata in schema_fields:
print_debug(f"- {field_name}")
return schema_fields
except Exception as e:
print(f"Error retrieving schema fields: {e}")
return []
def generate_schema_instances(schema_struct: Any) -> List[Any]:
"""
Generate instances for all fields in a given schema.
Args:
schema_struct: The Cap'n Proto schema structure
Returns:
A list of schema instances
"""
schema_fields = get_schema_fields(schema_struct)
instances = []
for field_prop in schema_fields:
try:
instance = create_schema_instance(schema_struct, field_prop)
if instance is not None:
instances.append(instance)
except Exception as e:
print(f"Skipping field due to error: {e}")
print(f"Generated {len(instances)} schema instances")
return instances
def persist_instances(instances: List[Any], filename: str) -> None:
"""
Write schema instances to a binary file.
Args:
instances: List of schema instances
filename: Output file path
"""
try:
with open(filename, 'wb') as f:
for instance in instances:
f.write(instance.to_bytes())
print(f"Successfully wrote {len(instances)} instances to {filename}")
except Exception as e:
print(f"Error persisting instances: {e}")
sys.exit(1)
def read_instances(filename: str, schema_type: Any) -> List[Any]:
"""
Read schema instances from a binary file.
Args:
filename: Input file path
schema_type: The schema type to use for reading
Returns:
A list of read schema instances
"""
try:
with open(filename, 'rb') as f:
data = f.read()
instances = list(schema_type.read_multiple_bytes(data))
print(f"Read {len(instances)} instances from {filename}")
return instances
except Exception as e:
print(f"Error reading instances: {e}")
sys.exit(1)
def compare_schemas(original_instances: List[Any], read_instances: List[Any]) -> bool:
"""
Compare original and read-back instances to detect potential breaking changes.
Args:
original_instances: List of originally generated instances
read_instances: List of instances read back from file
Returns:
Boolean indicating whether schemas appear compatible
"""
if len(original_instances) != len(read_instances):
print("❌ Schema Compatibility Warning: Instance count mismatch")
return False
compatible = True
for struct in read_instances:
try:
getattr(struct, struct.which()) # Attempting to access the field to validate readability
except Exception as e:
print(f"❌ Structural change detected: {struct.which()} is not readable.\nFull error: {e}")
compatible = False
return compatible
def main():
"""
CLI entry point for schema compatibility testing.
"""
# Setup argument parser
parser = argparse.ArgumentParser(
description='Cap\'n Proto Schema Compatibility Testing Tool',
epilog='Test schema compatibility by generating and reading back instances.'
)
# Add mutually exclusive group for generation or reading mode
mode_group = parser.add_mutually_exclusive_group(required=True)
mode_group.add_argument('-g', '--generate', action='store_true',
help='Generate schema instances')
mode_group.add_argument('-r', '--read', action='store_true',
help='Read and validate schema instances')
# Common arguments
parser.add_argument('-f', '--file',
default='schema_instances.bin',
help='Output/input binary file (default: schema_instances.bin)')
# Parse arguments
args = parser.parse_args()
# Import the schema dynamically
try:
from cereal import log
schema_type = log.Event
except ImportError:
print("Error: Unable to import schema. Ensure 'cereal' is installed.")
sys.exit(1)
# Execute based on mode
if args.generate:
print("🔧 Generating Schema Instances")
instances = generate_schema_instances(schema_type)
persist_instances(instances, args.file)
print("✅ Instance generation complete")
elif args.read:
print("🔍 Reading and Validating Schema Instances")
generated_instances = generate_schema_instances(schema_type)
read_back_instances = read_instances(args.file, schema_type)
# Compare schemas
if compare_schemas(generated_instances, read_back_instances):
print("✅ Schema Compatibility: No breaking changes detected")
sys.exit(0)
else:
print("❌ Potential Schema Breaking Changes Detected")
sys.exit(1)
if __name__ == "__main__":
main()
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
from enum import IntEnum
from typing import Optional
# TODO: this should be automatically determined using the capnp schema
@@ -10,7 +11,7 @@ class QueueSize(IntEnum):
class Service:
def __init__(self, should_log: bool, frequency: float, decimation: int | None = None,
def __init__(self, should_log: bool, frequency: float, decimation: Optional[int] = None,
queue_size: QueueSize = QueueSize.SMALL):
self.should_log = should_log
self.frequency = frequency
@@ -23,9 +24,11 @@ _services: dict[str, tuple] = {
# note: the "EncodeIdx" packets will still be in the log
"gyroscope": (True, 104., 104),
"accelerometer": (True, 104., 104),
"magnetometer": (True, 25.),
"lightSensor": (True, 100., 100),
"temperatureSensor": (True, 2., 200),
"gpsNMEA": (True, 9.),
"deviceState": (True, 2., 1),
"chestnutState": (True, 10., 10),
"touch": (True, 20., 1),
"can": (True, 100., 2053, QueueSize.BIG), # decimation gives ~3 msgs in a full segment
"controlsState": (True, 100., 10, QueueSize.MEDIUM),
@@ -33,37 +36,37 @@ _services: dict[str, tuple] = {
"pandaStates": (True, 10., 1),
"peripheralState": (True, 2., 1),
"radarState": (True, 20., 5),
"narrowRoadEncodeIdx": (False, 20., 1),
"radarTracks": (True, 20.),
"roadEncodeIdx": (False, 20., 1),
"liveTracks": (True, 20.),
"sendcan": (True, 100., 139, QueueSize.MEDIUM),
"logMessage": (True, 0., None, QueueSize.BIG),
"errorLogMessage": (True, 0., 1, QueueSize.BIG),
"extrinsicsCalibration": (True, 4., 4),
"lateralTorqueParameters": (True, 4., 1),
"lateralDelay": (True, 4., 1),
"operatingSystemLog": (True, 0.),
"logMessage": (True, 0.),
"errorLogMessage": (True, 0., 1),
"liveCalibration": (True, 4., 4),
"liveTorqueParameters": (True, 4., 1),
"liveDelay": (True, 4., 1),
"androidLog": (True, 0.),
"carState": (True, 100., 10),
"carControl": (True, 100., 10),
"carOutput": (True, 100., 10),
"longitudinalPlan": (True, 20., 10),
"lateralManeuverPlan": (True, 20.),
"driverAssistance": (True, 20., 20),
"procLog": (True, 0.5, 15, QueueSize.BIG),
"gpsLocationExternal": (True, 10., 10),
"gpsLocation": (True, 1., 1),
"ubloxGnss": (True, 10.),
"qcomGnss": (True, 2.),
"gnssMeasurements": (True, 10., 10),
"clocks": (True, 0.1, 1),
"ubloxRaw": (True, 20.),
"deviceMotion": (True, 20., 4),
"vehicleParameters": (True, 20., 5),
"livePose": (True, 20., 4),
"liveParameters": (True, 20., 5),
"cameraOdometry": (True, 20., 10),
"thumbnail": (True, 1 / 60., 1),
"onroadEvents": (True, 1., 1),
"carParams": (True, 0.02, 1),
"narrowRoadCameraState": (True, 20., 20),
"cabinCameraState": (True, 20., 20),
"cabinEncodeIdx": (False, 20., 1),
"roadCameraState": (True, 20., 20),
"driverCameraState": (True, 20., 20),
"driverEncodeIdx": (False, 20., 1),
"driverStateV2": (True, 20., 10),
"driverMonitoringState": (True, 20., 10),
"wideRoadEncodeIdx": (False, 20., 1),
@@ -71,15 +74,20 @@ _services: dict[str, tuple] = {
"drivingModelData": (True, 20., 10),
"modelV2": (True, 20., None, QueueSize.BIG),
"managerState": (True, 2., 1),
"qNarrowRoadEncodeIdx": (False, 20.),
"uploaderState": (True, 0., 1),
"navInstruction": (True, 1., 10),
"navRoute": (True, 0.),
"navThumbnail": (True, 0.),
"qRoadEncodeIdx": (False, 20.),
"userBookmark": (True, 0., 1),
"soundPressure": (True, 10., 10),
"rawAudioData": (False, 20.),
"bookmarkButton": (True, 0., 1),
"narrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
"cabinEncodeData": (False, 20., None, QueueSize.BIG),
"audioFeedback": (True, 0., 1),
"roadEncodeData": (False, 20., None, QueueSize.BIG),
"driverEncodeData": (False, 20., None, QueueSize.BIG),
"wideRoadEncodeData": (False, 20., None, QueueSize.BIG),
"qNarrowRoadEncodeData": (False, 20., None, QueueSize.BIG),
"qRoadEncodeData": (False, 20., None, QueueSize.BIG),
# sunnypilot
"modelManagerSP": (False, 1., 1, QueueSize.BIG),
@@ -99,12 +107,14 @@ _services: dict[str, tuple] = {
"testJoystick": (True, 0.),
"alertDebug": (True, 20., 5),
"livestreamWideRoadEncodeIdx": (False, 20.),
"livestreamNarrowRoadEncodeIdx": (False, 20.),
"livestreamCabinEncodeIdx": (False, 20.),
"livestreamRoadEncodeIdx": (False, 20.),
"livestreamDriverEncodeIdx": (False, 20.),
"livestreamWideRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamNarrowRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamCabinEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamRoadEncodeData": (False, 20., None, QueueSize.MEDIUM),
"livestreamDriverEncodeData": (False, 20., None, QueueSize.MEDIUM),
"customReservedRawData0": (True, 0.),
"customReservedRawData1": (True, 0.),
"customReservedRawData2": (True, 0.),
}
SERVICE_LIST = {name: Service(*vals) for
idx, (name, vals) in enumerate(_services.items())}
@@ -124,7 +134,8 @@ def build_header():
for k, v in SERVICE_LIST.items():
should_log = "true" if v.should_log else "false"
decimation = -1 if v.decimation is None else v.decimation
h += f' {{ "{k}", {{"{k}", {should_log}, {v.frequency:f}, {decimation:d}, {v.queue_size:d}}}}},\n'
h += ' { "%s", {"%s", %s, %f, %d, %d}},\n' % \
(k, k, should_log, v.frequency, decimation, v.queue_size)
h += "};\n"
h += "#endif\n"
+1
View File
@@ -0,0 +1 @@
*.cpp
+23
View File
@@ -0,0 +1,23 @@
Import('env', 'envCython', 'arch')
common_libs = [
'params.cc',
'swaglog.cc',
'util.cc',
'ratekeeper.cc',
]
_common = env.Library('common', common_libs, LIBS="json11")
Export('_common')
if GetOption('extras'):
env.Program('tests/test_common',
['tests/test_runner.cc', 'tests/test_params.cc', 'tests/test_util.cc', 'tests/test_swaglog.cc'],
LIBS=[_common, 'json11', 'zmq', 'pthread'])
# Cython bindings
params_python = envCython.Program('params_pyx.so', 'params_pyx.pyx', LIBS=envCython['LIBS'] + [_common, 'zmq', 'json11'])
common_python = [params_python]
Export('common_python')
@@ -3,8 +3,8 @@ import os
import requests
import unicodedata
from datetime import datetime, timedelta, UTC
from openpilot.common.hardware.hw import Paths
from openpilot.common.version import get_version
from openpilot.system.hardware.hw import Paths
from openpilot.system.version import get_version
# name: jwt signature algorithm
KEYS = {"id_rsa": "RS256",
@@ -38,7 +38,6 @@ class BaseApi:
}
if payload_extra is not None:
payload.update(payload_extra)
assert self.private_key is not None
token = jwt.encode(payload, self.private_key, algorithm=self.jwt_algorithm)
if isinstance(token, bytes):
token = token.decode('utf8')
@@ -1,4 +1,4 @@
import os
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../.."))
BASEDIR = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../"))
+37
View File
@@ -0,0 +1,37 @@
import math
import os
from pathlib import Path
CHUNK_SIZE = 45 * 1024 * 1024 # 45MB, under GitHub's 50MB limit
def get_chunk_name(name, idx, num_chunks):
return f"{name}.chunk{idx+1:02d}of{num_chunks:02d}"
def get_manifest_path(name):
return f"{name}.chunkmanifest"
def get_chunk_paths(path, file_size):
num_chunks = math.ceil(file_size / CHUNK_SIZE)
return [get_manifest_path(path)] + [get_chunk_name(path, i, num_chunks) for i in range(num_chunks)]
def chunk_file(path, targets):
manifest_path, *chunk_paths = targets
with open(path, 'rb') as f:
data = f.read()
actual_num_chunks = max(1, math.ceil(len(data) / CHUNK_SIZE))
assert len(chunk_paths) >= actual_num_chunks, f"Allowed {len(chunk_paths)} chunks but needs at least {actual_num_chunks}, for path {path}"
for i, chunk_path in enumerate(chunk_paths):
with open(chunk_path, 'wb') as f:
f.write(data[i * CHUNK_SIZE:(i + 1) * CHUNK_SIZE])
Path(manifest_path).write_text(str(len(chunk_paths)))
os.remove(path)
def read_file_chunked(path):
manifest_path = get_manifest_path(path)
if os.path.isfile(manifest_path):
num_chunks = int(Path(manifest_path).read_text().strip())
return b''.join(Path(get_chunk_name(path, i, num_chunks)).read_bytes() for i in range(num_chunks))
if os.path.isfile(path):
return Path(path).read_bytes()
raise FileNotFoundError(path)
@@ -28,7 +28,7 @@ class BounceFilter(FirstOrderFilter):
scale = self.dt / (1.0 / 60.0) # tuned at 60 fps
self.velocity.x += (x - self.x) * self.bounce * scale * self.dt
self.velocity.update(0.0)
if abs(self.velocity.x) < 1e-3:
if abs(self.velocity.x) < 1e-5:
self.velocity.x = 0.0
self.x += self.velocity.x
return self.x
@@ -31,3 +31,12 @@ def get_origin(cwd: str | None = None) -> str:
return run_cmd(["git", "config", "remote." + tracking_remote + ".url"], cwd=cwd)
except subprocess.CalledProcessError: # Not on a branch, fallback
return run_cmd_default(["git", "config", "--get", "remote.origin.url"], cwd=cwd)
@cache
def get_normalized_origin(cwd: str | None = None) -> str:
return get_origin(cwd) \
.replace("git@", "", 1) \
.replace(".git", "", 1) \
.replace("https://", "", 1) \
.replace(":", "/", 1)
@@ -1,19 +1,19 @@
"""
Utilities for generating mock messages for testing.
example in openpilot/common/tests/test_mock.py
example in common/tests/test_mock.py
"""
import functools
import threading
from openpilot.cereal.messaging import PubMaster
from openpilot.cereal.services import SERVICE_LIST
from openpilot.common.mock.generators import generate_deviceMotion
from cereal.messaging import PubMaster
from cereal.services import SERVICE_LIST
from openpilot.common.mock.generators import generate_livePose
from openpilot.common.realtime import Ratekeeper
MOCK_GENERATOR = {
"deviceMotion": generate_deviceMotion
"livePose": generate_livePose
}
+14
View File
@@ -0,0 +1,14 @@
from cereal import messaging
def generate_livePose():
msg = messaging.new_message('livePose')
meas = {'x': 0.0, 'y': 0.0, 'z': 0.0, 'xStd': 0.0, 'yStd': 0.0, 'zStd': 0.0, 'valid': True}
msg.livePose.orientationNED = meas
msg.livePose.velocityDevice = meas
msg.livePose.angularVelocityDevice = meas
msg.livePose.accelerationDevice = meas
msg.livePose.inputsOK = True
msg.livePose.posenetOK = True
msg.livePose.sensorsOK = True
return msg
+1
View File
@@ -0,0 +1 @@
#define DEFAULT_MODEL "CD210 (Default)"
+47
View File
@@ -0,0 +1,47 @@
import sys
import pytest
import inspect
class parameterized:
@staticmethod
def expand(cases):
cases = list(cases)
if not cases:
return lambda func: pytest.mark.skip("no parameterized cases")(func)
def decorator(func):
params = [p for p in inspect.signature(func).parameters if p != 'self']
normalized = [c if isinstance(c, tuple) else (c,) for c in cases]
# Infer arg count from first case so extra params (e.g. from @given) are left untouched
expand_params = params[: len(normalized[0])]
if len(expand_params) == 1:
return pytest.mark.parametrize(expand_params[0], [c[0] for c in normalized])(func)
return pytest.mark.parametrize(', '.join(expand_params), normalized)(func)
return decorator
def parameterized_class(attrs, input_list=None):
if isinstance(attrs, list) and (not attrs or isinstance(attrs[0], dict)):
params_list = attrs
else:
assert input_list is not None
attr_names = (attrs,) if isinstance(attrs, str) else tuple(attrs)
params_list = [dict(zip(attr_names, v if isinstance(v, (tuple, list)) else (v,), strict=False)) for v in input_list]
def decorator(cls):
globs = sys._getframe(1).f_globals
for i, params in enumerate(params_list):
name = f"{cls.__name__}_{i}"
new_cls = type(name, (cls,), dict(params))
new_cls.__module__ = cls.__module__
new_cls.__test__ = True # override inherited False so pytest collects this subclass
globs[name] = new_cls
# Don't collect the un-parametrised base, but return it so outer decorators
# (e.g. @pytest.mark.skip) land on it and propagate to subclasses via MRO.
cls.__test__ = False
return cls
return decorator
@@ -12,7 +12,7 @@
#include "common/queue.h"
#include "common/swaglog.h"
#include "common/util.h"
#include "common/hardware/hw.h"
#include "system/hardware/hw.h"
namespace {
+19
View File
@@ -0,0 +1,19 @@
from openpilot.common.params_pyx import Params, ParamKeyFlag, ParamKeyType, UnknownKeyName
assert Params
assert ParamKeyFlag
assert ParamKeyType
assert UnknownKeyName
if __name__ == "__main__":
import sys
params = Params()
key = sys.argv[1]
assert params.check_key(key), f"unknown param: {key}"
if len(sys.argv) == 3:
val = sys.argv[2]
print(f"SET: {key} = {val}")
params.put(key, val)
elif len(sys.argv) == 2:
print(f"GET: {key} = {params.get(key)}")
@@ -3,7 +3,7 @@
#include <string>
#include <unordered_map>
#include "openpilot/cereal/gen/cpp/log.capnp.h"
#include "cereal/gen/cpp/log.capnp.h"
inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"AccessToken", {CLEAR_ON_MANAGER_START | DONT_LOG, STRING}},
@@ -37,7 +37,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"DoShutdown", {CLEAR_ON_MANAGER_START, BOOL}},
{"DoUninstall", {CLEAR_ON_MANAGER_START, BOOL}},
{"DriverTooDistracted", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, BOOL}},
{"DriverLockoutCount", {CLEAR_ON_MANAGER_START | CLEAR_ON_IGNITION_ON, INT, "0"}},
{"AlphaLongitudinalEnabled", {PERSISTENT | DEVELOPMENT_ONLY | BACKUP, BOOL}},
{"ExperimentalMode", {PERSISTENT | BACKUP, BOOL}},
{"ExperimentalModeConfirmed", {PERSISTENT | BACKUP, BOOL}},
@@ -59,11 +58,12 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"IsDriverViewEnabled", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsEngaged", {PERSISTENT, BOOL}},
{"IsLdwEnabled", {PERSISTENT | BACKUP, BOOL}},
{"IsLiveStreaming", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsMetric", {PERSISTENT | BACKUP, BOOL}},
{"IsOffroad", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsOnroad", {PERSISTENT, BOOL}},
{"IsRhdDetected", {PERSISTENT, BOOL}},
{"IsReleaseBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsTakingSnapshot", {CLEAR_ON_MANAGER_START, BOOL}},
{"IsTestedBranch", {CLEAR_ON_MANAGER_START, BOOL}},
{"JoystickDebugMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LanguageSetting", {PERSISTENT | BACKUP, STRING, "en"}},
@@ -80,11 +80,8 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"LiveDelay", {PERSISTENT | BACKUP, BYTES}},
{"LiveParameters", {PERSISTENT, JSON}},
{"LiveParametersV2", {PERSISTENT, BYTES}},
{"LivestreamEncoderBitrate", {CLEAR_ON_MANAGER_START | DONT_LOG, INT}},
{"LivestreamRequestKeyframe", {CLEAR_ON_MANAGER_START | DONT_LOG, BOOL}},
{"LiveTorqueParameters", {PERSISTENT | DONT_LOG, BYTES}},
{"LocationFilterInitialState", {PERSISTENT, BYTES}},
{"LateralManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LongitudinalManeuverMode", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"LongitudinalPersonality", {PERSISTENT | BACKUP, INT, std::to_string(static_cast<int>(cereal::LongitudinalPersonality::STANDARD))}},
{"NetworkMetered", {PERSISTENT | BACKUP, BOOL}},
@@ -94,6 +91,7 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"Offroad_ConnectivityNeeded", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_ConnectivityNeededPrompt", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_ExcessiveActuation", {PERSISTENT, JSON}},
{"Offroad_IsTakingSnapshot", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_NeosUpdate", {CLEAR_ON_MANAGER_START, JSON}},
{"Offroad_NoFirmware", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
{"Offroad_Recalibration", {CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION, JSON}},
@@ -104,8 +102,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"OnroadCycleRequested", {CLEAR_ON_MANAGER_START, BOOL}},
{"OpenpilotEnabledToggle", {PERSISTENT | BACKUP, BOOL, "1"}},
{"PandaHeartbeatLost", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"PandaSomResetTriggered", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION, BOOL}},
{"PandaSignatures", {CLEAR_ON_MANAGER_START, BYTES}},
{"PrimeType", {PERSISTENT, INT}},
{"RecordAudio", {PERSISTENT | BACKUP, BOOL}},
{"RecordAudioFeedback", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RecordFront", {PERSISTENT | BACKUP, BOOL}},
{"RecordFrontLock", {PERSISTENT, BOOL}}, // for the internal fleet
{"SecOCKey", {PERSISTENT | DONT_LOG | BACKUP, STRING}},
@@ -130,8 +131,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"UpdaterLastFetchTime", {PERSISTENT, TIME}},
{"UptimeOffroad", {PERSISTENT, FLOAT, "0.0"}},
{"UptimeOnroad", {PERSISTENT, FLOAT, "0.0"}},
{"UsbGpuActive", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
{"UsbGpuLoading", {CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION | CLEAR_ON_IGNITION_ON, BOOL}},
{"Version", {PERSISTENT, STRING}},
// --- sunnypilot params --- //
@@ -173,15 +172,11 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"OnroadScreenOffBrightness", {PERSISTENT | BACKUP, INT, "0"}},
{"OnroadScreenOffBrightnessMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
{"OnroadScreenOffTimer", {PERSISTENT | BACKUP, INT, "15"}},
{"OnroadScreenOffTimerMigrated", {PERSISTENT | BACKUP, STRING, "0.0"}},
{"OnroadUploads", {PERSISTENT | BACKUP, BOOL, "1"}},
{"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}},
{"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RoadEdgeLaneChangeEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
{"RocketFuel", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ScreenSaverEnabled", {PERSISTENT | BACKUP, BOOL, "1"}},
{"ScreenSaverTimeout", {PERSISTENT | BACKUP, INT, "300"}},
{"ShowAdvancedControls", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ShowTurnSignals", {PERSISTENT | BACKUP, BOOL, "0"}},
{"StandstillTimer", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -207,7 +202,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
// sunnylink params
{"EnableSunnylinkUploader", {PERSISTENT | BACKUP, BOOL}},
{"LastSunnylinkPingTime", {CLEAR_ON_MANAGER_START, INT}},
{"ParamsVersion", {PERSISTENT, INT}},
{"SunnylinkCache_Roles", {PERSISTENT, STRING}},
{"SunnylinkCache_Users", {PERSISTENT, STRING}},
{"SunnylinkDongleId", {PERSISTENT, STRING}},
@@ -224,7 +218,6 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
{"SubaruStopAndGo", {PERSISTENT | BACKUP, BOOL, "0"}},
{"SubaruStopAndGoManualParkingBrake", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaCoopSteering", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TeslaMadsScreenButton", {PERSISTENT | BACKUP, INT, "0"}},
{"ToyotaEnforceStockLongitudinal", {PERSISTENT | BACKUP, BOOL, "0"}},
{"ToyotaStopAndGoHack", {PERSISTENT | BACKUP, BOOL, "0"}},
@@ -275,10 +268,9 @@ inline static std::unordered_map<std::string, ParamKeyAttributes> keys = {
// Torque lateral control custom params
{"CustomTorqueParams", {PERSISTENT | BACKUP , BOOL}},
{"EnforceTorqueControl", {PERSISTENT | BACKUP, BOOL}},
{"LateralJerkTorqueController", {PERSISTENT | BACKUP, BOOL, "0"}},
{"LiveTorqueParamsToggle", {PERSISTENT | BACKUP , BOOL}},
{"LiveTorqueParamsRelaxedToggle", {PERSISTENT | BACKUP , BOOL}},
{"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT, "0.0"}},
{"TorqueControlTune", {PERSISTENT | BACKUP, FLOAT}},
{"TorqueParamsOverrideEnabled", {PERSISTENT | BACKUP, BOOL, "0"}},
{"TorqueParamsOverrideFriction", {PERSISTENT | BACKUP, FLOAT, "0.1"}},
{"TorqueParamsOverrideLatAccelFactor", {PERSISTENT | BACKUP, FLOAT, "2.5"}},
+197
View File
@@ -0,0 +1,197 @@
# distutils: language = c++
# cython: language_level = 3
import builtins
import datetime
import json
from libcpp cimport bool
from libcpp.string cimport string
from libcpp.vector cimport vector
from libcpp.optional cimport optional
from openpilot.common.swaglog import cloudlog
cdef extern from "common/params.h":
cpdef enum ParamKeyFlag:
PERSISTENT
CLEAR_ON_MANAGER_START
CLEAR_ON_ONROAD_TRANSITION
CLEAR_ON_OFFROAD_TRANSITION
DEVELOPMENT_ONLY
CLEAR_ON_IGNITION_ON
BACKUP
ALL
cpdef enum ParamKeyType:
STRING
BOOL
INT
FLOAT
TIME
JSON
BYTES
cdef cppclass c_Params "Params":
c_Params(string) except + nogil
string get(string, bool) nogil
bool getBool(string, bool) nogil
int remove(string) nogil
int put(string, string) nogil
void putNonBlocking(string, string) nogil
void putBoolNonBlocking(string, bool) nogil
int putBool(string, bool) nogil
bool checkKey(string) nogil
ParamKeyType getKeyType(string) nogil
optional[string] getKeyDefaultValue(string) nogil
string getParamPath(string) nogil
void clearAll(ParamKeyFlag)
vector[string] allKeys(ParamKeyFlag)
PYTHON_2_CPP = {
(str, STRING): lambda v: v,
(builtins.bool, BOOL): lambda v: "1" if v else "0",
(int, INT): str,
(float, FLOAT): str,
(datetime.datetime, TIME): lambda v: v.isoformat(),
(dict, JSON): json.dumps,
(list, JSON): json.dumps,
(bytes, BYTES): lambda v: v,
}
CPP_2_PYTHON = {
STRING: lambda v: v.decode("utf-8"),
BOOL: lambda v: v == b"1",
INT: int,
FLOAT: float,
TIME: lambda v: datetime.datetime.fromisoformat(v.decode("utf-8")),
JSON: json.loads,
BYTES: lambda v: v,
}
def ensure_bytes(v):
return v.encode() if isinstance(v, str) else v
class UnknownKeyName(Exception):
pass
cdef class Params:
cdef c_Params* p
cdef str d
def __cinit__(self, d=""):
cdef string path = <string>d.encode()
with nogil:
self.p = new c_Params(path)
self.d = d
def __reduce__(self):
return (type(self), (self.d,))
def __dealloc__(self):
del self.p
def clear_all(self, tx_flag=ParamKeyFlag.ALL):
self.p.clearAll(tx_flag)
def check_key(self, key):
key = ensure_bytes(key)
if not self.p.checkKey(key):
raise UnknownKeyName(key)
return key
def python2cpp(self, proposed_type, expected_type, value, key):
cast = PYTHON_2_CPP.get((proposed_type, expected_type))
if cast:
return cast(value)
raise TypeError(f"Type mismatch while writing param {key}: {proposed_type=} {expected_type=} {value=}")
def _cpp2python(self, t, value, default, key):
if value is None:
return None
try:
return CPP_2_PYTHON[t](value)
except (KeyError, TypeError, ValueError):
cloudlog.warning(f"Failed to cast param {key} with {value=} from type {t=}")
return self._cpp2python(t, default, None, key)
def get(self, key, bool block=False, bool return_default=False):
cdef string k = self.check_key(key)
cdef ParamKeyType t = self.p.getKeyType(k)
cdef optional[string] default = self.p.getKeyDefaultValue(k)
cdef string val
with nogil:
val = self.p.get(k, block)
default_val = (default.value() if default.has_value() else None) if return_default else None
if val == b"":
if block:
# If we got no value while running in blocked mode
# it means we got an interrupt while waiting
raise KeyboardInterrupt
else:
return self._cpp2python(t, default_val, None, key)
return self._cpp2python(t, val, default_val, key)
def get_bool(self, key, bool block=False):
cdef string k = self.check_key(key)
cdef bool r
with nogil:
r = self.p.getBool(k, block)
return r
def _put_cast(self, key, dat):
cdef string k = self.check_key(key)
cdef ParamKeyType t = self.p.getKeyType(k)
return ensure_bytes(self.python2cpp(type(dat), t, dat, key))
def put(self, key, dat):
"""
Warning: This function blocks until the param is written to disk!
In very rare cases this can take over a second, and your code will hang.
Use the put_nonblocking, put_bool_nonblocking in time sensitive code, but
in general try to avoid writing params as much as possible.
"""
cdef string k = self.check_key(key)
cdef string dat_bytes = self._put_cast(key, dat)
with nogil:
self.p.put(k, dat_bytes)
def put_bool(self, key, bool val):
cdef string k = self.check_key(key)
with nogil:
self.p.putBool(k, val)
def put_nonblocking(self, key, dat):
cdef string k = self.check_key(key)
cdef string dat_bytes = self._put_cast(key, dat)
with nogil:
self.p.putNonBlocking(k, dat_bytes)
def put_bool_nonblocking(self, key, bool val):
cdef string k = self.check_key(key)
with nogil:
self.p.putBoolNonBlocking(k, val)
def remove(self, key):
cdef string k = self.check_key(key)
with nogil:
self.p.remove(k)
def get_param_path(self, key=""):
cdef string key_bytes = ensure_bytes(key)
return self.p.getParamPath(key_bytes).decode("utf-8")
def get_type(self, key):
return self.p.getKeyType(self.check_key(key))
def all_keys(self, flag=ParamKeyFlag.ALL):
return self.p.allKeys(flag)
def get_default_value(self, key):
cdef string k = self.check_key(key)
cdef ParamKeyType t = self.p.getKeyType(k)
cdef optional[string] default = self.p.getKeyDefaultValue(k)
return self._cpp2python(t, default.value(), None, key) if default.has_value() else None
def cpp2python(self, key, value):
cdef string k = self.check_key(key)
cdef ParamKeyType t = self.p.getKeyType(k)
return self._cpp2python(t, value, None, key)
+5 -7
View File
@@ -1,13 +1,11 @@
import numpy as np
from collections.abc import Sequence
Gain = int | float | tuple[Sequence[float], Sequence[float]] | list[list[float]]
from numbers import Number
class PIDController:
def __init__(self, k_p: Gain, k_i: Gain, k_d: Gain = 0., pos_limit=1e308, neg_limit=-1e308, rate=100):
self._k_p = ([0], [k_p]) if isinstance(k_p, (int, float)) else k_p
self._k_i = ([0], [k_i]) if isinstance(k_i, (int, float)) else k_i
self._k_d = ([0], [k_d]) if isinstance(k_d, (int, float)) else k_d
def __init__(self, k_p, k_i, k_d=0., pos_limit=1e308, neg_limit=-1e308, rate=100):
self._k_p: list[list[float]] = [[0], [k_p]] if isinstance(k_p, Number) else k_p
self._k_i: list[list[float]] = [[0], [k_i]] if isinstance(k_i, Number) else k_i
self._k_d: list[list[float]] = [[0], [k_d]] if isinstance(k_d, Number) else k_d
self.set_limits(pos_limit, neg_limit)
@@ -5,7 +5,7 @@
#include "common/params.h"
#include "common/util.h"
#include "common/hardware/hw.h"
#include "system/hardware/hw.h"
class OpenpilotPrefix {
public:
@@ -5,9 +5,9 @@ import uuid
from openpilot.common.params import Params
from openpilot.common.hardware import PC
from openpilot.common.hardware.hw import Paths
from openpilot.common.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
from openpilot.system.hardware import PC
from openpilot.system.hardware.hw import Paths
from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
class OpenpilotPrefix:
def __init__(self, prefix: str | None = None, create_dirs_on_enter: bool = True, clean_dirs_on_exit: bool = True, shared_download_cache: bool = False):
@@ -7,7 +7,7 @@ import time
from setproctitle import getproctitle
from openpilot.common.utils import MovingAverage
from openpilot.common.hardware import PC
from openpilot.system.hardware import PC
# time step for each process
@@ -28,11 +28,6 @@ class Priority:
CTRL_HIGH = 53
def drop_realtime() -> None:
if sys.platform == 'linux' and not PC:
os.sched_setscheduler(0, os.SCHED_OTHER, os.sched_param(0))
def set_core_affinity(cores: list[int]) -> None:
if sys.platform == 'linux' and not PC:
os.sched_setaffinity(0, cores)
@@ -8,7 +8,7 @@ class Spinner:
try:
self.spinner_proc = subprocess.Popen(["./spinner.py"],
stdin=subprocess.PIPE,
cwd=os.path.join(BASEDIR, "openpilot/system", "ui"),
cwd=os.path.join(BASEDIR, "system", "ui"),
close_fds=True)
except OSError:
self.spinner_proc = None
@@ -48,6 +48,9 @@ class RunningStat:
def std(self):
return np.sqrt(self.variance())
def params_to_save(self):
return [self.M, self.S, self.n]
class RunningStatFilter:
def __init__(self, raw_priors=None, filtered_priors=None, max_trackable=-1):
self.raw_stat = RunningStat(raw_priors, -1)
@@ -11,9 +11,9 @@
#include <zmq.h>
#include <stdarg.h>
#include "json11/json11.hpp"
#include "third_party/json11/json11.hpp"
#include "common/version.h"
#include "common/hardware/hw.h"
#include "system/hardware/hw.h"
#include "sunnypilot/common/version.h"
@@ -8,7 +8,7 @@ from logging.handlers import BaseRotatingHandler
import zmq
from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
from openpilot.common.hardware.hw import Paths
from openpilot.system.hardware.hw import Paths
def get_file_handler():
@@ -27,7 +27,7 @@ class SwaglogRotatingFileHandler(BaseRotatingHandler):
self.log_files = self.get_existing_logfiles()
log_indexes = [f.split(".")[-1] for f in self.log_files]
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
self.last_rollover = 0.0
self.last_rollover = None
self.doRollover()
def _open(self):
@@ -39,14 +39,13 @@ class SwaglogRotatingFileHandler(BaseRotatingHandler):
return stream
def get_existing_logfiles(self):
log_files = []
log_files = list()
base_dir = os.path.dirname(self.base_filename)
for fn in os.listdir(base_dir):
fp = os.path.join(base_dir, fn)
if fp.startswith(self.base_filename) and os.path.isfile(fp):
log_files.append(fp)
# newest first, matching _open()'s insert(0, ...) so doRollover()'s pop() deletes the oldest
return sorted(log_files, reverse=True)
return sorted(log_files)
def shouldRollover(self, record):
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
+1
View File
@@ -0,0 +1 @@
test_common
@@ -1,11 +1,10 @@
import os
from uuid import uuid4
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.utils import atomic_write
class TestFileHelpers(OpenpilotTestCase):
class TestFileHelpers:
def run_atomic_write_func(self, atomic_write_func):
path = f"/tmp/tmp{uuid4()}"
with atomic_write_func(path) as f:
@@ -1,11 +1,10 @@
import os
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.basedir import BASEDIR
from openpilot.common.markdown import parse_markdown
class TestMarkdown(OpenpilotTestCase):
class TestMarkdown:
def test_all_release_notes(self):
with open(os.path.join(BASEDIR, "CHANGELOG.md")) as f:
release_notes = f.read().split("\n\n")
+27
View File
@@ -0,0 +1,27 @@
#include "catch2/catch.hpp"
#define private public
#include "common/params.h"
#include "common/util.h"
TEST_CASE("params_nonblocking_put") {
char tmp_path[] = "/tmp/asyncWriter_XXXXXX";
const std::string param_path = mkdtemp(tmp_path);
auto param_names = {"CarParams", "IsMetric"};
{
Params params(param_path);
for (const auto &name : param_names) {
params.putNonBlocking(name, "1");
// param is empty
REQUIRE(params.get(name).empty());
}
// check if thread is running
REQUIRE(params.future.valid());
REQUIRE(params.future.wait_for(std::chrono::milliseconds(0)) == std::future_status::timeout);
}
// check results
Params p(param_path);
for (const auto &name : param_names) {
REQUIRE(p.get(name) == "1");
}
}
@@ -1,28 +1,28 @@
import pytest
import datetime
import os
import threading
import time
import uuid
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.params import Params, ParamKeyFlag, UnknownKeyName
class TestParams(OpenpilotTestCase):
class TestParams:
def setup_method(self):
self.params = Params()
def test_params_put_and_get(self):
self.params.put("DongleId", "cb38263377b873ee", block=True)
self.params.put("DongleId", "cb38263377b873ee")
assert self.params.get("DongleId") == "cb38263377b873ee"
def test_params_non_ascii(self):
st = b"\xe1\x90\xff"
self.params.put("CarParams", st, block=True)
self.params.put("CarParams", st)
assert self.params.get("CarParams") == st
def test_params_get_cleared_manager_start(self):
self.params.put("CarParams", b"test", block=True)
self.params.put("DongleId", "cb38263377b873ee", block=True)
self.params.put("CarParams", b"test")
self.params.put("DongleId", "cb38263377b873ee")
assert self.params.get("CarParams") == b"test"
undefined_param = self.params.get_param_path(uuid.uuid4().hex)
@@ -36,36 +36,31 @@ class TestParams(OpenpilotTestCase):
assert not os.path.isfile(undefined_param)
def test_params_two_things(self):
self.params.put("DongleId", "bob", block=True)
self.params.put("AthenadPid", 123, block=True)
self.params.put("DongleId", "bob")
self.params.put("AthenadPid", 123)
assert self.params.get("DongleId") == "bob"
assert self.params.get("AthenadPid") == 123
def test_params_get_block(self):
def _delayed_writer():
time.sleep(0.1)
self.params.put("CarParams", b"test", block=True)
self.params.put("CarParams", b"test")
threading.Thread(target=_delayed_writer).start()
assert self.params.get("CarParams") is None
assert self.params.get("CarParams", block=True) == b"test"
def test_params_unknown_key_fails(self):
with self.assertRaises(UnknownKeyName):
with pytest.raises(UnknownKeyName):
self.params.get("swag")
with self.assertRaises(UnknownKeyName):
with pytest.raises(UnknownKeyName):
self.params.get_bool("swag")
with self.assertRaises(UnknownKeyName):
self.params.put("swag", "abc", block=True)
with pytest.raises(UnknownKeyName):
self.params.put("swag", "abc")
with self.assertRaises(UnknownKeyName):
self.params.put_bool("swag", True, block=True)
with self.assertRaises(UnknownKeyName):
self.params.put(b"DongleId\0suffix", "abc", block=True)
assert self.params.get_param_path(b"key\0suffix").endswith("/key\0suffix")
with pytest.raises(UnknownKeyName):
self.params.put_bool("swag", True)
def test_remove_not_there(self):
assert self.params.get("CarParams") is None
@@ -76,23 +71,23 @@ class TestParams(OpenpilotTestCase):
self.params.remove("IsMetric")
assert not self.params.get_bool("IsMetric")
self.params.put_bool("IsMetric", True, block=True)
self.params.put_bool("IsMetric", True)
assert self.params.get_bool("IsMetric")
self.params.put_bool("IsMetric", False, block=True)
self.params.put_bool("IsMetric", False)
assert not self.params.get_bool("IsMetric")
self.params.put("IsMetric", True, block=True)
self.params.put("IsMetric", True)
assert self.params.get_bool("IsMetric")
self.params.put("IsMetric", False, block=True)
self.params.put("IsMetric", False)
assert not self.params.get_bool("IsMetric")
def test_put_non_blocking_with_get_block(self):
q = Params()
def _delayed_writer():
time.sleep(0.1)
Params().put("CarParams", b"test")
Params().put_nonblocking("CarParams", b"test")
threading.Thread(target=_delayed_writer).start()
assert q.get("CarParams") is None
assert q.get("CarParams", True) == b"test"
@@ -101,7 +96,7 @@ class TestParams(OpenpilotTestCase):
q = Params()
def _delayed_writer():
time.sleep(0.1)
Params().put_bool("CarParams", True)
Params().put_bool_nonblocking("CarParams", True)
threading.Thread(target=_delayed_writer).start()
assert q.get("CarParams") is None
assert q.get("CarParams", True) == b"1"
@@ -117,30 +112,30 @@ class TestParams(OpenpilotTestCase):
def test_params_default_value(self):
self.params.remove("LanguageSetting")
self.params.remove("LongitudinalPersonality")
self.params.remove("LiveParametersV2")
self.params.remove("LiveParameters")
assert self.params.get("LanguageSetting") is None
assert self.params.get("LanguageSetting", return_default=False) is None
assert isinstance(self.params.get("LanguageSetting", return_default=True), str)
assert isinstance(self.params.get("LongitudinalPersonality", return_default=True), int)
assert self.params.get("LiveParametersV2") is None
assert self.params.get("LiveParametersV2", return_default=True) is None
assert self.params.get("LiveParameters") is None
assert self.params.get("LiveParameters", return_default=True) is None
def test_params_get_type(self):
# json
self.params.put("ApiCache_FirehoseStats", {"a": 0}, block=True)
self.params.put("ApiCache_FirehoseStats", {"a": 0})
assert self.params.get("ApiCache_FirehoseStats") == {"a": 0}
# int
self.params.put("BootCount", 1441, block=True)
self.params.put("BootCount", 1441)
assert self.params.get("BootCount") == 1441
# bool
self.params.put("AdbEnabled", True, block=True)
self.params.put("AdbEnabled", True)
assert self.params.get("AdbEnabled")
assert isinstance(self.params.get("AdbEnabled"), bool)
# time
now = datetime.datetime.now(datetime.UTC)
self.params.put("InstallDate", now, block=True)
self.params.put("InstallDate", now)
assert self.params.get("InstallDate") == now
+2
View File
@@ -0,0 +1,2 @@
#define CATCH_CONFIG_MAIN
#include "catch2/catch.hpp"
@@ -1,8 +1,7 @@
from openpilot.common.test import OpenpilotTestCase
from openpilot.common.simple_kalman import KF1D
class TestSimpleKalman(OpenpilotTestCase):
class TestSimpleKalman:
def setup_method(self):
dt = 0.01
x0_0 = 0.0

Some files were not shown because too many files have changed in this diff Show More