diff --git a/.github/labeler.yaml b/.github/labeler.yaml new file mode 100644 index 0000000000..4c4d371d43 --- /dev/null +++ b/.github/labeler.yaml @@ -0,0 +1,61 @@ +CI / testing: + - all: + - changed-files: ['.github/**'] + - all: + - changed-files: ['**/test_*'] + +car: + - all: + - changed-files: ['selfdrive/car/**'] + +body: + - all: + - changed-files: ['selfdrive/car/body/*'] +chrysler: + - all: + - changed-files: ['selfdrive/car/chrysler/*'] +ford: + - all: + - changed-files: ['selfdrive/car/ford/*'] +gm: + - all: + - changed-files: ['selfdrive/car/gm/*'] +honda: + - all: + - changed-files: ['selfdrive/car/honda/*'] +hyundai: + - all: + - changed-files: ['selfdrive/car/hyundai/*'] +mazda: + - all: + - changed-files: ['selfdrive/car/mazda/*'] +nissan: + - all: + - changed-files: ['selfdrive/car/nissan/*'] +subaru: + - all: + - changed-files: ['selfdrive/car/subaru/*'] +tesla: + - all: + - changed-files: ['selfdrive/car/tesla/*'] +toyota: + - all: + - changed-files: ['selfdrive/car/toyota/*'] +volkswagen: + - all: + - changed-files: ['selfdrive/car/volkswagen/*'] + +simulation: + - all: + - changed-files: ['tools/sim/**'] +ui: + - all: + - changed-files: ['selfdrive/ui/**'] +tools: + - all: + - changed-files: ['tools/**'] + +multilanguage: + - all: + - changed-files: ['selfdrive/ui/translations/**'] + diff --git a/.github/workflows/compile-openpilot/action.yaml b/.github/workflows/compile-openpilot/action.yaml new file mode 100644 index 0000000000..8775c96262 --- /dev/null +++ b/.github/workflows/compile-openpilot/action.yaml @@ -0,0 +1,27 @@ +name: 'compile openpilot' + +inputs: + cache_key_prefix: + description: 'Prefix for caching key' + required: false + default: 'scons' + +runs: + using: "composite" + steps: + - shell: bash + name: Build openpilot with all flags + run: | + ${{ env.RUN }} "scons -j$(nproc)" + ${{ env.RUN }} "release/check-dirty.sh" + - shell: bash + name: Cleanup scons cache and rebuild + run: | + ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ + scons -j$(nproc) --cache-populate" + - name: Save scons cache + uses: actions/cache/save@v3 + if: github.ref == 'refs/heads/master' + with: + path: .ci_cache/scons_cache + key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} diff --git a/.github/workflows/labeler.yaml b/.github/workflows/labeler.yaml new file mode 100644 index 0000000000..20f7260ef7 --- /dev/null +++ b/.github/workflows/labeler.yaml @@ -0,0 +1,18 @@ +name: "Pull Request Labeler" +on: + pull_request_target: + +jobs: + labeler: + permissions: + contents: read + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + submodules: false + - uses: actions/labeler@v5.0.0-alpha.1 + with: + dot: true + configuration-path: .github/labeler.yaml \ No newline at end of file diff --git a/.github/workflows/prebuilt.yaml b/.github/workflows/prebuilt.yaml index 8b16ea90b9..3558d7eb7e 100644 --- a/.github/workflows/prebuilt.yaml +++ b/.github/workflows/prebuilt.yaml @@ -5,7 +5,7 @@ on: workflow_dispatch: env: - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh prebuilt jobs: @@ -29,5 +29,5 @@ jobs: submodules: true - name: Build and Push docker image run: | - $DOCKER_LOGIN + $DOCKER_GHCR_LOGIN eval "$BUILD" diff --git a/.github/workflows/selfdrive_tests.yaml b/.github/workflows/selfdrive_tests.yaml index a631069726..f7aba648a4 100644 --- a/.github/workflows/selfdrive_tests.yaml +++ b/.github/workflows/selfdrive_tests.yaml @@ -16,7 +16,8 @@ env: CL_BASE_IMAGE: openpilot-base-cl AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }} - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + DOCKER_HUB_LOGIN: docker login -u adeebshihadeh -p ${{ secrets.DOCKER_HUB_PAT }} BUILD: selfdrive/test/docker_build.sh base RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/sh -c @@ -67,30 +68,27 @@ jobs: cd $STRIPPED_DIR ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all" - build_all: - name: build all - runs-on: ubuntu-20.04 + build: + strategy: + matrix: + arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} steps: - uses: actions/checkout@v3 with: submodules: true + # login only on arm machines, due to buildjet rate limits + - name: Setup docker + if: contains(runner.name, 'buildjet') + run: | + $DOCKER_HUB_LOGIN - uses: ./.github/workflows/setup-with-retry - - name: Build openpilot with all flags - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 12 || 30) }} # allow more time when we missed the scons cache - run: | - ${{ env.RUN }} "scons -j$(nproc)" - ${{ env.RUN }} "release/check-dirty.sh" - - name: Cleanup scons cache and rebuild - timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 2 || 30) }} # allow more time when we missed the scons cache - run: | - ${{ env.RUN }} "rm -rf /tmp/scons_cache/* && \ - scons -j$(nproc) --cache-populate" - - name: Save scons cache - uses: actions/cache/save@v3 - if: github.ref == 'refs/heads/master' with: - path: .ci_cache/scons_cache - key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + cache_key_prefix: scons_${{ matrix.arch }} + - uses: ./.github/workflows/compile-openpilot + timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache + with: + cache_key_prefix: scons_${{ matrix.arch }} build_mac: name: build macos @@ -185,8 +183,10 @@ jobs: sudo rm -rf /Applications/ArmGNUToolchain/*/*/.fseventsd docker_push: - name: docker push - runs-on: ubuntu-20.04 + strategy: + matrix: + arch: ${{ fromJson( (github.repository == 'commaai/openpilot') && '["x86_64", "aarch64"]' || '["x86_64"]' ) }} + runs-on: ${{ (matrix.arch == 'aarch64') && 'buildjet-2vcpu-ubuntu-2204-arm' || 'ubuntu-20.04' }} if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' steps: - uses: actions/checkout@v3 @@ -195,14 +195,39 @@ jobs: - name: Setup to push to repo run: | echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN + echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV" + $DOCKER_GHCR_LOGIN + # login only on arm machines, due to buildjet rate limits + - name: Additional setup for buildjet + if: contains(runner.name, 'buildjet') + run: | + $DOCKER_HUB_LOGIN - uses: ./.github/workflows/setup-with-retry with: git-lfs: false - name: Build and push CL Docker image + if: matrix.arch == 'x86_64' run: | + unset TARGET_ARCHITECTURE eval "$BUILD_CL" + docker_push_multiarch: + name: docker push multiarch tag + runs-on: ubuntu-20.04 + if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' + needs: [docker_push] + steps: + - uses: actions/checkout@v3 + with: + submodules: false + - name: Setup docker + run: | + $DOCKER_GHCR_LOGIN + - name: Merge x64 and arm64 tags + run: | + export PUSH_IMAGE=true + selfdrive/test/docker_tag_multiarch.sh base x86_64 aarch64 + static_analysis: name: static analysis runs-on: ubuntu-20.04 @@ -249,18 +274,10 @@ jobs: timeout-minutes: 40 run: | ${{ env.RUN }} "export SKIP_LONG_TESTS=1 && \ - $PYTEST && \ - selfdrive/locationd/test/_test_locationd_lib.py && \ - ./system/ubloxd/tests/test_glonass_runner && \ + $PYTEST -n auto --dist=loadscope --timeout 30 -o cpp_files=test_* && \ ./selfdrive/ui/tests/create_test_translations.sh && \ QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \ ./selfdrive/ui/tests/test_translations.py && \ - ./common/tests/test_common && \ - ./selfdrive/boardd/tests/test_boardd_usbprotocol && \ - ./system/loggerd/tests/test_logger &&\ - ./system/proclogd/tests/test_proclog && \ - ./tools/replay/tests/test_replay && \ - ./tools/cabana/tests/test_cabana && \ ./system/camerad/test/ae_gray_test && \ ./selfdrive/test/process_replay/test_fuzzy.py" - name: "Upload coverage to Codecov" @@ -324,7 +341,7 @@ jobs: ${{ env.RUN }} "scons -j$(nproc)" # PYTHONWARNINGS triggers a SyntaxError in onnxruntime - name: Run model replay with ONNX - timeout-minutes: 2 + timeout-minutes: 3 run: | ${{ env.RUN_CL }} "unset PYTHONWARNINGS && \ ONNXCPU=1 CI=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \ diff --git a/.github/workflows/setup-with-retry/action.yaml b/.github/workflows/setup-with-retry/action.yaml index 427b5a45ac..e4c97af3ce 100644 --- a/.github/workflows/setup-with-retry/action.yaml +++ b/.github/workflows/setup-with-retry/action.yaml @@ -5,9 +5,14 @@ inputs: description: 'Whether or not to pull the git lfs' required: false default: 'true' - -env: - SLEEP_TIME: 30 # Time to sleep between retries + cache_key_prefix: + description: 'Prefix for caching key' + required: false + default: 'scons_x86_64' + sleep_time: + description: 'Time to sleep between retries' + required: false + default: 30 runs: using: "composite" @@ -17,23 +22,26 @@ runs: continue-on-error: true with: git_lfs: ${{ inputs.git_lfs }} + cache_key_prefix: ${{ inputs.cache_key_prefix }} is_retried: true - if: steps.setup1.outcome == 'failure' shell: bash - run: sleep ${{ env.SLEEP_TIME }} + run: sleep ${{ inputs.sleep_time }} - id: setup2 if: steps.setup1.outcome == 'failure' uses: ./.github/workflows/setup continue-on-error: true with: git_lfs: ${{ inputs.git_lfs }} + cache_key_prefix: ${{ inputs.cache_key_prefix }} is_retried: true - if: steps.setup2.outcome == 'failure' shell: bash - run: sleep ${{ env.SLEEP_TIME }} + run: sleep ${{ inputs.sleep_time }} - id: setup3 if: steps.setup2.outcome == 'failure' uses: ./.github/workflows/setup with: git_lfs: ${{ inputs.git_lfs }} - is_retried: true \ No newline at end of file + cache_key_prefix: ${{ inputs.cache_key_prefix }} + is_retried: true diff --git a/.github/workflows/setup/action.yaml b/.github/workflows/setup/action.yaml index 74b82f9158..eebc376346 100644 --- a/.github/workflows/setup/action.yaml +++ b/.github/workflows/setup/action.yaml @@ -5,6 +5,10 @@ inputs: description: 'Whether or not to pull the git lfs' required: false default: 'true' + cache_key_prefix: + description: 'Prefix for caching key' + required: false + default: 'scons_x86_64' is_retried: description: 'A mock param that asserts that we use the setup-with-retry instead of this action directly' required: false @@ -35,10 +39,10 @@ runs: uses: actions/cache/restore@v3 with: path: .ci_cache/scons_cache - key: scons-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} + key: ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }} restore-keys: | - scons-${{ env.CACHE_COMMIT_DATE }}- - scons- + ${{ inputs.cache_key_prefix }}-${{ env.CACHE_COMMIT_DATE }}- + ${{ inputs.cache_key_prefix }}- # if we didn't get a cache hit, make the directory manually so it doesn't fail on future steps - id: scons-cache-setup shell: bash diff --git a/.github/workflows/tools_tests.yaml b/.github/workflows/tools_tests.yaml index c7a5f80c3b..27c362fcd6 100644 --- a/.github/workflows/tools_tests.yaml +++ b/.github/workflows/tools_tests.yaml @@ -13,7 +13,7 @@ concurrency: env: BASE_IMAGE: openpilot-base CL_BASE_IMAGE: openpilot-base-cl - DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} + DOCKER_GHCR_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh base @@ -58,7 +58,7 @@ jobs: if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot' run: | echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN + $DOCKER_GHCR_LOGIN - name: Build and push sim image run: | selfdrive/test/docker_build.sh sim @@ -78,7 +78,7 @@ jobs: if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot' run: | echo "PUSH_IMAGE=true" >> "$GITHUB_ENV" - $DOCKER_LOGIN + $DOCKER_GHCR_LOGIN - name: Build and push docs image run: | selfdrive/test/docker_build.sh docs diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6f6e7285d1..a5da03b266 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,9 +1,89 @@ +exclude: '^(tinygrad_repo)' repos: +- repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-ast + exclude: '^(third_party)/' + - id: check-json + - id: check-toml + - id: check-xml + - id: check-yaml + - id: check-merge-conflict + - id: check-symlinks + - id: check-executables-have-shebangs + - id: check-shebang-scripts-are-executable + - id: check-added-large-files + args: ['--maxkb=100'] +- repo: https://github.com/codespell-project/codespell + rev: v2.2.5 + hooks: + - id: codespell + exclude: '^(third_party/)|(body/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(opendbc/)|(laika_repo/)|(rednose_repo/)|(selfdrive/ui/translations/.*.ts)|(poetry.lock)' + args: + # if you've got a short variable name that's getting flagged, add it here + - -L bu,ro,te,ue,alo,hda,ois,nam,nams,ned,som,parm,setts,inout,warmup,bumb,nd,sie + - --builtins clear,rare,informal,usage,code,names,en-GB_to_en-US +- repo: local + hooks: + - id: mypy + name: mypy + entry: mypy + language: system + types: [python] + args: ['--explicit-package-bases'] + exclude: '^(third_party/)|(cereal/)|(opendbc/)|(panda/)|(laika/)|(laika_repo/)|(rednose/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)|(xx/)' +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.0.288 + hooks: + - id: ruff + exclude: '^(third_party/)|(cereal/)|(rednose/)|(panda/)|(laika/)|(laika_repo/)|(rednose_repo/)|(tinygrad/)|(tinygrad_repo/)' +- repo: local + hooks: + - id: cppcheck + name: cppcheck + entry: cppcheck + language: system + types: [c++] + exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(tools/)|(selfdrive/modeld/thneed/debug/)|(selfdrive/modeld/test/)|(selfdrive/camerad/test/)|(installer/)' + args: + - --error-exitcode=1 + - --language=c++ + - --quiet + - --force + - -j8 +- repo: https://github.com/cpplint/cpplint + rev: 1.6.1 + hooks: + - id: cpplint + exclude: '^(third_party/)|(cereal/)|(body/)|(rednose/)|(rednose_repo/)|(opendbc/)|(panda/)|(generated/)' + args: + - --quiet + - --counting=total + - --linelength=240 + # https://google.github.io/styleguide/cppguide.html + # relevant rules are whitelisted, see all options with: cpplint --filter= + - --filter=-build,-legal,-readability,-runtime,-whitespace,+build/include_subdir,+build/forward_decl,+build/include_what_you_use,+build/deprecated,+whitespace/comma,+whitespace/line_length,+whitespace/empty_if_body,+whitespace/empty_loop_body,+whitespace/empty_conditional_body,+whitespace/forcolon,+whitespace/parens,+whitespace/semicolon,+whitespace/tab,+readability/braces +- repo: local + hooks: + - id: test_translations + name: test translations + entry: selfdrive/ui/tests/test_translations.py + language: script + pass_filenames: false - repo: https://github.com/python-poetry/poetry - rev: '1.5.0' + rev: '1.6.0' hooks: - id: poetry-check - id: poetry-lock name: validate poetry lock args: - --check +- repo: https://github.com/python-jsonschema/check-jsonschema + rev: 0.26.3 + hooks: + - id: check-github-workflows diff --git a/Jenkinsfile b/Jenkinsfile index c0bdd53253..176c82bcc9 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,4 +1,4 @@ -def phone(String ip, String step_label, String cmd) { +def device(String ip, String step_label, String cmd) { withCredentials([file(credentialsId: 'id_rsa', variable: 'key_file')]) { def ssh_cmd = """ ssh -tt -o StrictHostKeyChecking=no -i ${key_file} 'comma@${ip}' /usr/bin/bash <<'END' @@ -51,226 +51,186 @@ END""" } } -def phone_steps(String device_type, steps) { - lock(resource: "", label: device_type, inversePrecedence: true, variable: 'device_ip', quantity: 1) { - timeout(time: 20, unit: 'MINUTES') { - phone(device_ip, "git checkout", readFile("selfdrive/test/setup_device_ci.sh"),) - steps.each { item -> - phone(device_ip, item[0], item[1]) +def deviceStage(String stageName, String deviceType, List env, def steps) { + stage(stageName) { + if (currentBuild.result != null) { + return + } + + def extra = env.collect { "export ${it}" }.join('\n'); + + docker.image('ghcr.io/commaai/alpine-ssh').inside('--user=root') { + lock(resource: "", label: deviceType, inversePrecedence: true, variable: 'device_ip', quantity: 1) { + timeout(time: 20, unit: 'MINUTES') { + device(device_ip, "git checkout", extra + "\n" + readFile("selfdrive/test/setup_device_ci.sh")) + steps.each { item -> + device(device_ip, item[0], item[1]) + } + } } } } } -pipeline { - agent none - environment { - CI = "1" - PYTHONWARNINGS = "error" - TEST_DIR = "/data/openpilot" - SOURCE_DIR = "/data/openpilot_source/" - AZURE_TOKEN = credentials('azure_token') - MAPBOX_TOKEN = credentials('mapbox_token') +def pcStage(String stageName, Closure body) { + node { + stage(stageName) { + if (currentBuild.result != null) { + return + } + + checkout scm + + def dockerArgs = '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache -v /tmp/scons_cache:/tmp/scons_cache'; + docker.build("openpilot-base:build-${env.GIT_COMMIT}", "-f Dockerfile.openpilot_base .").inside(dockerArgs) { + timeout(time: 20, unit: 'MINUTES') { + try { + sh "git config --global --add safe.directory '*'" + sh "git submodule update --init --recursive" + sh "git lfs pull" + body() + } finally { + sh "rm -rf ${env.WORKSPACE}/* || true" + sh "rm -rf .* || true" + } + } + } } - options { - timeout(time: 3, unit: 'HOURS') - disableConcurrentBuilds(abortPrevious: env.BRANCH_NAME != 'master') - } - - stages { - stage('build release3-staging') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - when { - branch 'devel-staging' - } - steps { - phone_steps("tici-needs-can", [ - ["build release3-staging & dashcam3-staging", "RELEASE_BRANCH=release3-staging DASHCAM_BRANCH=dashcam3-staging $SOURCE_DIR/release/build_release.sh"], - ]) - } - } - - stage('build nightly') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - when { - branch 'master-ci' - } - steps { - phone_steps("tici-needs-can", [ - ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"], - ]) - } - } - - stage('openpilot tests') { - when { - not { - anyOf { - branch 'master-ci'; branch 'devel'; branch 'devel-staging'; - branch 'release3'; branch 'release3-staging'; branch 'dashcam3'; branch 'dashcam3-staging'; - branch 'testing-closet*'; branch 'hotfix-*' - } - } - } - - parallel { - - /* - stage('simulator') { - agent { - dockerfile { - filename 'Dockerfile.sim_nvidia' - dir 'tools/sim' - args '--user=root' - } - } - steps { - sh "git config --global --add safe.directory '*'" - sh "git submodule update --init --recursive" - sh "git lfs pull" - lock(resource: "", label: "simulator", inversePrecedence: true, quantity: 1) { - sh "${WORKSPACE}/tools/sim/build_container.sh" - sh "DETACH=1 ${WORKSPACE}/tools/sim/start_carla.sh" - sh "${WORKSPACE}/tools/sim/start_openpilot_docker.sh" - } - } - - post { - always { - sh "docker kill carla_sim || true" - sh "rm -rf ${WORKSPACE}/* || true" - sh "rm -rf .* || true" - } - } - } - */ - - stage('PC tests') { - agent { - dockerfile { - filename 'Dockerfile.openpilot_base' - args '--user=root -v /tmp/comma_download_cache:/tmp/comma_download_cache' - } - } - steps { - sh "git config --global --add safe.directory '*'" - sh "git submodule update --init --depth=1 --recursive" - sh "git lfs pull" - // tests that our build system's dependencies are configured properly, needs a machine with lots of cores - sh "scons --clean && scons --no-cache --random -j42" - sh "INTERNAL_SEG_CNT=500 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \ - pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py" - sh "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py" - } - - post { - always { - sh "rm -rf ${WORKSPACE}/* || true" - sh "rm -rf .* || true" - } - } - } - - stage('tizi-tests') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tizi", [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], - ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], - ["test rawgpsd", "pytest system/sensord/rawgps/test_rawgps.py"], - ]) - } - } - - stage('build') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - environment { - R3_PUSH = "${env.BRANCH_NAME == 'master' ? '1' : ' '}" - } - steps { - phone_steps("tici-needs-can", [ - ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR ./build_devel.sh"], - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["check dirty", "release/check-dirty.sh"], - ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"], - ["time to onroad", "cd selfdrive/test/ && pytest test_time_to_onroad.py"], - ]) - } - } - - stage('loopback-tests') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tici-loopback", [ - ["build openpilot", "cd selfdrive/manager && ./build.py"], - ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], - ]) - } - } - - stage('HW + Unit Tests') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tici-common", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], - ["test power draw", "pytest system/hardware/tici/tests/test_power_draw.py"], - ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], - ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"], - ["test manager", "pytest selfdrive/manager/test/test_manager.py"], - ["test nav", "pytest selfdrive/navd/tests/"], - ]) - } - } - - stage('camerad') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tici-ar0231", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - phone_steps("tici-ox03c10", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test camerad", "pytest system/camerad/test/test_camerad.py"], - ["test exposure", "pytest system/camerad/test/test_exposure.py"], - ]) - } - } - - stage('sensord') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tici-lsmc", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], - ]) - phone_steps("tici-bmx-lsm", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], - ]) - } - } - - stage('replay') { - agent { docker { image 'ghcr.io/commaai/alpine-ssh'; args '--user=root' } } - steps { - phone_steps("tici-replay", [ - ["build", "cd selfdrive/manager && ./build.py"], - ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"], - ]) - } - } - - } - } - } } + +def setupCredentials() { + withCredentials([ + string(credentialsId: 'azure_token', variable: 'AZURE_TOKEN'), + string(credentialsId: 'mapbox_token', variable: 'MAPBOX_TOKEN') + ]) { + env.AZURE_TOKEN = "${AZURE_TOKEN}" + env.MAPBOX_TOKEN = "${MAPBOX_TOKEN}" + } +} + +node { + env.CI = "1" + env.PYTHONWARNINGS = "error" + env.TEST_DIR = "/data/openpilot" + env.SOURCE_DIR = "/data/openpilot_source/" + setupCredentials() + + env.GIT_BRANCH = checkout(scm).GIT_BRANCH + env.GIT_COMMIT = checkout(scm).GIT_COMMIT + + def excludeBranches = ['master-ci', 'devel', 'devel-staging', 'release3', 'release3-staging', + 'dashcam3', 'dashcam3-staging', 'testing-closet*', 'hotfix-*'] + def excludeRegex = excludeBranches.join('|').replaceAll('\\*', '.*') + + if (env.BRANCH_NAME != 'master') { + properties([ + disableConcurrentBuilds(abortPrevious: true) + ]) + } + + try { + if (env.BRANCH_NAME == 'devel-staging') { + deviceStage("build release3-staging", "tici-needs-can", [], [ + ["build release3-staging & dashcam3-staging", "RELEASE_BRANCH=release3-staging DASHCAM_BRANCH=dashcam3-staging $SOURCE_DIR/release/build_release.sh"], + ]) + } + + if (env.BRANCH_NAME == 'master-ci') { + deviceStage("build nightly", "tici-needs-can", [], [ + ["build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"], + ]) + } + + if (!env.BRANCH_NAME.matches(excludeRegex)) { + parallel ( + // tici tests + 'onroad tests': { + deviceStage("onroad", "tici-needs-can", ["SKIP_COPY=1"], [ + ["build master-ci", "cd $SOURCE_DIR/release && TARGET_DIR=$TEST_DIR ./build_devel.sh"], + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["check dirty", "release/check-dirty.sh"], + ["onroad tests", "cd selfdrive/test/ && ./test_onroad.py"], + ["time to onroad", "cd selfdrive/test/ && pytest test_time_to_onroad.py"], + ]) + }, + 'HW + Unit Tests': { + deviceStage("tici", "tici-common", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test power draw", "./system/hardware/tici/tests/test_power_draw.py"], + ["test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest system/loggerd/tests/test_encoder.py"], + ["test pigeond", "pytest system/sensord/tests/test_pigeond.py"], + ["test manager", "pytest selfdrive/manager/test/test_manager.py"], + ["test nav", "pytest selfdrive/navd/tests/"], + ]) + }, + 'loopback': { + deviceStage("tici", "tici-loopback", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ]) + }, + 'camerad': { + deviceStage("AR0231", "tici-ar0231", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + deviceStage("OX03C10", "tici-ox03c10", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test camerad", "pytest system/camerad/test/test_camerad.py"], + ["test exposure", "pytest system/camerad/test/test_exposure.py"], + ]) + }, + 'sensord': { + deviceStage("LSM + MMC", "tici-lsmc", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], + ]) + deviceStage("BMX + LSM", "tici-bmx-lsm", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["test sensord", "cd system/sensord/tests && pytest test_sensord.py"], + ]) + }, + 'replay': { + deviceStage("tici", "tici-replay", ["UNSAFE=1"], [ + ["build", "cd selfdrive/manager && ./build.py"], + ["model replay", "cd selfdrive/test/process_replay && ./model_replay.py"], + ]) + }, + 'tizi': { + deviceStage("tizi", "tizi", ["UNSAFE=1"], [ + ["build openpilot", "cd selfdrive/manager && ./build.py"], + ["test boardd loopback", "SINGLE_PANDA=1 pytest selfdrive/boardd/tests/test_boardd_loopback.py"], + ["test pandad", "pytest selfdrive/boardd/tests/test_pandad.py"], + ["test amp", "pytest system/hardware/tici/tests/test_amplifier.py"], + ["test hw", "pytest system/hardware/tici/tests/test_hardware.py"], + ["test rawgpsd", "pytest system/sensord/rawgps/test_rawgps.py"], + ]) + }, + + // *** PC tests *** + 'PC tests': { + pcStage("PC tests") { + // tests that our build system's dependencies are configured properly, + // needs a machine with lots of cores + sh label: "test multi-threaded build", script: "scons --no-cache --random -j42" + } + }, + 'car tests': { + pcStage("car tests") { + sh "scons -j30" + sh label: "test_models.py", script: "INTERNAL_SEG_CNT=250 INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt FILEREADER_CACHE=1 \ + pytest -n42 --dist=loadscope selfdrive/car/tests/test_models.py" + sh label: "test_car_interfaces.py", script: "MAX_EXAMPLES=100 pytest -n42 selfdrive/car/tests/test_car_interfaces.py" + } + }, + + ) + } + } catch (Exception e) { + currentBuild.result = 'FAILED' + throw e + } +} \ No newline at end of file diff --git a/RELEASES.md b/RELEASES.md index 0a7925200e..89f864dd71 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -1,7 +1,11 @@ -Version 0.9.5 (202X-XX-XX) +Version 0.9.5 (2023-09-27) ======================== +* New driving model + * Improved navigate on openpilot performance using navigation instructions as an additional model input +* Hyundai Azera 2022 support thanks to sunnyhaibin! * Hyundai Ioniq 6 2023 support thanks to sunnyhaibin, alamo3, and sshane! * Hyundai Kona Electric 2023 (Korean version) support thanks to sunnyhaibin and haram-KONA! +* Kia K8 Hybrid (with HDA II) 2023 support thanks to sunnyhaibin! * Kia Sorento Hybrid 2023 support thanks to sunnyhaibin! * Lexus IS 2023 support thanks to L3R5! diff --git a/SConstruct b/SConstruct index f213a5e0bd..a086f8448a 100644 --- a/SConstruct +++ b/SConstruct @@ -244,18 +244,6 @@ def progress_function(node): if os.environ.get('SCONS_PROGRESS'): Progress(progress_function, interval=node_interval) -SHARED = False - -# TODO: this can probably be removed -def abspath(x): - if arch == 'aarch64': - pth = os.path.join("/data/pythonpath", x[0].path) - env.Depends(pth, x) - return File(pth) - else: - # rpath works elsewhere - return x[0].path.rsplit("/", 1)[1][:-3] - # Cython build environment py_include = sysconfig.get_paths()['include'] envCython = env.Clone() @@ -337,34 +325,35 @@ if GetOption("clazy"): qt_env['ENV']['CLAZY_IGNORE_DIRS'] = qt_dirs[0] qt_env['ENV']['CLAZY_CHECKS'] = ','.join(checks) -Export('env', 'qt_env', 'arch', 'real_arch', 'SHARED') +Export('env', 'qt_env', 'arch', 'real_arch') +# Build common module SConscript(['common/SConscript']) Import('_common', '_gpucommon') -if SHARED: - common, gpucommon = abspath(common), abspath(gpucommon) -else: - common = [_common, 'json11'] - gpucommon = [_gpucommon] +common = [_common, 'json11'] +gpucommon = [_gpucommon] Export('common', 'gpucommon') -# cereal and messaging are shared with the system +# Build cereal and messaging SConscript(['cereal/SConscript']) -if SHARED: - cereal = abspath([File('cereal/libcereal_shared.so')]) - messaging = abspath([File('cereal/libmessaging_shared.so')]) -else: - cereal = [File('#cereal/libcereal.a')] - messaging = [File('#cereal/libmessaging.a')] - visionipc = [File('#cereal/libvisionipc.a')] + +cereal = [File('#cereal/libcereal.a')] +messaging = [File('#cereal/libmessaging.a')] +visionipc = [File('#cereal/libvisionipc.a')] messaging_python = [File('#cereal/messaging/messaging_pyx.so')] Export('cereal', 'messaging', 'messaging_python', 'visionipc') -# Build rednose library and ekf models +# Build other submodules +SConscript([ + 'body/board/SConscript', + 'opendbc/can/SConscript', + 'panda/SConscript', +]) +# Build rednose library and ekf models rednose_deps = [ "#selfdrive/locationd/models/constants.py", "#selfdrive/locationd/models/gnss_helpers.py", @@ -406,15 +395,6 @@ if arch != "Darwin": ]) # Build openpilot - -# build submodules -SConscript([ - 'body/board/SConscript', - 'cereal/SConscript', - 'opendbc/can/SConscript', - 'panda/SConscript', -]) - SConscript(['third_party/SConscript']) SConscript(['selfdrive/boardd/SConscript']) diff --git a/cereal b/cereal index fae6aed120..bf00063015 160000 --- a/cereal +++ b/cereal @@ -1 +1 @@ -Subproject commit fae6aed120b5b1b22e21785ecfc416fe1ed959d2 +Subproject commit bf00063015ef0cfc517b762d6f901f178256f520 diff --git a/common/SConscript b/common/SConscript index d22aca128b..97322b248d 100644 --- a/common/SConscript +++ b/common/SConscript @@ -1,9 +1,4 @@ -Import('env', 'envCython', 'arch', 'SHARED') - -if SHARED: - fxn = env.SharedLibrary -else: - fxn = env.Library +Import('env', 'envCython', 'arch') common_libs = [ 'params.cc', @@ -18,13 +13,13 @@ common_libs = [ if arch != "Darwin": common_libs.append('gpio.cc') -_common = fxn('common', common_libs, LIBS="json11") +_common = env.Library('common', common_libs, LIBS="json11") files = [ 'clutil.cc', ] -_gpucommon = fxn('gpucommon', files) +_gpucommon = env.Library('gpucommon', files) Export('_common', '_gpucommon') if GetOption('extras'): diff --git a/common/params.h b/common/params.h index 24b1bffeb1..fbe0bba6b0 100644 --- a/common/params.h +++ b/common/params.h @@ -15,7 +15,11 @@ enum ParamKeyType { class Params { public: - Params(const std::string &path = {}); + explicit Params(const std::string &path = {}); + // Not copyable. + Params(const Params&) = delete; + Params& operator=(const Params&) = delete; + std::vector allKeys() const; bool checkKey(const std::string &key); ParamKeyType getKeyType(const std::string &key); diff --git a/common/prefix.py b/common/prefix.py index 702b0feae4..6b7e7e2fd7 100644 --- a/common/prefix.py +++ b/common/prefix.py @@ -5,10 +5,11 @@ import uuid from typing import Optional from openpilot.common.params import Params +from openpilot.system.hardware.hw import Paths class OpenpilotPrefix: def __init__(self, prefix: Optional[str] = None, clean_dirs_on_exit: bool = True): - self.prefix = prefix if prefix else str(uuid.uuid4()) + self.prefix = prefix if prefix else str(uuid.uuid4().hex[0:15]) self.msgq_path = os.path.join('/dev/shm', self.prefix) self.clean_dirs_on_exit = clean_dirs_on_exit @@ -18,6 +19,7 @@ class OpenpilotPrefix: os.mkdir(self.msgq_path) except FileExistsError: pass + os.makedirs(Paths.log_root(), exist_ok=True) return self @@ -36,3 +38,6 @@ class OpenpilotPrefix: shutil.rmtree(os.path.realpath(symlink_path), ignore_errors=True) os.remove(symlink_path) shutil.rmtree(self.msgq_path, ignore_errors=True) + shutil.rmtree(Paths.log_root(), ignore_errors=True) + shutil.rmtree(Paths.download_cache_root(), ignore_errors=True) + shutil.rmtree(Paths.comma_home(), ignore_errors=True) diff --git a/common/swaglog.cc b/common/swaglog.cc index 923f701ab2..bcd597a257 100644 --- a/common/swaglog.cc +++ b/common/swaglog.cc @@ -20,7 +20,7 @@ class SwaglogState : public LogState { public: - SwaglogState() : LogState("ipc:///tmp/logmessage") {} + SwaglogState() : LogState(Path::swaglog_ipc().c_str()) {} json11::Json::object ctx_j; diff --git a/common/tests/test_ratekeeper.cc b/common/tests/test_ratekeeper.cc index f5aa16a859..32c7dfe58c 100644 --- a/common/tests/test_ratekeeper.cc +++ b/common/tests/test_ratekeeper.cc @@ -6,11 +6,18 @@ TEST_CASE("RateKeeper") { float freq = GENERATE(10, 50, 100); RateKeeper rk("Test RateKeeper", freq); + + int lags = 0; + int bad_keep_times = 0; for (int i = 0; i < freq; ++i) { double begin = seconds_since_boot(); util::sleep_for(util::random_int(0, 1000.0 / freq - 1)); bool lagged = rk.keepTime(); - REQUIRE(std::abs(seconds_since_boot() - begin - (1 / freq)) < 1e-3); - REQUIRE(lagged == false); + lags += lagged; + bad_keep_times += (seconds_since_boot() - begin - (1 / freq)) > 1e-3; } + + // need a tolerance here due to scheduling + REQUIRE(lags < 5); + REQUIRE(bad_keep_times < 5); } diff --git a/common/tests/test_swaglog.cc b/common/tests/test_swaglog.cc index 322354d730..09bc4c3795 100644 --- a/common/tests/test_swaglog.cc +++ b/common/tests/test_swaglog.cc @@ -9,7 +9,6 @@ #include "system/hardware/hw.h" #include "third_party/json11/json11.hpp" -const char *SWAGLOG_ADDR = "ipc:///tmp/logmessage"; std::string daemon_name = "testy"; std::string dongle_id = "test_dongle_id"; int LINE_NO = 0; @@ -25,7 +24,7 @@ void log_thread(int thread_id, int msg_cnt) { void recv_log(int thread_cnt, int thread_msg_cnt) { void *zctx = zmq_ctx_new(); void *sock = zmq_socket(zctx, ZMQ_PULL); - zmq_bind(sock, SWAGLOG_ADDR); + zmq_bind(sock, Path::swaglog_ipc().c_str()); std::vector thread_msgs(thread_cnt); int total_count = 0; diff --git a/common/tests/test_util.cc b/common/tests/test_util.cc index 68fced19c2..de87fa3e06 100644 --- a/common/tests/test_util.cc +++ b/common/tests/test_util.cc @@ -38,7 +38,8 @@ TEST_CASE("util::read_file") { std::string content = random_bytes(64 * 1024); write(fd, content.c_str(), content.size()); std::string ret = util::read_file(filename); - REQUIRE(ret == content); + bool equal = (ret == content); + REQUIRE(equal); close(fd); } SECTION("read directory") { @@ -108,7 +109,8 @@ TEST_CASE("util::safe_fwrite") { REQUIRE(ret == 0); ret = fclose(f); REQUIRE(ret == 0); - REQUIRE(dat == util::read_file(filename)); + bool equal = (dat == util::read_file(filename)); + REQUIRE(equal); } TEST_CASE("util::create_directories") { diff --git a/common/util.h b/common/util.h index 30f514f8dc..5c0ef7fec5 100644 --- a/common/util.h +++ b/common/util.h @@ -188,9 +188,9 @@ class LogState { void *zctx = nullptr; void *sock = nullptr; int print_level; - const char* endpoint; + std::string endpoint; - LogState(const char* _endpoint) { + LogState(std::string _endpoint) { endpoint = _endpoint; } @@ -202,7 +202,7 @@ class LogState { int timeout = 100; zmq_setsockopt(sock, ZMQ_LINGER, &timeout, sizeof(timeout)); - zmq_connect(sock, endpoint); + zmq_connect(sock, endpoint.c_str()); initialized = true; } diff --git a/docs/CARS.md b/docs/CARS.md index e99cdcf071..8bb318c13e 100644 --- a/docs/CARS.md +++ b/docs/CARS.md @@ -4,7 +4,7 @@ A supported vehicle is one that just works when you install a comma device. All supported cars provide a better experience than any stock system. Supported vehicles reference the US market unless otherwise specified. -# 260 Supported Cars +# 262 Supported Cars |Make|Model|Supported Package|ACC|No ACC accel below|No ALC below|Steering Torque|Resume from stop|Hardware Needed
 |Video| |---|---|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:| @@ -71,6 +71,7 @@ A supported vehicle is one that just works when you install a comma device. All |Honda|Passport 2019-23|All|openpilot|25 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Honda Nidec connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Honda|Pilot 2016-22|Honda Sensing|openpilot|25 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Honda Nidec connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Honda|Ridgeline 2017-23|Honda Sensing|openpilot|25 mph|12 mph|[![star](assets/icon-star-empty.svg)](##)|[![star](assets/icon-star-empty.svg)](##)|
Parts- 1 Honda Nidec connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Hyundai|Azera 2022|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai K connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Elantra 2017-19|Smart Cruise Control (SCC)|Stock|19 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai B connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Elantra 2021-23|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai K connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Elantra GT 2017-19|Smart Cruise Control (SCC)|Stock|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai E connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| @@ -88,8 +89,8 @@ A supported vehicle is one that just works when you install a comma device. All |Hyundai|Ioniq Plug-in Hybrid 2019|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|32 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai C connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Ioniq Plug-in Hybrid 2020-22|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai H connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Kona 2020|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai B connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| -|Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai G connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| -|Hyundai|Kona Electric 2022|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai O connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Hyundai|Kona Electric 2018-21|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai G connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Hyundai|Kona Electric 2022-23|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai O connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Kona Electric (with HDA II, Korea only) 2023[6](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai R connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Kona Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai I connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Hyundai|Palisade 2020-22|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai H connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| @@ -109,7 +110,7 @@ A supported vehicle is one that just works when you install a comma device. All |Hyundai|Veloster 2019-20|Smart Cruise Control (SCC)|Stock|5 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai E connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Jeep|Grand Cherokee 2016-18|Adaptive Cruise Control (ACC)|Stock|0 mph|9 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 FCA connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Jeep|Grand Cherokee 2019-21|Adaptive Cruise Control (ACC)|Stock|0 mph|39 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 FCA connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| -|Kia|Carnival 2023[6](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai A connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Kia|Carnival 2023-24[6](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai A connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|Carnival (China only) 2023[6](#footnotes)|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai K connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|Ceed 2019|Smart Cruise Control (SCC)|Stock|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai E connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|EV6 (Southeast Asia only) 2022-23[6](#footnotes)|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai P connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| @@ -119,6 +120,7 @@ A supported vehicle is one that just works when you install a comma device. All |Kia|Forte 2023|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai E connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|K5 2021-22|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai A connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|K5 Hybrid 2020|Smart Cruise Control (SCC)|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai A connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| +|Kia|K8 Hybrid (with HDA II) 2023[6](#footnotes)|Highway Driving Assist II|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai Q connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|Niro EV 2019|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai H connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|Niro EV 2020|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai F connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| |Kia|Niro EV 2021|All|openpilot available[1](#footnotes)|0 mph|0 mph|[![star](assets/icon-star-full.svg)](##)|[![star](assets/icon-star-full.svg)](##)|
Parts- 1 Hyundai C connector
- 1 RJ45 cable (7 ft)
- 1 comma power v2
- 1 comma three
- 1 harness box
- 1 mount
- 1 right angle OBD-C cable (1.5 ft)
Buy Here
|| diff --git a/opendbc b/opendbc index d292afe444..ab68094723 160000 --- a/opendbc +++ b/opendbc @@ -1 +1 @@ -Subproject commit d292afe444489f172ad2837c80b292fcc6aa6113 +Subproject commit ab68094723d9a615ca940eece04db2f2f755c4ff diff --git a/panda b/panda index be34fc4c02..329e049732 160000 --- a/panda +++ b/panda @@ -1 +1 @@ -Subproject commit be34fc4c0206a8d92e4c30660cbb1193f0ebc6b8 +Subproject commit 329e049732b316c3bf1a23b1895e229a5bfbc9cd diff --git a/poetry.lock b/poetry.lock index b1bd2df7c6..2c134ec32d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -314,13 +314,13 @@ files = [ [[package]] name = "azure-core" -version = "1.29.3" +version = "1.29.4" description = "Microsoft Azure Core Library for Python" optional = false python-versions = ">=3.7" files = [ - {file = "azure-core-1.29.3.tar.gz", hash = "sha256:c92700af982e71c8c73de9f4c20da8b3f03ce2c22d13066e4d416b4629c87903"}, - {file = "azure_core-1.29.3-py3-none-any.whl", hash = "sha256:f8b2910f92b66293d93bd00564924ad20ad48f4a1e150577cf18d1e7d4f9263c"}, + {file = "azure-core-1.29.4.tar.gz", hash = "sha256:500b3aa9bf2e90c5ccc88bb105d056114ca0ce7d0ce73afb8bc4d714b2fc7568"}, + {file = "azure_core-1.29.4-py3-none-any.whl", hash = "sha256:b03261bcba22c0b9290faf9999cedd23e849ed2577feee90515694cea6bc74bf"}, ] [package.dependencies] @@ -813,63 +813,63 @@ test = ["pytest", "pytest-timeout"] [[package]] name = "coverage" -version = "7.3.0" +version = "7.3.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.8" files = [ - {file = "coverage-7.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db76a1bcb51f02b2007adacbed4c88b6dee75342c37b05d1822815eed19edee5"}, - {file = "coverage-7.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c02cfa6c36144ab334d556989406837336c1d05215a9bdf44c0bc1d1ac1cb637"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:477c9430ad5d1b80b07f3c12f7120eef40bfbf849e9e7859e53b9c93b922d2af"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ce2ee86ca75f9f96072295c5ebb4ef2a43cecf2870b0ca5e7a1cbdd929cf67e1"}, - {file = "coverage-7.3.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:68d8a0426b49c053013e631c0cdc09b952d857efa8f68121746b339912d27a12"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:b3eb0c93e2ea6445b2173da48cb548364f8f65bf68f3d090404080d338e3a689"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:90b6e2f0f66750c5a1178ffa9370dec6c508a8ca5265c42fbad3ccac210a7977"}, - {file = "coverage-7.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:96d7d761aea65b291a98c84e1250cd57b5b51726821a6f2f8df65db89363be51"}, - {file = "coverage-7.3.0-cp310-cp310-win32.whl", hash = "sha256:63c5b8ecbc3b3d5eb3a9d873dec60afc0cd5ff9d9f1c75981d8c31cfe4df8527"}, - {file = "coverage-7.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:97c44f4ee13bce914272589b6b41165bbb650e48fdb7bd5493a38bde8de730a1"}, - {file = "coverage-7.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:74c160285f2dfe0acf0f72d425f3e970b21b6de04157fc65adc9fd07ee44177f"}, - {file = "coverage-7.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b543302a3707245d454fc49b8ecd2c2d5982b50eb63f3535244fd79a4be0c99d"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ad0f87826c4ebd3ef484502e79b39614e9c03a5d1510cfb623f4a4a051edc6fd"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:13c6cbbd5f31211d8fdb477f0f7b03438591bdd077054076eec362cf2207b4a7"}, - {file = "coverage-7.3.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fac440c43e9b479d1241fe9d768645e7ccec3fb65dc3a5f6e90675e75c3f3e3a"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:3c9834d5e3df9d2aba0275c9f67989c590e05732439b3318fa37a725dff51e74"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:4c8e31cf29b60859876474034a83f59a14381af50cbe8a9dbaadbf70adc4b214"}, - {file = "coverage-7.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:7a9baf8e230f9621f8e1d00c580394a0aa328fdac0df2b3f8384387c44083c0f"}, - {file = "coverage-7.3.0-cp311-cp311-win32.whl", hash = "sha256:ccc51713b5581e12f93ccb9c5e39e8b5d4b16776d584c0f5e9e4e63381356482"}, - {file = "coverage-7.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:887665f00ea4e488501ba755a0e3c2cfd6278e846ada3185f42d391ef95e7e70"}, - {file = "coverage-7.3.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d000a739f9feed900381605a12a61f7aaced6beae832719ae0d15058a1e81c1b"}, - {file = "coverage-7.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:59777652e245bb1e300e620ce2bef0d341945842e4eb888c23a7f1d9e143c446"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9737bc49a9255d78da085fa04f628a310c2332b187cd49b958b0e494c125071"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5247bab12f84a1d608213b96b8af0cbb30d090d705b6663ad794c2f2a5e5b9fe"}, - {file = "coverage-7.3.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2ac9a1de294773b9fa77447ab7e529cf4fe3910f6a0832816e5f3d538cfea9a"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:85b7335c22455ec12444cec0d600533a238d6439d8d709d545158c1208483873"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:36ce5d43a072a036f287029a55b5c6a0e9bd73db58961a273b6dc11a2c6eb9c2"}, - {file = "coverage-7.3.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:211a4576e984f96d9fce61766ffaed0115d5dab1419e4f63d6992b480c2bd60b"}, - {file = "coverage-7.3.0-cp312-cp312-win32.whl", hash = "sha256:56afbf41fa4a7b27f6635bc4289050ac3ab7951b8a821bca46f5b024500e6321"}, - {file = "coverage-7.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:7f297e0c1ae55300ff688568b04ff26b01c13dfbf4c9d2b7d0cb688ac60df479"}, - {file = "coverage-7.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ac0dec90e7de0087d3d95fa0533e1d2d722dcc008bc7b60e1143402a04c117c1"}, - {file = "coverage-7.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:438856d3f8f1e27f8e79b5410ae56650732a0dcfa94e756df88c7e2d24851fcd"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1084393c6bda8875c05e04fce5cfe1301a425f758eb012f010eab586f1f3905e"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49ab200acf891e3dde19e5aa4b0f35d12d8b4bd805dc0be8792270c71bd56c54"}, - {file = "coverage-7.3.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a67e6bbe756ed458646e1ef2b0778591ed4d1fcd4b146fc3ba2feb1a7afd4254"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:8f39c49faf5344af36042b293ce05c0d9004270d811c7080610b3e713251c9b0"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:7df91fb24c2edaabec4e0eee512ff3bc6ec20eb8dccac2e77001c1fe516c0c84"}, - {file = "coverage-7.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:34f9f0763d5fa3035a315b69b428fe9c34d4fc2f615262d6be3d3bf3882fb985"}, - {file = "coverage-7.3.0-cp38-cp38-win32.whl", hash = "sha256:bac329371d4c0d456e8d5f38a9b0816b446581b5f278474e416ea0c68c47dcd9"}, - {file = "coverage-7.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:b859128a093f135b556b4765658d5d2e758e1fae3e7cc2f8c10f26fe7005e543"}, - {file = "coverage-7.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fc0ed8d310afe013db1eedd37176d0839dc66c96bcfcce8f6607a73ffea2d6ba"}, - {file = "coverage-7.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:e61260ec93f99f2c2d93d264b564ba912bec502f679793c56f678ba5251f0393"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:97af9554a799bd7c58c0179cc8dbf14aa7ab50e1fd5fa73f90b9b7215874ba28"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3558e5b574d62f9c46b76120a5c7c16c4612dc2644c3d48a9f4064a705eaee95"}, - {file = "coverage-7.3.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37d5576d35fcb765fca05654f66aa71e2808d4237d026e64ac8b397ffa66a56a"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:07ea61bcb179f8f05ffd804d2732b09d23a1238642bf7e51dad62082b5019b34"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:80501d1b2270d7e8daf1b64b895745c3e234289e00d5f0e30923e706f110334e"}, - {file = "coverage-7.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4eddd3153d02204f22aef0825409091a91bf2a20bce06fe0f638f5c19a85de54"}, - {file = "coverage-7.3.0-cp39-cp39-win32.whl", hash = "sha256:2d22172f938455c156e9af2612650f26cceea47dc86ca048fa4e0b2d21646ad3"}, - {file = "coverage-7.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:60f64e2007c9144375dd0f480a54d6070f00bb1a28f65c408370544091c9bc9e"}, - {file = "coverage-7.3.0-pp38.pp39.pp310-none-any.whl", hash = "sha256:5492a6ce3bdb15c6ad66cb68a0244854d9917478877a25671d70378bdc8562d0"}, - {file = "coverage-7.3.0.tar.gz", hash = "sha256:49dbb19cdcafc130f597d9e04a29d0a032ceedf729e41b181f51cd170e6ee865"}, + {file = "coverage-7.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cd0f7429ecfd1ff597389907045ff209c8fdb5b013d38cfa7c60728cb484b6e3"}, + {file = "coverage-7.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:966f10df9b2b2115da87f50f6a248e313c72a668248be1b9060ce935c871f276"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0575c37e207bb9b98b6cf72fdaaa18ac909fb3d153083400c2d48e2e6d28bd8e"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:245c5a99254e83875c7fed8b8b2536f040997a9b76ac4c1da5bff398c06e860f"}, + {file = "coverage-7.3.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c96dd7798d83b960afc6c1feb9e5af537fc4908852ef025600374ff1a017392"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:de30c1aa80f30af0f6b2058a91505ea6e36d6535d437520067f525f7df123887"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:50dd1e2dd13dbbd856ffef69196781edff26c800a74f070d3b3e3389cab2600d"}, + {file = "coverage-7.3.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:b9c0c19f70d30219113b18fe07e372b244fb2a773d4afde29d5a2f7930765136"}, + {file = "coverage-7.3.1-cp310-cp310-win32.whl", hash = "sha256:770f143980cc16eb601ccfd571846e89a5fe4c03b4193f2e485268f224ab602f"}, + {file = "coverage-7.3.1-cp310-cp310-win_amd64.whl", hash = "sha256:cdd088c00c39a27cfa5329349cc763a48761fdc785879220d54eb785c8a38520"}, + {file = "coverage-7.3.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:74bb470399dc1989b535cb41f5ca7ab2af561e40def22d7e188e0a445e7639e3"}, + {file = "coverage-7.3.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:025ded371f1ca280c035d91b43252adbb04d2aea4c7105252d3cbc227f03b375"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a6191b3a6ad3e09b6cfd75b45c6aeeffe7e3b0ad46b268345d159b8df8d835f9"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7eb0b188f30e41ddd659a529e385470aa6782f3b412f860ce22b2491c89b8593"}, + {file = "coverage-7.3.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75c8f0df9dfd8ff745bccff75867d63ef336e57cc22b2908ee725cc552689ec8"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:7eb3cd48d54b9bd0e73026dedce44773214064be93611deab0b6a43158c3d5a0"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:ac3c5b7e75acac31e490b7851595212ed951889918d398b7afa12736c85e13ce"}, + {file = "coverage-7.3.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5b4ee7080878077af0afa7238df1b967f00dc10763f6e1b66f5cced4abebb0a3"}, + {file = "coverage-7.3.1-cp311-cp311-win32.whl", hash = "sha256:229c0dd2ccf956bf5aeede7e3131ca48b65beacde2029f0361b54bf93d36f45a"}, + {file = "coverage-7.3.1-cp311-cp311-win_amd64.whl", hash = "sha256:c6f55d38818ca9596dc9019eae19a47410d5322408140d9a0076001a3dcb938c"}, + {file = "coverage-7.3.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:5289490dd1c3bb86de4730a92261ae66ea8d44b79ed3cc26464f4c2cde581fbc"}, + {file = "coverage-7.3.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ca833941ec701fda15414be400c3259479bfde7ae6d806b69e63b3dc423b1832"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd694e19c031733e446c8024dedd12a00cda87e1c10bd7b8539a87963685e969"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aab8e9464c00da5cb9c536150b7fbcd8850d376d1151741dd0d16dfe1ba4fd26"}, + {file = "coverage-7.3.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d38444efffd5b056fcc026c1e8d862191881143c3aa80bb11fcf9dca9ae204"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:8a07b692129b8a14ad7a37941a3029c291254feb7a4237f245cfae2de78de037"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:2829c65c8faaf55b868ed7af3c7477b76b1c6ebeee99a28f59a2cb5907a45760"}, + {file = "coverage-7.3.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1f111a7d85658ea52ffad7084088277135ec5f368457275fc57f11cebb15607f"}, + {file = "coverage-7.3.1-cp312-cp312-win32.whl", hash = "sha256:c397c70cd20f6df7d2a52283857af622d5f23300c4ca8e5bd8c7a543825baa5a"}, + {file = "coverage-7.3.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ae4c6da8b3d123500f9525b50bf0168023313963e0e2e814badf9000dd6ef92"}, + {file = "coverage-7.3.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ca70466ca3a17460e8fc9cea7123c8cbef5ada4be3140a1ef8f7b63f2f37108f"}, + {file = "coverage-7.3.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f2781fd3cabc28278dc982a352f50c81c09a1a500cc2086dc4249853ea96b981"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6407424621f40205bbe6325686417e5e552f6b2dba3535dd1f90afc88a61d465"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:04312b036580ec505f2b77cbbdfb15137d5efdfade09156961f5277149f5e344"}, + {file = "coverage-7.3.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac9ad38204887349853d7c313f53a7b1c210ce138c73859e925bc4e5d8fc18e7"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:53669b79f3d599da95a0afbef039ac0fadbb236532feb042c534fbb81b1a4e40"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:614f1f98b84eb256e4f35e726bfe5ca82349f8dfa576faabf8a49ca09e630086"}, + {file = "coverage-7.3.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f1a317fdf5c122ad642db8a97964733ab7c3cf6009e1a8ae8821089993f175ff"}, + {file = "coverage-7.3.1-cp38-cp38-win32.whl", hash = "sha256:defbbb51121189722420a208957e26e49809feafca6afeef325df66c39c4fdb3"}, + {file = "coverage-7.3.1-cp38-cp38-win_amd64.whl", hash = "sha256:f4f456590eefb6e1b3c9ea6328c1e9fa0f1006e7481179d749b3376fc793478e"}, + {file = "coverage-7.3.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:f12d8b11a54f32688b165fd1a788c408f927b0960984b899be7e4c190ae758f1"}, + {file = "coverage-7.3.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:f09195dda68d94a53123883de75bb97b0e35f5f6f9f3aa5bf6e496da718f0cb6"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c6601a60318f9c3945be6ea0f2a80571f4299b6801716f8a6e4846892737ebe4"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:07d156269718670d00a3b06db2288b48527fc5f36859425ff7cec07c6b367745"}, + {file = "coverage-7.3.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:636a8ac0b044cfeccae76a36f3b18264edcc810a76a49884b96dd744613ec0b7"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:5d991e13ad2ed3aced177f524e4d670f304c8233edad3210e02c465351f785a0"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:586649ada7cf139445da386ab6f8ef00e6172f11a939fc3b2b7e7c9082052fa0"}, + {file = "coverage-7.3.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4aba512a15a3e1e4fdbfed2f5392ec221434a614cc68100ca99dcad7af29f3f8"}, + {file = "coverage-7.3.1-cp39-cp39-win32.whl", hash = "sha256:6bc6f3f4692d806831c136c5acad5ccedd0262aa44c087c46b7101c77e139140"}, + {file = "coverage-7.3.1-cp39-cp39-win_amd64.whl", hash = "sha256:553d7094cb27db58ea91332e8b5681bac107e7242c23f7629ab1316ee73c4981"}, + {file = "coverage-7.3.1-pp38.pp39.pp310-none-any.whl", hash = "sha256:220eb51f5fb38dfdb7e5d54284ca4d0cd70ddac047d750111a68ab1798945194"}, + {file = "coverage-7.3.1.tar.gz", hash = "sha256:6cb7fe1581deb67b782c153136541e20901aa312ceedaf1467dcb35255787952"}, ] [package.extras] @@ -1595,13 +1595,13 @@ zoneinfo = ["backports.zoneinfo (>=0.2.1)", "tzdata (>=2022.1)"] [[package]] name = "identify" -version = "2.5.27" +version = "2.5.28" description = "File identification library for Python" optional = false python-versions = ">=3.8" files = [ - {file = "identify-2.5.27-py2.py3-none-any.whl", hash = "sha256:fdb527b2dfe24602809b2201e033c2a113d7bdf716db3ca8e3243f735dcecaba"}, - {file = "identify-2.5.27.tar.gz", hash = "sha256:287b75b04a0e22d727bc9a41f0d4f3c1bcada97490fa6eabb5b28f0e9097e733"}, + {file = "identify-2.5.28-py2.py3-none-any.whl", hash = "sha256:87816de144bf46d161bd5b3e8f5596b16cade3b80be537087334b26bc5c177f3"}, + {file = "identify-2.5.28.tar.gz", hash = "sha256:94bb59643083ebd60dc996d043497479ee554381fbc5307763915cda49b0e78f"}, ] [package.extras] @@ -2144,6 +2144,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -2178,52 +2188,58 @@ files = [ [[package]] name = "matplotlib" -version = "3.7.2" +version = "3.7.3" description = "Python plotting package" optional = false python-versions = ">=3.8" files = [ - {file = "matplotlib-3.7.2-cp310-cp310-macosx_10_12_universal2.whl", hash = "sha256:2699f7e73a76d4c110f4f25be9d2496d6ab4f17345307738557d345f099e07de"}, - {file = "matplotlib-3.7.2-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:a8035ba590658bae7562786c9cc6ea1a84aa49d3afab157e414c9e2ea74f496d"}, - {file = "matplotlib-3.7.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2f8e4a49493add46ad4a8c92f63e19d548b2b6ebbed75c6b4c7f46f57d36cdd1"}, - {file = "matplotlib-3.7.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71667eb2ccca4c3537d9414b1bc00554cb7f91527c17ee4ec38027201f8f1603"}, - {file = "matplotlib-3.7.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:152ee0b569a37630d8628534c628456b28686e085d51394da6b71ef84c4da201"}, - {file = "matplotlib-3.7.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:070f8dddd1f5939e60aacb8fa08f19551f4b0140fab16a3669d5cd6e9cb28fc8"}, - {file = "matplotlib-3.7.2-cp310-cp310-win32.whl", hash = "sha256:fdbb46fad4fb47443b5b8ac76904b2e7a66556844f33370861b4788db0f8816a"}, - {file = "matplotlib-3.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:23fb1750934e5f0128f9423db27c474aa32534cec21f7b2153262b066a581fd1"}, - {file = "matplotlib-3.7.2-cp311-cp311-macosx_10_12_universal2.whl", hash = "sha256:30e1409b857aa8a747c5d4f85f63a79e479835f8dffc52992ac1f3f25837b544"}, - {file = "matplotlib-3.7.2-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:50e0a55ec74bf2d7a0ebf50ac580a209582c2dd0f7ab51bc270f1b4a0027454e"}, - {file = "matplotlib-3.7.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac60daa1dc83e8821eed155796b0f7888b6b916cf61d620a4ddd8200ac70cd64"}, - {file = "matplotlib-3.7.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:305e3da477dc8607336ba10bac96986d6308d614706cae2efe7d3ffa60465b24"}, - {file = "matplotlib-3.7.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c308b255efb9b06b23874236ec0f10f026673ad6515f602027cc8ac7805352d"}, - {file = "matplotlib-3.7.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60c521e21031632aa0d87ca5ba0c1c05f3daacadb34c093585a0be6780f698e4"}, - {file = "matplotlib-3.7.2-cp311-cp311-win32.whl", hash = "sha256:26bede320d77e469fdf1bde212de0ec889169b04f7f1179b8930d66f82b30cbc"}, - {file = "matplotlib-3.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:af4860132c8c05261a5f5f8467f1b269bf1c7c23902d75f2be57c4a7f2394b3e"}, - {file = "matplotlib-3.7.2-cp38-cp38-macosx_10_12_universal2.whl", hash = "sha256:a1733b8e84e7e40a9853e505fe68cc54339f97273bdfe6f3ed980095f769ddc7"}, - {file = "matplotlib-3.7.2-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:d9881356dc48e58910c53af82b57183879129fa30492be69058c5b0d9fddf391"}, - {file = "matplotlib-3.7.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f081c03f413f59390a80b3e351cc2b2ea0205839714dbc364519bcf51f4b56ca"}, - {file = "matplotlib-3.7.2-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1cd120fca3407a225168238b790bd5c528f0fafde6172b140a2f3ab7a4ea63e9"}, - {file = "matplotlib-3.7.2-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a2c1590b90aa7bd741b54c62b78de05d4186271e34e2377e0289d943b3522273"}, - {file = "matplotlib-3.7.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6d2ff3c984b8a569bc1383cd468fc06b70d7b59d5c2854ca39f1436ae8394117"}, - {file = "matplotlib-3.7.2-cp38-cp38-win32.whl", hash = "sha256:5dea00b62d28654b71ca92463656d80646675628d0828e08a5f3b57e12869e13"}, - {file = "matplotlib-3.7.2-cp38-cp38-win_amd64.whl", hash = "sha256:0f506a1776ee94f9e131af1ac6efa6e5bc7cb606a3e389b0ccb6e657f60bb676"}, - {file = "matplotlib-3.7.2-cp39-cp39-macosx_10_12_universal2.whl", hash = "sha256:6515e878f91894c2e4340d81f0911857998ccaf04dbc1bba781e3d89cbf70608"}, - {file = "matplotlib-3.7.2-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:71f7a8c6b124e904db550f5b9fe483d28b896d4135e45c4ea381ad3b8a0e3256"}, - {file = "matplotlib-3.7.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:12f01b92ecd518e0697da4d97d163b2b3aa55eb3eb4e2c98235b3396d7dad55f"}, - {file = "matplotlib-3.7.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7e28d6396563955f7af437894a36bf2b279462239a41028323e04b85179058b"}, - {file = "matplotlib-3.7.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dbcf59334ff645e6a67cd5f78b4b2cdb76384cdf587fa0d2dc85f634a72e1a3e"}, - {file = "matplotlib-3.7.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:318c89edde72ff95d8df67d82aca03861240512994a597a435a1011ba18dbc7f"}, - {file = "matplotlib-3.7.2-cp39-cp39-win32.whl", hash = "sha256:ce55289d5659b5b12b3db4dc9b7075b70cef5631e56530f14b2945e8836f2d20"}, - {file = "matplotlib-3.7.2-cp39-cp39-win_amd64.whl", hash = "sha256:2ecb5be2b2815431c81dc115667e33da0f5a1bcf6143980d180d09a717c4a12e"}, - {file = "matplotlib-3.7.2-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:fdcd28360dbb6203fb5219b1a5658df226ac9bebc2542a9e8f457de959d713d0"}, - {file = "matplotlib-3.7.2-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0c3cca3e842b11b55b52c6fb8bd6a4088693829acbfcdb3e815fa9b7d5c92c1b"}, - {file = "matplotlib-3.7.2-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebf577c7a6744e9e1bd3fee45fc74a02710b214f94e2bde344912d85e0c9af7c"}, - {file = "matplotlib-3.7.2-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:936bba394682049919dda062d33435b3be211dc3dcaa011e09634f060ec878b2"}, - {file = "matplotlib-3.7.2-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:bc221ffbc2150458b1cd71cdd9ddd5bb37962b036e41b8be258280b5b01da1dd"}, - {file = "matplotlib-3.7.2-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:35d74ebdb3f71f112b36c2629cf32323adfbf42679e2751252acd468f5001c07"}, - {file = "matplotlib-3.7.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:717157e61b3a71d3d26ad4e1770dc85156c9af435659a25ee6407dc866cb258d"}, - {file = "matplotlib-3.7.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:20f844d6be031948148ba49605c8b96dfe7d3711d1b63592830d650622458c11"}, - {file = "matplotlib-3.7.2.tar.gz", hash = "sha256:a8cdb91dddb04436bd2f098b8fdf4b81352e68cf4d2c6756fcc414791076569b"}, + {file = "matplotlib-3.7.3-cp310-cp310-macosx_10_12_universal2.whl", hash = "sha256:085c33b27561d9c04386789d5aa5eb4a932ddef43cfcdd0e01735f9a6e85ce0c"}, + {file = "matplotlib-3.7.3-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:c568e80e1c17f68a727f30f591926751b97b98314d8e59804f54f86ae6fa6a22"}, + {file = "matplotlib-3.7.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:7baf98c5ad59c5c4743ea884bb025cbffa52dacdfdac0da3e6021a285a90377e"}, + {file = "matplotlib-3.7.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:236024f582e40dac39bca592258888b38ae47a9fed7b8de652d68d3d02d47d2b"}, + {file = "matplotlib-3.7.3-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12b4f6795efea037ce2d41e7c417ad8bd02d5719c6ad4a8450a0708f4a1cfb89"}, + {file = "matplotlib-3.7.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78b2136cc6c5415b78977e0e8c608647d597204b05b1d9089ccf513c7d913733"}, + {file = "matplotlib-3.7.3-cp310-cp310-win32.whl", hash = "sha256:122dcbf9be0086e2a95d9e5e0632dbf3bd5b65eaa68c369363310a6c87753059"}, + {file = "matplotlib-3.7.3-cp310-cp310-win_amd64.whl", hash = "sha256:4aab27d9e33293389e3c1d7c881d414a72bdfda0fedc3a6bf46c6fa88d9b8015"}, + {file = "matplotlib-3.7.3-cp311-cp311-macosx_10_12_universal2.whl", hash = "sha256:d5adc743de91e8e0b13df60deb1b1c285b8effea3d66223afceb14b63c9b05de"}, + {file = "matplotlib-3.7.3-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:55de4cf7cd0071b8ebf203981b53ab64f988a0a1f897a2dff300a1124e8bcd8b"}, + {file = "matplotlib-3.7.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:ac03377fd908aaee2312d0b11735753e907adb6f4d1d102de5e2425249693f6c"}, + {file = "matplotlib-3.7.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:755bafc10a46918ce9a39980009b54b02dd249594e5adf52f9c56acfddb5d0b7"}, + {file = "matplotlib-3.7.3-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1a6094c6f8e8d18db631754df4fe9a34dec3caf074f6869a7db09f18f9b1d6b2"}, + {file = "matplotlib-3.7.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:272dba2f1b107790ed78ebf5385b8d14b27ad9e90419de340364b49fe549a993"}, + {file = "matplotlib-3.7.3-cp311-cp311-win32.whl", hash = "sha256:591c123bed1cb4b9996fb60b41a6d89c2ec4943244540776c5f1283fb6960a53"}, + {file = "matplotlib-3.7.3-cp311-cp311-win_amd64.whl", hash = "sha256:3bf3a178c6504694cee8b88b353df0051583f2f6f8faa146f67115c27c856881"}, + {file = "matplotlib-3.7.3-cp312-cp312-macosx_10_12_universal2.whl", hash = "sha256:edf54cac8ee3603f3093616b40a931e8c063969756a4d78a86e82c2fea9659f7"}, + {file = "matplotlib-3.7.3-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:91e36a85ea639a1ba9f91427041eac064b04829945fe331a92617b6cb21d27e5"}, + {file = "matplotlib-3.7.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:caf5eaaf7c68f8d7df269dfbcaf46f48a70ff482bfcebdcc97519671023f2a7d"}, + {file = "matplotlib-3.7.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:74bf57f505efea376097e948b7cdd87191a7ce8180616390aef496639edf601f"}, + {file = "matplotlib-3.7.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ee152a88a0da527840a426535514b6ed8ac4240eb856b1da92cf48124320e346"}, + {file = "matplotlib-3.7.3-cp312-cp312-win_amd64.whl", hash = "sha256:67a410a9c9e07cbc83581eeea144bbe298870bf0ac0ee2f2e10a015ab7efee19"}, + {file = "matplotlib-3.7.3-cp38-cp38-macosx_10_12_universal2.whl", hash = "sha256:259999c05285cb993d7f2a419cea547863fa215379eda81f7254c9e932963729"}, + {file = "matplotlib-3.7.3-cp38-cp38-macosx_10_12_x86_64.whl", hash = "sha256:3f4e7fd5a6157e1d018ce2166ec8e531a481dd4a36f035b5c23edfe05a25419a"}, + {file = "matplotlib-3.7.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:faa3d12d8811d08d14080a8b7b9caea9a457dc495350166b56df0db4b9909ef5"}, + {file = "matplotlib-3.7.3-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:336e88900c11441e458da01c8414fc57e04e17f9d3bb94958a76faa2652bcf6b"}, + {file = "matplotlib-3.7.3-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:12f4c0dd8aa280d796c8772ea8265a14f11a04319baa3a16daa5556065e8baea"}, + {file = "matplotlib-3.7.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1990955b11e7918d256cf3b956b10997f405b7917a3f1c7d8e69c1d15c7b1930"}, + {file = "matplotlib-3.7.3-cp38-cp38-win32.whl", hash = "sha256:e78707b751260b42b721507ad7aa60fe4026d7f51c74cca6b9cd8b123ebb633a"}, + {file = "matplotlib-3.7.3-cp38-cp38-win_amd64.whl", hash = "sha256:e594ee43c59ea39ca5c6244667cac9d017a3527febc31f5532ad9135cf7469ec"}, + {file = "matplotlib-3.7.3-cp39-cp39-macosx_10_12_universal2.whl", hash = "sha256:6eaa1cf0e94c936a26b78f6d756c5fbc12e0a58c8a68b7248a2a31456ce4e234"}, + {file = "matplotlib-3.7.3-cp39-cp39-macosx_10_12_x86_64.whl", hash = "sha256:0a97af9d22e8ebedc9f00b043d9bbd29a375e9e10b656982012dded44c10fd77"}, + {file = "matplotlib-3.7.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1f9c6c16597af660433ab330b59ee2934b832ee1fabcaf5cbde7b2add840f31e"}, + {file = "matplotlib-3.7.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a7240259b4b9cbc62381f6378cff4d57af539162a18e832c1e48042fabc40b6b"}, + {file = "matplotlib-3.7.3-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:747c6191d2e88ae854809e69aa358dbf852ff1a5738401b85c1cc9012309897a"}, + {file = "matplotlib-3.7.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec726b08a5275d827aa91bb951e68234a4423adb91cf65bc0fcdc0f2777663f7"}, + {file = "matplotlib-3.7.3-cp39-cp39-win32.whl", hash = "sha256:40e3b9b450c6534f07278310c4e34caff41c2a42377e4b9d47b0f8d3ac1083a2"}, + {file = "matplotlib-3.7.3-cp39-cp39-win_amd64.whl", hash = "sha256:dfc118642903a23e309b1da32886bb39a4314147d013e820c86b5fb4cb2e36d0"}, + {file = "matplotlib-3.7.3-pp38-pypy38_pp73-macosx_10_12_x86_64.whl", hash = "sha256:165c8082bf8fc0360c24aa4724a22eaadbfd8c28bf1ccf7e94d685cad48261e4"}, + {file = "matplotlib-3.7.3-pp38-pypy38_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ebd8470cc2a3594746ff0513aecbfa2c55ff6f58e6cef2efb1a54eb87c88ffa2"}, + {file = "matplotlib-3.7.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7153453669c9672b52095119fd21dd032d19225d48413a2871519b17db4b0fde"}, + {file = "matplotlib-3.7.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:498a08267dc69dd8f24c4b5d7423fa584d7ce0027ba71f7881df05fc09b89bb7"}, + {file = "matplotlib-3.7.3-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:d48999c4b19b5a0c058c9cd828ff6fc7748390679f6cf9a2ad653a3e802c87d3"}, + {file = "matplotlib-3.7.3-pp39-pypy39_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22d65d18b4ee8070a5fea5761d59293f1f9e2fac37ec9ce090463b0e629432fd"}, + {file = "matplotlib-3.7.3-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c40cde976c36693cc0767e27cf5f443f91c23520060bd9496678364adfafe9c"}, + {file = "matplotlib-3.7.3-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:39018a2b17592448fbfdf4b8352955e6c3905359939791d4ff429296494d1a0c"}, + {file = "matplotlib-3.7.3.tar.gz", hash = "sha256:f09b3dd6bdeb588de91f853bbb2d6f0ff8ab693485b0c49035eaa510cb4f142e"}, ] [package.dependencies] @@ -2231,11 +2247,12 @@ contourpy = ">=1.0.1" cycler = ">=0.10" fonttools = ">=4.22.0" kiwisolver = ">=1.0.1" -numpy = ">=1.20" +numpy = ">=1.20,<2" packaging = ">=20.0" pillow = ">=6.2.0" -pyparsing = ">=2.3.1,<3.1" +pyparsing = ">=2.3.1" python-dateutil = ">=2.7" +setuptools_scm = ">=7" [[package]] name = "mdit-py-plugins" @@ -2766,14 +2783,7 @@ files = [ ] [package.dependencies] -numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, - {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""}, - {version = ">=1.17.0", markers = "python_version >= \"3.7\""}, - {version = ">=1.17.3", markers = "python_version >= \"3.8\""}, -] +numpy = {version = ">=1.23.5", markers = "python_version >= \"3.11\""} [[package]] name = "opencv-python-headless" @@ -2792,14 +2802,7 @@ files = [ ] [package.dependencies] -numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, - {version = ">=1.21.4", markers = "python_version >= \"3.10\" and platform_system == \"Darwin\""}, - {version = ">=1.23.5", markers = "python_version >= \"3.11\""}, - {version = ">=1.19.3", markers = "python_version >= \"3.6\" and platform_system == \"Linux\" and platform_machine == \"aarch64\" or python_version >= \"3.9\""}, - {version = ">=1.17.0", markers = "python_version >= \"3.7\""}, - {version = ">=1.17.3", markers = "python_version >= \"3.8\""}, -] +numpy = {version = ">=1.23.5", markers = "python_version >= \"3.11\""} [[package]] name = "packaging" @@ -3194,24 +3197,24 @@ files = [ [[package]] name = "protobuf" -version = "4.24.2" +version = "4.24.3" description = "" optional = false python-versions = ">=3.7" files = [ - {file = "protobuf-4.24.2-cp310-abi3-win32.whl", hash = "sha256:58e12d2c1aa428ece2281cef09bbaa6938b083bcda606db3da4e02e991a0d924"}, - {file = "protobuf-4.24.2-cp310-abi3-win_amd64.whl", hash = "sha256:77700b55ba41144fc64828e02afb41901b42497b8217b558e4a001f18a85f2e3"}, - {file = "protobuf-4.24.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:237b9a50bd3b7307d0d834c1b0eb1a6cd47d3f4c2da840802cd03ea288ae8880"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:25ae91d21e3ce8d874211110c2f7edd6384816fb44e06b2867afe35139e1fd1c"}, - {file = "protobuf-4.24.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:c00c3c7eb9ad3833806e21e86dca448f46035242a680f81c3fe068ff65e79c74"}, - {file = "protobuf-4.24.2-cp37-cp37m-win32.whl", hash = "sha256:4e69965e7e54de4db989289a9b971a099e626f6167a9351e9d112221fc691bc1"}, - {file = "protobuf-4.24.2-cp37-cp37m-win_amd64.whl", hash = "sha256:c5cdd486af081bf752225b26809d2d0a85e575b80a84cde5172a05bbb1990099"}, - {file = "protobuf-4.24.2-cp38-cp38-win32.whl", hash = "sha256:6bd26c1fa9038b26c5c044ee77e0ecb18463e957fefbaeb81a3feb419313a54e"}, - {file = "protobuf-4.24.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb7aa97c252279da65584af0456f802bd4b2de429eb945bbc9b3d61a42a8cd16"}, - {file = "protobuf-4.24.2-cp39-cp39-win32.whl", hash = "sha256:2b23bd6e06445699b12f525f3e92a916f2dcf45ffba441026357dea7fa46f42b"}, - {file = "protobuf-4.24.2-cp39-cp39-win_amd64.whl", hash = "sha256:839952e759fc40b5d46be319a265cf94920174d88de31657d5622b5d8d6be5cd"}, - {file = "protobuf-4.24.2-py3-none-any.whl", hash = "sha256:3b7b170d3491ceed33f723bbf2d5a260f8a4e23843799a3906f16ef736ef251e"}, - {file = "protobuf-4.24.2.tar.gz", hash = "sha256:7fda70797ddec31ddfa3576cbdcc3ddbb6b3078b737a1a87ab9136af0570cd6e"}, + {file = "protobuf-4.24.3-cp310-abi3-win32.whl", hash = "sha256:20651f11b6adc70c0f29efbe8f4a94a74caf61b6200472a9aea6e19898f9fcf4"}, + {file = "protobuf-4.24.3-cp310-abi3-win_amd64.whl", hash = "sha256:3d42e9e4796a811478c783ef63dc85b5a104b44aaaca85d4864d5b886e4b05e3"}, + {file = "protobuf-4.24.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:6e514e8af0045be2b56e56ae1bb14f43ce7ffa0f68b1c793670ccbe2c4fc7d2b"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:ba53c2f04798a326774f0e53b9c759eaef4f6a568ea7072ec6629851c8435959"}, + {file = "protobuf-4.24.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:f6ccbcf027761a2978c1406070c3788f6de4a4b2cc20800cc03d52df716ad675"}, + {file = "protobuf-4.24.3-cp37-cp37m-win32.whl", hash = "sha256:1b182c7181a2891e8f7f3a1b5242e4ec54d1f42582485a896e4de81aa17540c2"}, + {file = "protobuf-4.24.3-cp37-cp37m-win_amd64.whl", hash = "sha256:b0271a701e6782880d65a308ba42bc43874dabd1a0a0f41f72d2dac3b57f8e76"}, + {file = "protobuf-4.24.3-cp38-cp38-win32.whl", hash = "sha256:e29d79c913f17a60cf17c626f1041e5288e9885c8579832580209de8b75f2a52"}, + {file = "protobuf-4.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:067f750169bc644da2e1ef18c785e85071b7c296f14ac53e0900e605da588719"}, + {file = "protobuf-4.24.3-cp39-cp39-win32.whl", hash = "sha256:2da777d34b4f4f7613cdf85c70eb9a90b1fbef9d36ae4a0ccfe014b0b07906f1"}, + {file = "protobuf-4.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:f631bb982c5478e0c1c70eab383af74a84be66945ebf5dd6b06fc90079668d0b"}, + {file = "protobuf-4.24.3-py3-none-any.whl", hash = "sha256:f6f8dc65625dadaad0c8545319c2e2f0424fede988368893ca3844261342c11a"}, + {file = "protobuf-4.24.3.tar.gz", hash = "sha256:12e9ad2ec079b833176d2921be2cb24281fa591f0b119b208b788adc48c2561d"}, ] [[package]] @@ -3623,13 +3626,13 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] [[package]] name = "pyparsing" -version = "3.0.9" +version = "3.1.1" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.6.8" files = [ - {file = "pyparsing-3.0.9-py3-none-any.whl", hash = "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc"}, - {file = "pyparsing-3.0.9.tar.gz", hash = "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb"}, + {file = "pyparsing-3.1.1-py3-none-any.whl", hash = "sha256:32c7c0b711493c72ff18a981d24f28aaf9c1fb7ed5e9667c9e84e3db623bdbfb"}, + {file = "pyparsing-3.1.1.tar.gz", hash = "sha256:ede28a1a32462f5a9705e07aea48001a08f7cf81a021585011deba701581a0db"}, ] [package.extras] @@ -3759,13 +3762,13 @@ cp2110 = ["hidapi"] [[package]] name = "pytest" -version = "7.4.1" +version = "7.4.2" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.7" files = [ - {file = "pytest-7.4.1-py3-none-any.whl", hash = "sha256:460c9a59b14e27c602eb5ece2e47bec99dc5fc5f6513cf924a7d03a578991b1f"}, - {file = "pytest-7.4.1.tar.gz", hash = "sha256:2f2301e797521b23e4d2585a0a3d7b5e50fdddaaf7e7d6773ea26ddb17c213ab"}, + {file = "pytest-7.4.2-py3-none-any.whl", hash = "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002"}, + {file = "pytest-7.4.2.tar.gz", hash = "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"}, ] [package.dependencies] @@ -3795,6 +3798,21 @@ pytest = ">=4.6" [package.extras] testing = ["fields", "hunter", "process-tests", "pytest-xdist", "six", "virtualenv"] +[[package]] +name = "pytest-cpp" +version = "2.4.0" +description = "Use pytest's runner to discover and execute C++ tests" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-cpp-2.4.0.tar.gz", hash = "sha256:dcb8b09d4c714fc7f778dc40a7c2c47e73cc50280d7f9bba3bdd3ca98abd4685"}, + {file = "pytest_cpp-2.4.0-py3-none-any.whl", hash = "sha256:e7c5f40b70b00cc09d672a1584e35beb9b9553512cdd4b1d1f99468747b3bc31"}, +] + +[package.dependencies] +colorama = "*" +pytest = ">=7.0" + [[package]] name = "pytest-subtests" version = "0.11.0" @@ -4333,19 +4351,39 @@ test = ["pytest"] [[package]] name = "setuptools" -version = "68.1.2" +version = "68.2.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-68.1.2-py3-none-any.whl", hash = "sha256:3d8083eed2d13afc9426f227b24fd1659489ec107c0e86cec2ffdde5c92e790b"}, - {file = "setuptools-68.1.2.tar.gz", hash = "sha256:3d4dfa6d95f1b101d695a6160a7626e15583af71a5f52176efa5d39a054d475d"}, + {file = "setuptools-68.2.1-py3-none-any.whl", hash = "sha256:eff96148eb336377ab11beee0c73ed84f1709a40c0b870298b0d058828761bae"}, + {file = "setuptools-68.2.1.tar.gz", hash = "sha256:56ee14884fd8d0cd015411f4a13f40b4356775a0aefd9ebc1d3bfb9a1acb32f1"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5,<=7.1.2)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.1)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + +[[package]] +name = "setuptools-scm" +version = "7.1.0" +description = "the blessed package to manage your versions by scm tags" +optional = false +python-versions = ">=3.7" +files = [ + {file = "setuptools_scm-7.1.0-py3-none-any.whl", hash = "sha256:73988b6d848709e2af142aa48c986ea29592bbcfca5375678064708205253d8e"}, + {file = "setuptools_scm-7.1.0.tar.gz", hash = "sha256:6c508345a771aad7d56ebff0e70628bf2b0ec7573762be9960214730de278f27"}, +] + +[package.dependencies] +packaging = ">=20.0" +setuptools = "*" +typing-extensions = "*" + +[package.extras] +test = ["pytest (>=6.2)", "virtualenv (>20)"] +toml = ["setuptools (>=42)"] [[package]] name = "shapely" @@ -4894,13 +4932,13 @@ zstd = ["zstandard (>=0.18.0)"] [[package]] name = "virtualenv" -version = "20.24.4" +version = "20.24.5" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.7" files = [ - {file = "virtualenv-20.24.4-py3-none-any.whl", hash = "sha256:29c70bb9b88510f6414ac3e55c8b413a1f96239b6b789ca123437d5e892190cb"}, - {file = "virtualenv-20.24.4.tar.gz", hash = "sha256:772b05bfda7ed3b8ecd16021ca9716273ad9f4467c801f27e83ac73430246dca"}, + {file = "virtualenv-20.24.5-py3-none-any.whl", hash = "sha256:b80039f280f4919c77b30f1c23294ae357c4c8701042086e3fc005963e4e537b"}, + {file = "virtualenv-20.24.5.tar.gz", hash = "sha256:e8361967f6da6fbdf1426483bfe9fca8287c242ac0bc30429905721cefbff752"}, ] [package.dependencies] @@ -4914,13 +4952,13 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess [[package]] name = "websocket-client" -version = "1.6.2" +version = "1.6.3" description = "WebSocket client for Python with low level API options" optional = false python-versions = ">=3.8" files = [ - {file = "websocket-client-1.6.2.tar.gz", hash = "sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa"}, - {file = "websocket_client-1.6.2-py3-none-any.whl", hash = "sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"}, + {file = "websocket-client-1.6.3.tar.gz", hash = "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f"}, + {file = "websocket_client-1.6.3-py3-none-any.whl", hash = "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"}, ] [package.extras] @@ -5049,4 +5087,4 @@ testing = ["big-O", "jaraco.functools", "jaraco.itertools", "more-itertools", "p [metadata] lock-version = "2.0" python-versions = "~3.11" -content-hash = "0e7f2a907e73eaa31137d70be3a56d350d6fc6dbc54811c80d6031409fddcac1" +content-hash = "7258a24274936ccdafb930a683f4ea197e82ab187698d0723bb4683ca41e6d26" diff --git a/pyproject.toml b/pyproject.toml index 1c7d857652..5d4bcafa7a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.pytest.ini_options] minversion = "6.0" -addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers" +addopts = "--ignore=openpilot/ --ignore=cereal/ --ignore=opendbc/ --ignore=panda/ --ignore=rednose_repo/ --ignore=tinygrad_repo/ --ignore=laika_repo/ -Werror --strict-config --strict-markers --durations=10" +#cpp_files = "test_*" # uncomment when agnos has pytest-cpp and remove from CI python_files = "test_*.py" #timeout = "30" # you get this long by default markers = [ @@ -18,9 +19,12 @@ testpaths = [ "selfdrive/test/longitudinal_maneuvers", "system/hardware/tici", "system/loggerd", + "system/proclogd", "system/tests", "system/ubloxd", - "tools/lib/tests" + "tools/lib/tests", + "tools/replay", + "tools/cabana" ] [tool.mypy] @@ -142,6 +146,7 @@ pygame = "*" pyprof2calltree = "*" pytest = "*" pytest-cov = "*" +pytest-cpp = "*" pytest-subtests = "*" pytest-xdist = "*" pytest-timeout = "*" @@ -167,7 +172,7 @@ build-backend = "poetry.core.masonry.api" # https://beta.ruff.rs/docs/configuration/#using-pyprojecttoml [tool.ruff] -select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A", "B"] +select = ["E", "F", "W", "PIE", "C4", "ISC", "RUF100", "A", "B", "TID251"] ignore = ["W292", "E741", "E402", "C408", "ISC003", "B027", "B024"] line-length = 160 target-version="py311" @@ -180,3 +185,9 @@ exclude = [ "third_party", ] flake8-implicit-str-concat.allow-multiline=false +[tool.ruff.flake8-tidy-imports.banned-api] +"selfdrive".msg = "Use openpilot.selfdrive" +"common".msg = "Use openpilot.common" +"system".msg = "Use openpilot.system" +"third_party".msg = "Use openpilot.third_party" +"tools".msg = "Use openpilot.tools" \ No newline at end of file diff --git a/release/files_common b/release/files_common index 81623a4942..e4d77a1ef6 100644 --- a/release/files_common +++ b/release/files_common @@ -212,6 +212,7 @@ system/hardware/__init__.py system/hardware/base.h system/hardware/base.py system/hardware/hw.h +system/hardware/hw.py system/hardware/tici/__init__.py system/hardware/tici/hardware.h system/hardware/tici/hardware.py @@ -363,12 +364,10 @@ selfdrive/modeld/.gitignore selfdrive/modeld/__init__.py selfdrive/modeld/SConscript selfdrive/modeld/modeld.py -selfdrive/modeld/navmodeld.cc -selfdrive/modeld/dmonitoringmodeld.cc +selfdrive/modeld/navmodeld.py +selfdrive/modeld/dmonitoringmodeld.py selfdrive/modeld/constants.py selfdrive/modeld/modeld -selfdrive/modeld/navmodeld -selfdrive/modeld/dmonitoringmodeld selfdrive/modeld/models/__init__.py selfdrive/modeld/models/*.pxd @@ -381,12 +380,8 @@ selfdrive/modeld/models/driving.cc selfdrive/modeld/models/driving.h selfdrive/modeld/models/supercombo.onnx -selfdrive/modeld/models/dmonitoring.cc -selfdrive/modeld/models/dmonitoring.h selfdrive/modeld/models/dmonitoring_model_q.dlc -selfdrive/modeld/models/nav.cc -selfdrive/modeld/models/nav.h selfdrive/modeld/models/navmodel_q.dlc selfdrive/modeld/transforms/loadyuv.cc diff --git a/selfdrive/athena/athenad.py b/selfdrive/athena/athenad.py index 49b214f5e9..899605d989 100755 --- a/selfdrive/athena/athenad.py +++ b/selfdrive/athena/athenad.py @@ -36,11 +36,11 @@ from openpilot.common.file_helpers import CallbackReader from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware import HARDWARE, PC, AGNOS -from openpilot.system.loggerd.config import ROOT from openpilot.system.loggerd.xattr_cache import getxattr, setxattr from openpilot.selfdrive.statsd import STATS_DIR -from openpilot.system.swaglog import SWAGLOG_DIR, cloudlog +from openpilot.system.swaglog import cloudlog from openpilot.system.version import get_commit, get_origin, get_short_branch, get_version +from openpilot.selfdrive.hardware.hw import Paths # TODO: use socket constant when mypy recognizes this as a valid attribute @@ -119,12 +119,11 @@ class AbortTransferException(Exception): class UploadQueueCache: - params = Params() @staticmethod def initialize(upload_queue: Queue[UploadItem]) -> None: try: - upload_queue_json = UploadQueueCache.params.get("AthenadUploadQueue") + upload_queue_json = Params().get("AthenadUploadQueue") if upload_queue_json is not None: for item in json.loads(upload_queue_json): upload_queue.put(UploadItem.from_dict(item)) @@ -136,7 +135,7 @@ class UploadQueueCache: try: queue: List[Optional[UploadItem]] = list(upload_queue.queue) items = [asdict(i) for i in queue if i is not None and (i.id not in cancelled_uploads)] - UploadQueueCache.params.put("AthenadUploadQueue", json.dumps(items)) + Params().put("AthenadUploadQueue", json.dumps(items)) except Exception: cloudlog.exception("athena.UploadQueueCache.cache.exception") @@ -352,7 +351,7 @@ def scan_dir(path: str, prefix: str) -> List[str]: # (glob and friends traverse entire dir tree) with os.scandir(path) as i: for e in i: - rel_path = os.path.relpath(e.path, ROOT) + rel_path = os.path.relpath(e.path, Paths.log_root()) if e.is_dir(follow_symlinks=False): # add trailing slash rel_path = os.path.join(rel_path, '') @@ -367,7 +366,7 @@ def scan_dir(path: str, prefix: str) -> List[str]: @dispatcher.add_method def listDataDirectory(prefix='') -> List[str]: - return scan_dir(ROOT, prefix) + return scan_dir(Paths.log_root(), prefix) @dispatcher.add_method @@ -408,7 +407,7 @@ def uploadFilesToUrls(files_data: List[UploadFileDict]) -> UploadFilesToUrlRespo failed.append(file.fn) continue - path = os.path.join(ROOT, file.fn) + path = os.path.join(Paths.log_root(), file.fn) if not os.path.exists(path) and not os.path.exists(strip_bz2_extension(path)): failed.append(file.fn) continue @@ -572,8 +571,8 @@ def get_logs_to_send_sorted() -> List[str]: # TODO: scan once then use inotify to detect file creation/deletion curr_time = int(time.time()) logs = [] - for log_entry in os.listdir(SWAGLOG_DIR): - log_path = os.path.join(SWAGLOG_DIR, log_entry) + for log_entry in os.listdir(Paths.swaglog_root()): + log_path = os.path.join(Paths.swaglog_root(), log_entry) time_sent = 0 try: value = getxattr(log_path, LOG_ATTR_NAME) @@ -608,7 +607,7 @@ def log_handler(end_event: threading.Event) -> None: cloudlog.debug(f"athena.log_handler.forward_request {log_entry}") try: curr_time = int(time.time()) - log_path = os.path.join(SWAGLOG_DIR, log_entry) + log_path = os.path.join(Paths.swaglog_root(), log_entry) setxattr(log_path, LOG_ATTR_NAME, int.to_bytes(curr_time, 4, sys.byteorder)) with open(log_path) as f: jsonrpc = { @@ -635,7 +634,7 @@ def log_handler(end_event: threading.Event) -> None: log_success = "result" in log_resp and log_resp["result"].get("success") cloudlog.debug(f"athena.log_handler.forward_response {log_entry} {log_success}") if log_entry and log_success: - log_path = os.path.join(SWAGLOG_DIR, log_entry) + log_path = os.path.join(Paths.swaglog_root(), log_entry) try: setxattr(log_path, LOG_ATTR_NAME, LOG_ATTR_VALUE_MAX_UNIX_TIME) except OSError: diff --git a/selfdrive/athena/tests/test_athenad.py b/selfdrive/athena/tests/test_athenad.py index e088bcf356..f4c229188e 100755 --- a/selfdrive/athena/tests/test_athenad.py +++ b/selfdrive/athena/tests/test_athenad.py @@ -3,7 +3,6 @@ import json import os import requests import shutil -import tempfile import time import threading import queue @@ -18,11 +17,11 @@ from unittest import mock from websocket import ABNF from websocket._exceptions import WebSocketConnectionClosedException -from openpilot.system import swaglog from openpilot.selfdrive.athena import athenad from openpilot.selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher from openpilot.selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server from cereal import messaging +from openpilot.selfdrive.hardware.hw import Paths class TestAthenadMethods(unittest.TestCase): @@ -30,8 +29,6 @@ class TestAthenadMethods(unittest.TestCase): def setUpClass(cls): cls.SOCKET_PORT = 45454 athenad.Params = MockParams - athenad.ROOT = tempfile.mkdtemp() - athenad.SWAGLOG_DIR = swaglog.SWAGLOG_DIR = tempfile.mkdtemp() athenad.Api = MockApi athenad.LOCAL_PORT_WHITELIST = {cls.SOCKET_PORT} @@ -41,8 +38,8 @@ class TestAthenadMethods(unittest.TestCase): athenad.cur_upload_items.clear() athenad.cancelled_uploads.clear() - for i in os.listdir(athenad.ROOT): - p = os.path.join(athenad.ROOT, i) + for i in os.listdir(Paths.log_root()): + p = os.path.join(Paths.log_root(), i) if os.path.isdir(p): shutil.rmtree(p) else: @@ -61,7 +58,7 @@ class TestAthenadMethods(unittest.TestCase): @staticmethod def _create_file(file: str, parent: Optional[str] = None) -> str: - fn = os.path.join(athenad.ROOT if parent is None else parent, file) + fn = os.path.join(Paths.log_root() if parent is None else parent, file) os.makedirs(os.path.dirname(fn), exist_ok=True) Path(fn).touch() return fn @@ -418,7 +415,7 @@ class TestAthenadMethods(unittest.TestCase): fl = list() for i in range(10): file = f'swaglog.{i:010}' - self._create_file(file, athenad.SWAGLOG_DIR) + self._create_file(file, Paths.swaglog_root()) fl.append(file) # ensure the list is all logs except most recent diff --git a/selfdrive/car/gm/carstate.py b/selfdrive/car/gm/carstate.py index ec3d79173c..b3aabce1c2 100644 --- a/selfdrive/car/gm/carstate.py +++ b/selfdrive/car/gm/carstate.py @@ -99,7 +99,7 @@ class CarState(CarStateBase): ret.leftBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 1 ret.rightBlinker = pt_cp.vl["BCMTurnSignals"]["TurnSignals"] == 2 - ret.parkingBrake = pt_cp.vl["VehicleIgnitionAlt"]["ParkBrake"] == 1 + ret.parkingBrake = pt_cp.vl["BCMGeneralPlatformStatus"]["ParkBrakeSwActive"] == 1 ret.cruiseState.available = pt_cp.vl["ECMEngineStatus"]["CruiseMainOn"] != 0 ret.espDisabled = pt_cp.vl["ESPStatus"]["TractionControlOn"] != 1 ret.accFaulted = (pt_cp.vl["AcceleratorPedal2"]["CruiseState"] == AccState.FAULTED or @@ -136,7 +136,7 @@ class CarState(CarStateBase): ("PSCMStatus", 10), ("ESPStatus", 10), ("BCMDoorBeltStatus", 10), - ("VehicleIgnitionAlt", 10), + ("BCMGeneralPlatformStatus", 10), ("EBCMWheelSpdFront", 20), ("EBCMWheelSpdRear", 20), ("EBCMFrictionBrakeStatus", 20), diff --git a/selfdrive/car/hyundai/carstate.py b/selfdrive/car/hyundai/carstate.py index 9fad0db7f4..2edeebff31 100644 --- a/selfdrive/car/hyundai/carstate.py +++ b/selfdrive/car/hyundai/carstate.py @@ -111,10 +111,12 @@ class CarState(CarStateBase): ret.cruiseState.available = cp.vl["TCS13"]["ACCEnable"] == 0 ret.cruiseState.enabled = cp.vl["TCS13"]["ACC_REQ"] == 1 ret.cruiseState.standstill = False + ret.cruiseState.nonAdaptive = False else: ret.cruiseState.available = cp_cruise.vl["SCC11"]["MainMode_ACC"] == 1 ret.cruiseState.enabled = cp_cruise.vl["SCC12"]["ACCMode"] != 0 ret.cruiseState.standstill = cp_cruise.vl["SCC11"]["SCCInfoDisplay"] == 4. + ret.cruiseState.nonAdaptive = cp_cruise.vl["SCC11"]["SCCInfoDisplay"] == 2. # Shows 'Cruise Control' on dash ret.cruiseState.speed = cp_cruise.vl["SCC11"]["VSetDis"] * speed_conv # TODO: Find brake pressure @@ -251,6 +253,13 @@ class CarState(CarStateBase): ret.cruiseState.speed = cp_cruise_info.vl["SCC_CONTROL"]["VSetDis"] * speed_factor self.cruise_info = copy.copy(cp_cruise_info.vl["SCC_CONTROL"]) + # Manual Speed Limit Assist is a feature that replaces non-adaptive cruise control on EV CAN FD platforms. + # It limits the vehicle speed, overridable by pressing the accelerator past a certain point. + # The car will brake, but does not respect positive acceleration commands in this mode + # TODO: find this message on ICE & HYBRID cars + cruise control signals (if exists) + if self.CP.carFingerprint in EV_CAR: + ret.cruiseState.nonAdaptive = cp.vl["MANUAL_SPEED_LIMIT_ASSIST"]["MSLA_ENABLED"] == 1 + self.prev_cruise_buttons = self.cruise_buttons[-1] self.cruise_buttons.extend(cp.vl_all[self.cruise_btns_msg_canfd]["CRUISE_BUTTONS"]) self.main_buttons.extend(cp.vl_all[self.cruise_btns_msg_canfd]["ADAPTIVE_CRUISE_MAIN_BTN"]) @@ -363,6 +372,11 @@ class CarState(CarStateBase): ("DOORS_SEATBELTS", 4), ] + if CP.carFingerprint in EV_CAR: + messages += [ + ("MANUAL_SPEED_LIMIT_ASSIST", 10), + ] + if not (CP.flags & HyundaiFlags.CANFD_ALT_BUTTONS): messages += [ ("CRUISE_BUTTONS", 50) diff --git a/selfdrive/car/hyundai/hyundaican.py b/selfdrive/car/hyundai/hyundaican.py index ccacb206bf..e26dbeb48b 100644 --- a/selfdrive/car/hyundai/hyundaican.py +++ b/selfdrive/car/hyundai/hyundaican.py @@ -37,7 +37,8 @@ def create_lkas11(packer, frame, car_fingerprint, apply_steer, steer_req, CAR.IONIQ_EV_2020, CAR.IONIQ_PHEV, CAR.KIA_SELTOS, CAR.ELANTRA_2021, CAR.GENESIS_G70_2020, CAR.ELANTRA_HEV_2021, CAR.SONATA_HYBRID, CAR.KONA_EV, CAR.KONA_HEV, CAR.KONA_EV_2022, CAR.SANTA_FE_2022, CAR.KIA_K5_2021, CAR.IONIQ_HEV_2022, CAR.SANTA_FE_HEV_2022, - CAR.SANTA_FE_PHEV_2022, CAR.KIA_STINGER_2022, CAR.KIA_K5_HEV_2020, CAR.KIA_CEED): + CAR.SANTA_FE_PHEV_2022, CAR.KIA_STINGER_2022, CAR.KIA_K5_HEV_2020, CAR.KIA_CEED, + CAR.AZERA_6TH_GEN): values["CF_Lkas_LdwsActivemode"] = int(left_lane) + (int(right_lane) << 1) values["CF_Lkas_LdwsOpt_USM"] = 2 diff --git a/selfdrive/car/hyundai/interface.py b/selfdrive/car/hyundai/interface.py index 1cf5a4b26e..1109c52277 100644 --- a/selfdrive/car/hyundai/interface.py +++ b/selfdrive/car/hyundai/interface.py @@ -68,7 +68,11 @@ class CarInterface(CarInterfaceBase): ret.steerLimitTimer = 0.4 CarInterfaceBase.configure_torque_tune(candidate, ret.lateralTuning) - if candidate in (CAR.SANTA_FE, CAR.SANTA_FE_2022, CAR.SANTA_FE_HEV_2022, CAR.SANTA_FE_PHEV_2022): + if candidate == CAR.AZERA_6TH_GEN: + ret.mass = 1540. # average + ret.wheelbase = 2.885 + ret.steerRatio = 14.5 + elif candidate in (CAR.SANTA_FE, CAR.SANTA_FE_2022, CAR.SANTA_FE_HEV_2022, CAR.SANTA_FE_PHEV_2022): ret.mass = 3982. * CV.LB_TO_KG ret.wheelbase = 2.766 # Values from optimizer @@ -215,6 +219,10 @@ class CarInterface(CarInterfaceBase): ret.mass = 2087. ret.wheelbase = 3.09 ret.steerRatio = 14.23 + elif candidate == CAR.KIA_K8_HEV_1ST_GEN: + ret.mass = 1630. # https://carprices.ae/brands/kia/2023/k8/1.6-turbo-hybrid + ret.wheelbase = 2.895 + ret.steerRatio = 13.27 # guesstimate from K5 platform # Genesis elif candidate == CAR.GENESIS_GV60_EV_1ST_GEN: diff --git a/selfdrive/car/hyundai/tests/test_hyundai.py b/selfdrive/car/hyundai/tests/test_hyundai.py index 39a9aaf627..11268913aa 100755 --- a/selfdrive/car/hyundai/tests/test_hyundai.py +++ b/selfdrive/car/hyundai/tests/test_hyundai.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +from hypothesis import settings, given, strategies as st import unittest from cereal import car @@ -66,6 +67,14 @@ class TestHyundaiFingerprint(unittest.TestCase): part = code.split(b"-")[1] self.assertFalse(part.startswith(b'CW'), "Car has bad part number") + @settings(max_examples=100) + @given(data=st.data()) + def test_platform_codes_fuzzy_fw(self, data): + """Ensure function doesn't raise an exception""" + fw_strategy = st.lists(st.binary()) + fws = data.draw(fw_strategy) + get_platform_codes(fws) + # Tests for platform codes, part numbers, and FW dates which Hyundai will use to fuzzy # fingerprint in the absence of full FW matches: def test_platform_code_ecus_available(self): diff --git a/selfdrive/car/hyundai/values.py b/selfdrive/car/hyundai/values.py index ce451601af..e02e4561cd 100644 --- a/selfdrive/car/hyundai/values.py +++ b/selfdrive/car/hyundai/values.py @@ -75,6 +75,7 @@ class HyundaiFlagsSP(IntFlag): class CAR: # Hyundai + AZERA_6TH_GEN = "HYUNDAI AZERA 6TH GEN" ELANTRA = "HYUNDAI ELANTRA 2017" ELANTRA_2021 = "HYUNDAI ELANTRA 2021" ELANTRA_2022_NON_SCC = "HYUNDAI ELANTRA SE 2022 NON-SCC" @@ -113,6 +114,7 @@ class CAR: KIA_FORTE_2021_NON_SCC = "KIA FORTE LXS 2021 NON-SCC" KIA_K5_2021 = "KIA K5 2021" KIA_K5_HEV_2020 = "KIA K5 HYBRID 2020" + KIA_K8_HEV_1ST_GEN = "KIA K8 HYBRID 1ST GEN" KIA_NIRO_EV = "KIA NIRO EV 2020" KIA_NIRO_EV_2ND_GEN = "KIA NIRO EV 2ND GEN" KIA_NIRO_PHEV = "KIA NIRO HYBRID 2019" @@ -162,6 +164,7 @@ class HyundaiCarInfo(CarInfo): CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = { + CAR.AZERA_6TH_GEN: HyundaiCarInfo("Hyundai Azera 2022", "All", car_parts=CarParts.common([CarHarness.hyundai_k])), CAR.ELANTRA: [ HyundaiCarInfo("Hyundai Elantra 2017-19", min_enable_speed=19 * CV.MPH_TO_MS, car_parts=CarParts.common([CarHarness.hyundai_b])), HyundaiCarInfo("Hyundai Elantra GT 2017-19", car_parts=CarParts.common([CarHarness.hyundai_e])), @@ -183,7 +186,7 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = { CAR.IONIQ_PHEV: HyundaiCarInfo("Hyundai Ioniq Plug-in Hybrid 2020-22", "All", car_parts=CarParts.common([CarHarness.hyundai_h])), CAR.KONA: HyundaiCarInfo("Hyundai Kona 2020", car_parts=CarParts.common([CarHarness.hyundai_b])), CAR.KONA_EV: HyundaiCarInfo("Hyundai Kona Electric 2018-21", car_parts=CarParts.common([CarHarness.hyundai_g])), - CAR.KONA_EV_2022: HyundaiCarInfo("Hyundai Kona Electric 2022", car_parts=CarParts.common([CarHarness.hyundai_o])), + CAR.KONA_EV_2022: HyundaiCarInfo("Hyundai Kona Electric 2022-23", car_parts=CarParts.common([CarHarness.hyundai_o])), CAR.KONA_HEV: HyundaiCarInfo("Hyundai Kona Hybrid 2020", video_link="https://youtu.be/0dwpAHiZgFo", car_parts=CarParts.common([CarHarness.hyundai_i])), # TODO: check packages CAR.KONA_EV_2ND_GEN: HyundaiCarInfo("Hyundai Kona Electric (with HDA II, Korea only) 2023", video_link="https://www.youtube.com/watch?v=U2fOCmcQ8hw", car_parts=CarParts.common([CarHarness.hyundai_r])), @@ -227,6 +230,7 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = { ], CAR.KIA_K5_2021: HyundaiCarInfo("Kia K5 2021-22", car_parts=CarParts.common([CarHarness.hyundai_a])), CAR.KIA_K5_HEV_2020: HyundaiCarInfo("Kia K5 Hybrid 2020", car_parts=CarParts.common([CarHarness.hyundai_a])), + CAR.KIA_K8_HEV_1ST_GEN: HyundaiCarInfo("Kia K8 Hybrid (with HDA II) 2023", "Highway Driving Assist II", car_parts=CarParts.common([CarHarness.hyundai_q])), CAR.KIA_NIRO_EV: [ HyundaiCarInfo("Kia Niro EV 2019", "All", video_link="https://www.youtube.com/watch?v=lT7zcG6ZpGo", car_parts=CarParts.common([CarHarness.hyundai_h])), HyundaiCarInfo("Kia Niro EV 2020", "All", video_link="https://www.youtube.com/watch?v=lT7zcG6ZpGo", car_parts=CarParts.common([CarHarness.hyundai_f])), @@ -270,7 +274,7 @@ CAR_INFO: Dict[str, Optional[Union[HyundaiCarInfo, List[HyundaiCarInfo]]]] = { HyundaiCarInfo("Kia EV6 (with HDA II) 2022-23", "Highway Driving Assist II", car_parts=CarParts.common([CarHarness.hyundai_p])) ], CAR.KIA_CARNIVAL_4TH_GEN: [ - HyundaiCarInfo("Kia Carnival 2023", car_parts=CarParts.common([CarHarness.hyundai_a])), + HyundaiCarInfo("Kia Carnival 2023-24", car_parts=CarParts.common([CarHarness.hyundai_a])), HyundaiCarInfo("Kia Carnival (China only) 2023", car_parts=CarParts.common([CarHarness.hyundai_k])) ], @@ -475,6 +479,7 @@ PART_NUMBER_FW_PATTERN = re.compile(b'(?<=[0-9][.,][0-9]{2} )([0-9]{5}[-/]?[A-Z] # TODO: use abs, it has the platform code and part number on many platforms PLATFORM_CODE_ECUS = [Ecu.fwdRadar, Ecu.fwdCamera, Ecu.eps] # So far we've only seen dates in fwdCamera +# TODO: there are date codes in the ABS firmware versions in hex DATE_FW_ECUS = [Ecu.fwdCamera] FW_QUERY_CONFIG = FwQueryConfig( @@ -538,6 +543,23 @@ FW_QUERY_CONFIG = FwQueryConfig( ) FW_VERSIONS = { + CAR.AZERA_6TH_GEN: { + (Ecu.fwdRadar, 0x7d0, None): [ + b'\xf1\x00IG__ SCC F-CU- 1.00 1.00 99110-G8100 ', + ], + (Ecu.eps, 0x7d4, None): [ + b'\xf1\x00IG MDPS C 1.00 1.02 56310G8510\x00 4IGSC103', + ], + (Ecu.fwdCamera, 0x7c4, None): [ + b'\xf1\x00IG MFC AT MES LHD 1.00 1.04 99211-G8100 200511', + ], + (Ecu.transmission, 0x7e1, None): [ + b'\xf1\x00bcsh8p54 U912\x00\x00\x00\x00\x00\x00SIG0M35MH0\xa4 |.', + ], + (Ecu.engine, 0x7e0, None): [ + b'\xf1\x81641KA051\x00\x00\x00\x00\x00\x00\x00\x00', + ], + }, CAR.HYUNDAI_GENESIS: { (Ecu.fwdCamera, 0x7c4, None): [ b'\xf1\x00DH LKAS 1.1 -150210', @@ -1478,10 +1500,7 @@ FW_VERSIONS = { b'\xf1\x8758520-K4010\xf1\x00OS IEB \x04 101 \x11\x13 58520-K4010', b'\xf1\x8758520-K4010\xf1\x00OS IEB \x03 101 \x11\x13 58520-K4010', b'\xf1\x00OS IEB \r 102"\x05\x16 58520-K4010', - # TODO: these return from the MULTI request, above return from LONG - b'\x01\x04\x7f\xff\xff\xf8\xff\xff\x00\x00\x01\xd3\x00\x00\x00\x00\xff\xb7\xff\xee\xff\xe0\x00\xc0\xc0\xfc\xd5\xfc\x00\x00U\x10\xffP\xf5\xff\xfd\x00\x00\x00\x00\xfc\x00\x01', - b'\x01\x04\x7f\xff\xff\xf8\xff\xff\x00\x00\x01\xdb\x00\x00\x00\x00\xff\xb1\xff\xd9\xff\xd2\x00\xc0\xc0\xfc\xd5\xfc\x00\x00U\x10\xff\xd6\xf5\x00\x06\x00\x00\x00\x14\xfd\x00\x04', - b'\x01\x04\x7f\xff\xff\xf8\xff\xff\x00\x00\x01\xd3\x00\x00\x00\x00\xff\xb7\xff\xf4\xff\xd9\x00\xc0', + b'\xf1\x00OS IEB \x02 102"\x05\x16 58520-K4010', ], (Ecu.fwdCamera, 0x7C4, None): [ b'\xf1\x00OSP LKA AT CND LHD 1.00 1.02 99211-J9110 802', @@ -1489,12 +1508,14 @@ FW_VERSIONS = { b'\xf1\x00OSP LKA AT AUS RHD 1.00 1.04 99211-J9200 904', b'\xf1\x00OSP LKA AT EUR LHD 1.00 1.04 99211-J9200 904', b'\xf1\x00OSP LKA AT EUR RHD 1.00 1.04 99211-J9200 904', + b'\xf1\x00OSP LKA AT USA LHD 1.00 1.04 99211-J9200 904', ], (Ecu.eps, 0x7D4, None): [ b'\xf1\x00OSP MDPS C 1.00 1.02 56310K4260\x00 4OEPC102', b'\xf1\x00OSP MDPS C 1.00 1.02 56310/K4970 4OEPC102', b'\xf1\x00OSP MDPS C 1.00 1.02 56310/K4271 4OEPC102', b'\xf1\x00OSP MDPS C 1.00 1.02 56310K4971\x00 4OEPC102', + b'\xf1\x00OSP MDPS C 1.00 1.02 56310K4261\x00 4OEPC102', ], (Ecu.fwdRadar, 0x7D0, None): [ b'\xf1\x00YB__ FCA ----- 1.00 1.01 99110-K4500 \x00\x00\x00', @@ -2005,6 +2026,7 @@ FW_VERSIONS = { (Ecu.fwdCamera, 0x7c4, None): [ b'\xf1\x00KA4 MFC AT USA LHD 1.00 1.06 99210-R0000 220221', b'\xf1\x00KA4CMFC AT CHN LHD 1.00 1.01 99211-I4000 210525', + b'\xf1\x00KA4 MFC AT USA LHD 1.00 1.00 99210-R0100 230105', ], (Ecu.fwdRadar, 0x7d0, None): [ b'\xf1\x00KA4_ SCC FHCUP 1.00 1.03 99110-R0000 ', @@ -2020,6 +2042,14 @@ FW_VERSIONS = { b'\xf1\x00MQhe SCC FHCUP 1.00 1.07 99110-P4000 ', ], }, + CAR.KIA_K8_HEV_1ST_GEN: { + (Ecu.fwdCamera, 0x7c4, None): [ + b'\xf1\x00GL3HMFC AT KOR LHD 1.00 1.03 99211-L8000 210907', + ], + (Ecu.fwdRadar, 0x7d0, None): [ + b'\xf1\x00GL3_ RDR ----- 1.00 1.02 99110-L8000 ', + ], + }, } CHECKSUM = { @@ -2042,7 +2072,7 @@ CAN_GEARS = { CANFD_CAR = {CAR.KIA_EV6, CAR.IONIQ_5, CAR.IONIQ_6, CAR.TUCSON_4TH_GEN, CAR.TUCSON_HYBRID_4TH_GEN, CAR.KIA_SPORTAGE_HYBRID_5TH_GEN, CAR.SANTA_CRUZ_1ST_GEN, CAR.KIA_SPORTAGE_5TH_GEN, CAR.GENESIS_GV70_1ST_GEN, CAR.KIA_SORENTO_PHEV_4TH_GEN, CAR.GENESIS_GV60_EV_1ST_GEN, CAR.KIA_SORENTO_4TH_GEN, CAR.KIA_NIRO_HEV_2ND_GEN, CAR.KIA_NIRO_EV_2ND_GEN, - CAR.GENESIS_GV80, CAR.KIA_CARNIVAL_4TH_GEN, CAR.KIA_SORENTO_HEV_4TH_GEN, CAR.KONA_EV_2ND_GEN} + CAR.GENESIS_GV80, CAR.KIA_CARNIVAL_4TH_GEN, CAR.KIA_SORENTO_HEV_4TH_GEN, CAR.KONA_EV_2ND_GEN, CAR.KIA_K8_HEV_1ST_GEN} # The radar does SCC on these cars when HDA I, rather than the camera CANFD_RADAR_SCC_CAR = {CAR.GENESIS_GV70_1ST_GEN, CAR.KIA_SORENTO_PHEV_4TH_GEN, CAR.KIA_SORENTO_4TH_GEN, CAR.GENESIS_GV80, @@ -2055,13 +2085,13 @@ CAMERA_SCC_CAR = {CAR.KONA_EV_2022, } HYBRID_CAR = {CAR.IONIQ_PHEV, CAR.ELANTRA_HEV_2021, CAR.KIA_NIRO_PHEV, CAR.KIA_NIRO_HEV_2021, CAR.SONATA_HYBRID, CAR.KONA_HEV, CAR.IONIQ, CAR.IONIQ_HEV_2022, CAR.SANTA_FE_HEV_2022, CAR.SANTA_FE_PHEV_2022, CAR.IONIQ_PHEV_2019, CAR.TUCSON_HYBRID_4TH_GEN, CAR.KIA_SPORTAGE_HYBRID_5TH_GEN, CAR.KIA_SORENTO_PHEV_4TH_GEN, CAR.KIA_K5_HEV_2020, CAR.KIA_NIRO_HEV_2ND_GEN, - CAR.KIA_SORENTO_HEV_4TH_GEN, CAR.KIA_OPTIMA_H} + CAR.KIA_SORENTO_HEV_4TH_GEN, CAR.KIA_OPTIMA_H, CAR.KIA_K8_HEV_1ST_GEN} EV_CAR = {CAR.IONIQ_EV_2020, CAR.IONIQ_EV_LTD, CAR.KONA_EV, CAR.KIA_NIRO_EV, CAR.KIA_NIRO_EV_2ND_GEN, CAR.KONA_EV_2022, CAR.KIA_EV6, CAR.IONIQ_5, CAR.IONIQ_6, CAR.GENESIS_GV60_EV_1ST_GEN, CAR.KONA_EV_2ND_GEN} # these cars require a special panda safety mode due to missing counters and checksums in the messages -LEGACY_SAFETY_MODE_CAR = {CAR.HYUNDAI_GENESIS, CAR.IONIQ_EV_LTD, CAR.KONA_EV, CAR.KIA_OPTIMA_G4, +LEGACY_SAFETY_MODE_CAR = {CAR.HYUNDAI_GENESIS, CAR.IONIQ_EV_LTD, CAR.KIA_OPTIMA_G4, CAR.VELOSTER, CAR.GENESIS_G70, CAR.GENESIS_G80, CAR.KIA_CEED, CAR.ELANTRA, CAR.IONIQ_HEV_2022, CAR.KIA_OPTIMA_H} @@ -2075,6 +2105,7 @@ NON_SCC_RADAR_FCA_CAR = {CAR.GENESIS_G70_2021_NON_SCC, } # If 0x500 is present on bus 1 it probably has a Mando radar outputting radar points. # If no points are outputted by default it might be possible to turn it on using selfdrive/debug/hyundai_enable_radar_points.py DBC = { + CAR.AZERA_6TH_GEN: dbc_dict('hyundai_kia_generic', None), CAR.ELANTRA: dbc_dict('hyundai_kia_generic', None), CAR.ELANTRA_2021: dbc_dict('hyundai_kia_generic', None), CAR.ELANTRA_2022_NON_SCC: dbc_dict('hyundai_kia_generic', None), @@ -2139,4 +2170,5 @@ DBC = { CAR.KIA_CARNIVAL_4TH_GEN: dbc_dict('hyundai_canfd', None), CAR.KIA_SORENTO_HEV_4TH_GEN: dbc_dict('hyundai_canfd', None), CAR.KONA_EV_2ND_GEN: dbc_dict('hyundai_canfd', None), + CAR.KIA_K8_HEV_1ST_GEN: dbc_dict('hyundai_canfd', None), } diff --git a/selfdrive/car/mock/interface.py b/selfdrive/car/mock/interface.py index d36f972a87..c7821bdb97 100755 --- a/selfdrive/car/mock/interface.py +++ b/selfdrive/car/mock/interface.py @@ -1,61 +1,35 @@ #!/usr/bin/env python3 from cereal import car -from openpilot.system.swaglog import cloudlog import cereal.messaging as messaging -from openpilot.selfdrive.car import get_safety_config from openpilot.selfdrive.car.interfaces import CarInterfaceBase - -# mocked car interface to work with chffrplus +# mocked car interface for dashcam mode class CarInterface(CarInterfaceBase): def __init__(self, CP, CarController, CarState): super().__init__(CP, CarController, CarState) - cloudlog.debug("Using Mock Car Interface") - - self.sm = messaging.SubMaster(['gpsLocation', 'gpsLocationExternal']) - self.speed = 0. - self.prev_speed = 0. + self.sm = messaging.SubMaster(['gpsLocation', 'gpsLocationExternal']) @staticmethod def _get_params(ret, candidate, fingerprint, car_fw, experimental_long, docs): ret.carName = "mock" - ret.safetyConfigs = [get_safety_config(car.CarParams.SafetyModel.noOutput)] ret.mass = 1700. ret.wheelbase = 2.70 ret.centerToFront = ret.wheelbase * 0.5 - ret.steerRatio = 13. # reasonable - + ret.steerRatio = 13. return ret - # returns a car.CarState def _update(self, c): self.sm.update(0) gps_sock = 'gpsLocationExternal' if self.sm.rcv_frame['gpsLocationExternal'] > 1 else 'gpsLocation' - if self.sm.updated[gps_sock]: - self.prev_speed = self.speed - self.speed = self.sm[gps_sock].speed - # create message ret = car.CarState.new_message() - - # speeds - ret.vEgo = self.speed - ret.vEgoRaw = self.speed - - ret.aEgo = self.speed - self.prev_speed - ret.brakePressed = ret.aEgo < -0.5 - - ret.standstill = self.speed < 0.01 - ret.wheelSpeeds.fl = self.speed - ret.wheelSpeeds.fr = self.speed - ret.wheelSpeeds.rl = self.speed - ret.wheelSpeeds.rr = self.speed + ret.vEgo = self.sm[gps_sock].speed + ret.vEgoRaw = self.sm[gps_sock].speed return ret def apply(self, c, now_nanos): - # in mock no carcontrols actuators = car.CarControl.Actuators.new_message() return actuators, [] diff --git a/selfdrive/car/mock/radar_interface.py b/selfdrive/car/mock/radar_interface.py old mode 100755 new mode 100644 index b461fcd5f8..e654bd61fd --- a/selfdrive/car/mock/radar_interface.py +++ b/selfdrive/car/mock/radar_interface.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python3 from openpilot.selfdrive.car.interfaces import RadarInterfaceBase class RadarInterface(RadarInterfaceBase): diff --git a/selfdrive/car/subaru/carcontroller.py b/selfdrive/car/subaru/carcontroller.py index 5553fda0d8..19304efdb1 100644 --- a/selfdrive/car/subaru/carcontroller.py +++ b/selfdrive/car/subaru/carcontroller.py @@ -1,8 +1,13 @@ from openpilot.common.numpy_fast import clip, interp from opendbc.can.packer import CANPacker -from openpilot.selfdrive.car import apply_driver_steer_torque_limits +from openpilot.selfdrive.car import apply_driver_steer_torque_limits, common_fault_avoidance from openpilot.selfdrive.car.subaru import subarucan -from openpilot.selfdrive.car.subaru.values import DBC, GLOBAL_GEN2, PREGLOBAL_CARS, HYBRID_CARS, CanBus, CarControllerParams, SubaruFlags +from openpilot.selfdrive.car.subaru.values import DBC, GLOBAL_GEN2, PREGLOBAL_CARS, HYBRID_CARS, STEER_RATE_LIMITED, CanBus, CarControllerParams, SubaruFlags + +# FIXME: These limits aren't exact. The real limit is more than likely over a larger time period and +# involves the total steering angle change rather than rate, but these limits work well for now +MAX_STEER_RATE = 25 # deg/s +MAX_STEER_RATE_FRAMES = 7 # tx control frames needed before torque can be cut class CarController: @@ -12,6 +17,7 @@ class CarController: self.frame = 0 self.cruise_button_prev = 0 + self.steer_rate_counter = 0 self.p = CarControllerParams(CP) self.packer = CANPacker(DBC[CP.carFingerprint]['pt']) @@ -38,7 +44,15 @@ class CarController: if self.CP.carFingerprint in PREGLOBAL_CARS: can_sends.append(subarucan.create_preglobal_steering_control(self.packer, self.frame // self.p.STEER_STEP, apply_steer, CC.latActive)) else: - can_sends.append(subarucan.create_steering_control(self.packer, apply_steer, CC.latActive)) + apply_steer_req = CC.latActive + + if self.CP.carFingerprint in STEER_RATE_LIMITED: + # Steering rate fault prevention + self.steer_rate_counter, apply_steer_req = \ + common_fault_avoidance(abs(CS.out.steeringRateDeg) > MAX_STEER_RATE, apply_steer_req, + self.steer_rate_counter, MAX_STEER_RATE_FRAMES) + + can_sends.append(subarucan.create_steering_control(self.packer, apply_steer, apply_steer_req)) self.apply_steer_last = apply_steer diff --git a/selfdrive/car/subaru/values.py b/selfdrive/car/subaru/values.py index c3cdd7ad13..1547876008 100644 --- a/selfdrive/car/subaru/values.py +++ b/selfdrive/car/subaru/values.py @@ -710,3 +710,7 @@ LKAS_ANGLE = {CAR.FORESTER_2022, CAR.OUTBACK_2023, CAR.ASCENT_2023} GLOBAL_GEN2 = {CAR.OUTBACK, CAR.LEGACY, CAR.OUTBACK_2023, CAR.ASCENT_2023} PREGLOBAL_CARS = {CAR.FORESTER_PREGLOBAL, CAR.LEGACY_PREGLOBAL, CAR.OUTBACK_PREGLOBAL, CAR.OUTBACK_PREGLOBAL_2018} HYBRID_CARS = {CAR.CROSSTREK_HYBRID, CAR.FORESTER_HYBRID} + +# Cars that temporarily fault when steering angle rate is greater than some threshold. +# Appears to be all cars that started production after 2020 +STEER_RATE_LIMITED = GLOBAL_GEN2 | {CAR.IMPREZA_2020} diff --git a/selfdrive/car/tests/routes.py b/selfdrive/car/tests/routes.py index 8299d15d52..f7ad219f63 100755 --- a/selfdrive/car/tests/routes.py +++ b/selfdrive/car/tests/routes.py @@ -25,7 +25,6 @@ non_tested_cars = [ HONDA.ODYSSEY_CHN, VOLKSWAGEN.CRAFTER_MK2, # need a route from an ACC-equipped Crafter TOYOTA.RAV4_TSS2_2023, - TOYOTA.RAV4H_TSS2_2023, SUBARU.FORESTER_HYBRID, ] @@ -94,6 +93,7 @@ routes = [ CarTestRoute("2d5808fae0b38ac6|2021-09-01--17-14-11", HONDA.HONDA_E), CarTestRoute("f44aa96ace22f34a|2021-12-22--06-22-31", HONDA.CIVIC_2022), + CarTestRoute("87d7f06ade479c2e|2023-09-11--23-30-11", HYUNDAI.AZERA_6TH_GEN), CarTestRoute("6fe86b4e410e4c37|2020-07-22--16-27-13", HYUNDAI.HYUNDAI_GENESIS), CarTestRoute("b5d6dc830ad63071|2022-12-12--21-28-25", HYUNDAI.GENESIS_GV60_EV_1ST_GEN, segment=12), CarTestRoute("70c5bec28ec8e345|2020-08-08--12-22-23", HYUNDAI.GENESIS_G70), @@ -134,6 +134,7 @@ routes = [ CarTestRoute("ab59fe909f626921|2021-10-18--18-34-28", HYUNDAI.IONIQ_HEV_2022), CarTestRoute("22d955b2cd499c22|2020-08-10--19-58-21", HYUNDAI.KONA), CarTestRoute("efc48acf44b1e64d|2021-05-28--21-05-04", HYUNDAI.KONA_EV), + CarTestRoute("f90d3cd06caeb6fa|2023-09-06--17-15-47", HYUNDAI.KONA_EV), # openpilot longitudinal enabled CarTestRoute("ff973b941a69366f|2022-07-28--22-01-19", HYUNDAI.KONA_EV_2022, segment=11), CarTestRoute("1618132d68afc876|2023-08-27--09-32-14", HYUNDAI.KONA_EV_2ND_GEN, segment=13), CarTestRoute("49f3c13141b6bc87|2021-07-28--08-05-13", HYUNDAI.KONA_HEV), @@ -145,6 +146,7 @@ routes = [ CarTestRoute("9b25e8c1484a1b67|2023-04-13--10-41-45", HYUNDAI.KIA_EV6), CarTestRoute("007d5e4ad9f86d13|2021-09-30--15-09-23", HYUNDAI.KIA_K5_2021), CarTestRoute("c58dfc9fc16590e0|2023-01-14--13-51-48", HYUNDAI.KIA_K5_HEV_2020), + CarTestRoute("78ad5150de133637|2023-09-13--16-15-57", HYUNDAI.KIA_K8_HEV_1ST_GEN), CarTestRoute("50c6c9b85fd1ff03|2020-10-26--17-56-06", HYUNDAI.KIA_NIRO_EV), CarTestRoute("b153671049a867b3|2023-04-05--10-00-30", HYUNDAI.KIA_NIRO_EV_2ND_GEN), CarTestRoute("173219cf50acdd7b|2021-07-05--10-27-41", HYUNDAI.KIA_NIRO_PHEV), @@ -181,6 +183,7 @@ routes = [ CarTestRoute("a5c341bb250ca2f0|2022-05-18--16-05-17", TOYOTA.RAV4_TSS2_2022), CarTestRoute("7e34a988419b5307|2019-12-18--19-13-30", TOYOTA.RAV4H_TSS2), CarTestRoute("2475fb3eb2ffcc2e|2022-04-29--12-46-23", TOYOTA.RAV4H_TSS2_2022), + CarTestRoute("49e041422a032273|2023-09-14--09-21-32", TOYOTA.RAV4H_TSS2_2023), CarTestRoute("7a31f030957b9c85|2023-04-01--14-12-51", TOYOTA.LEXUS_ES), CarTestRoute("e6a24be49a6cd46e|2019-10-29--10-52-42", TOYOTA.LEXUS_ES_TSS2), CarTestRoute("da23c367491f53e2|2021-05-21--09-09-11", TOYOTA.LEXUS_CTH, segment=3), diff --git a/selfdrive/car/tests/test_car_interfaces.py b/selfdrive/car/tests/test_car_interfaces.py index 0c970f5c75..787b5d6316 100755 --- a/selfdrive/car/tests/test_car_interfaces.py +++ b/selfdrive/car/tests/test_car_interfaces.py @@ -87,9 +87,6 @@ class TestCarInterfaces(unittest.TestCase): self.assertTrue(not math.isnan(tune.torque.kf) and tune.torque.kf > 0) self.assertTrue(not math.isnan(tune.torque.friction) and tune.torque.friction > 0) - elif tune.which() == 'indi': - self.assertTrue(len(tune.indi.outerLoopGainV)) - cc_msg = FuzzyGenerator.get_random_msg(data.draw, car.CarControl, real_floats=True) # Run car interface now_nanos = 0 diff --git a/selfdrive/car/tests/test_fw_fingerprint.py b/selfdrive/car/tests/test_fw_fingerprint.py index 7d5bd85e99..378b68e43e 100755 --- a/selfdrive/car/tests/test_fw_fingerprint.py +++ b/selfdrive/car/tests/test_fw_fingerprint.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import pytest import random import time import unittest @@ -224,6 +225,7 @@ class TestFwFingerprintTiming(unittest.TestCase): self._assert_timing(vin_time / self.N, vin_ref_time) print(f'get_vin, query time={vin_time / self.N} seconds') + @pytest.mark.timeout(60) def test_fw_query_timing(self): total_ref_time = 6.07 brand_ref_times = { diff --git a/selfdrive/car/tests/test_lateral_limits.py b/selfdrive/car/tests/test_lateral_limits.py index 9de6063f3d..1fc626972f 100755 --- a/selfdrive/car/tests/test_lateral_limits.py +++ b/selfdrive/car/tests/test_lateral_limits.py @@ -32,7 +32,7 @@ ABOVE_LIMITS_CARS = [ car_model_jerks: DefaultDict[str, Dict[str, float]] = defaultdict(dict) -@parameterized_class('car_model', [(c,) for c in CAR_MODELS]) +@parameterized_class('car_model', [(c,) for c in sorted(CAR_MODELS)]) class TestLateralLimits(unittest.TestCase): car_model: str diff --git a/selfdrive/car/tests/test_models.py b/selfdrive/car/tests/test_models.py index 6cebdde77b..e6460d491e 100755 --- a/selfdrive/car/tests/test_models.py +++ b/selfdrive/car/tests/test_models.py @@ -9,6 +9,7 @@ from parameterized import parameterized_class from cereal import log, car from openpilot.common.basedir import BASEDIR +from openpilot.common.params import Params from openpilot.common.realtime import DT_CTRL from openpilot.selfdrive.car.fingerprints import all_known_cars from openpilot.selfdrive.car.car_helpers import FRAME_FINGERPRINT, interfaces @@ -16,6 +17,7 @@ from openpilot.selfdrive.car.gm.values import CAR as GM from openpilot.selfdrive.car.honda.values import CAR as HONDA, HONDA_BOSCH from openpilot.selfdrive.car.hyundai.values import CAR as HYUNDAI from openpilot.selfdrive.car.tests.routes import non_tested_cars, routes, CarTestRoute +from openpilot.selfdrive.controls.controlsd import Controls from openpilot.selfdrive.test.openpilotci import get_url from openpilot.tools.lib.logreader import LogReader from openpilot.tools.lib.route import Route, SegmentName, RouteName @@ -23,6 +25,7 @@ from openpilot.tools.lib.route import Route, SegmentName, RouteName from panda.tests.libpanda import libpanda_py from openpilot.selfdrive.test.helpers import SKIP_ENV_VAR +EventName = car.CarEvent.EventName PandaType = log.PandaState.PandaType SafetyModel = car.CarParams.SafetyModel @@ -163,9 +166,11 @@ class TestCarModelBase(unittest.TestCase): del cls.can_msgs def setUp(self): - self.CI = self.CarInterface(self.CP, self.CarController, self.CarState) + self.CI = self.CarInterface(self.CP.copy(), self.CarController, self.CarState) assert self.CI + Params().put_bool("OpenpilotEnabledToggle", self.openpilot_enabled) + # TODO: check safetyModel is in release panda build self.safety = libpanda_py.libpanda @@ -187,8 +192,6 @@ class TestCarModelBase(unittest.TestCase): self.assertTrue(len(self.CP.lateralTuning.pid.kpV)) elif tuning == 'torque': self.assertTrue(self.CP.lateralTuning.torque.kf > 0) - elif tuning == 'indi': - self.assertTrue(len(self.CP.lateralTuning.indi.outerLoopGainV)) else: raise Exception("unknown tuning") @@ -317,6 +320,8 @@ class TestCarModelBase(unittest.TestCase): controls_allowed_prev = False CS_prev = car.CarState.new_message() checks = defaultdict(lambda: 0) + controlsd = Controls(CI=self.CI) + controlsd.initialized = True for idx, can in enumerate(self.can_msgs): CS = self.CI.update(CC, (can.as_builder().to_bytes(), )) for msg in filter(lambda m: m.src in range(64), can.can): @@ -361,7 +366,10 @@ class TestCarModelBase(unittest.TestCase): checks['cruiseState'] += CS.cruiseState.enabled != self.safety.get_cruise_engaged_prev() else: # Check for enable events on rising edge of controls allowed - button_enable = any(evt.enable for evt in CS.events) + controlsd.update_events(CS) + controlsd.CS_prev = CS + button_enable = (any(evt.enable for evt in CS.events) and + not any(evt == EventName.pedalPressed for evt in controlsd.events.names)) mismatch = button_enable != (self.safety.get_controls_allowed() and not controls_allowed_prev) checks['controlsAllowed'] += mismatch controls_allowed_prev = self.safety.get_controls_allowed() diff --git a/selfdrive/car/torque_data/override.yaml b/selfdrive/car/torque_data/override.yaml index 149ae6ee97..7ff7fad19a 100644 --- a/selfdrive/car/torque_data/override.yaml +++ b/selfdrive/car/torque_data/override.yaml @@ -12,10 +12,6 @@ SUBARU FORESTER 2022: [.nan, 3.0, .nan] SUBARU OUTBACK 7TH GEN: [.nan, 3.0, .nan] SUBARU ASCENT 2023: [.nan, 3.0, .nan] -# Toyota LTA also has torque -TOYOTA RAV4 2023: [.nan, 3.0, .nan] -TOYOTA RAV4 HYBRID 2023: [.nan, 3.0, .nan] - # Tesla has high torque TESLA AP1 MODEL S: [.nan, 2.5, .nan] TESLA AP2 MODEL S: [.nan, 2.5, .nan] @@ -60,6 +56,10 @@ KIA SORENTO HYBRID 4TH GEN: [2.5, 2.5, 0.1] HYUNDAI KONA ELECTRIC 2ND GEN: [2.5, 2.5, 0.1] HYUNDAI IONIQ 6 2023: [2.5, 2.5, 0.1] HONDA ACCORD 4CYL 9TH GEN: [1.7, 0.35, 0.21] +HYUNDAI AZERA 6TH GEN: [1.8, 1.8, 0.1] +KIA K8 HYBRID 1ST GEN: [2.5, 2.5, 0.1] +TOYOTA RAV4 2023: [2.5, 2.5, 0.1] +TOYOTA RAV4 HYBRID 2023: [2.5, 2.5, 0.1] # Dashcam or fallback configured as ideal car mock: [10.0, 10, 0.0] diff --git a/selfdrive/car/toyota/interface.py b/selfdrive/car/toyota/interface.py index d98c1ad425..9822d8ac1b 100644 --- a/selfdrive/car/toyota/interface.py +++ b/selfdrive/car/toyota/interface.py @@ -28,7 +28,15 @@ class CarInterface(CarInterfaceBase): if DBC[candidate]["pt"] == "toyota_new_mc_pt_generated": ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_ALT_BRAKE - if candidate in ANGLE_CONTROL_CAR: + # Allow angle control cars with whitelisted EPSs to use torque control (made in Japan) + # So far only hybrid RAV4 2023 has been seen with this FW version + angle_car_torque_fw = any(fw.ecu == "eps" and fw.fwVersion == b'8965B42371\x00\x00\x00\x00\x00\x00' for fw in car_fw) + if candidate not in ANGLE_CONTROL_CAR or (angle_car_torque_fw and candidate == CAR.RAV4H_TSS2_2023): + CarInterfaceBase.configure_torque_tune(candidate, ret.lateralTuning) + + ret.steerActuatorDelay = 0.12 # Default delay, Prius has larger delay + ret.steerLimitTimer = 0.4 + else: ret.dashcamOnly = True ret.steerControlType = SteerControlType.angle ret.safetyConfigs[0].safetyParam |= Panda.FLAG_TOYOTA_LTA @@ -36,11 +44,6 @@ class CarInterface(CarInterfaceBase): # LTA control can be more delayed and winds up more often ret.steerActuatorDelay = 0.25 ret.steerLimitTimer = 0.8 - else: - CarInterfaceBase.configure_torque_tune(candidate, ret.lateralTuning) - - ret.steerActuatorDelay = 0.12 # Default delay, Prius has larger delay - ret.steerLimitTimer = 0.4 ret.stoppingControl = False # Toyota starts braking more when it thinks you want to stop @@ -122,21 +125,25 @@ class CarInterface(CarInterfaceBase): ret.steerRatio = 14.3 ret.tireStiffnessFactor = 0.7933 ret.mass = 3585. * CV.LB_TO_KG # Average between ICE and Hybrid - ret.lateralTuning.init('pid') - ret.lateralTuning.pid.kiBP = [0.0] - ret.lateralTuning.pid.kpBP = [0.0] - ret.lateralTuning.pid.kpV = [0.6] - ret.lateralTuning.pid.kiV = [0.1] - ret.lateralTuning.pid.kf = 0.00007818594 - # 2019+ RAV4 TSS2 uses two different steering racks and specific tuning seems to be necessary. - # See https://github.com/commaai/openpilot/pull/21429#issuecomment-873652891 - for fw in car_fw: - if fw.ecu == "eps" and (fw.fwVersion.startswith(b'\x02') or fw.fwVersion in [b'8965B42181\x00\x00\x00\x00\x00\x00']): - ret.lateralTuning.pid.kpV = [0.15] - ret.lateralTuning.pid.kiV = [0.05] - ret.lateralTuning.pid.kf = 0.00004 - break + # Only specific EPS FW accept torque on 2023 RAV4, so they likely are all the same + # TODO: revisit this disparity if there is a divide for 2023 + if candidate not in (CAR.RAV4_TSS2_2023, CAR.RAV4H_TSS2_2023): + ret.lateralTuning.init('pid') + ret.lateralTuning.pid.kiBP = [0.0] + ret.lateralTuning.pid.kpBP = [0.0] + ret.lateralTuning.pid.kpV = [0.6] + ret.lateralTuning.pid.kiV = [0.1] + ret.lateralTuning.pid.kf = 0.00007818594 + + # 2019+ RAV4 TSS2 uses two different steering racks and specific tuning seems to be necessary. + # See https://github.com/commaai/openpilot/pull/21429#issuecomment-873652891 + for fw in car_fw: + if fw.ecu == "eps" and (fw.fwVersion.startswith(b'\x02') or fw.fwVersion in [b'8965B42181\x00\x00\x00\x00\x00\x00']): + ret.lateralTuning.pid.kpV = [0.15] + ret.lateralTuning.pid.kiV = [0.05] + ret.lateralTuning.pid.kf = 0.00004 + break elif candidate in (CAR.COROLLA_TSS2, CAR.COROLLAH_TSS2): ret.wheelbase = 2.67 # Average between 2.70 for sedan and 2.64 for hatchback diff --git a/selfdrive/car/toyota/tests/print_platform_codes.py b/selfdrive/car/toyota/tests/print_platform_codes.py new file mode 100755 index 0000000000..94badc5cde --- /dev/null +++ b/selfdrive/car/toyota/tests/print_platform_codes.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +from cereal import car +from openpilot.selfdrive.car.toyota.values import FW_VERSIONS, PLATFORM_CODE_ECUS, get_platform_codes + +Ecu = car.CarParams.Ecu +ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} + +if __name__ == "__main__": + for car_model, ecus in FW_VERSIONS.items(): + print() + print(car_model) + for ecu in sorted(ecus, key=lambda x: int(x[0])): + if ecu[0] not in PLATFORM_CODE_ECUS: + continue + + platform_codes = get_platform_codes(ecus[ecu]) + codes = {code for code, _ in platform_codes} + dates = {date for _, date in platform_codes if date is not None} + print(f' (Ecu.{ECU_NAME[ecu[0]]}, {hex(ecu[1])}, {ecu[2]}):') + print(f' Codes: {codes}') + print(f' Versions: {dates}') diff --git a/selfdrive/car/toyota/tests/test_toyota.py b/selfdrive/car/toyota/tests/test_toyota.py index c621734e2b..0c503ebbf0 100755 --- a/selfdrive/car/toyota/tests/test_toyota.py +++ b/selfdrive/car/toyota/tests/test_toyota.py @@ -1,10 +1,13 @@ #!/usr/bin/env python3 -from cereal import car +from hypothesis import given, settings, strategies as st import unittest -from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS +from cereal import car +from openpilot.selfdrive.car.toyota.values import CAR, DBC, TSS2_CAR, ANGLE_CONTROL_CAR, RADAR_ACC_CAR, FW_VERSIONS, \ + get_platform_codes Ecu = car.CarParams.Ecu +ECU_NAME = {v: k for k, v in Ecu.schema.enumerants.items()} class TestToyotaInterfaces(unittest.TestCase): @@ -39,5 +42,22 @@ class TestToyotaInterfaces(unittest.TestCase): self.assertIn(Ecu.eps, present_ecus) +class TestToyotaFingerprint(unittest.TestCase): + @settings(max_examples=100) + @given(data=st.data()) + def test_platform_codes_fuzzy_fw(self, data): + fw_strategy = st.lists(st.binary()) + fws = data.draw(fw_strategy) + get_platform_codes(fws) + + def test_fw_pattern(self): + """Asserts all ECUs can be parsed""" + for ecus in FW_VERSIONS.values(): + for fws in ecus.values(): + for fw in fws: + ret = get_platform_codes([fw]) + self.assertTrue(len(ret)) + + if __name__ == "__main__": unittest.main() diff --git a/selfdrive/car/toyota/values.py b/selfdrive/car/toyota/values.py index b0f7a70985..fd0a54b349 100644 --- a/selfdrive/car/toyota/values.py +++ b/selfdrive/car/toyota/values.py @@ -1,7 +1,8 @@ +import re from collections import defaultdict from dataclasses import dataclass, field from enum import Enum, IntFlag -from typing import Dict, List, Union +from typing import Dict, List, Set, Tuple, Union from cereal import car from openpilot.common.conversions import Conversions as CV @@ -234,6 +235,71 @@ STATIC_DSU_MSGS = [ CAR.SIENNA, CAR.LEXUS_CTH, CAR.LEXUS_ES, CAR.LEXUS_ESH, CAR.LEXUS_RX, CAR.PRIUS_V), 0, 100, b'\x0c\x00\x00\x00\x00\x00\x00\x00'), ] + +def get_platform_codes(fw_versions: List[bytes]) -> Set[Tuple[bytes, bytes]]: + codes = set() # (Optional[part]-platform-major_version, minor_version) + for fw in fw_versions: + # FW versions returned from UDS queries can return multiple fields/chunks of data (different ECU calibrations, different data?) + # and are prefixed with a byte that describes how many chunks of data there are. + # But FW returned from KWP requires querying of each sub-data id and does not have a length prefix. + + length_code = 1 + length_code_match = FW_LEN_CODE.search(fw) + if length_code_match is not None: + length_code = length_code_match.group()[0] + fw = fw[1:] + + # fw length should be multiple of 16 bytes (per chunk, even if no length code), skip parsing if unexpected length + if length_code * FW_CHUNK_LEN != len(fw): + continue + + chunks = [fw[FW_CHUNK_LEN * i:FW_CHUNK_LEN * i + FW_CHUNK_LEN].strip(b'\x00 ') for i in range(length_code)] + + # only first is considered for now since second is commonly shared (TODO: understand that) + first_chunk = chunks[0] + if len(first_chunk) == 8: + # TODO: no part number, but some short chunks have it in subsequent chunks + fw_match = SHORT_FW_PATTERN.search(first_chunk) + if fw_match is not None: + platform, major_version, sub_version = fw_match.groups() + # codes.add((platform + b'-' + major_version, sub_version)) + codes.add((b'-'.join((platform, major_version)), sub_version)) + + elif len(first_chunk) == 10: + fw_match = MEDIUM_FW_PATTERN.search(first_chunk) + if fw_match is not None: + part, platform, major_version, sub_version = fw_match.groups() + codes.add((b'-'.join((part, platform, major_version)), sub_version)) + + elif len(first_chunk) == 12: + fw_match = LONG_FW_PATTERN.search(first_chunk) + if fw_match is not None: + part, platform, major_version, sub_version = fw_match.groups() + codes.add((b'-'.join((part, platform, major_version)), sub_version)) + + return codes + + +# Regex patterns for parsing more general platform-specific identifiers from FW versions. +# - Part number: Toyota part number (usually last character needs to be ignored to find a match). +# - Platform: usually multiple codes per an openpilot platform, however this has the less variability and +# is usually shared across ECUs and model years signifying this describes something about the specific platform. +# - Major version: second least variable part of the FW version. Seen splitting cars by model year such as RAV4 2022/2023 and Prius. +# It is important to note that these aren't always consecutive, for example: +# Prius TSS-P has these major versions over 16 FW: 2, 3, 4, 6, 8 while Prius TSS2 has: 5 +# - Sub version: exclusive to major version, but shared with other cars. Should only be used for further filtering, +# more exploration is needed. +SHORT_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{4})') +MEDIUM_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{1})(?P[A-Z0-9]{2})') +LONG_FW_PATTERN = re.compile(b'(?P[A-Z0-9]{5})(?P[A-Z0-9]{2})(?P[A-Z0-9]{2})(?P[A-Z0-9]{3})') +FW_LEN_CODE = re.compile(b'^[\x01-\x05]') # 5 chunks max. highest seen is 3 chunks, 16 bytes each +FW_CHUNK_LEN = 16 + +# List of ECUs expected to have platform codes +# TODO: use hybrid ECU, splits many similar ICE and hybrid variants +PLATFORM_CODE_ECUS = [Ecu.abs, Ecu.engine, Ecu.eps, Ecu.dsu, Ecu.fwdCamera, Ecu.fwdRadar] + + # Some ECUs that use KWP2000 have their FW versions on non-standard data identifiers. # Toyota diagnostic software first gets the supported data ids, then queries them one by one. # For example, sends: 0x1a8800, receives: 0x1a8800010203, queries: 0x1a8801, 0x1a8802, 0x1a8803 @@ -2323,7 +2389,8 @@ UNSUPPORTED_DSU_CAR = {CAR.LEXUS_IS, CAR.LEXUS_RC} # these cars have a radar which sends ACC messages instead of the camera RADAR_ACC_CAR = {CAR.RAV4H_TSS2_2022, CAR.RAV4_TSS2_2022, CAR.RAV4H_TSS2_2023, CAR.RAV4_TSS2_2023, CAR.CHR_TSS2, CAR.CHRH_TSS2} -# these cars use the Lane Tracing Assist (LTA) message for lateral control +# these cars manufactured in U.S., Canada have EPSs that reject Lane Keep Assist (LKA, torque) messages and require +# Lane Tracing Assist (LTA, angle) to steer properly. cars manufactured in Japan still work with the older LKA messages which is detected ANGLE_CONTROL_CAR = {CAR.RAV4H_TSS2_2023, CAR.RAV4_TSS2_2023} EV_HYBRID_CAR = {CAR.AVALONH_2019, CAR.AVALONH_TSS2, CAR.CAMRYH, CAR.CAMRYH_TSS2, CAR.CHRH, CAR.CHRH_TSS2, CAR.COROLLAH_TSS2, diff --git a/selfdrive/car/volkswagen/values.py b/selfdrive/car/volkswagen/values.py index ebb873c8be..494d59bad2 100644 --- a/selfdrive/car/volkswagen/values.py +++ b/selfdrive/car/volkswagen/values.py @@ -461,6 +461,7 @@ FW_VERSIONS = { b'\xf1\x878V0906259K \xf1\x890003', b'\xf1\x878V0906259P \xf1\x890001', b'\xf1\x878V0906259Q \xf1\x890002', + b'\xf1\x878V0906259R \xf1\x890002', b'\xf1\x878V0906264F \xf1\x890003', b'\xf1\x878V0906264L \xf1\x890002', b'\xf1\x878V0906264M \xf1\x890001', @@ -502,6 +503,7 @@ FW_VERSIONS = { b'\xf1\x870GC300012A \xf1\x891401', b'\xf1\x870GC300012A \xf1\x891403', b'\xf1\x870GC300014B \xf1\x892401', + b'\xf1\x870GC300014B \xf1\x892403', b'\xf1\x870GC300014B \xf1\x892405', b'\xf1\x870GC300020G \xf1\x892401', b'\xf1\x870GC300020G \xf1\x892403', @@ -521,6 +523,7 @@ FW_VERSIONS = { b'\xf1\x875Q0959655BS\xf1\x890403\xf1\x82\x1314160011123300314240012250229333463100', b'\xf1\x875Q0959655BT\xf1\x890403\xf1\x82\x13141600111233003142404A2251229333463100', b'\xf1\x875Q0959655BT\xf1\x890403\xf1\x82\x13141600111233003142404A2252229333463100', + b'\xf1\x875Q0959655BT\xf1\x890403\xf1\x82\x13141600111233003142405A2251229333463100', b'\xf1\x875Q0959655BT\xf1\x890403\xf1\x82\x13141600111233003142405A2252229333463100', b'\xf1\x875Q0959655C \xf1\x890361\xf1\x82\x111413001112120004110415121610169112', b'\xf1\x875Q0959655D \xf1\x890388\xf1\x82\x111413001113120006110417121A101A9113', @@ -547,6 +550,7 @@ FW_VERSIONS = { b'\xf1\x873Q0909144K \xf1\x895072\xf1\x82\x0571A0J714A1', b'\xf1\x873Q0909144L \xf1\x895081\xf1\x82\x0571A0JA15A1', b'\xf1\x873Q0909144M \xf1\x895082\xf1\x82\x0571A01A18A1', + b'\xf1\x873Q0909144M \xf1\x895082\xf1\x82\x0571A02A16A1', b'\xf1\x873Q0909144M \xf1\x895082\xf1\x82\x0571A0JA16A1', b'\xf1\x873QM909144 \xf1\x895072\xf1\x82\x0571A01714A1', b'\xf1\x875Q0909143K \xf1\x892033\xf1\x820519A9040203', @@ -1323,6 +1327,7 @@ FW_VERSIONS = { b'\xf1\x8704E906027BT\xf1\x899042', b'\xf1\x8704L906026ET\xf1\x891343', b'\xf1\x8704L906026FP\xf1\x891196', + b'\xf1\x8704L906026KA\xf1\x896014', b'\xf1\x8704L906026KB\xf1\x894071', b'\xf1\x8704L906026KD\xf1\x894798', b'\xf1\x8704L906026MT\xf1\x893076', @@ -1340,6 +1345,7 @@ FW_VERSIONS = { b'\xf1\x870D9300014K \xf1\x895006', b'\xf1\x870D9300041H \xf1\x894905', b'\xf1\x870D9300043F \xf1\x895202', + b'\xf1\x870GC300013K \xf1\x892403', b'\xf1\x870GC300014M \xf1\x892801', b'\xf1\x870GC300019G \xf1\x892803', b'\xf1\x870GC300043 \xf1\x892301', @@ -1352,6 +1358,7 @@ FW_VERSIONS = { b'\xf1\x875Q0959655BH\xf1\x890336\xf1\x82\02331310031313100313131013141319331413100', b'\xf1\x875Q0959655BK\xf1\x890336\xf1\x82\x1331310031313100313131013141319331413100', b'\xf1\x875Q0959655CA\xf1\x890403\xf1\x82\x1331310031313100313151013141319331423100', + b'\xf1\x875Q0959655CA\xf1\x890403\xf1\x82\x1331310031313100313151823143319331423100', b'\xf1\x875Q0959655CH\xf1\x890421\xf1\x82\x1333310031313100313152025350539331463100', b'\xf1\x875Q0959655CH\xf1\x890421\xf1\x82\x1333310031313100313152855372539331463100', ], diff --git a/selfdrive/controls/controlsd.py b/selfdrive/controls/controlsd.py index ab26928d9c..d339ce226a 100755 --- a/selfdrive/controls/controlsd.py +++ b/selfdrive/controls/controlsd.py @@ -381,17 +381,19 @@ class Controls: else: self.logged_comm_issue = None - if not self.sm['liveParameters'].valid and not TESTING_CLOSET and (not SIMULATION or REPLAY): - self.events.add(EventName.vehicleModelInvalid) if not self.sm['lateralPlan'].mpcSolutionValid: self.events.add(EventName.plannerError) - if not (self.sm['liveParameters'].sensorValid or self.sm['liveLocationKalman'].sensorsOK) and not NOSENSOR: - if self.sm.frame > 5 / DT_CTRL: # Give locationd some time to receive all the inputs - self.events.add(EventName.sensorDataInvalid) if not self.sm['liveLocationKalman'].posenetOK: self.events.add(EventName.posenetInvalid) if not self.sm['liveLocationKalman'].deviceStable: self.events.add(EventName.deviceFalling) + if not (self.sm['liveParameters'].sensorValid or self.sm['liveLocationKalman'].sensorsOK): + if self.sm.frame > 5 / DT_CTRL: # Give locationd some time to receive sensor inputs + self.events.add(EventName.sensorDataInvalid) + if not self.sm['liveLocationKalman'].inputsOK: + self.events.add(EventName.locationdTemporaryError) + if not self.sm['liveParameters'].valid and not TESTING_CLOSET and (not SIMULATION or REPLAY): + self.events.add(EventName.paramsdTemporaryError) if not REPLAY: # Check for mismatch between openpilot and car's PCM @@ -427,8 +429,6 @@ class Controls: if self.sm['modelV2'].frameDropPerc > 20: self.events.add(EventName.modeldLagging) - if self.sm['liveLocationKalman'].excessiveResets: - self.events.add(EventName.localizerMalfunction) def data_sample(self): """Receive data from sockets and update carState""" @@ -814,8 +814,6 @@ class Controls: controlsState.lateralControlState.pidState = lac_log elif lat_tuning == 'torque': controlsState.lateralControlState.torqueState = lac_log - elif lat_tuning == 'indi': - controlsState.lateralControlState.indiState = lac_log self.pm.send('controlsState', dat) diff --git a/selfdrive/controls/lib/events.py b/selfdrive/controls/lib/events.py index 6d55c2875d..83609f7713 100755 --- a/selfdrive/controls/lib/events.py +++ b/selfdrive/controls/lib/events.py @@ -434,19 +434,6 @@ EVENTS: Dict[int, Dict[str, Union[Alert, AlertCallbackType]]] = { # ********** events only containing alerts that display while engaged ********** - # openpilot tries to learn certain parameters about your car by observing - # how the car behaves to steering inputs from both human and openpilot driving. - # This includes: - # - steer ratio: gear ratio of the steering rack. Steering angle divided by tire angle - # - tire stiffness: how much grip your tires have - # - angle offset: most steering angle sensors are offset and measure a non zero angle when driving straight - # This alert is thrown when any of these values exceed a sanity check. This can be caused by - # bad alignment or bad sensor data. If this happens consistently consider creating an issue on GitHub - EventName.vehicleModelInvalid: { - ET.NO_ENTRY: NoEntryAlert("Vehicle Parameter Identification Failed"), - ET.SOFT_DISABLE: soft_disable_alert("Vehicle Parameter Identification Failed"), - }, - EventName.steerTempUnavailableSilent: { ET.WARNING: Alert( "Steering Temporarily Unavailable", @@ -606,11 +593,34 @@ EVENTS: Dict[int, Dict[str, Union[Alert, AlertCallbackType]]] = { ET.PERMANENT: NormalPermanentAlert("GPS Malfunction", "Likely Hardware Issue"), }, - # When the GPS position and localizer diverge the localizer is reset to the - # current GPS position. This alert is thrown when the localizer is reset - # more often than expected. - EventName.localizerMalfunction: { - # ET.PERMANENT: NormalPermanentAlert("Sensor Malfunction", "Hardware Malfunction"), + EventName.locationdTemporaryError: { + ET.NO_ENTRY: NoEntryAlert("locationd Temporary Error"), + ET.SOFT_DISABLE: soft_disable_alert("locationd Temporary Error"), + }, + + EventName.locationdPermanentError: { + ET.NO_ENTRY: NoEntryAlert("locationd Permanent Error"), + ET.IMMEDIATE_DISABLE: ImmediateDisableAlert("locationd Permanent Error"), + ET.PERMANENT: NormalPermanentAlert("locationd Permanent Error"), + }, + + # openpilot tries to learn certain parameters about your car by observing + # how the car behaves to steering inputs from both human and openpilot driving. + # This includes: + # - steer ratio: gear ratio of the steering rack. Steering angle divided by tire angle + # - tire stiffness: how much grip your tires have + # - angle offset: most steering angle sensors are offset and measure a non zero angle when driving straight + # This alert is thrown when any of these values exceed a sanity check. This can be caused by + # bad alignment or bad sensor data. If this happens consistently consider creating an issue on GitHub + EventName.paramsdTemporaryError: { + ET.NO_ENTRY: NoEntryAlert("paramsd Temporary Error"), + ET.SOFT_DISABLE: soft_disable_alert("paramsd Temporary Error"), + }, + + EventName.paramsdPermanentError: { + ET.NO_ENTRY: NoEntryAlert("paramsd Permanent Error"), + ET.IMMEDIATE_DISABLE: ImmediateDisableAlert("paramsd Permanent Error"), + ET.PERMANENT: NormalPermanentAlert("paramsd Permanent Error"), }, EventName.speedLimitActive: { diff --git a/selfdrive/controls/radard.py b/selfdrive/controls/radard.py index 8a21fdd8ab..ec42ae66f2 100755 --- a/selfdrive/controls/radard.py +++ b/selfdrive/controls/radard.py @@ -50,7 +50,8 @@ class KalmanParams: class Track: - def __init__(self, v_lead: float, kalman_params: KalmanParams): + def __init__(self, identifier: int, v_lead: float, kalman_params: KalmanParams): + self.identifier = identifier self.cnt = 0 self.aLeadTau = _LEAD_ACCEL_TAU self.K_A = kalman_params.A @@ -98,11 +99,12 @@ class Track: "vLead": float(self.vLead), "vLeadK": float(self.vLeadK), "aLeadK": float(self.aLeadK), + "aLeadTau": float(self.aLeadTau), "status": True, "fcw": self.is_potential_fcw(model_prob), "modelProb": model_prob, "radar": True, - "aLeadTau": float(self.aLeadTau) + "radarTrackId": self.identifier, } def potential_low_speed_lead(self, v_ego: float): @@ -158,8 +160,9 @@ def get_RadarState_from_vision(lead_msg: capnp._DynamicStructReader, v_ego: floa "aLeadTau": 0.3, "fcw": False, "modelProb": float(lead_msg.prob), + "status": True, "radar": False, - "status": True + "radarTrackId": -1, } @@ -237,7 +240,7 @@ class RadarD: # create the track if it doesn't exist or it's a new track if ids not in self.tracks: - self.tracks[ids] = Track(v_lead, self.kalman_params) + self.tracks[ids] = Track(ids, v_lead, self.kalman_params) self.tracks[ids].update(rpt[0], rpt[1], rpt[2], v_lead, rpt[3]) # *** publish radarState *** diff --git a/selfdrive/controls/tests/test_cruise_speed.py b/selfdrive/controls/tests/test_cruise_speed.py index 1d43b49ccf..76a2222e85 100755 --- a/selfdrive/controls/tests/test_cruise_speed.py +++ b/selfdrive/controls/tests/test_cruise_speed.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import itertools import numpy as np import unittest @@ -31,21 +32,19 @@ def run_cruise_simulation(cruise, e2e, t_end=20.): return output[-1, 3] +@parameterized_class(("e2e", "personality", "speed"), itertools.product( + [True, False], # e2e + log.LongitudinalPersonality.schema.enumerants, # personality + [5,35])) # speed class TestCruiseSpeed(unittest.TestCase): def test_cruise_speed(self): params = Params() - personalities = [log.LongitudinalPersonality.relaxed, - log.LongitudinalPersonality.standard, - log.LongitudinalPersonality.aggressive] - for personality in personalities: - params.put("LongitudinalPersonality", str(personality)) - for e2e in [False, True]: - for speed in [5,35]: - print(f'Testing {speed} m/s') - cruise_speed = float(speed) + params.put("LongitudinalPersonality", str(self.personality)) + print(f'Testing {self.speed} m/s') + cruise_speed = float(self.speed) - simulation_steady_state = run_cruise_simulation(cruise_speed, e2e) - self.assertAlmostEqual(simulation_steady_state, cruise_speed, delta=.01, msg=f'Did not reach {speed} m/s') + simulation_steady_state = run_cruise_simulation(cruise_speed, self.e2e) + self.assertAlmostEqual(simulation_steady_state, cruise_speed, delta=.01, msg=f'Did not reach {self.speed} m/s') # TODO: test pcmCruise diff --git a/selfdrive/controls/tests/test_leads.py b/selfdrive/controls/tests/test_leads.py index bd8baf096e..268d9c47a7 100755 --- a/selfdrive/controls/tests/test_leads.py +++ b/selfdrive/controls/tests/test_leads.py @@ -3,8 +3,8 @@ import unittest import cereal.messaging as messaging -from selfdrive.test.process_replay import replay_process_with_name -from selfdrive.car.toyota.values import CAR as TOYOTA +from openpilot.selfdrive.test.process_replay import replay_process_with_name +from openpilot.selfdrive.car.toyota.values import CAR as TOYOTA class TestLeads(unittest.TestCase): diff --git a/selfdrive/locationd/SConscript b/selfdrive/locationd/SConscript index 740f827a49..a6febe0170 100644 --- a/selfdrive/locationd/SConscript +++ b/selfdrive/locationd/SConscript @@ -7,8 +7,4 @@ locationd_sources = ["locationd.cc", "models/live_kf.cc", ekf_sym_cc] lenv = env.Clone() lenv["_LIBFLAGS"] += f' {libkf[0].get_labspath()}' locationd = lenv.Program("locationd", locationd_sources, LIBS=loc_libs + transformations) -lenv.Depends(locationd, libkf) - -if File("liblocationd.cc").exists(): - liblocationd = lenv.SharedLibrary("liblocationd", ["liblocationd.cc"] + locationd_sources, LIBS=loc_libs + transformations) - lenv.Depends(liblocationd, libkf) \ No newline at end of file +lenv.Depends(locationd, libkf) \ No newline at end of file diff --git a/selfdrive/locationd/laikad.py b/selfdrive/locationd/laikad.py index 3d190fb00a..f00595618e 100755 --- a/selfdrive/locationd/laikad.py +++ b/selfdrive/locationd/laikad.py @@ -24,11 +24,11 @@ from laika.opt import calc_pos_fix, get_posfix_sympy_fun, calc_vel_fix, get_velf from openpilot.selfdrive.locationd.models.constants import GENERATED_DIR, ObservationKind from openpilot.selfdrive.locationd.models.gnss_kf import GNSSKalman from openpilot.selfdrive.locationd.models.gnss_kf import States as GStates +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import cloudlog MAX_TIME_GAP = 10 EPHEMERIS_CACHE = 'LaikadEphemerisV3' -DOWNLOADS_CACHE_FOLDER = "/tmp/comma_download_cache/" CACHE_VERSION = 0.2 POS_FIX_RESIDUAL_THRESHOLD = 100.0 @@ -83,7 +83,7 @@ class Laikad: save_ephemeris: If true saves and loads nav and orbit ephemeris to cache. """ self.astro_dog = AstroDog(valid_const=valid_const, auto_update=auto_update, valid_ephem_types=valid_ephem_types, - clear_old_ephemeris=True, cache_dir=DOWNLOADS_CACHE_FOLDER) + clear_old_ephemeris=True, cache_dir=Paths.download_cache_root()) self.gnss_kf = GNSSKalman(GENERATED_DIR, cython=True, erratic_clock=use_qcom) self.auto_fetch_navs = auto_fetch_navs @@ -435,9 +435,9 @@ def kf_add_observations(gnss_kf: GNSSKalman, t: float, measurements: List[GNSSMe def clear_tmp_cache(): - if os.path.exists(DOWNLOADS_CACHE_FOLDER): - shutil.rmtree(DOWNLOADS_CACHE_FOLDER) - os.mkdir(DOWNLOADS_CACHE_FOLDER) + if os.path.exists(Paths.download_cache_root()): + shutil.rmtree(Paths.download_cache_root()) + os.mkdir(Paths.download_cache_root()) def main(sm=None, pm=None): diff --git a/selfdrive/locationd/liblocationd.cc b/selfdrive/locationd/liblocationd.cc deleted file mode 100644 index e2b85d93a1..0000000000 --- a/selfdrive/locationd/liblocationd.cc +++ /dev/null @@ -1,41 +0,0 @@ -#include "selfdrive/locationd/locationd.h" - -extern "C" { - typedef Localizer* Localizer_t; - - Localizer *localizer_init(bool has_ublox) { - return new Localizer(has_ublox ? LocalizerGnssSource::UBLOX : LocalizerGnssSource::QCOM); - } - - void localizer_get_message_bytes(Localizer *localizer, bool inputsOK, bool sensorsOK, bool gpsOK, bool msgValid, - char *buff, size_t buff_size) { - MessageBuilder msg_builder; - kj::ArrayPtr arr = localizer->get_message_bytes(msg_builder, inputsOK, sensorsOK, gpsOK, msgValid).asChars(); - assert(buff_size >= arr.size()); - memcpy(buff, arr.begin(), arr.size()); - } - - void localizer_handle_msg_bytes(Localizer *localizer, const char *data, size_t size) { - localizer->handle_msg_bytes(data, size); - } - - void get_filter_internals(Localizer *localizer, double *state_buff, double *std_buff){ - Eigen::VectorXd state = localizer->get_state(); - memcpy(state_buff, state.data(), sizeof(double) * state.size()); - Eigen::VectorXd stdev = localizer->get_stdev(); - memcpy(std_buff, stdev.data(), sizeof(double) * stdev.size()); - } - - bool is_gps_ok(Localizer *localizer){ - return localizer->is_gps_ok(); - } - - bool are_inputs_ok(Localizer *localizer){ - return localizer->are_inputs_ok(); - } - - void observation_timings_invalid_reset(Localizer *localizer){ - localizer->observation_timings_invalid_reset(); - } - -} diff --git a/selfdrive/locationd/locationd.cc b/selfdrive/locationd/locationd.cc index 401de0cfdd..600ba46853 100644 --- a/selfdrive/locationd/locationd.cc +++ b/selfdrive/locationd/locationd.cc @@ -21,9 +21,11 @@ const double VALID_TIME_SINCE_RESET = 1.0; // s const double VALID_POS_STD = 50.0; // m const double MAX_RESET_TRACKER = 5.0; const double SANE_GPS_UNCERTAINTY = 1500.0; // m -const double INPUT_INVALID_THRESHOLD = 5.0; // same as reset tracker -const double DECAY = 0.99995; // same as reset tracker +const double INPUT_INVALID_THRESHOLD = 0.5; // same as reset tracker +const double RESET_TRACKER_DECAY = 0.99995; +const double DECAY = 0.9993; // ~10 secs to resume after a bad input const double MAX_FILTER_REWIND_TIME = 0.8; // s +const double YAWRATE_CROSS_ERR_CHECK_FACTOR = 30; // TODO: GPS sensor time offsets are empirically calculated // They should be replaced with synced time from a real clock @@ -256,7 +258,13 @@ void Localizer::handle_sensor(double current_time, const cereal::SensorEventData if (log.getSensor() == SENSOR_GYRO_UNCALIBRATED && log.getType() == SENSOR_TYPE_GYROSCOPE_UNCALIBRATED) { auto v = log.getGyroUncalibrated().getV(); auto meas = Vector3d(-v[2], -v[1], -v[0]); - if (meas.norm() < ROTATION_SANITY_CHECK) { + + VectorXd gyro_bias = this->kf->get_x().segment(STATE_GYRO_BIAS_START); + float gyro_camodo_yawrate_err = std::abs((meas[2] - gyro_bias[2]) - this->camodo_yawrate_distribution[0]); + float gyro_camodo_yawrate_err_threshold = YAWRATE_CROSS_ERR_CHECK_FACTOR * this->camodo_yawrate_distribution[1]; + bool gyro_valid = gyro_camodo_yawrate_err < gyro_camodo_yawrate_err_threshold; + + if ((meas.norm() < ROTATION_SANITY_CHECK) && gyro_valid) { this->kf->predict_and_observe(sensor_time, OBSERVATION_PHONE_GYRO, { meas }); this->observation_values_invalid["gyroscope"] *= DECAY; } else { @@ -483,6 +491,7 @@ void Localizer::handle_cam_odo(double current_time, const cereal::CameraOdometry this->kf->predict_and_observe(current_time, OBSERVATION_CAMERA_ODO_TRANSLATION, { trans_device }, { trans_device_cov }); this->observation_values_invalid["cameraOdometry"] *= DECAY; + this->camodo_yawrate_distribution = Vector2d(rot_device[2], rotate_std(this->device_from_calib, rot_calib_std)[2]); } void Localizer::handle_live_calib(double current_time, const cereal::LiveCalibrationData::Reader& log) { @@ -538,7 +547,7 @@ void Localizer::time_check(double current_time) { void Localizer::update_reset_tracker() { // reset tracker is tuned to trigger when over 1reset/10s over 2min period if (this->is_gps_ok()) { - this->reset_tracker *= DECAY; + this->reset_tracker *= RESET_TRACKER_DECAY; } else { this->reset_tracker = 0.0; } diff --git a/selfdrive/locationd/locationd.h b/selfdrive/locationd/locationd.h index f3069048cd..47c8bf5627 100644 --- a/selfdrive/locationd/locationd.h +++ b/selfdrive/locationd/locationd.h @@ -94,6 +94,7 @@ private: float gps_variance_factor; float gps_vertical_variance_factor; double gps_time_offset; + Eigen::VectorXd camodo_yawrate_distribution = Eigen::Vector2d(0.0, 10.0); // mean, std void configure_gnss_source(const LocalizerGnssSource &source); }; diff --git a/selfdrive/locationd/test/_test_locationd_lib.py b/selfdrive/locationd/test/_test_locationd_lib.py deleted file mode 100755 index bf4fcbdbe9..0000000000 --- a/selfdrive/locationd/test/_test_locationd_lib.py +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python3 -"""This test can't be run together with other locationd tests. -cffi.dlopen breaks the list of registered filters.""" -import os -import random -import unittest - -from cffi import FFI - -import cereal.messaging as messaging -from cereal import log - -from openpilot.common.ffi_wrapper import suffix - -SENSOR_DECIMATION = 1 -VISION_DECIMATION = 1 - -LIBLOCATIOND_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), '../liblocationd' + suffix())) - - -class TestLocationdLib(unittest.TestCase): - def setUp(self): - header = '''typedef ...* Localizer_t; -Localizer_t localizer_init(bool has_ublox); -void localizer_get_message_bytes(Localizer_t localizer, bool inputsOK, bool sensorsOK, bool gpsOK, bool msgValid, char *buff, size_t buff_size); -void localizer_handle_msg_bytes(Localizer_t localizer, const char *data, size_t size);''' - - self.ffi = FFI() - self.ffi.cdef(header) - self.lib = self.ffi.dlopen(LIBLOCATIOND_PATH) - - self.localizer = self.lib.localizer_init(True) # default to ublox - - self.buff_size = 2048 - self.msg_buff = self.ffi.new(f'char[{self.buff_size}]') - - def localizer_handle_msg(self, msg_builder): - bytstr = msg_builder.to_bytes() - self.lib.localizer_handle_msg_bytes(self.localizer, self.ffi.from_buffer(bytstr), len(bytstr)) - - def localizer_get_msg(self, t=0, inputsOK=True, sensorsOK=True, gpsOK=True, msgValid=True): - self.lib.localizer_get_message_bytes(self.localizer, inputsOK, sensorsOK, gpsOK, msgValid, self.ffi.addressof(self.msg_buff, 0), self.buff_size) - with log.Event.from_bytes(self.ffi.buffer(self.msg_buff), nesting_limit=self.buff_size // 8) as log_evt: - return log_evt - - def test_liblocalizer(self): - msg = messaging.new_message('liveCalibration') - msg.liveCalibration.validBlocks = random.randint(1, 10) - msg.liveCalibration.rpyCalib = [random.random() / 10 for _ in range(3)] - - self.localizer_handle_msg(msg) - liveloc = self.localizer_get_msg() - self.assertTrue(liveloc is not None) - - @unittest.skip("temporarily disabled due to false positives") - def test_device_fell(self): - msg = messaging.new_message('accelerometer') - msg.accelerometer.sensor = 1 - msg.accelerometer.timestamp = msg.logMonoTime - msg.accelerometer.type = 1 - msg.accelerometer.init('acceleration') - msg.accelerometer.acceleration.v = [10.0, 0.0, 0.0] # zero with gravity - self.localizer_handle_msg(msg) - - ret = self.localizer_get_msg() - self.assertTrue(ret.liveLocationKalman.deviceStable) - - msg = messaging.new_message('accelerometer') - msg.accelerometer.sensor = 1 - msg.accelerometer.timestamp = msg.logMonoTime - msg.accelerometer.type = 1 - msg.accelerometer.init('acceleration') - msg.accelerometer.acceleration.v = [50.1, 0.0, 0.0] # more than 40 m/s**2 - self.localizer_handle_msg(msg) - - ret = self.localizer_get_msg() - self.assertFalse(ret.liveLocationKalman.deviceStable) - - def test_posenet_spike(self): - for _ in range(SENSOR_DECIMATION): - msg = messaging.new_message('carState') - msg.carState.vEgo = 6.0 # more than 5 m/s - self.localizer_handle_msg(msg) - - ret = self.localizer_get_msg() - self.assertTrue(ret.liveLocationKalman.posenetOK) - - for _ in range(20 * VISION_DECIMATION): # size of hist_old - msg = messaging.new_message('cameraOdometry') - msg.cameraOdometry.rot = [0.0, 0.0, 0.0] - msg.cameraOdometry.rotStd = [0.1, 0.1, 0.1] - msg.cameraOdometry.trans = [0.0, 0.0, 0.0] - msg.cameraOdometry.transStd = [2.0, 0.1, 0.1] - self.localizer_handle_msg(msg) - - for _ in range(20 * VISION_DECIMATION): # size of hist_new - msg = messaging.new_message('cameraOdometry') - msg.cameraOdometry.rot = [0.0, 0.0, 0.0] - msg.cameraOdometry.rotStd = [1.0, 1.0, 1.0] - msg.cameraOdometry.trans = [0.0, 0.0, 0.0] - msg.cameraOdometry.transStd = [10.1, 0.1, 0.1] # more than 4 times larger - self.localizer_handle_msg(msg) - - ret = self.localizer_get_msg() - self.assertFalse(ret.liveLocationKalman.posenetOK) - -if __name__ == "__main__": - unittest.main() - diff --git a/selfdrive/locationd/test/test_locationd_scenarios.py b/selfdrive/locationd/test/test_locationd_scenarios.py new file mode 100755 index 0000000000..d2455ef9e0 --- /dev/null +++ b/selfdrive/locationd/test/test_locationd_scenarios.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python3 +import unittest +import numpy as np +from collections import defaultdict +from enum import Enum + + +from openpilot.selfdrive.test.openpilotci import get_url +from openpilot.tools.lib.logreader import LogReader +from openpilot.selfdrive.test.process_replay.process_replay import replay_process_with_name + +TEST_ROUTE, TEST_SEG_NUM = "ff2bd20623fcaeaa|2023-09-05--10-14-54", 4 +GPS_MESSAGES = ['gpsLocationExternal', 'gpsLocation'] +SELECT_COMPARE_FIELDS = { + 'yaw_rate': ['angularVelocityCalibrated', 'value', 2], + 'roll': ['orientationNED', 'value', 0], + 'gps_flag': ['gpsOK'], + 'inputs_flag': ['inputsOK'], + 'sensors_flag': ['sensorsOK'], +} +JUNK_IDX = 100 + + +class Scenario(Enum): + BASE = 'base' + GPS_OFF = 'gps_off' + GPS_OFF_MIDWAY = 'gps_off_midway' + GPS_ON_MIDWAY = 'gps_on_midway' + GPS_TUNNEL = 'gps_tunnel' + GYRO_OFF = 'gyro_off' + GYRO_SPIKE_MIDWAY = 'gyro_spike_midway' + ACCEL_OFF = 'accel_off' + ACCEL_SPIKE_MIDWAY = 'accel_spike_midway' + + +def get_select_fields_data(logs): + def get_nested_keys(msg, keys): + val = None + for key in keys: + val = getattr(msg if val is None else val, key) if isinstance(key, str) else val[key] + return val + llk = [x.liveLocationKalman for x in logs if x.which() == 'liveLocationKalman'] + data = defaultdict(list) + for msg in llk: + for key, fields in SELECT_COMPARE_FIELDS.items(): + data[key].append(get_nested_keys(msg, fields)) + for key in data: + data[key] = np.array(data[key][JUNK_IDX:], dtype=float) + return data + + +def run_scenarios(scenario, logs): + if scenario == Scenario.BASE: + pass + + elif scenario == Scenario.GPS_OFF: + logs = sorted([x for x in logs if x.which() not in GPS_MESSAGES], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.GPS_OFF_MIDWAY: + non_gps = [x for x in logs if x.which() not in GPS_MESSAGES] + gps = [x for x in logs if x.which() in GPS_MESSAGES] + logs = sorted(non_gps + gps[: len(gps) // 2], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.GPS_ON_MIDWAY: + non_gps = [x for x in logs if x.which() not in GPS_MESSAGES] + gps = [x for x in logs if x.which() in GPS_MESSAGES] + logs = sorted(non_gps + gps[len(gps) // 2:], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.GPS_TUNNEL: + non_gps = [x for x in logs if x.which() not in GPS_MESSAGES] + gps = [x for x in logs if x.which() in GPS_MESSAGES] + logs = sorted(non_gps + gps[:len(gps) // 4] + gps[-len(gps) // 4:], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.GYRO_OFF: + logs = sorted([x for x in logs if x.which() != 'gyroscope'], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.GYRO_SPIKE_MIDWAY: + non_gyro = [x for x in logs if x.which() not in 'gyroscope'] + gyro = [x for x in logs if x.which() in 'gyroscope'] + temp = gyro[len(gyro) // 2].as_builder() + temp.gyroscope.gyroUncalibrated.v[0] += 3.0 + gyro[len(gyro) // 2] = temp.as_reader() + logs = sorted(non_gyro + gyro, key=lambda x: x.logMonoTime) + + elif scenario == Scenario.ACCEL_OFF: + logs = sorted([x for x in logs if x.which() != 'accelerometer'], key=lambda x: x.logMonoTime) + + elif scenario == Scenario.ACCEL_SPIKE_MIDWAY: + non_accel = [x for x in logs if x.which() not in 'accelerometer'] + accel = [x for x in logs if x.which() in 'accelerometer'] + temp = accel[len(accel) // 2].as_builder() + temp.accelerometer.acceleration.v[0] += 10.0 + accel[len(accel) // 2] = temp.as_reader() + logs = sorted(non_accel + accel, key=lambda x: x.logMonoTime) + + replayed_logs = replay_process_with_name(name='locationd', lr=logs) + return get_select_fields_data(logs), get_select_fields_data(replayed_logs) + + +class TestLocationdScenarios(unittest.TestCase): + """ + Test locationd with different scenarios. In all these scenarios, we expect the following: + - locationd kalman filter should never go unstable (we care mostly about yaw_rate, roll, gpsOK, inputsOK, sensorsOK) + - faulty values should be ignored, with appropriate flags set + """ + + @classmethod + def setUpClass(cls): + cls.logs = list(LogReader(get_url(TEST_ROUTE, TEST_SEG_NUM))) + + def test_base(self): + """ + Test: unchanged log + Expected Result: + - yaw_rate: unchanged + - roll: unchanged + """ + orig_data, replayed_data = run_scenarios(Scenario.BASE, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + + def test_gps_off(self): + """ + Test: no GPS message for the entire segment + Expected Result: + - yaw_rate: unchanged + - roll: + - gpsOK: False + """ + orig_data, replayed_data = run_scenarios(Scenario.GPS_OFF, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + self.assertTrue(np.all(replayed_data['gps_flag'] == 0.0)) + + def test_gps_off_midway(self): + """ + Test: no GPS message for the second half of the segment + Expected Result: + - yaw_rate: unchanged + - roll: + - gpsOK: True for the first half, False for the second half + """ + orig_data, replayed_data = run_scenarios(Scenario.GPS_OFF_MIDWAY, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + self.assertTrue(np.diff(replayed_data['gps_flag'])[512] == -1.0) + + def test_gps_on_midway(self): + """ + Test: no GPS message for the first half of the segment + Expected Result: + - yaw_rate: unchanged + - roll: + - gpsOK: False for the first half, True for the second half + """ + orig_data, replayed_data = run_scenarios(Scenario.GPS_ON_MIDWAY, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(1.5))) + self.assertTrue(np.diff(replayed_data['gps_flag'])[505] == 1.0) + + def test_gps_tunnel(self): + """ + Test: no GPS message for the middle section of the segment + Expected Result: + - yaw_rate: unchanged + - roll: + - gpsOK: False for the middle section, True for the rest + """ + orig_data, replayed_data = run_scenarios(Scenario.GPS_TUNNEL, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + self.assertTrue(np.diff(replayed_data['gps_flag'])[213] == -1.0) + self.assertTrue(np.diff(replayed_data['gps_flag'])[805] == 1.0) + + def test_gyro_off(self): + """ + Test: no gyroscope message for the entire segment + Expected Result: + - yaw_rate: 0 + - roll: 0 + - sensorsOK: False + """ + _, replayed_data = run_scenarios(Scenario.GYRO_OFF, self.logs) + self.assertTrue(np.allclose(replayed_data['yaw_rate'], 0.0)) + self.assertTrue(np.allclose(replayed_data['roll'], 0.0)) + self.assertTrue(np.all(replayed_data['sensors_flag'] == 0.0)) + + def test_gyro_spikes(self): + """ + Test: a gyroscope spike in the middle of the segment + Expected Result: + - yaw_rate: unchanged + - roll: unchanged + - inputsOK: False for some time after the spike, True for the rest + """ + orig_data, replayed_data = run_scenarios(Scenario.GYRO_SPIKE_MIDWAY, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + self.assertTrue(np.diff(replayed_data['inputs_flag'])[500] == -1.0) + self.assertTrue(np.diff(replayed_data['inputs_flag'])[694] == 1.0) + + def test_accel_off(self): + """ + Test: no accelerometer message for the entire segment + Expected Result: + - yaw_rate: 0 + - roll: 0 + - sensorsOK: False + """ + _, replayed_data = run_scenarios(Scenario.ACCEL_OFF, self.logs) + self.assertTrue(np.allclose(replayed_data['yaw_rate'], 0.0)) + self.assertTrue(np.allclose(replayed_data['roll'], 0.0)) + self.assertTrue(np.all(replayed_data['sensors_flag'] == 0.0)) + + def test_accel_spikes(self): + """ + ToDo: + Test: an accelerometer spike in the middle of the segment + Expected Result: Right now, the kalman filter is not robust to small spikes like it is to gyroscope spikes. + """ + orig_data, replayed_data = run_scenarios(Scenario.ACCEL_SPIKE_MIDWAY, self.logs) + self.assertTrue(np.allclose(orig_data['yaw_rate'], replayed_data['yaw_rate'], atol=np.radians(0.2))) + self.assertTrue(np.allclose(orig_data['roll'], replayed_data['roll'], atol=np.radians(0.5))) + + +if __name__ == "__main__": + unittest.main() diff --git a/selfdrive/manager/process_config.py b/selfdrive/manager/process_config.py index 61bdc8fbf4..3e91aff565 100644 --- a/selfdrive/manager/process_config.py +++ b/selfdrive/manager/process_config.py @@ -52,13 +52,13 @@ procs = [ PythonProcess("micd", "system.micd", iscar), PythonProcess("timezoned", "system.timezoned", always_run, enabled=not PC), - NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], driverview, enabled=(not PC or WEBCAM)), + PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)), NativeProcess("encoderd", "system/loggerd", ["./encoderd"], only_onroad), NativeProcess("stream_encoderd", "system/loggerd", ["./encoderd", "--stream"], notcar), NativeProcess("loggerd", "system/loggerd", ["./loggerd"], logging), NativeProcess("modeld", "selfdrive/modeld", ["./modeld"], only_onroad), NativeProcess("mapsd", "selfdrive/navd", ["./mapsd"], only_onroad), - NativeProcess("navmodeld", "selfdrive/modeld", ["./navmodeld"], only_onroad), + PythonProcess("navmodeld", "selfdrive.modeld.navmodeld", only_onroad), NativeProcess("sensord", "system/sensord", ["./sensord"], only_onroad, enabled=not PC), NativeProcess("ui", "selfdrive/ui", ["./ui"], always_run, watchdog_max_dt=(5 if not PC else None)), NativeProcess("soundd", "selfdrive/ui/soundd", ["./soundd"], only_onroad), diff --git a/selfdrive/modeld/SConscript b/selfdrive/modeld/SConscript index 03836b6593..63f3b2b97a 100644 --- a/selfdrive/modeld/SConscript +++ b/selfdrive/modeld/SConscript @@ -32,14 +32,8 @@ if arch == "Darwin": else: libs += ['OpenCL'] -# Use onnx on PC -if arch != "larch64" and not GetOption('snpe'): - common_src += ['runners/onnxmodel.cc'] - lenv['CFLAGS'].append("-DUSE_ONNX_MODEL") - lenv['CXXFLAGS'].append("-DUSE_ONNX_MODEL") - # Set path definitions -for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transforms/loadyuv.cl', 'ONNXRUNNER': 'runners/onnx_runner.py'}.items(): +for pathdef, fn in {'TRANSFORM': 'transforms/transform.cl', 'LOADYUV': 'transforms/loadyuv.cl'}.items(): for xenv in (lenv, lenvCython): xenv['CXXFLAGS'].append(f'-D{pathdef}_PATH=\\"{File(fn).abspath}\\"') @@ -49,31 +43,15 @@ snpe_rpath_pc = f"{Dir('#').abspath}/third_party/snpe/x86_64-linux-clang" snpe_rpath = lenvCython['RPATH'] + [snpe_rpath_qcom if arch == "larch64" else snpe_rpath_pc] cython_libs = envCython["LIBS"] + libs -onnxmodel_lib = lenv.Library('onnxmodel', ['runners/onnxmodel.cc']) snpemodel_lib = lenv.Library('snpemodel', ['runners/snpemodel.cc']) commonmodel_lib = lenv.Library('commonmodel', common_src) driving_lib = lenv.Library('driving', ['models/driving.cc']) lenvCython.Program('runners/runmodel_pyx.so', 'runners/runmodel_pyx.pyx', LIBS=cython_libs, FRAMEWORKS=frameworks) -lenvCython.Program('runners/onnxmodel_pyx.so', 'runners/onnxmodel_pyx.pyx', LIBS=[onnxmodel_lib, *cython_libs], FRAMEWORKS=frameworks) lenvCython.Program('runners/snpemodel_pyx.so', 'runners/snpemodel_pyx.pyx', LIBS=[snpemodel_lib, snpe_lib, *cython_libs], FRAMEWORKS=frameworks, RPATH=snpe_rpath) lenvCython.Program('models/commonmodel_pyx.so', 'models/commonmodel_pyx.pyx', LIBS=[commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) lenvCython.Program('models/driving_pyx.so', 'models/driving_pyx.pyx', LIBS=[driving_lib, commonmodel_lib, *cython_libs], FRAMEWORKS=frameworks) -# Compile binaries -lenv['FRAMEWORKS'] = frameworks -common_model = lenv.Object(common_src) - -lenv.Program('_dmonitoringmodeld', [ - "dmonitoringmodeld.cc", - "models/dmonitoring.cc", - ]+common_model, LIBS=libs + snpe_lib) - -lenv.Program('_navmodeld', [ - "navmodeld.cc", - "models/nav.cc", - ]+common_model, LIBS=libs + snpe_lib) - # Build thneed model if arch == "larch64" or GetOption('pc_thneed'): fn = File("models/supercombo").abspath diff --git a/selfdrive/modeld/dmonitoringmodeld b/selfdrive/modeld/dmonitoringmodeld deleted file mode 100755 index fc007470b2..0000000000 --- a/selfdrive/modeld/dmonitoringmodeld +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -if [ -f /TICI ]; then - export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/third_party/snpe/larch64:$LD_LIBRARY_PATH" - export ADSP_LIBRARY_PATH="/data/pythonpath/third_party/snpe/dsp/" -else - export LD_LIBRARY_PATH="$DIR/../../third_party/snpe/x86_64-linux-clang:$DIR/../../openpilot/third_party/snpe/x86_64:$LD_LIBRARY_PATH" -fi -exec ./_dmonitoringmodeld diff --git a/selfdrive/modeld/dmonitoringmodeld.cc b/selfdrive/modeld/dmonitoringmodeld.cc deleted file mode 100644 index 2890b44fe0..0000000000 --- a/selfdrive/modeld/dmonitoringmodeld.cc +++ /dev/null @@ -1,69 +0,0 @@ -#include -#include - -#include -#include - -#include "cereal/visionipc/visionipc_client.h" -#include "common/params.h" -#include "common/swaglog.h" -#include "common/util.h" -#include "selfdrive/modeld/models/dmonitoring.h" - -ExitHandler do_exit; - -void run_model(DMonitoringModelState &model, VisionIpcClient &vipc_client) { - PubMaster pm({"driverStateV2"}); - SubMaster sm({"liveCalibration"}); - float calib[CALIB_LEN] = {0}; - // double last = 0; - - while (!do_exit) { - VisionIpcBufExtra extra = {}; - VisionBuf *buf = vipc_client.recv(&extra); - if (buf == nullptr) continue; - - sm.update(0); - if (sm.updated("liveCalibration")) { - auto calib_msg = sm["liveCalibration"].getLiveCalibration().getRpyCalib(); - for (int i = 0; i < CALIB_LEN; i++) { - calib[i] = calib_msg[i]; - } - } - - double t1 = millis_since_boot(); - DMonitoringModelResult model_res = dmonitoring_eval_frame(&model, buf->addr, buf->width, buf->height, buf->stride, buf->uv_offset, calib); - double t2 = millis_since_boot(); - - // send dm packet - dmonitoring_publish(pm, extra.frame_id, model_res, (t2 - t1) / 1000.0, model.output); - - // printf("dmonitoring process: %.2fms, from last %.2fms\n", t2 - t1, t1 - last); - // last = t1; - } -} - -int main(int argc, char **argv) { - setpriority(PRIO_PROCESS, 0, -15); - - // init the models - DMonitoringModelState model; - dmonitoring_init(&model); - - Params().putBool("DmModelInitialized", true); - - LOGW("connecting to driver stream"); - VisionIpcClient vipc_client = VisionIpcClient("camerad", VISION_STREAM_DRIVER, true); - while (!do_exit && !vipc_client.connect(false)) { - util::sleep_for(100); - } - - // run the models - if (vipc_client.connected) { - LOGW("connected with buffer size: %zu", vipc_client.buffers[0].len); - run_model(model, vipc_client); - } - - dmonitoring_free(&model); - return 0; -} diff --git a/selfdrive/modeld/dmonitoringmodeld.py b/selfdrive/modeld/dmonitoringmodeld.py new file mode 100755 index 0000000000..53c0af0ff3 --- /dev/null +++ b/selfdrive/modeld/dmonitoringmodeld.py @@ -0,0 +1,157 @@ +#!/usr/bin/env python3 +import os +import gc +import math +import time +import ctypes +import numpy as np +from pathlib import Path +from typing import Tuple, Dict + +from cereal import messaging +from cereal.messaging import PubMaster, SubMaster +from cereal.visionipc import VisionIpcClient, VisionStreamType, VisionBuf +from openpilot.system.swaglog import cloudlog +from openpilot.common.params import Params +from openpilot.common.realtime import set_realtime_priority +from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime +from openpilot.selfdrive.modeld.models.commonmodel_pyx import sigmoid + +CALIB_LEN = 3 +REG_SCALE = 0.25 +MODEL_WIDTH = 1440 +MODEL_HEIGHT = 960 +OUTPUT_SIZE = 84 +SEND_RAW_PRED = os.getenv('SEND_RAW_PRED') +MODEL_PATHS = { + ModelRunner.SNPE: Path(__file__).parent / 'models/dmonitoring_model_q.dlc', + ModelRunner.ONNX: Path(__file__).parent / 'models/dmonitoring_model.onnx'} + +class DriverStateResult(ctypes.Structure): + _fields_ = [ + ("face_orientation", ctypes.c_float*3), + ("face_position", ctypes.c_float*3), + ("face_orientation_std", ctypes.c_float*3), + ("face_position_std", ctypes.c_float*3), + ("face_prob", ctypes.c_float), + ("_unused_a", ctypes.c_float*8), + ("left_eye_prob", ctypes.c_float), + ("_unused_b", ctypes.c_float*8), + ("right_eye_prob", ctypes.c_float), + ("left_blink_prob", ctypes.c_float), + ("right_blink_prob", ctypes.c_float), + ("sunglasses_prob", ctypes.c_float), + ("occluded_prob", ctypes.c_float), + ("ready_prob", ctypes.c_float*4), + ("not_ready_prob", ctypes.c_float*2)] + +class DMonitoringModelResult(ctypes.Structure): + _fields_ = [ + ("driver_state_lhd", DriverStateResult), + ("driver_state_rhd", DriverStateResult), + ("poor_vision_prob", ctypes.c_float), + ("wheel_on_right_prob", ctypes.c_float)] + +class ModelState: + inputs: Dict[str, np.ndarray] + output: np.ndarray + model: ModelRunner + + def __init__(self): + assert ctypes.sizeof(DMonitoringModelResult) == OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) + self.output = np.zeros(OUTPUT_SIZE, dtype=np.float32) + self.inputs = { + 'input_img': np.zeros(MODEL_HEIGHT * MODEL_WIDTH, dtype=np.uint8), + 'calib': np.zeros(CALIB_LEN, dtype=np.float32)} + + self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None) + self.model.addInput("input_img", None) + self.model.addInput("calib", self.inputs['calib']) + + def run(self, buf:VisionBuf, calib:np.ndarray) -> Tuple[np.ndarray, float]: + self.inputs['calib'][:] = calib + + v_offset = buf.height - MODEL_HEIGHT + h_offset = (buf.width - MODEL_WIDTH) // 2 + buf_data = buf.data.reshape(-1, buf.stride) + input_data = self.inputs['input_img'].reshape(MODEL_HEIGHT, MODEL_WIDTH) + input_data[:] = buf_data[v_offset:v_offset+MODEL_HEIGHT, h_offset:h_offset+MODEL_WIDTH] + + t1 = time.perf_counter() + self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32)) + self.model.execute() + t2 = time.perf_counter() + return self.output, t2 - t1 + + +def fill_driver_state(msg, ds_result: DriverStateResult): + msg.faceOrientation = [x * REG_SCALE for x in ds_result.face_orientation] + msg.faceOrientationStd = [math.exp(x) for x in ds_result.face_orientation_std] + msg.facePosition = [x * REG_SCALE for x in ds_result.face_position[:2]] + msg.facePositionStd = [math.exp(x) for x in ds_result.face_position_std[:2]] + msg.faceProb = sigmoid(ds_result.face_prob) + msg.leftEyeProb = sigmoid(ds_result.left_eye_prob) + msg.rightEyeProb = sigmoid(ds_result.right_eye_prob) + msg.leftBlinkProb = sigmoid(ds_result.left_blink_prob) + msg.rightBlinkProb = sigmoid(ds_result.right_blink_prob) + msg.sunglassesProb = sigmoid(ds_result.sunglasses_prob) + msg.occludedProb = sigmoid(ds_result.occluded_prob) + msg.readyProb = [sigmoid(x) for x in ds_result.ready_prob] + msg.notReadyProb = [sigmoid(x) for x in ds_result.not_ready_prob] + +def get_driverstate_packet(model_output: np.ndarray, frame_id: int, location_ts: int, execution_time: float, dsp_execution_time: float): + model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(DMonitoringModelResult)).contents + msg = messaging.new_message('driverStateV2') + ds = msg.driverStateV2 + ds.frameId = frame_id + ds.modelExecutionTime = execution_time + ds.dspExecutionTime = dsp_execution_time + ds.poorVisionProb = sigmoid(model_result.poor_vision_prob) + ds.wheelOnRightProb = sigmoid(model_result.wheel_on_right_prob) + ds.rawPredictions = model_output.tobytes() if SEND_RAW_PRED else b'' + fill_driver_state(ds.leftDriverData, model_result.driver_state_lhd) + fill_driver_state(ds.rightDriverData, model_result.driver_state_rhd) + return msg + + +def main(): + gc.disable() + set_realtime_priority(1) + + model = ModelState() + cloudlog.warning("models loaded, dmonitoringmodeld starting") + Params().put_bool("DmModelInitialized", True) + + cloudlog.warning("connecting to driver stream") + vipc_client = VisionIpcClient("camerad", VisionStreamType.VISION_STREAM_DRIVER, True) + while not vipc_client.connect(False): + time.sleep(0.1) + assert vipc_client.is_connected() + cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}") + + sm = SubMaster(["liveCalibration"]) + pm = PubMaster(["driverStateV2"]) + + calib = np.zeros(CALIB_LEN, dtype=np.float32) + # last = 0 + + while True: + buf = vipc_client.recv() + if buf is None: + continue + + sm.update(0) + if sm.updated["liveCalibration"]: + calib[:] = np.array(sm["liveCalibration"].rpyCalib) + + t1 = time.perf_counter() + model_output, dsp_execution_time = model.run(buf, calib) + t2 = time.perf_counter() + + pm.send("driverStateV2", get_driverstate_packet(model_output, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, dsp_execution_time)) + # print("dmonitoring process: %.2fms, from last %.2fms\n" % (t2 - t1, t1 - last)) + # last = t1 + + +if __name__ == "__main__": + main() diff --git a/selfdrive/modeld/modeld.py b/selfdrive/modeld/modeld.py index ece706cfd9..3c416ce939 100755 --- a/selfdrive/modeld/modeld.py +++ b/selfdrive/modeld/modeld.py @@ -46,11 +46,11 @@ class ModelState: self.prev_desire = np.zeros(DESIRE_LEN, dtype=np.float32) self.output = np.zeros(NET_OUTPUT_SIZE, dtype=np.float32) self.inputs = { - 'desire_pulse': np.zeros(DESIRE_LEN * (HISTORY_BUFFER_LEN+1), dtype=np.float32), + 'desire': np.zeros(DESIRE_LEN * (HISTORY_BUFFER_LEN+1), dtype=np.float32), 'traffic_convention': np.zeros(TRAFFIC_CONVENTION_LEN, dtype=np.float32), 'nav_features': np.zeros(NAV_FEATURE_LEN, dtype=np.float32), 'nav_instructions': np.zeros(NAV_INSTRUCTION_LEN, dtype=np.float32), - 'feature_buffer': np.zeros(HISTORY_BUFFER_LEN * FEATURE_LEN, dtype=np.float32), + 'features_buffer': np.zeros(HISTORY_BUFFER_LEN * FEATURE_LEN, dtype=np.float32), } self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.GPU, False, context) @@ -62,10 +62,10 @@ class ModelState: def run(self, buf: VisionBuf, wbuf: VisionBuf, transform: np.ndarray, transform_wide: np.ndarray, inputs: Dict[str, np.ndarray], prepare_only: bool) -> Optional[np.ndarray]: # Model decides when action is completed, so desire input is just a pulse triggered on rising edge - inputs['desire_pulse'][0] = 0 - self.inputs['desire_pulse'][:-DESIRE_LEN] = self.inputs['desire_pulse'][DESIRE_LEN:] - self.inputs['desire_pulse'][-DESIRE_LEN:] = np.where(inputs['desire_pulse'] - self.prev_desire > .99, inputs['desire_pulse'], 0) - self.prev_desire[:] = inputs['desire_pulse'] + inputs['desire'][0] = 0 + self.inputs['desire'][:-DESIRE_LEN] = self.inputs['desire'][DESIRE_LEN:] + self.inputs['desire'][-DESIRE_LEN:] = np.where(inputs['desire'] - self.prev_desire > .99, inputs['desire'], 0) + self.prev_desire[:] = inputs['desire'] self.inputs['traffic_convention'][:] = inputs['traffic_convention'] self.inputs['nav_features'][:] = inputs['nav_features'] @@ -81,8 +81,8 @@ class ModelState: return None self.model.execute() - self.inputs['feature_buffer'][:-FEATURE_LEN] = self.inputs['feature_buffer'][FEATURE_LEN:] - self.inputs['feature_buffer'][-FEATURE_LEN:] = self.output[OUTPUT_SIZE:OUTPUT_SIZE+FEATURE_LEN] + self.inputs['features_buffer'][:-FEATURE_LEN] = self.inputs['features_buffer'][FEATURE_LEN:] + self.inputs['features_buffer'][-FEATURE_LEN:] = self.output[OUTPUT_SIZE:OUTPUT_SIZE+FEATURE_LEN] return self.output @@ -232,7 +232,7 @@ def main(): cloudlog.error(f"skipping model eval. Dropped {vipc_dropped_frames} frames") inputs:Dict[str, np.ndarray] = { - 'desire_pulse': vec_desire, + 'desire': vec_desire, 'traffic_convention': traffic_convention, 'driving_style': driving_style, 'nav_features': nav_features, diff --git a/selfdrive/modeld/models/dmonitoring.cc b/selfdrive/modeld/models/dmonitoring.cc deleted file mode 100644 index eb5239bef0..0000000000 --- a/selfdrive/modeld/models/dmonitoring.cc +++ /dev/null @@ -1,133 +0,0 @@ -#include - -#include "common/mat.h" -#include "common/modeldata.h" -#include "common/params.h" -#include "common/timing.h" -#include "system/hardware/hw.h" - -#include "selfdrive/modeld/models/dmonitoring.h" - -constexpr int MODEL_WIDTH = 1440; -constexpr int MODEL_HEIGHT = 960; - -template -static inline T *get_buffer(std::vector &buf, const size_t size) { - if (buf.size() < size) buf.resize(size); - return buf.data(); -} - -void dmonitoring_init(DMonitoringModelState* s) { - -#ifdef USE_ONNX_MODEL - s->m = new ONNXModel("models/dmonitoring_model.onnx", &s->output[0], OUTPUT_SIZE, USE_DSP_RUNTIME, true); -#else - s->m = new SNPEModel("models/dmonitoring_model_q.dlc", &s->output[0], OUTPUT_SIZE, USE_DSP_RUNTIME, true); -#endif - - s->m->addInput("input_imgs", NULL, 0); - s->m->addInput("calib", s->calib, CALIB_LEN); -} - -void parse_driver_data(DriverStateResult &ds_res, const DMonitoringModelState* s, int out_idx_offset) { - for (int i = 0; i < 3; ++i) { - ds_res.face_orientation[i] = s->output[out_idx_offset+i] * REG_SCALE; - ds_res.face_orientation_std[i] = exp(s->output[out_idx_offset+6+i]); - } - for (int i = 0; i < 2; ++i) { - ds_res.face_position[i] = s->output[out_idx_offset+3+i] * REG_SCALE; - ds_res.face_position_std[i] = exp(s->output[out_idx_offset+9+i]); - } - for (int i = 0; i < 4; ++i) { - ds_res.ready_prob[i] = sigmoid(s->output[out_idx_offset+35+i]); - } - for (int i = 0; i < 2; ++i) { - ds_res.not_ready_prob[i] = sigmoid(s->output[out_idx_offset+39+i]); - } - ds_res.face_prob = sigmoid(s->output[out_idx_offset+12]); - ds_res.left_eye_prob = sigmoid(s->output[out_idx_offset+21]); - ds_res.right_eye_prob = sigmoid(s->output[out_idx_offset+30]); - ds_res.left_blink_prob = sigmoid(s->output[out_idx_offset+31]); - ds_res.right_blink_prob = sigmoid(s->output[out_idx_offset+32]); - ds_res.sunglasses_prob = sigmoid(s->output[out_idx_offset+33]); - ds_res.occluded_prob = sigmoid(s->output[out_idx_offset+34]); -} - -void fill_driver_data(cereal::DriverStateV2::DriverData::Builder ddata, const DriverStateResult &ds_res) { - ddata.setFaceOrientation(ds_res.face_orientation); - ddata.setFaceOrientationStd(ds_res.face_orientation_std); - ddata.setFacePosition(ds_res.face_position); - ddata.setFacePositionStd(ds_res.face_position_std); - ddata.setFaceProb(ds_res.face_prob); - ddata.setLeftEyeProb(ds_res.left_eye_prob); - ddata.setRightEyeProb(ds_res.right_eye_prob); - ddata.setLeftBlinkProb(ds_res.left_blink_prob); - ddata.setRightBlinkProb(ds_res.right_blink_prob); - ddata.setSunglassesProb(ds_res.sunglasses_prob); - ddata.setOccludedProb(ds_res.occluded_prob); - ddata.setReadyProb(ds_res.ready_prob); - ddata.setNotReadyProb(ds_res.not_ready_prob); -} - -DMonitoringModelResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_buf, int width, int height, int stride, int uv_offset, float *calib) { - int v_off = height - MODEL_HEIGHT; - int h_off = (width - MODEL_WIDTH) / 2; - int yuv_buf_len = MODEL_WIDTH * MODEL_HEIGHT; - - uint8_t *raw_buf = (uint8_t *) stream_buf; - // vertical crop free - uint8_t *raw_y_start = raw_buf + stride * v_off; - - uint8_t *net_input_buf = get_buffer(s->net_input_buf, yuv_buf_len); - - // here makes a uint8 copy - for (int r = 0; r < MODEL_HEIGHT; ++r) { - memcpy(net_input_buf + r * MODEL_WIDTH, raw_y_start + r * stride + h_off, MODEL_WIDTH); - } - - // printf("preprocess completed. %d \n", yuv_buf_len); - // FILE *dump_yuv_file = fopen("/tmp/rawdump.yuv", "wb"); - // fwrite(net_input_buf, yuv_buf_len, sizeof(uint8_t), dump_yuv_file); - // fclose(dump_yuv_file); - - double t1 = millis_since_boot(); - s->m->setInputBuffer("input_imgs", (float*)net_input_buf, yuv_buf_len / sizeof(float)); - for (int i = 0; i < CALIB_LEN; i++) { - s->calib[i] = calib[i]; - } - s->m->execute(); - double t2 = millis_since_boot(); - - DMonitoringModelResult model_res = {0}; - parse_driver_data(model_res.driver_state_lhd, s, 0); - parse_driver_data(model_res.driver_state_rhd, s, 41); - model_res.poor_vision_prob = sigmoid(s->output[82]); - model_res.wheel_on_right_prob = sigmoid(s->output[83]); - model_res.dsp_execution_time = (t2 - t1) / 1000.; - - return model_res; -} - -void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringModelResult &model_res, float execution_time, kj::ArrayPtr raw_pred) { - // make msg - MessageBuilder msg; - auto framed = msg.initEvent().initDriverStateV2(); - framed.setFrameId(frame_id); - framed.setModelExecutionTime(execution_time); - framed.setDspExecutionTime(model_res.dsp_execution_time); - - framed.setPoorVisionProb(model_res.poor_vision_prob); - framed.setWheelOnRightProb(model_res.wheel_on_right_prob); - fill_driver_data(framed.initLeftDriverData(), model_res.driver_state_lhd); - fill_driver_data(framed.initRightDriverData(), model_res.driver_state_rhd); - - if (send_raw_pred) { - framed.setRawPredictions(raw_pred.asBytes()); - } - - pm.send("driverStateV2", msg); -} - -void dmonitoring_free(DMonitoringModelState* s) { - delete s->m; -} diff --git a/selfdrive/modeld/models/dmonitoring.h b/selfdrive/modeld/models/dmonitoring.h deleted file mode 100644 index ae2bf05394..0000000000 --- a/selfdrive/modeld/models/dmonitoring.h +++ /dev/null @@ -1,50 +0,0 @@ -#pragma once - -#include - -#include "cereal/messaging/messaging.h" -#include "common/util.h" -#include "selfdrive/modeld/models/commonmodel.h" -#include "selfdrive/modeld/runners/run.h" - -#define CALIB_LEN 3 - -#define OUTPUT_SIZE 84 -#define REG_SCALE 0.25f - -typedef struct DriverStateResult { - float face_orientation[3]; - float face_orientation_std[3]; - float face_position[2]; - float face_position_std[2]; - float face_prob; - float left_eye_prob; - float right_eye_prob; - float left_blink_prob; - float right_blink_prob; - float sunglasses_prob; - float occluded_prob; - float ready_prob[4]; - float not_ready_prob[2]; -} DriverStateResult; - -typedef struct DMonitoringModelResult { - DriverStateResult driver_state_lhd; - DriverStateResult driver_state_rhd; - float poor_vision_prob; - float wheel_on_right_prob; - float dsp_execution_time; -} DMonitoringModelResult; - -typedef struct DMonitoringModelState { - RunModel *m; - float output[OUTPUT_SIZE]; - std::vector net_input_buf; - float calib[CALIB_LEN]; -} DMonitoringModelState; - -void dmonitoring_init(DMonitoringModelState* s); -DMonitoringModelResult dmonitoring_eval_frame(DMonitoringModelState* s, void* stream_buf, int width, int height, int stride, int uv_offset, float *calib); -void dmonitoring_publish(PubMaster &pm, uint32_t frame_id, const DMonitoringModelResult &model_res, float execution_time, kj::ArrayPtr raw_pred); -void dmonitoring_free(DMonitoringModelState* s); - diff --git a/selfdrive/modeld/models/driving.h b/selfdrive/modeld/models/driving.h index f13cd155f1..de0250d212 100644 --- a/selfdrive/modeld/models/driving.h +++ b/selfdrive/modeld/models/driving.h @@ -6,13 +6,16 @@ #include "cereal/messaging/messaging.h" #include "common/modeldata.h" #include "common/util.h" -#include "selfdrive/modeld/models/nav.h" +#include "selfdrive/modeld/models/commonmodel.h" +#include "selfdrive/modeld/runners/run.h" constexpr int FEATURE_LEN = 128; constexpr int HISTORY_BUFFER_LEN = 99; constexpr int DESIRE_LEN = 8; constexpr int DESIRE_PRED_LEN = 4; constexpr int TRAFFIC_CONVENTION_LEN = 2; +constexpr int NAV_FEATURE_LEN = 256; +constexpr int NAV_INSTRUCTION_LEN = 150; constexpr int DRIVING_STYLE_LEN = 12; constexpr int MODEL_FREQ = 20; diff --git a/selfdrive/modeld/models/nav.cc b/selfdrive/modeld/models/nav.cc deleted file mode 100644 index 8208b0bfef..0000000000 --- a/selfdrive/modeld/models/nav.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "selfdrive/modeld/models/nav.h" - -#include -#include - -#include "common/mat.h" -#include "common/modeldata.h" -#include "common/timing.h" - - -void navmodel_init(NavModelState* s) { - #ifdef USE_ONNX_MODEL - s->m = new ONNXModel("models/navmodel.onnx", &s->output[0], NAV_NET_OUTPUT_SIZE, USE_DSP_RUNTIME, true); - #else - s->m = new SNPEModel("models/navmodel_q.dlc", &s->output[0], NAV_NET_OUTPUT_SIZE, USE_DSP_RUNTIME, true); - #endif - - s->m->addInput("map", NULL, 0); -} - -NavModelResult* navmodel_eval_frame(NavModelState* s, VisionBuf* buf) { - memcpy(s->net_input_buf, buf->addr, NAV_INPUT_SIZE); - - double t1 = millis_since_boot(); - s->m->setInputBuffer("map", (float*)s->net_input_buf, NAV_INPUT_SIZE/sizeof(float)); - s->m->execute(); - double t2 = millis_since_boot(); - - NavModelResult *model_res = (NavModelResult*)&s->output; - model_res->dsp_execution_time = (t2 - t1) / 1000.; - return model_res; -} - -void fill_plan(cereal::NavModelData::Builder &framed, const NavModelOutputPlan &plan) { - std::array pos_x, pos_y; - std::array pos_x_std, pos_y_std; - - for (int i=0; im; -} diff --git a/selfdrive/modeld/models/nav.h b/selfdrive/modeld/models/nav.h deleted file mode 100644 index 800abec252..0000000000 --- a/selfdrive/modeld/models/nav.h +++ /dev/null @@ -1,57 +0,0 @@ -#pragma once - -#include "cereal/messaging/messaging.h" -#include "cereal/visionipc/visionipc_client.h" -#include "common/util.h" -#include "common/modeldata.h" -#include "selfdrive/modeld/models/commonmodel.h" -#include "selfdrive/modeld/runners/run.h" - -constexpr int NAV_INPUT_SIZE = 256*256; -constexpr int NAV_FEATURE_LEN = 256; -constexpr int NAV_INSTRUCTION_LEN = 150; -constexpr int NAV_DESIRE_LEN = 32; - -struct NavModelOutputXY { - float x; - float y; -}; -static_assert(sizeof(NavModelOutputXY) == sizeof(float)*2); - -struct NavModelOutputPlan { - std::array mean; - std::array std; -}; -static_assert(sizeof(NavModelOutputPlan) == sizeof(NavModelOutputXY)*TRAJECTORY_SIZE*2); - -struct NavModelOutputDesirePrediction { - std::array values; -}; -static_assert(sizeof(NavModelOutputDesirePrediction) == sizeof(float)*NAV_DESIRE_LEN); - -struct NavModelOutputFeatures { - std::array values; -}; -static_assert(sizeof(NavModelOutputFeatures) == sizeof(float)*NAV_FEATURE_LEN); - -struct NavModelResult { - const NavModelOutputPlan plan; - const NavModelOutputDesirePrediction desire_pred; - const NavModelOutputFeatures features; - float dsp_execution_time; -}; -static_assert(sizeof(NavModelResult) == sizeof(NavModelOutputPlan) + sizeof(NavModelOutputDesirePrediction) + sizeof(NavModelOutputFeatures) + sizeof(float)); - -constexpr int NAV_OUTPUT_SIZE = sizeof(NavModelResult) / sizeof(float); -constexpr int NAV_NET_OUTPUT_SIZE = NAV_OUTPUT_SIZE - 1; - -struct NavModelState { - RunModel *m; - uint8_t net_input_buf[NAV_INPUT_SIZE]; - float output[NAV_OUTPUT_SIZE]; -}; - -void navmodel_init(NavModelState* s); -NavModelResult* navmodel_eval_frame(NavModelState* s, VisionBuf* buf); -void navmodel_publish(PubMaster &pm, VisionIpcBufExtra &extra, const NavModelResult &model_res, float execution_time, bool route_valid); -void navmodel_free(NavModelState* s); diff --git a/selfdrive/modeld/navmodeld b/selfdrive/modeld/navmodeld deleted file mode 100755 index 58215c2e9d..0000000000 --- a/selfdrive/modeld/navmodeld +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR - -if [ -f /TICI ]; then - export LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu:/data/pythonpath/third_party/snpe/larch64:$LD_LIBRARY_PATH" - export ADSP_LIBRARY_PATH="/data/pythonpath/third_party/snpe/dsp/" -else - export LD_LIBRARY_PATH="$DIR/../../third_party/snpe/x86_64-linux-clang:$DIR/../../openpilot/third_party/snpe/x86_64:$LD_LIBRARY_PATH" -fi -exec ./_navmodeld diff --git a/selfdrive/modeld/navmodeld.cc b/selfdrive/modeld/navmodeld.cc deleted file mode 100644 index 4610f503d1..0000000000 --- a/selfdrive/modeld/navmodeld.cc +++ /dev/null @@ -1,70 +0,0 @@ -#include -#include - -#include -#include - -#include "cereal/visionipc/visionipc_client.h" -#include "common/params.h" -#include "common/swaglog.h" -#include "common/util.h" -#include "selfdrive/modeld/models/nav.h" - -ExitHandler do_exit; - -void run_model(NavModelState &model, VisionIpcClient &vipc_client) { - SubMaster sm({"navInstruction"}); - PubMaster pm({"navModel"}); - - //double last_ts = 0; - //uint32_t last_frame_id = 0; - VisionIpcBufExtra extra = {}; - - while (!do_exit) { - VisionBuf *buf = vipc_client.recv(&extra); - if (buf == nullptr) continue; - - sm.update(0); - - double t1 = millis_since_boot(); - NavModelResult *model_res = navmodel_eval_frame(&model, buf); - double t2 = millis_since_boot(); - - // send navmodel packet - navmodel_publish(pm, extra, *model_res, (t2 - t1) / 1000.0, sm["navInstruction"].getValid()); - - //printf("navmodel process: %.2fms, from last %.2fms\n", t2 - t1, t1 - last_ts); - //last_ts = t1; - //last_frame_id = extra.frame_id; - } -} - -int main(int argc, char **argv) { - setpriority(PRIO_PROCESS, 0, -15); - - // there exists a race condition when two processes try to create a - // SNPE model runner at the same time, wait for dmonitoringmodeld to finish - LOGW("waiting for dmonitoringmodeld to initialize"); - if (!Params().getBool("DmModelInitialized", true)) { - return 0; - } - - // init the models - NavModelState model; - navmodel_init(&model); - LOGW("models loaded, navmodeld starting"); - - VisionIpcClient vipc_client = VisionIpcClient("navd", VISION_STREAM_MAP, true); - while (!do_exit && !vipc_client.connect(false)) { - util::sleep_for(100); - } - - // run the models - if (vipc_client.connected) { - LOGW("connected with buffer size: %zu", vipc_client.buffers[0].len); - run_model(model, vipc_client); - } - - navmodel_free(&model); - return 0; -} diff --git a/selfdrive/modeld/navmodeld.py b/selfdrive/modeld/navmodeld.py new file mode 100755 index 0000000000..f5dc51ed20 --- /dev/null +++ b/selfdrive/modeld/navmodeld.py @@ -0,0 +1,118 @@ +#!/usr/bin/env python3 +import gc +import math +import time +import ctypes +import numpy as np +from pathlib import Path +from typing import Tuple, Dict + +from cereal import messaging +from cereal.messaging import PubMaster, SubMaster +from cereal.visionipc import VisionIpcClient, VisionStreamType +from openpilot.system.swaglog import cloudlog +from openpilot.common.params import Params +from openpilot.common.realtime import set_realtime_priority +from openpilot.selfdrive.modeld.constants import IDX_N +from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime + +NAV_INPUT_SIZE = 256*256 +NAV_FEATURE_LEN = 256 +NAV_DESIRE_LEN = 32 +NAV_OUTPUT_SIZE = 2*2*IDX_N + NAV_DESIRE_LEN + NAV_FEATURE_LEN +MODEL_PATHS = { + ModelRunner.SNPE: Path(__file__).parent / 'models/navmodel_q.dlc', + ModelRunner.ONNX: Path(__file__).parent / 'models/navmodel.onnx'} + +class NavModelOutputXY(ctypes.Structure): + _fields_ = [ + ("x", ctypes.c_float), + ("y", ctypes.c_float)] + +class NavModelOutputPlan(ctypes.Structure): + _fields_ = [ + ("mean", NavModelOutputXY*IDX_N), + ("std", NavModelOutputXY*IDX_N)] + +class NavModelResult(ctypes.Structure): + _fields_ = [ + ("plan", NavModelOutputPlan), + ("desire_pred", ctypes.c_float*NAV_DESIRE_LEN), + ("features", ctypes.c_float*NAV_FEATURE_LEN)] + +class ModelState: + inputs: Dict[str, np.ndarray] + output: np.ndarray + model: ModelRunner + + def __init__(self): + assert ctypes.sizeof(NavModelResult) == NAV_OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) + self.output = np.zeros(NAV_OUTPUT_SIZE, dtype=np.float32) + self.inputs = {'input_img': np.zeros(NAV_INPUT_SIZE, dtype=np.uint8)} + self.model = ModelRunner(MODEL_PATHS, self.output, Runtime.DSP, True, None) + self.model.addInput("input_img", None) + + def run(self, buf:np.ndarray) -> Tuple[np.ndarray, float]: + self.inputs['input_img'][:] = buf + + t1 = time.perf_counter() + self.model.setInputBuffer("input_img", self.inputs['input_img'].view(np.float32)) + self.model.execute() + t2 = time.perf_counter() + return self.output, t2 - t1 + +def get_navmodel_packet(model_output: np.ndarray, valid: bool, frame_id: int, location_ts: int, execution_time: float, dsp_execution_time: float): + model_result = ctypes.cast(model_output.ctypes.data, ctypes.POINTER(NavModelResult)).contents + msg = messaging.new_message('navModel') + msg.valid = valid + msg.navModel.frameId = frame_id + msg.navModel.locationMonoTime = location_ts + msg.navModel.modelExecutionTime = execution_time + msg.navModel.dspExecutionTime = dsp_execution_time + msg.navModel.features = model_result.features[:] + msg.navModel.desirePrediction = model_result.desire_pred[:] + msg.navModel.position.x = [p.x for p in model_result.plan.mean] + msg.navModel.position.y = [p.y for p in model_result.plan.mean] + msg.navModel.position.xStd = [math.exp(p.x) for p in model_result.plan.std] + msg.navModel.position.yStd = [math.exp(p.y) for p in model_result.plan.std] + return msg + + +def main(): + gc.disable() + set_realtime_priority(1) + + # there exists a race condition when two processes try to create a + # SNPE model runner at the same time, wait for dmonitoringmodeld to finish + cloudlog.warning("waiting for dmonitoringmodeld to initialize") + if not Params().get_bool("DmModelInitialized", True): + return + + model = ModelState() + cloudlog.warning("models loaded, navmodeld starting") + + vipc_client = VisionIpcClient("navd", VisionStreamType.VISION_STREAM_MAP, True) + while not vipc_client.connect(False): + time.sleep(0.1) + assert vipc_client.is_connected() + cloudlog.warning(f"connected with buffer size: {vipc_client.buffer_len}") + + sm = SubMaster(["navInstruction"]) + pm = PubMaster(["navModel"]) + + while True: + buf = vipc_client.recv() + if buf is None: + continue + + sm.update(0) + t1 = time.perf_counter() + model_output, dsp_execution_time = model.run(buf.data[:buf.uv_offset]) + t2 = time.perf_counter() + + valid = vipc_client.valid and sm.valid["navInstruction"] + pm.send("navModel", get_navmodel_packet(model_output, valid, vipc_client.frame_id, vipc_client.timestamp_sof, t2 - t1, dsp_execution_time)) + + +if __name__ == "__main__": + main() diff --git a/selfdrive/modeld/runners/__init__.py b/selfdrive/modeld/runners/__init__.py index 9193d63e2b..4c29bf3f1c 100644 --- a/selfdrive/modeld/runners/__init__.py +++ b/selfdrive/modeld/runners/__init__.py @@ -19,7 +19,7 @@ class ModelRunner(RunModel): from openpilot.selfdrive.modeld.runners.snpemodel_pyx import SNPEModel as Runner runner_type = ModelRunner.SNPE elif ModelRunner.ONNX in paths: - from openpilot.selfdrive.modeld.runners.onnxmodel_pyx import ONNXModel as Runner + from openpilot.selfdrive.modeld.runners.onnxmodel import ONNXModel as Runner runner_type = ModelRunner.ONNX else: raise Exception("Couldn't select a model runner, make sure to pass at least one valid model path") diff --git a/selfdrive/modeld/runners/onnx_runner.py b/selfdrive/modeld/runners/onnx_runner.py deleted file mode 100755 index 9e53874578..0000000000 --- a/selfdrive/modeld/runners/onnx_runner.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python3 - -import os -import sys -import numpy as np -from typing import Tuple, Dict, Union, Any - -os.environ["OMP_NUM_THREADS"] = "4" -os.environ["OMP_WAIT_POLICY"] = "PASSIVE" - -import onnxruntime as ort - -ORT_TYPES_TO_NP_TYPES = {'tensor(float16)': np.float16, 'tensor(float)': np.float32, 'tensor(uint8)': np.uint8} - -def read(sz, tf8=False): - dd = [] - gt = 0 - szof = 1 if tf8 else 4 - while gt < sz * szof: - st = os.read(0, sz * szof - gt) - assert(len(st) > 0) - dd.append(st) - gt += len(st) - r = np.frombuffer(b''.join(dd), dtype=np.uint8 if tf8 else np.float32) - if tf8: - r = r / 255. - return r - -def write(d): - os.write(1, d.tobytes()) - -def run_loop(m, tf8_input=False): - ishapes = [[1]+ii.shape[1:] for ii in m.get_inputs()] - keys = [x.name for x in m.get_inputs()] - itypes = [ORT_TYPES_TO_NP_TYPES[x.type] for x in m.get_inputs()] - - # run once to initialize CUDA provider - if "CUDAExecutionProvider" in m.get_providers(): - m.run(None, dict(zip(keys, [np.zeros(shp, dtype=itp) for shp, itp in zip(ishapes, itypes, strict=True)], strict=True))) - - print("ready to run onnx model", keys, ishapes, file=sys.stderr) - while 1: - inputs = [] - for k, shp, itp in zip(keys, ishapes, itypes, strict=True): - ts = np.product(shp) - #print("reshaping %s with offset %d" % (str(shp), offset), file=sys.stderr) - inputs.append(read(ts, (k=='input_img' and tf8_input)).reshape(shp).astype(itp)) - ret = m.run(None, dict(zip(keys, inputs, strict=True))) - #print(ret, file=sys.stderr) - for r in ret: - write(r.astype(np.float32)) - - -if __name__ == "__main__": - print(sys.argv, file=sys.stderr) - print("Onnx available providers: ", ort.get_available_providers(), file=sys.stderr) - options = ort.SessionOptions() - options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL - - provider: Union[str, Tuple[str, Dict[Any, Any]]] - if 'OpenVINOExecutionProvider' in ort.get_available_providers() and 'ONNXCPU' not in os.environ: - provider = 'OpenVINOExecutionProvider' - elif 'CUDAExecutionProvider' in ort.get_available_providers() and 'ONNXCPU' not in os.environ: - options.intra_op_num_threads = 2 - provider = ('CUDAExecutionProvider', {'cudnn_conv_algo_search': 'DEFAULT'}) - else: - options.intra_op_num_threads = 2 - options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL - options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL - provider = 'CPUExecutionProvider' - - try: - print("Onnx selected provider: ", [provider], file=sys.stderr) - ort_session = ort.InferenceSession(sys.argv[1], options, providers=[provider]) - print("Onnx using ", ort_session.get_providers(), file=sys.stderr) - run_loop(ort_session, tf8_input=("--use_tf8" in sys.argv)) - except KeyboardInterrupt: - pass diff --git a/selfdrive/modeld/runners/onnxmodel.cc b/selfdrive/modeld/runners/onnxmodel.cc deleted file mode 100644 index ebe6ad847c..0000000000 --- a/selfdrive/modeld/runners/onnxmodel.cc +++ /dev/null @@ -1,88 +0,0 @@ -#include "selfdrive/modeld/runners/onnxmodel.h" - -#include -#include -#include -#include -#include - -#include "common/util.h" - -ONNXModel::ONNXModel(const std::string path, float *_output, size_t _output_size, int runtime, bool _use_tf8, cl_context context) { - LOGD("loading model %s", path.c_str()); - - output = _output; - output_size = _output_size; - use_tf8 = _use_tf8; - - int err = pipe(pipein); - assert(err == 0); - err = pipe(pipeout); - assert(err == 0); - - std::string onnx_runner = ONNXRUNNER_PATH; - std::string tf8_arg = use_tf8 ? "--use_tf8" : ""; - - proc_pid = fork(); - if (proc_pid == 0) { - LOGD("spawning onnx process %s", onnx_runner.c_str()); - char *argv[] = {(char*)onnx_runner.c_str(), (char*)path.c_str(), (char*)tf8_arg.c_str(), nullptr}; - dup2(pipein[0], 0); - dup2(pipeout[1], 1); - close(pipein[0]); - close(pipein[1]); - close(pipeout[0]); - close(pipeout[1]); - execvp(onnx_runner.c_str(), argv); - exit(1); // exit if the exec fails - } - - // parent - close(pipein[0]); - close(pipeout[1]); -} - -ONNXModel::~ONNXModel() { - close(pipein[1]); - close(pipeout[0]); - kill(proc_pid, SIGTERM); -} - -void ONNXModel::pwrite(float *buf, int size) { - char *cbuf = (char *)buf; - int tw = size*sizeof(float); - while (tw > 0) { - int err = write(pipein[1], cbuf, tw); - //printf("host write %d\n", err); - assert(err >= 0); - cbuf += err; - tw -= err; - } - LOGD("host write of size %d done", size); -} - -void ONNXModel::pread(float *buf, int size) { - char *cbuf = (char *)buf; - int tr = size*sizeof(float); - struct pollfd fds[1]; - fds[0].fd = pipeout[0]; - fds[0].events = POLLIN; - while (tr > 0) { - int err; - err = poll(fds, 1, 10000); // 10 second timeout - assert(err == 1 || (err == -1 && errno == EINTR)); - LOGD("host read remaining %d/%lu poll %d", tr, size*sizeof(float), err); - err = read(pipeout[0], cbuf, tr); - assert(err > 0 || (err == 0 && errno == EINTR)); - cbuf += err; - tr -= err; - } - LOGD("host read done"); -} - -void ONNXModel::execute() { - for (auto &input : inputs) { - pwrite(input->buffer, input->size); - } - pread(output, output_size); -} diff --git a/selfdrive/modeld/runners/onnxmodel.h b/selfdrive/modeld/runners/onnxmodel.h deleted file mode 100644 index f9bae1a0a9..0000000000 --- a/selfdrive/modeld/runners/onnxmodel.h +++ /dev/null @@ -1,23 +0,0 @@ -#pragma once - -#include - -#include "selfdrive/modeld/runners/runmodel.h" - -class ONNXModel : public RunModel { -public: - ONNXModel(const std::string path, float *output, size_t output_size, int runtime, bool _use_tf8 = false, cl_context context = NULL); - ~ONNXModel(); - void execute(); -private: - int proc_pid; - float *output; - size_t output_size; - bool use_tf8; - - // pipe to communicate to onnx_runner subprocess - void pread(float *buf, int size); - void pwrite(float *buf, int size); - int pipein[2]; - int pipeout[2]; -}; diff --git a/selfdrive/modeld/runners/onnxmodel.pxd b/selfdrive/modeld/runners/onnxmodel.pxd deleted file mode 100644 index 165f043da8..0000000000 --- a/selfdrive/modeld/runners/onnxmodel.pxd +++ /dev/null @@ -1,9 +0,0 @@ -# distutils: language = c++ - -from libcpp.string cimport string - -from cereal.visionipc.visionipc cimport cl_context - -cdef extern from "selfdrive/modeld/runners/onnxmodel.h": - cdef cppclass ONNXModel: - ONNXModel(string, float*, size_t, int, bool, cl_context) diff --git a/selfdrive/modeld/runners/onnxmodel.py b/selfdrive/modeld/runners/onnxmodel.py new file mode 100644 index 0000000000..3c20a39760 --- /dev/null +++ b/selfdrive/modeld/runners/onnxmodel.py @@ -0,0 +1,70 @@ +import os +import sys +import numpy as np +from typing import Tuple, Dict, Union, Any + +from openpilot.selfdrive.modeld.runners.runmodel_pyx import RunModel + +ORT_TYPES_TO_NP_TYPES = {'tensor(float16)': np.float16, 'tensor(float)': np.float32, 'tensor(uint8)': np.uint8} + +def create_ort_session(path): + os.environ["OMP_NUM_THREADS"] = "4" + os.environ["OMP_WAIT_POLICY"] = "PASSIVE" + + import onnxruntime as ort + print("Onnx available providers: ", ort.get_available_providers(), file=sys.stderr) + options = ort.SessionOptions() + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_DISABLE_ALL + + provider: Union[str, Tuple[str, Dict[Any, Any]]] + if 'OpenVINOExecutionProvider' in ort.get_available_providers() and 'ONNXCPU' not in os.environ: + provider = 'OpenVINOExecutionProvider' + elif 'CUDAExecutionProvider' in ort.get_available_providers() and 'ONNXCPU' not in os.environ: + options.intra_op_num_threads = 2 + provider = ('CUDAExecutionProvider', {'cudnn_conv_algo_search': 'DEFAULT'}) + else: + options.intra_op_num_threads = 2 + options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + options.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL + provider = 'CPUExecutionProvider' + + print("Onnx selected provider: ", [provider], file=sys.stderr) + ort_session = ort.InferenceSession(path, options, providers=[provider]) + print("Onnx using ", ort_session.get_providers(), file=sys.stderr) + return ort_session + + +class ONNXModel(RunModel): + def __init__(self, path, output, runtime, use_tf8, cl_context): + self.inputs = {} + self.output = output + self.use_tf8 = use_tf8 + + self.session = create_ort_session(path) + self.input_names = [x.name for x in self.session.get_inputs()] + self.input_shapes = {x.name: [1, *x.shape[1:]] for x in self.session.get_inputs()} + self.input_dtypes = {x.name: ORT_TYPES_TO_NP_TYPES[x.type] for x in self.session.get_inputs()} + + # run once to initialize CUDA provider + if "CUDAExecutionProvider" in self.session.get_providers(): + self.session.run(None, {k: np.zeros(self.input_shapes[k], dtype=self.input_dtypes[k]) for k in self.input_names}) + print("ready to run onnx model", self.input_shapes, file=sys.stderr) + + def addInput(self, name, buffer): + assert name in self.input_names + self.inputs[name] = buffer + + def setInputBuffer(self, name, buffer): + assert name in self.inputs + self.inputs[name] = buffer + + def getCLBuffer(self, name): + return None + + def execute(self): + inputs = {k: (v.view(np.uint8) / 255. if self.use_tf8 and k == 'input_img' else v) for k,v in self.inputs.items()} + inputs = {k: v.reshape(self.input_shapes[k]).astype(self.input_dtypes[k]) for k,v in inputs.items()} + outputs = self.session.run(None, inputs) + assert len(outputs) == 1, "Only single model outputs are supported" + self.output[:] = outputs[0] + return self.output diff --git a/selfdrive/modeld/runners/onnxmodel_pyx.pyx b/selfdrive/modeld/runners/onnxmodel_pyx.pyx deleted file mode 100644 index 27aa22d1c6..0000000000 --- a/selfdrive/modeld/runners/onnxmodel_pyx.pyx +++ /dev/null @@ -1,14 +0,0 @@ -# distutils: language = c++ -# cython: c_string_encoding=ascii - -from libcpp cimport bool -from libcpp.string cimport string - -from .onnxmodel cimport ONNXModel as cppONNXModel -from selfdrive.modeld.models.commonmodel_pyx cimport CLContext -from selfdrive.modeld.runners.runmodel_pyx cimport RunModel -from selfdrive.modeld.runners.runmodel cimport RunModel as cppRunModel - -cdef class ONNXModel(RunModel): - def __cinit__(self, string path, float[:] output, int runtime, bool use_tf8, CLContext context): - self.model = new cppONNXModel(path, &output[0], len(output), runtime, use_tf8, context.context) diff --git a/selfdrive/modeld/runners/run.h b/selfdrive/modeld/runners/run.h index cae2f5b27a..36ad262a5b 100644 --- a/selfdrive/modeld/runners/run.h +++ b/selfdrive/modeld/runners/run.h @@ -2,7 +2,3 @@ #include "selfdrive/modeld/runners/runmodel.h" #include "selfdrive/modeld/runners/snpemodel.h" - -#if defined(USE_ONNX_MODEL) -#include "selfdrive/modeld/runners/onnxmodel.h" -#endif diff --git a/selfdrive/test/docker_build.sh b/selfdrive/test/docker_build.sh index 3851e2b1e3..e0ba54f058 100755 --- a/selfdrive/test/docker_build.sh +++ b/selfdrive/test/docker_build.sh @@ -1,45 +1,26 @@ -#!/bin/bash +#!/usr/bin/env bash set -e # To build sim and docs, you can run the following to mount the scons cache to the same place as in CI: # mkdir -p .ci_cache/scons_cache # sudo mount --bind /tmp/scons_cache/ .ci_cache/scons_cache -if [ $1 = "base" ]; then - export DOCKER_IMAGE=openpilot-base - export DOCKER_FILE=Dockerfile.openpilot_base -elif [ $1 = "docs" ]; then - export DOCKER_IMAGE=openpilot-docs - export DOCKER_FILE=docs/docker/Dockerfile -elif [ $1 = "sim" ]; then - export DOCKER_IMAGE=openpilot-sim - export DOCKER_FILE=tools/sim/Dockerfile.sim -elif [ $1 = "prebuilt" ]; then - export DOCKER_IMAGE=openpilot-prebuilt - export DOCKER_FILE=Dockerfile.openpilot -elif [ $1 = "cl" ]; then - export DOCKER_IMAGE=openpilot-base-cl - export DOCKER_FILE=Dockerfile.openpilot_base_cl -else - echo "Invalid docker build image $1" - exit 1 -fi - -export DOCKER_REGISTRY=ghcr.io/commaai -export COMMIT_SHA=$(git rev-parse HEAD) - -LOCAL_TAG=$DOCKER_IMAGE -REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG -REMOTE_SHA_TAG=$REMOTE_TAG:$COMMIT_SHA - SCRIPT_DIR=$(dirname "$0") OPENPILOT_DIR=$SCRIPT_DIR/../../ +if [ -n "$TARGET_ARCHITECTURE" ]; then + PLATFORM="linux/$TARGET_ARCHITECTURE" + TAG_SUFFIX="-$TARGET_ARCHITECTURE" +else + PLATFORM="linux/$(uname -m)" + TAG_SUFFIX="" +fi -DOCKER_BUILDKIT=1 docker buildx build --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR +source $SCRIPT_DIR/docker_common.sh $1 "$TAG_SUFFIX" -if [[ ! -z "$PUSH_IMAGE" ]]; -then +DOCKER_BUILDKIT=1 docker buildx build --platform $PLATFORM --load --cache-to type=inline --cache-from type=registry,ref=$REMOTE_TAG -t $REMOTE_TAG -t $LOCAL_TAG -f $OPENPILOT_DIR/$DOCKER_FILE $OPENPILOT_DIR + +if [ -n "$PUSH_IMAGE" ]; then docker push $REMOTE_TAG docker tag $REMOTE_TAG $REMOTE_SHA_TAG docker push $REMOTE_SHA_TAG -fi \ No newline at end of file +fi diff --git a/selfdrive/test/docker_common.sh b/selfdrive/test/docker_common.sh new file mode 100644 index 0000000000..68ea472d26 --- /dev/null +++ b/selfdrive/test/docker_common.sh @@ -0,0 +1,27 @@ +if [ $1 = "base" ]; then + export DOCKER_IMAGE=openpilot-base + export DOCKER_FILE=Dockerfile.openpilot_base +elif [ $1 = "docs" ]; then + export DOCKER_IMAGE=openpilot-docs + export DOCKER_FILE=docs/docker/Dockerfile +elif [ $1 = "sim" ]; then + export DOCKER_IMAGE=openpilot-sim + export DOCKER_FILE=tools/sim/Dockerfile.sim +elif [ $1 = "prebuilt" ]; then + export DOCKER_IMAGE=openpilot-prebuilt + export DOCKER_FILE=Dockerfile.openpilot +elif [ $1 = "cl" ]; then + export DOCKER_IMAGE=openpilot-base-cl + export DOCKER_FILE=Dockerfile.openpilot_base_cl +else + echo "Invalid docker build image $1" + exit 1 +fi + +export DOCKER_REGISTRY=ghcr.io/commaai +export COMMIT_SHA=$(git rev-parse HEAD) + +TAG_SUFFIX=$2 +LOCAL_TAG=$DOCKER_IMAGE$TAG_SUFFIX +REMOTE_TAG=$DOCKER_REGISTRY/$LOCAL_TAG +REMOTE_SHA_TAG=$DOCKER_REGISTRY/$LOCAL_TAG:$COMMIT_SHA diff --git a/selfdrive/test/docker_tag_multiarch.sh b/selfdrive/test/docker_tag_multiarch.sh new file mode 100755 index 0000000000..c1761802c7 --- /dev/null +++ b/selfdrive/test/docker_tag_multiarch.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +set -e + +if [ $# -lt 2 ]; then + echo "Usage: $0 ..." + exit 1 +fi + +SCRIPT_DIR=$(dirname "$0") +ARCHS=("${@:2}") + +source $SCRIPT_DIR/docker_common.sh $1 + +MANIFEST_AMENDS="" +for ARCH in ${ARCHS[@]}; do + MANIFEST_AMENDS="$MANIFEST_AMENDS --amend $REMOTE_TAG-$ARCH:$COMMIT_SHA" +done + +docker manifest create $REMOTE_TAG $MANIFEST_AMENDS +docker manifest create $REMOTE_SHA_TAG $MANIFEST_AMENDS + +if [[ -n "$PUSH_IMAGE" ]]; then + docker manifest push $REMOTE_TAG + docker manifest push $REMOTE_SHA_TAG +fi diff --git a/selfdrive/test/helpers.py b/selfdrive/test/helpers.py index 1fbf2c4d20..eb07432c40 100644 --- a/selfdrive/test/helpers.py +++ b/selfdrive/test/helpers.py @@ -1,9 +1,6 @@ import os import time -import tempfile -from typing import List, Union -from unittest import mock from functools import wraps import cereal.messaging as messaging @@ -72,46 +69,3 @@ def with_processes(processes, init_time=0, ignore_stopped=None): return wrap return wrapper - - -def temporary_mock_dir(mock_paths_in: Union[List[str], str], kwarg: Union[str, None] = None, generator = tempfile.TemporaryDirectory): - """ - mock_paths_in: string or string list representing the full path of the variable you want to mock. - kwarg: str or None representing the kwarg that gets passed into the test function, in case the test needs access to the temporary directory. - generator: a context to use to generate the temporary directory - """ - def wrapper(func): - @wraps(func) - def wrap(*args, **kwargs): - mock_paths = mock_paths_in if isinstance(mock_paths_in, list) else [mock_paths_in] - with generator() as temp_dir: - mocks = [] - for mock_path in mock_paths: - mocks.append(mock.patch(mock_path, str(temp_dir))) - [mock.start() for mock in mocks] - try: - if kwarg is not None: - kwargs[kwarg] = temp_dir - func(*args, **kwargs) - finally: - [mock.stop() for mock in mocks] - - return wrap - return wrapper - -def string_context(context): - class StringContext: - def __enter__(self): - return context - - def __exit__(self, *args): - pass - - return StringContext - -temporary_dir = temporary_mock_dir([], "temp_dir") -temporary_cache_dir = temporary_mock_dir("openpilot.tools.lib.url_file.CACHE_DIR") -temporary_swaglog_dir = temporary_mock_dir("openpilot.system.swaglog.SWAGLOG_DIR", "temp_dir") -temporary_laikad_downloads_dir = temporary_mock_dir("openpilot.selfdrive.locationd.laikad.DOWNLOADS_CACHE_FOLDER") -temporary_swaglog_ipc = temporary_mock_dir(["openpilot.system.swaglog.SWAGLOG_IPC", "system.logmessaged.SWAGLOG_IPC"], - generator=string_context("/tmp/test_swaglog_ipc")) \ No newline at end of file diff --git a/selfdrive/test/process_replay/ref_commit b/selfdrive/test/process_replay/ref_commit index f5e098910f..c6a2d58fec 100644 --- a/selfdrive/test/process_replay/ref_commit +++ b/selfdrive/test/process_replay/ref_commit @@ -1 +1 @@ -649c512f95d9ad80e7c8c4d852cfa5f468a28fd5 \ No newline at end of file +b421ff389ce720b70a36dd2b3510af54eb484b5f \ No newline at end of file diff --git a/selfdrive/test/process_replay/test_processes.py b/selfdrive/test/process_replay/test_processes.py index ab51948dd0..973c16971e 100755 --- a/selfdrive/test/process_replay/test_processes.py +++ b/selfdrive/test/process_replay/test_processes.py @@ -20,8 +20,8 @@ source_segments = [ ("BODY", "937ccb7243511b65|2022-05-24--16-03-09--1"), # COMMA.BODY ("HYUNDAI", "02c45f73a2e5c6e9|2021-01-01--19-08-22--1"), # HYUNDAI.SONATA ("HYUNDAI2", "d545129f3ca90f28|2022-11-07--20-43-08--3"), # HYUNDAI.KIA_EV6 (+ QCOM GPS) - ("TOYOTA", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA.PRIUS (INDI) - ("TOYOTA2", "0982d79ebb0de295|2021-01-03--20-03-36--6"), # TOYOTA.RAV4 (LQR) + ("TOYOTA", "0982d79ebb0de295|2021-01-04--17-13-21--13"), # TOYOTA.PRIUS + ("TOYOTA2", "0982d79ebb0de295|2021-01-03--20-03-36--6"), # TOYOTA.RAV4 ("TOYOTA3", "f7d7e3538cda1a2a|2021-08-16--08-55-34--6"), # TOYOTA.COROLLA_TSS2 ("HONDA", "eb140f119469d9ab|2021-06-12--10-46-24--27"), # HONDA.CIVIC (NIDEC) ("HONDA2", "7d2244f34d1bbcda|2021-06-25--12-25-37--26"), # HONDA.ACCORD (BOSCH) diff --git a/selfdrive/test/setup_device_ci.sh b/selfdrive/test/setup_device_ci.sh index 1b99c31038..ca458f2a79 100755 --- a/selfdrive/test/setup_device_ci.sh +++ b/selfdrive/test/setup_device_ci.sh @@ -56,31 +56,68 @@ sleep infinity EOF chmod +x $CONTINUE_PATH +safe_checkout() { + # completely clean TEST_DIR + + cd $SOURCE_DIR + + # cleanup orphaned locks + find .git -type f -name "*.lock" -exec rm {} + + + git reset --hard + git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT + find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \; + git reset --hard $GIT_COMMIT + git checkout $GIT_COMMIT + git clean -xdff + git submodule sync + git submodule update --init --recursive + git submodule foreach --recursive "git reset --hard && git clean -xdff" + + git lfs pull + (ulimit -n 65535 && git lfs prune) + + echo "git checkout done, t=$SECONDS" + du -hs $SOURCE_DIR $SOURCE_DIR/.git + + if [ -z "SKIP_COPY" ]; then + rsync -a --delete $SOURCE_DIR $TEST_DIR + fi +} + +unsafe_checkout() { + # checkout directly in test dir, leave old build products + + cd $TEST_DIR + + # cleanup orphaned locks + find .git -type f -name "*.lock" -exec rm {} + + + git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin $GIT_COMMIT + git checkout --force --no-recurse-submodules $GIT_COMMIT + git reset --hard $GIT_COMMIT + git clean -df + git submodule sync + git submodule update --init --recursive + git submodule foreach --recursive "git reset --hard && git clean -df" + + git lfs pull + (ulimit -n 65535 && git lfs prune) +} + +export GIT_PACK_THREADS=8 + # set up environment if [ ! -d "$SOURCE_DIR" ]; then git clone https://github.com/commaai/openpilot.git $SOURCE_DIR fi -cd $SOURCE_DIR -# cleanup orphaned locks -find .git -type f -name "*.lock" -exec rm {} + - -git reset --hard -git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT -find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \; -git reset --hard $GIT_COMMIT -git checkout $GIT_COMMIT -git clean -xdff -git submodule sync -git submodule update --init --recursive -git submodule foreach --recursive "git reset --hard && git clean -xdff" - -git lfs pull -(ulimit -n 65535 && git lfs prune) - -echo "git checkout done, t=$SECONDS" -du -hs $SOURCE_DIR $SOURCE_DIR/.git - -rsync -a --delete $SOURCE_DIR $TEST_DIR +if [ ! -z "$UNSAFE" ]; then + echo "doing unsafe checkout" + unsafe_checkout +else + echo "doing safe checkout" + safe_checkout +fi echo "$TEST_DIR synced with $GIT_COMMIT, t=$SECONDS" diff --git a/selfdrive/test/test_onroad.py b/selfdrive/test/test_onroad.py index 22055d8f20..553ef1afa6 100755 --- a/selfdrive/test/test_onroad.py +++ b/selfdrive/test/test_onroad.py @@ -19,8 +19,8 @@ from openpilot.common.timeout import Timeout from openpilot.common.params import Params from openpilot.selfdrive.controls.lib.events import EVENTS, ET from openpilot.system.hardware import HARDWARE -from openpilot.system.loggerd.config import ROOT from openpilot.selfdrive.test.helpers import set_params_enabled, release_only +from openpilot.system.hardware.hw import Paths from openpilot.tools.lib.logreader import LogReader # Baseline CPU usage by process @@ -37,8 +37,8 @@ PROCS = { "./_sensord": 7.0, "selfdrive.controls.radard": 4.5, "selfdrive.modeld.modeld": 8.0, - "./_dmonitoringmodeld": 5.0, - "./_navmodeld": 1.0, + "selfdrive.modeld.dmonitoringmodeld": 8.0, + "selfdrive.modeld.navmodeld": 1.0, "selfdrive.thermald.thermald": 3.87, "selfdrive.locationd.calibrationd": 2.0, "selfdrive.locationd.torqued": 5.0, @@ -102,7 +102,7 @@ class TestOnroad(unittest.TestCase): @classmethod def setUpClass(cls): if "DEBUG" in os.environ: - segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(ROOT).iterdir()) + segs = filter(lambda x: os.path.exists(os.path.join(x, "rlog")), Path(Paths.log_root()).iterdir()) segs = sorted(segs, key=lambda x: x.stat().st_mtime) print(segs[-3]) cls.lr = list(LogReader(os.path.join(segs[-3], "rlog"))) @@ -115,8 +115,8 @@ class TestOnroad(unittest.TestCase): params.remove("CurrentRoute") set_params_enabled() os.environ['TESTING_CLOSET'] = '1' - if os.path.exists(ROOT): - shutil.rmtree(ROOT) + if os.path.exists(Paths.log_root()): + shutil.rmtree(Paths.log_root()) os.system("rm /dev/shm/*") # Make sure athena isn't running @@ -143,8 +143,8 @@ class TestOnroad(unittest.TestCase): while len(cls.segments) < 3: segs = set() - if Path(ROOT).exists(): - segs = set(Path(ROOT).glob(f"{route}--*")) + if Path(Paths.log_root()).exists(): + segs = set(Path(Paths.log_root()).glob(f"{route}--*")) cls.segments = sorted(segs, key=lambda s: int(str(s).rsplit('--')[-1])) time.sleep(2) diff --git a/selfdrive/tombstoned.py b/selfdrive/tombstoned.py index 3f99e16831..a1479003ee 100755 --- a/selfdrive/tombstoned.py +++ b/selfdrive/tombstoned.py @@ -10,8 +10,8 @@ import glob from typing import NoReturn from openpilot.common.file_helpers import mkdirs_exists_ok -from openpilot.system.loggerd.config import ROOT import openpilot.selfdrive.sentry as sentry +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import cloudlog from openpilot.system.version import get_commit @@ -130,7 +130,7 @@ def report_tombstone_apport(fn): new_fn = f"{date}_{get_commit(default='nocommit')[:8]}_{safe_fn(clean_path)}"[:MAX_TOMBSTONE_FN_LEN] - crashlog_dir = os.path.join(ROOT, "crash") + crashlog_dir = os.path.join(Paths.log_root(), "crash") mkdirs_exists_ok(crashlog_dir) # Files could be on different filesystems, copy, then delete diff --git a/selfdrive/ui/qt/onroad.cc b/selfdrive/ui/qt/onroad.cc index d9b02965b4..a571457021 100644 --- a/selfdrive/ui/qt/onroad.cc +++ b/selfdrive/ui/qt/onroad.cc @@ -278,7 +278,6 @@ void OnroadAlerts::paintEvent(QPaintEvent *event) { ExperimentalButton::ExperimentalButton(QWidget *parent) : experimental_mode(false), engageable(false), QPushButton(parent) { setFixedSize(btn_size, btn_size); - params = Params(); engage_img = loadPixmap("../assets/img_chffr_wheel.png", {img_size, img_size}); experimental_img = loadPixmap("../assets/img_experimental.svg", {img_size, img_size}); QObject::connect(this, &QPushButton::clicked, this, &ExperimentalButton::changeMode); diff --git a/system/hardware/hw.h b/system/hardware/hw.h index 1274851787..2f6ccfffda 100644 --- a/system/hardware/hw.h +++ b/system/hardware/hw.h @@ -14,16 +14,37 @@ #endif namespace Path { -inline std::string log_root() { - if (const char *env = getenv("LOG_ROOT")) { - return env; + inline std::string openpilot_prefix() { + return util::getenv("OPENPILOT_PREFIX", ""); + } + + inline std::string comma_home() { + return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix(); + } + + inline std::string log_root() { + if (const char *env = getenv("LOG_ROOT")) { + return env; + } + return Hardware::PC() ? Path::comma_home() + "/media/0/realdata" : "/data/media/0/realdata"; + } + + inline std::string params() { + return Hardware::PC() ? util::getenv("PARAMS_ROOT", Path::comma_home() + "/params") : "/data/params"; + } + + inline std::string rsa_file() { + return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa"; + } + + inline std::string swaglog_ipc() { + return "ipc:///tmp/logmessage" + Path::openpilot_prefix(); + } + + inline std::string download_cache_root() { + if (const char *env = getenv("COMMA_CACHE")) { + return env; + } + return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/"; } - return Hardware::PC() ? util::getenv("HOME") + "/.comma/media/0/realdata" : "/data/media/0/realdata"; -} -inline std::string params() { - return Hardware::PC() ? util::getenv("PARAMS_ROOT", util::getenv("HOME") + "/.comma/params") : "/data/params"; -} -inline std::string rsa_file() { - return Hardware::PC() ? util::getenv("HOME") + "/.comma/persist/comma/id_rsa" : "/persist/comma/id_rsa"; -} } // namespace Path diff --git a/system/hardware/hw.py b/system/hardware/hw.py new file mode 100644 index 0000000000..3f7eac02e9 --- /dev/null +++ b/system/hardware/hw.py @@ -0,0 +1,35 @@ +import os +from pathlib import Path + +from openpilot.selfdrive.hardware import PC + +class Paths: + @staticmethod + def comma_home() -> str: + return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", "")) + + @staticmethod + def log_root() -> str: + if os.environ.get('LOG_ROOT', False): + return os.environ['LOG_ROOT'] + elif PC: + return str(Path(Paths.comma_home()) / "media" / "0" / "realdata") + else: + return '/data/media/0/realdata/' + + @staticmethod + def swaglog_root() -> str: + if PC: + return os.path.join(Paths.comma_home(), "log") + else: + return "/data/log/" + + @staticmethod + def swaglog_ipc() -> str: + return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "") + + @staticmethod + def download_cache_root() -> str: + if os.environ.get('COMMA_CACHE', False): + return os.environ['COMMA_CACHE'] + return "/tmp/comma_download_cache" + os.environ.get("OPENPILOT_PREFIX", "") + "/" diff --git a/system/loggerd/bootlog.cc b/system/loggerd/bootlog.cc index becd293c02..771594d20c 100644 --- a/system/loggerd/bootlog.cc +++ b/system/loggerd/bootlog.cc @@ -50,11 +50,11 @@ static kj::Array build_boot_log() { int main(int argc, char** argv) { const std::string timestr = logger_get_route_name(); - const std::string path = LOG_ROOT + "/boot/" + timestr; + const std::string path = Path::log_root() + "/boot/" + timestr; LOGW("bootlog to %s", path.c_str()); // Open bootlog - bool r = util::create_directories(LOG_ROOT + "/boot/", 0775); + bool r = util::create_directories(Path::log_root() + "/boot/", 0775); assert(r); RawFile file(path.c_str()); diff --git a/system/loggerd/config.py b/system/loggerd/config.py index dfc117f2c1..0245585fc0 100644 --- a/system/loggerd/config.py +++ b/system/loggerd/config.py @@ -1,13 +1,7 @@ import os from pathlib import Path from openpilot.system.hardware import PC - -if os.environ.get('LOG_ROOT', False): - ROOT = os.environ['LOG_ROOT'] -elif PC: - ROOT = str(Path.home() / ".comma" / "media" / "0" / "realdata") -else: - ROOT = '/data/media/0/realdata/' +from openpilot.selfdrive.hardware.hw import Paths CAMERA_FPS = 20 @@ -23,7 +17,7 @@ STATS_FLUSH_TIME_S = 60 def get_available_percent(default=None): try: - statvfs = os.statvfs(ROOT) + statvfs = os.statvfs(Paths.log_root()) available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks except OSError: available_percent = default @@ -33,7 +27,7 @@ def get_available_percent(default=None): def get_available_bytes(default=None): try: - statvfs = os.statvfs(ROOT) + statvfs = os.statvfs(Paths.log_root()) available_bytes = statvfs.f_bavail * statvfs.f_frsize except OSError: available_bytes = default diff --git a/system/loggerd/deleter.py b/system/loggerd/deleter.py index ab4a0b97c1..b060232b6e 100755 --- a/system/loggerd/deleter.py +++ b/system/loggerd/deleter.py @@ -3,9 +3,9 @@ import os import shutil import threading from typing import List - +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import cloudlog -from openpilot.system.loggerd.config import ROOT, get_available_bytes, get_available_percent +from openpilot.system.loggerd.config import get_available_bytes, get_available_percent from openpilot.system.loggerd.uploader import listdir_by_creation from openpilot.system.loggerd.xattr_cache import getxattr @@ -20,7 +20,7 @@ PRESERVE_COUNT = 5 def has_preserve_xattr(d: str) -> bool: - return getxattr(os.path.join(ROOT, d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE + return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE def get_preserved_segments(dirs_by_creation: List[str]) -> List[str]: @@ -51,14 +51,14 @@ def deleter_thread(exit_event): out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT if out_of_percent or out_of_bytes: - dirs = listdir_by_creation(ROOT) + dirs = listdir_by_creation(Paths.log_root()) # skip deleting most recent N preserved segments (and their prior segment) preserved_dirs = get_preserved_segments(dirs) # remove the earliest directory we can for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)): - delete_path = os.path.join(ROOT, delete_dir) + delete_path = os.path.join(Paths.log_root(), delete_dir) if any(name.endswith(".lock") for name in os.listdir(delete_path)): continue diff --git a/system/loggerd/logger.h b/system/loggerd/logger.h index 36db1a5a74..06b11e72f9 100644 --- a/system/loggerd/logger.h +++ b/system/loggerd/logger.h @@ -16,8 +16,6 @@ #include "common/swaglog.h" #include "system/hardware/hw.h" -const std::string LOG_ROOT = Path::log_root(); - #define LOGGER_MAX_HANDLES 16 class RawFile { diff --git a/system/loggerd/loggerd.cc b/system/loggerd/loggerd.cc index 84d5632d5a..adb24c913c 100644 --- a/system/loggerd/loggerd.cc +++ b/system/loggerd/loggerd.cc @@ -24,7 +24,7 @@ struct LoggerdState { void logger_rotate(LoggerdState *s) { int segment = -1; - int err = logger_next(&s->logger, LOG_ROOT.c_str(), s->segment_path, sizeof(s->segment_path), &segment); + int err = logger_next(&s->logger, Path::log_root().c_str(), s->segment_path, sizeof(s->segment_path), &segment); assert(err == 0); s->rotate_segment = segment; s->ready_to_rotate = 0; diff --git a/system/loggerd/tests/fill.py b/system/loggerd/tests/fill.py index c749f42b97..5e6f887b12 100755 --- a/system/loggerd/tests/fill.py +++ b/system/loggerd/tests/fill.py @@ -3,7 +3,8 @@ from pathlib import Path -from openpilot.system.loggerd.config import ROOT, get_available_percent +from openpilot.system.hardware.hw import Paths +from openpilot.system.loggerd.config import get_available_percent from openpilot.system.loggerd.tests.loggerd_tests_common import create_random_file @@ -11,7 +12,7 @@ if __name__ == "__main__": segment_idx = 0 while True: seg_name = f"1970-01-01--00-00-00--{segment_idx}" - seg_path = Path(ROOT) / seg_name + seg_path = Path(Paths.log_root()) / seg_name print(seg_path) diff --git a/system/loggerd/tests/loggerd_tests_common.py b/system/loggerd/tests/loggerd_tests_common.py index 89ddbd658c..8bfb571861 100644 --- a/system/loggerd/tests/loggerd_tests_common.py +++ b/system/loggerd/tests/loggerd_tests_common.py @@ -1,11 +1,9 @@ import os -import errno -import shutil import random -import tempfile import unittest from pathlib import Path from typing import Optional +from openpilot.system.hardware.hw import Paths import openpilot.system.loggerd.deleter as deleter import openpilot.system.loggerd.uploader as uploader @@ -87,8 +85,6 @@ class UploaderTestCase(unittest.TestCase): uploader.Api = MockApiIgnore def setUp(self): - self.root = Path(tempfile.mkdtemp()) - uploader.ROOT = str(self.root) # Monkey patch root dir uploader.Api = MockApi uploader.Params = MockParams uploader.fake_upload = True @@ -99,16 +95,9 @@ class UploaderTestCase(unittest.TestCase): self.seg_format2 = "2019-05-18--11-22-33--{}" self.seg_dir = self.seg_format.format(self.seg_num) - def tearDown(self): - try: - shutil.rmtree(self.root) - except OSError as e: - if e.errno != errno.ENOENT: - raise - def make_file_with_data(self, f_dir: str, fn: str, size_mb: float = .1, lock: bool = False, upload_xattr: Optional[bytes] = None, preserve_xattr: Optional[bytes] = None) -> Path: - file_path = self.root / f_dir / fn + file_path = Path(Paths.log_root()) / f_dir / fn create_random_file(file_path, size_mb, lock, upload_xattr) if preserve_xattr is not None: diff --git a/system/loggerd/tests/test_deleter.py b/system/loggerd/tests/test_deleter.py index 86152b05f8..e4112b7b4e 100755 --- a/system/loggerd/tests/test_deleter.py +++ b/system/loggerd/tests/test_deleter.py @@ -22,7 +22,6 @@ class TestDeleter(UploaderTestCase): super().setUp() self.fake_stats = Stats(f_bavail=0, f_blocks=10, f_frsize=4096) deleter.os.statvfs = self.fake_statvfs - deleter.ROOT = str(self.root) def start_thread(self): self.end_event = threading.Event() diff --git a/system/loggerd/tests/test_encoder.py b/system/loggerd/tests/test_encoder.py index 41b83c0821..bec956a316 100755 --- a/system/loggerd/tests/test_encoder.py +++ b/system/loggerd/tests/test_encoder.py @@ -14,9 +14,9 @@ from tqdm import trange from openpilot.common.params import Params from openpilot.common.timeout import Timeout from openpilot.system.hardware import TICI -from openpilot.system.loggerd.config import ROOT from openpilot.selfdrive.manager.process_config import managed_processes from openpilot.tools.lib.logreader import LogReader +from openpilot.selfdrive.hardware.hw import Paths SEGMENT_LENGTH = 2 FULL_SIZE = 2507572 @@ -48,12 +48,12 @@ class TestEncoder(unittest.TestCase): self._clear_logs() def _clear_logs(self): - if os.path.exists(ROOT): - shutil.rmtree(ROOT) + if os.path.exists(Paths.log_root()): + shutil.rmtree(Paths.log_root()) def _get_latest_segment_path(self): - last_route = sorted(Path(ROOT).iterdir())[-1] - return os.path.join(ROOT, last_route) + last_route = sorted(Path(Paths.log_root()).iterdir())[-1] + return os.path.join(Paths.log_root(), last_route) # TODO: this should run faster than real time @parameterized.expand([(True, ), (False, )]) @@ -146,7 +146,7 @@ class TestEncoder(unittest.TestCase): for i in trange(num_segments): # poll for next segment with Timeout(int(SEGMENT_LENGTH*10), error_msg=f"timed out waiting for segment {i}"): - while Path(f"{route_prefix_path}--{i+1}") not in Path(ROOT).iterdir(): + while Path(f"{route_prefix_path}--{i+1}") not in Path(Paths.log_root()).iterdir(): time.sleep(0.1) check_seg(i) finally: diff --git a/system/loggerd/tests/test_logger.cc b/system/loggerd/tests/test_logger.cc index 9c82299091..9f815c2189 100644 --- a/system/loggerd/tests/test_logger.cc +++ b/system/loggerd/tests/test_logger.cc @@ -97,7 +97,7 @@ TEST_CASE("logger") { auto logging_thread = [&]() -> void { LoggerHandle *lh = logger_get_handle(&logger); - REQUIRE(lh != nullptr); + assert(lh != nullptr); int segment = main_segment; int delayed_cnt = 0; while (!do_exit) { diff --git a/system/loggerd/tests/test_loggerd.py b/system/loggerd/tests/test_loggerd.py index 816a4d4198..3ea29b0d82 100755 --- a/system/loggerd/tests/test_loggerd.py +++ b/system/loggerd/tests/test_loggerd.py @@ -16,7 +16,7 @@ from cereal.services import service_list from openpilot.common.basedir import BASEDIR from openpilot.common.params import Params from openpilot.common.timeout import Timeout -from openpilot.system.loggerd.config import ROOT +from openpilot.system.hardware.hw import Paths from openpilot.system.loggerd.xattr_cache import getxattr from openpilot.system.loggerd.deleter import PRESERVE_ATTR_NAME, PRESERVE_ATTR_VALUE from openpilot.selfdrive.manager.process_config import managed_processes @@ -32,8 +32,11 @@ CEREAL_SERVICES = [f for f in log.Event.schema.union_fields if f in service_list class TestLoggerd(unittest.TestCase): + def setUp(self): + os.environ.pop("LOG_ROOT", None) + def _get_latest_log_dir(self): - log_dirs = sorted(Path(ROOT).iterdir(), key=lambda f: f.stat().st_mtime) + log_dirs = sorted(Path(Paths.log_root()).iterdir(), key=lambda f: f.stat().st_mtime) return log_dirs[-1] def _get_log_dir(self, x): diff --git a/system/loggerd/tests/test_uploader.py b/system/loggerd/tests/test_uploader.py index 6e2f86d6ca..a7349fc5e4 100755 --- a/system/loggerd/tests/test_uploader.py +++ b/system/loggerd/tests/test_uploader.py @@ -7,6 +7,7 @@ import logging import json from pathlib import Path from typing import List, Optional +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import cloudlog from openpilot.system.loggerd.uploader import uploader_fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE @@ -84,7 +85,7 @@ class TestUploader(UploaderTestCase): self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") for f_path in exp_order: - self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") + self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") @@ -102,7 +103,7 @@ class TestUploader(UploaderTestCase): self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") for f_path in exp_order: - self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") + self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") @@ -121,7 +122,7 @@ class TestUploader(UploaderTestCase): self.assertFalse(len(log_handler.upload_ignored) < len(exp_order), "Some files failed to ignore") self.assertFalse(len(log_handler.upload_ignored) > len(exp_order), "Some files were ignored twice") for f_path in exp_order: - self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not ignored") + self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not ignored") self.assertTrue(log_handler.upload_ignored == exp_order, "Files ignored in wrong order") @@ -146,7 +147,7 @@ class TestUploader(UploaderTestCase): self.assertFalse(len(log_handler.upload_order) < len(exp_order), "Some files failed to upload") self.assertFalse(len(log_handler.upload_order) > len(exp_order), "Some files were uploaded twice") for f_path in exp_order: - self.assertEqual(os.getxattr((self.root / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") + self.assertEqual(os.getxattr((Path(Paths.log_root()) / f_path).with_suffix(""), UPLOAD_ATTR_NAME), UPLOAD_ATTR_VALUE, "All files not uploaded") self.assertTrue(log_handler.upload_order == exp_order, "Files uploaded in wrong order") diff --git a/system/loggerd/tools/mark_all_uploaded.py b/system/loggerd/tools/mark_all_uploaded.py index 84b9c20d78..0502184380 100644 --- a/system/loggerd/tools/mark_all_uploaded.py +++ b/system/loggerd/tools/mark_all_uploaded.py @@ -1,8 +1,8 @@ import os +from openpilot.system.hardware.hw import Paths from openpilot.system.loggerd.uploader import UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE -from openpilot.system.loggerd.config import ROOT -for folder in os.walk(ROOT): +for folder in os.walk(Paths.log_root()): for file1 in folder[2]: full_path = os.path.join(folder[0], file1) os.setxattr(full_path, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE) diff --git a/system/loggerd/uploader.py b/system/loggerd/uploader.py index 2d4505dc28..b520adc94c 100755 --- a/system/loggerd/uploader.py +++ b/system/loggerd/uploader.py @@ -17,8 +17,8 @@ from openpilot.common.api import Api from openpilot.common.params import Params from openpilot.common.realtime import set_core_affinity from openpilot.system.hardware import TICI +from openpilot.system.hardware.hw import Paths from openpilot.system.loggerd.xattr_cache import getxattr, setxattr -from openpilot.system.loggerd.config import ROOT from openpilot.system.swaglog import cloudlog NetworkType = log.DeviceState.NetworkType @@ -244,7 +244,7 @@ def uploader_fn(exit_event: threading.Event) -> None: except Exception: cloudlog.exception("failed to set core affinity") - clear_locks(ROOT) + clear_locks(Paths.log_root()) params = Params() dongle_id = params.get("DongleId", encoding='utf8') @@ -262,7 +262,7 @@ def uploader_fn(exit_event: threading.Event) -> None: sm = messaging.SubMaster(['deviceState']) pm = messaging.PubMaster(['uploaderState']) - uploader = Uploader(dongle_id, ROOT) + uploader = Uploader(dongle_id, Paths.log_root()) backoff = 0.1 while not exit_event.is_set(): diff --git a/system/logmessaged.py b/system/logmessaged.py index 4799348990..c53e20e483 100755 --- a/system/logmessaged.py +++ b/system/logmessaged.py @@ -4,8 +4,8 @@ from typing import NoReturn import cereal.messaging as messaging from openpilot.common.logging_extra import SwagLogFileFormatter +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import get_file_handler -from system.swaglog import SWAGLOG_IPC def main() -> NoReturn: @@ -15,7 +15,7 @@ def main() -> NoReturn: ctx = zmq.Context.instance() sock = ctx.socket(zmq.PULL) - sock.bind(f"ipc://{SWAGLOG_IPC}") + sock.bind(Paths.swaglog_ipc()) # and we publish them log_message_sock = messaging.pub_sock('logMessage') diff --git a/system/proclogd/tests/test_proclog.cc b/system/proclogd/tests/test_proclog.cc index affde2f320..33fccd4f30 100644 --- a/system/proclogd/tests/test_proclog.cc +++ b/system/proclogd/tests/test_proclog.cc @@ -140,7 +140,6 @@ TEST_CASE("buildProcLogerMessage") { REQUIRE(p.getName() == "test_proclog"); REQUIRE(p.getState() == 'R'); REQUIRE_THAT(p.getExe().cStr(), Catch::Matchers::Contains("test_proclog")); - REQUIRE(p.getCmdline().size() == 1); REQUIRE_THAT(p.getCmdline()[0], Catch::Matchers::Contains("test_proclog")); } else { std::string cmd_path = "/proc/" + std::to_string(p.getPid()) + "/cmdline"; diff --git a/system/swaglog.py b/system/swaglog.py index 6ccd0d9791..ba81318234 100644 --- a/system/swaglog.py +++ b/system/swaglog.py @@ -8,18 +8,12 @@ from logging.handlers import BaseRotatingHandler import zmq from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter -from openpilot.system.hardware import PC +from openpilot.system.hardware.hw import Paths -if PC: - SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log") -else: - SWAGLOG_DIR = "/data/log/" - -SWAGLOG_IPC = "/tmp/logmessage" def get_file_handler(): - Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True) - base_filename = os.path.join(SWAGLOG_DIR, "swaglog") + Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True) + base_filename = os.path.join(Paths.swaglog_root(), "swaglog") handler = SwaglogRotatingFileHandler(base_filename) return handler @@ -91,7 +85,7 @@ class UnixDomainSocketHandler(logging.Handler): self.zctx = zmq.Context() self.sock = self.zctx.socket(zmq.PUSH) self.sock.setsockopt(zmq.LINGER, 10) - self.sock.connect(f"ipc://{SWAGLOG_IPC}") + self.sock.connect(Paths.swaglog_ipc()) self.pid = os.getpid() def emit(self, record): diff --git a/system/tests/test_logmessaged.py b/system/tests/test_logmessaged.py index 0b70774bbb..5d0e1bfc82 100755 --- a/system/tests/test_logmessaged.py +++ b/system/tests/test_logmessaged.py @@ -6,23 +6,22 @@ import unittest import cereal.messaging as messaging from openpilot.selfdrive.manager.process_config import managed_processes +from openpilot.system.hardware.hw import Paths from openpilot.system.swaglog import cloudlog, ipchandler -from openpilot.selfdrive.test.helpers import temporary_swaglog_dir, temporary_swaglog_ipc class TestLogmessaged(unittest.TestCase): - def _setup(self, temp_dir): + def setUp(self): # clear the IPC buffer in case some other tests used cloudlog and filled it ipchandler.close() ipchandler.connect() - self.temp_dir = temp_dir managed_processes['logmessaged'].start() self.sock = messaging.sub_sock("logMessage", timeout=1000, conflate=False) self.error_sock = messaging.sub_sock("logMessage", timeout=1000, conflate=False) # ensure sockets are connected - time.sleep(0.2) + time.sleep(1) messaging.drain_sock(self.sock) messaging.drain_sock(self.error_sock) @@ -32,12 +31,9 @@ class TestLogmessaged(unittest.TestCase): managed_processes['logmessaged'].stop(block=True) def _get_log_files(self): - return list(glob.glob(os.path.join(self.temp_dir, "swaglog.*"))) + return list(glob.glob(os.path.join(Paths.swaglog_root(), "swaglog.*"))) - @temporary_swaglog_dir - @temporary_swaglog_ipc - def test_simple_log(self, temp_dir): - self._setup(temp_dir) + def test_simple_log(self): msgs = [f"abc {i}" for i in range(10)] for m in msgs: cloudlog.error(m) @@ -46,10 +42,7 @@ class TestLogmessaged(unittest.TestCase): assert len(m) == len(msgs) assert len(self._get_log_files()) >= 1 - @temporary_swaglog_dir - @temporary_swaglog_ipc - def test_big_log(self, temp_dir): - self._setup(temp_dir) + def test_big_log(self): n = 10 msg = "a"*3*1024*1024 for _ in range(n): diff --git a/system/ubloxd/tests/test_ublox_processing.py b/system/ubloxd/tests/test_ublox_processing.py index 01be79f0d4..311604881a 100755 --- a/system/ubloxd/tests/test_ublox_processing.py +++ b/system/ubloxd/tests/test_ublox_processing.py @@ -8,9 +8,9 @@ from laika.helpers import ConstellationId from laika.raw_gnss import correct_measurements, process_measurements, read_raw_ublox from laika.opt import calc_pos_fix from openpilot.selfdrive.test.openpilotci import get_url +from openpilot.system.hardware.hw import Paths from openpilot.tools.lib.logreader import LogReader from openpilot.selfdrive.test.helpers import with_processes -from openpilot.selfdrive.test.helpers import temporary_dir import cereal.messaging as messaging def get_gnss_measurements(log_reader): @@ -56,9 +56,8 @@ class TestUbloxProcessing(unittest.TestCase): self.assertEqual(count_gps, 5036) self.assertEqual(count_glonass, 3651) - @temporary_dir - def test_get_fix(self, temp_dir): - dog = AstroDog(cache_dir=temp_dir) + def test_get_fix(self): + dog = AstroDog(cache_dir=Paths.download_cache_root()) position_fix_found = 0 count_processed_measurements = 0 count_corrected_measurements = 0 @@ -100,7 +99,7 @@ class TestUbloxProcessing(unittest.TestCase): rcv_msgs = [] for msg in self.ublox_raw: ur_pm.send(msg.which(), msg.as_builder()) - time.sleep(0.01) + time.sleep(0.001) rcv_msgs += messaging.drain_sock(ugs) time.sleep(0.1) diff --git a/system/ubloxd/tests/ublox.py b/system/ubloxd/tests/ublox.py deleted file mode 100755 index 3243500e74..0000000000 --- a/system/ubloxd/tests/ublox.py +++ /dev/null @@ -1,938 +0,0 @@ -#!/usr/bin/env python3 -''' -UBlox binary protocol handling - -Copyright Andrew Tridgell, October 2012 -Released under GNU GPL version 3 or later - -WARNING: This code has originally intended for -ublox version 7, it has been adapted to work -for ublox version 8, not all functions may work. -''' - - -import struct -import time - -# protocol constants -PREAMBLE1 = 0xb5 -PREAMBLE2 = 0x62 - -# message classes -CLASS_NAV = 0x01 -CLASS_RXM = 0x02 -CLASS_INF = 0x04 -CLASS_ACK = 0x05 -CLASS_CFG = 0x06 -CLASS_MON = 0x0A -CLASS_AID = 0x0B -CLASS_TIM = 0x0D -CLASS_ESF = 0x10 - -# ACK messages -MSG_ACK_NACK = 0x00 -MSG_ACK_ACK = 0x01 - -# NAV messages -MSG_NAV_POSECEF = 0x1 -MSG_NAV_POSLLH = 0x2 -MSG_NAV_STATUS = 0x3 -MSG_NAV_DOP = 0x4 -MSG_NAV_SOL = 0x6 -MSG_NAV_PVT = 0x7 -MSG_NAV_POSUTM = 0x8 -MSG_NAV_VELNED = 0x12 -MSG_NAV_VELECEF = 0x11 -MSG_NAV_TIMEGPS = 0x20 -MSG_NAV_TIMEUTC = 0x21 -MSG_NAV_CLOCK = 0x22 -MSG_NAV_SVINFO = 0x30 -MSG_NAV_SAT = 0x35 -MSG_NAV_AOPSTATUS = 0x60 -MSG_NAV_DGPS = 0x31 -MSG_NAV_DOP = 0x04 -MSG_NAV_EKFSTATUS = 0x40 -MSG_NAV_SBAS = 0x32 -MSG_NAV_SOL = 0x06 -MSG_NAV_SAT = 0x35 - -# RXM messages -MSG_RXM_RAW = 0x15 -MSG_RXM_SFRB = 0x11 -MSG_RXM_SFRBX = 0x13 -MSG_RXM_SVSI = 0x20 -MSG_RXM_EPH = 0x31 -MSG_RXM_ALM = 0x30 -MSG_RXM_PMREQ = 0x41 - -# AID messages -MSG_AID_ALM = 0x30 -MSG_AID_EPH = 0x31 -MSG_AID_ALPSRV = 0x32 -MSG_AID_AOP = 0x33 -MSG_AID_DATA = 0x10 -MSG_AID_ALP = 0x50 -MSG_AID_DATA = 0x10 -MSG_AID_HUI = 0x02 -MSG_AID_INI = 0x01 -MSG_AID_REQ = 0x00 - -# CFG messages -MSG_CFG_PRT = 0x00 -MSG_CFG_ANT = 0x13 -MSG_CFG_DAT = 0x06 -MSG_CFG_EKF = 0x12 -MSG_CFG_ESFGWT = 0x29 -MSG_CFG_CFG = 0x09 -MSG_CFG_USB = 0x1b -MSG_CFG_RATE = 0x08 -MSG_CFG_SET_RATE = 0x01 -MSG_CFG_NAV5 = 0x24 -MSG_CFG_FXN = 0x0E -MSG_CFG_INF = 0x02 -MSG_CFG_ITFM = 0x39 -MSG_CFG_MSG = 0x01 -MSG_CFG_NAVX5 = 0x23 -MSG_CFG_NMEA = 0x17 -MSG_CFG_NVS = 0x22 -MSG_CFG_PM2 = 0x3B -MSG_CFG_PM = 0x32 -MSG_CFG_ITMF = 0x39 -MSG_CFG_RINV = 0x34 -MSG_CFG_RST = 0x04 -MSG_CFG_RXM = 0x11 -MSG_CFG_SBAS = 0x16 -MSG_CFG_TMODE2 = 0x3D -MSG_CFG_TMODE = 0x1D -MSG_CFG_TPS = 0x31 -MSG_CFG_TP = 0x07 -MSG_CFG_GNSS = 0x3E -MSG_CFG_ODO = 0x1E - -# ESF messages -MSG_ESF_MEAS = 0x02 -MSG_ESF_STATUS = 0x10 - -# INF messages -MSG_INF_DEBUG = 0x04 -MSG_INF_ERROR = 0x00 -MSG_INF_NOTICE = 0x02 -MSG_INF_TEST = 0x03 -MSG_INF_WARNING = 0x01 - -# MON messages -MSG_MON_SCHD = 0x01 -MSG_MON_HW = 0x09 -MSG_MON_HW2 = 0x0B -MSG_MON_IO = 0x02 -MSG_MON_MSGPP = 0x06 -MSG_MON_RXBUF = 0x07 -MSG_MON_RXR = 0x21 -MSG_MON_TXBUF = 0x08 -MSG_MON_VER = 0x04 - -# TIM messages -MSG_TIM_TP = 0x01 -MSG_TIM_TM2 = 0x03 -MSG_TIM_SVIN = 0x04 -MSG_TIM_VRFY = 0x06 - -# port IDs -PORT_DDC = 0 -PORT_SERIAL1 = 1 -PORT_SERIAL2 = 2 -PORT_USB = 3 -PORT_SPI = 4 - -# dynamic models -DYNAMIC_MODEL_PORTABLE = 0 -DYNAMIC_MODEL_STATIONARY = 2 -DYNAMIC_MODEL_PEDESTRIAN = 3 -DYNAMIC_MODEL_AUTOMOTIVE = 4 -DYNAMIC_MODEL_SEA = 5 -DYNAMIC_MODEL_AIRBORNE1G = 6 -DYNAMIC_MODEL_AIRBORNE2G = 7 -DYNAMIC_MODEL_AIRBORNE4G = 8 - -#reset items -RESET_HOT = 0 -RESET_WARM = 1 -RESET_COLD = 0xFFFF - -RESET_HW = 0 -RESET_SW = 1 -RESET_SW_GPS = 2 -RESET_HW_GRACEFUL = 4 -RESET_GPS_STOP = 8 -RESET_GPS_START = 9 - - -class UBloxError(Exception): - '''Ublox error class''' - - def __init__(self, msg): - Exception.__init__(self, msg) - self.message = msg - - -class UBloxAttrDict(dict): - '''allow dictionary members as attributes''' - - def __init__(self): - dict.__init__(self) - - def __getattr__(self, name): - try: - return self.__getitem__(name) - except KeyError as e: - raise RuntimeError(f"ublock invalid attr: {name}") from e - - def __setattr__(self, name, value): - if name in self.__dict__: - # allow set on normal attributes - dict.__setattr__(self, name, value) - else: - self.__setitem__(name, value) - - -def ArrayParse(field): - '''parse an array descriptor''' - arridx = field.find('[') - if arridx == -1: - return (field, -1) - alen = int(field[arridx + 1:-1]) - fieldname = field[:arridx] - return (fieldname, alen) - - -class UBloxDescriptor: - '''class used to describe the layout of a UBlox message''' - - def __init__(self, - name, - msg_format, - fields=None, - count_field=None, - format2=None, - fields2=None): - if fields is None: - fields = [] - - self.name = name - self.msg_format = msg_format - self.fields = fields - self.count_field = count_field - self.format2 = format2 - self.fields2 = fields2 - - def unpack(self, msg): - '''unpack a UBloxMessage, creating the .fields and ._recs attributes in msg''' - msg._fields = {} - - # unpack main message blocks. A comm - formats = self.msg_format.split(',') - buf = msg._buf[6:-2] - count = 0 - msg._recs = [] - fields = self.fields[:] - - for fmt in formats: - size1 = struct.calcsize(fmt) - if size1 > len(buf): - raise UBloxError("%s INVALID_SIZE1=%u" % (self.name, len(buf))) - f1 = list(struct.unpack(fmt, buf[:size1])) - i = 0 - while i < len(f1): - field = fields.pop(0) - (fieldname, alen) = ArrayParse(field) - if alen == -1: - msg._fields[fieldname] = f1[i] - if self.count_field == fieldname: - count = int(f1[i]) - i += 1 - else: - msg._fields[fieldname] = [0] * alen - for a in range(alen): - msg._fields[fieldname][a] = f1[i] - i += 1 - buf = buf[size1:] - if len(buf) == 0: - break - - if self.count_field == '_remaining': - count = len(buf) // struct.calcsize(self.format2) - - if count == 0: - msg._unpacked = True - if len(buf) != 0: - raise UBloxError("EXTRA_BYTES=%u" % len(buf)) - return - - size2 = struct.calcsize(self.format2) - for _ in range(count): - r = UBloxAttrDict() - if size2 > len(buf): - raise UBloxError("INVALID_SIZE=%u, " % len(buf)) - f2 = list(struct.unpack(self.format2, buf[:size2])) - for i in range(len(self.fields2)): - r[self.fields2[i]] = f2[i] - buf = buf[size2:] - msg._recs.append(r) - if len(buf) != 0: - raise UBloxError("EXTRA_BYTES=%u" % len(buf)) - msg._unpacked = True - - def pack(self, msg, msg_class=None, msg_id=None): - '''pack a UBloxMessage from the .fields and ._recs attributes in msg''' - f1 = [] - if msg_class is None: - msg_class = msg.msg_class() - if msg_id is None: - msg_id = msg.msg_id() - msg._buf = '' - - fields = self.fields[:] - for f in fields: - (fieldname, alen) = ArrayParse(f) - if fieldname not in msg._fields: - break - if alen == -1: - f1.append(msg._fields[fieldname]) - else: - for a in range(alen): - f1.append(msg._fields[fieldname][a]) - try: - # try full length message - fmt = self.msg_format.replace(',', '') - msg._buf = struct.pack(fmt, *tuple(f1)) - except Exception: - # try without optional part - fmt = self.msg_format.split(',')[0] - msg._buf = struct.pack(fmt, *tuple(f1)) - - length = len(msg._buf) - if msg._recs: - length += len(msg._recs) * struct.calcsize(self.format2) - header = struct.pack('= level: - print(msg) - - def unpack(self): - '''unpack a message''' - if not self.valid(): - raise UBloxError('INVALID MESSAGE') - msg_type = self.msg_type() - if msg_type not in msg_types: - raise UBloxError('Unknown message %s length=%u' % (str(msg_type), len(self._buf))) - msg_types[msg_type].unpack(self) - return self._fields, self._recs - - def pack(self): - '''pack a message''' - if not self.valid(): - raise UBloxError('INVALID MESSAGE') - msg_type = self.msg_type() - if msg_type not in msg_types: - raise UBloxError('Unknown message %s' % str(msg_type)) - msg_types[msg_type].pack(self) - - def name(self): - '''return the short string name for a message''' - if not self.valid(): - raise UBloxError('INVALID MESSAGE') - msg_type = self.msg_type() - if msg_type not in msg_types: - raise UBloxError('Unknown message %s length=%u' % (str(msg_types), len(self._buf))) - return msg_types[msg_type].name - - def msg_class(self): - '''return the message class''' - return self._buf[2] - - def msg_id(self): - '''return the message id within the class''' - return self._buf[3] - - def msg_type(self): - '''return the message type tuple (class, id)''' - return (self.msg_class(), self.msg_id()) - - def msg_length(self): - '''return the payload length''' - (payload_length, ) = struct.unpack(' 0 and self._buf[0] != PREAMBLE1: - return False - if len(self._buf) > 1 and self._buf[1] != PREAMBLE2: - self.debug(1, "bad pre2") - return False - if self.needed_bytes() == 0 and not self.valid(): - if len(self._buf) > 8: - self.debug(1, "bad checksum len=%u needed=%u" % (len(self._buf), - self.needed_bytes())) - else: - self.debug(1, "bad len len=%u needed=%u" % (len(self._buf), self.needed_bytes())) - return False - return True - - def add(self, data): - '''add some bytes to a message''' - self._buf += data - while not self.valid_so_far() and len(self._buf) > 0: - '''handle corrupted streams''' - self._buf = self._buf[1:] - if self.needed_bytes() < 0: - self._buf = "" - - def checksum(self, data=None): - '''return a checksum tuple for a message''' - if data is None: - data = self._buf[2:-2] - #cs = 0 - ck_a = 0 - ck_b = 0 - for i in data: - ck_a = (ck_a + i) & 0xFF - ck_b = (ck_b + ck_a) & 0xFF - return (ck_a, ck_b) - - def valid_checksum(self): - '''check if the checksum is OK''' - (ck_a, ck_b) = self.checksum() - #d = self._buf[2:-2] - (ck_a2, ck_b2) = struct.unpack('= 8 and self.needed_bytes() == 0 and self.valid_checksum() - - -class UBlox: - def __init__(self, dev, baudrate): - self.dev = dev - self.baudrate = baudrate - - self.use_sendrecv = False - self.read_only = False - self.debug_level = 0 - - self.logfile = None - self.log = None - self.preferred_dynamic_model = None - self.preferred_usePPP = None - self.preferred_dgps_timeout = None - - def close(self): - '''close the device''' - self.dev.close() - self.dev = None - - def set_debug(self, debug_level): - '''set debug level''' - self.debug_level = debug_level - - def debug(self, level, msg): - '''write a debug message''' - if self.debug_level >= level: - print(msg) - - def set_logfile(self, logfile, append=False): - '''setup logging to a file''' - if self.log is not None: - self.log.close() - self.log = None - self.logfile = logfile - if self.logfile is not None: - if append: - mode = 'ab' - else: - mode = 'wb' - self.log = open(self.logfile, mode=mode) - - def set_preferred_dynamic_model(self, model): - '''set the preferred dynamic model for receiver''' - self.preferred_dynamic_model = model - if model is not None: - self.configure_poll(CLASS_CFG, MSG_CFG_NAV5) - - def set_preferred_dgps_timeout(self, timeout): - '''set the preferred DGPS timeout for receiver''' - self.preferred_dgps_timeout = timeout - if timeout is not None: - self.configure_poll(CLASS_CFG, MSG_CFG_NAV5) - - def set_preferred_usePPP(self, usePPP): - '''set the preferred usePPP setting for the receiver''' - if usePPP is None: - self.preferred_usePPP = None - return - self.preferred_usePPP = int(usePPP) - self.configure_poll(CLASS_CFG, MSG_CFG_NAVX5) - - def nmea_checksum(self, msg): - d = msg[1:] - cs = 0 - for i in d: - cs ^= ord(i) - return cs - - def write(self, buf): - '''write some bytes''' - if not self.read_only: - if self.use_sendrecv: - return self.dev.send(buf) - if isinstance(buf, str): - return self.dev.write(str.encode(buf)) - else: - return self.dev.write(buf) - - def read(self, n): - '''read some bytes''' - if self.use_sendrecv: - try: - return self.dev.recv(n) - except OSError: - return '' - return self.dev.read(n) - - def send_nmea(self, msg): - if not self.read_only: - s = msg + "*%02X" % self.nmea_checksum(msg) + "\r\n" - self.write(s) - - def set_binary(self): - '''put a UBlox into binary mode using a NMEA string''' - if not self.read_only: - print("try set binary at %u" % self.baudrate) - self.send_nmea("$PUBX,41,0,0007,0001,%u,0" % self.baudrate) - self.send_nmea("$PUBX,41,1,0007,0001,%u,0" % self.baudrate) - self.send_nmea("$PUBX,41,2,0007,0001,%u,0" % self.baudrate) - self.send_nmea("$PUBX,41,3,0007,0001,%u,0" % self.baudrate) - self.send_nmea("$PUBX,41,4,0007,0001,%u,0" % self.baudrate) - self.send_nmea("$PUBX,41,5,0007,0001,%u,0" % self.baudrate) - - def disable_nmea(self): - ''' stop sending all types of nmea messages ''' - self.send_nmea("$PUBX,40,GSV,1,1,1,1,1,0") - self.send_nmea("$PUBX,40,GGA,0,0,0,0,0,0") - self.send_nmea("$PUBX,40,GSA,0,0,0,0,0,0") - self.send_nmea("$PUBX,40,VTG,0,0,0,0,0,0") - self.send_nmea("$PUBX,40,TXT,0,0,0,0,0,0") - self.send_nmea("$PUBX,40,RMC,0,0,0,0,0,0") - - def seek_percent(self, pct): - '''seek to the given percentage of a file''' - self.dev.seek(0, 2) - filesize = self.dev.tell() - self.dev.seek(pct * 0.01 * filesize) - - def special_handling(self, msg): - '''handle automatic configuration changes''' - if msg.name() == 'CFG_NAV5': - msg.unpack() - sendit = False - pollit = False - if self.preferred_dynamic_model is not None and msg.dynModel != self.preferred_dynamic_model: - msg.dynModel = self.preferred_dynamic_model - sendit = True - pollit = True - if self.preferred_dgps_timeout is not None and msg.dgpsTimeOut != self.preferred_dgps_timeout: - msg.dgpsTimeOut = self.preferred_dgps_timeout - self.debug(2, "Setting dgpsTimeOut=%u" % msg.dgpsTimeOut) - sendit = True - # we don't re-poll for this one, as some receivers refuse to set it - if sendit: - msg.pack() - self.send(msg) - if pollit: - self.configure_poll(CLASS_CFG, MSG_CFG_NAV5) - if msg.name() == 'CFG_NAVX5' and self.preferred_usePPP is not None: - msg.unpack() - if msg.usePPP != self.preferred_usePPP: - msg.usePPP = self.preferred_usePPP - msg.mask = 1 << 13 - msg.pack() - self.send(msg) - self.configure_poll(CLASS_CFG, MSG_CFG_NAVX5) - - def receive_message(self, ignore_eof=False): - '''blocking receive of one ublox message''' - msg = UBloxMessage() - while True: - n = msg.needed_bytes() - b = self.read(n) - if not b: - if ignore_eof: - time.sleep(0.01) - continue - if len(msg._buf) > 0: - self.debug(1, "dropping %d bytes" % len(msg._buf)) - return None - msg.add(b) - if self.log is not None: - self.log.write(b) - self.log.flush() - if msg.valid(): - self.special_handling(msg) - return msg - - def receive_message_noerror(self, ignore_eof=False): - '''blocking receive of one ublox message, ignoring errors''' - try: - return self.receive_message(ignore_eof=ignore_eof) - except UBloxError as e: - print(e) - return None - except OSError as e: - # Occasionally we get hit with 'resource temporarily unavailable' - # messages here on the serial device, catch them too. - print(e) - return None - - def send(self, msg): - '''send a preformatted ublox message''' - if not msg.valid(): - self.debug(1, "invalid send") - return - if not self.read_only: - self.write(msg._buf) - - def send_message(self, msg_class, msg_id, payload): - '''send a ublox message with class, id and payload''' - msg = UBloxMessage() - msg._buf = struct.pack('> (7 - j)) & 1) != 0 ? "1" : "0"; + int val = ((binary[i] >> (7 - j)) & 1) != 0 ? 1 : 0; // Bit update frequency based highlighting double offset = !item.sigs.empty() ? 50 : 0; auto n = last_msg.bit_change_counts[i][7 - j]; @@ -317,7 +317,7 @@ void BinaryViewModel::updateState() { color.setAlpha(alpha); updateItem(i, j, val, color); } - updateItem(i, 8, toHex(binary[i]), last_msg.colors[i]); + updateItem(i, 8, binary[i], last_msg.colors[i]); } } @@ -348,6 +348,13 @@ BinaryItemDelegate::BinaryItemDelegate(QObject *parent) : QStyledItemDelegate(pa small_font.setPixelSize(8); hex_font = QFontDatabase::systemFont(QFontDatabase::FixedFont); hex_font.setBold(true); + + bin_text_table[0].setText("0"); + bin_text_table[1].setText("1"); + for (int i = 0; i < 256; ++i) { + hex_text_table[i].setText(QStringLiteral("%1").arg(i, 2, 16, QLatin1Char('0')).toUpper()); + hex_text_table[i].prepare({}, hex_font); + } } bool BinaryItemDelegate::hasSignal(const QModelIndex &index, int dx, int dy, const cabana::Signal *sig) const { @@ -392,7 +399,9 @@ void BinaryItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op } else if (!item->valid) { painter->fillRect(option.rect, QBrush(Qt::darkGray, Qt::BDiagPattern)); } - painter->drawText(option.rect, Qt::AlignCenter, item->val); + if (item->valid) { + utils::drawStaticText(painter, option.rect, index.column() == 8 ? hex_text_table[item->val] : bin_text_table[item->val]); + } if (item->is_msb || item->is_lsb) { painter->setFont(small_font); painter->drawText(option.rect.adjusted(8, 0, -8, -3), Qt::AlignRight | Qt::AlignBottom, item->is_msb ? "M" : "L"); diff --git a/tools/cabana/binaryview.h b/tools/cabana/binaryview.h index 04e1d5b2af..584910dc83 100644 --- a/tools/cabana/binaryview.h +++ b/tools/cabana/binaryview.h @@ -19,6 +19,8 @@ public: void drawSignalCell(QPainter* painter, const QStyleOptionViewItem &option, const QModelIndex &index, const cabana::Signal *sig) const; QFont small_font, hex_font; + std::array hex_text_table; + std::array bin_text_table; }; class BinaryViewModel : public QAbstractTableModel { @@ -26,7 +28,7 @@ public: BinaryViewModel(QObject *parent) : QAbstractTableModel(parent) {} void refresh(); void updateState(); - void updateItem(int row, int col, const QString &val, const QColor &color); + void updateItem(int row, int col, uint8_t val, const QColor &color); QVariant headerData(int section, Qt::Orientation orientation, int role = Qt::DisplayRole) const override; QVariant data(const QModelIndex &index, int role = Qt::DisplayRole) const override; int rowCount(const QModelIndex &parent = QModelIndex()) const override { return row_count; } @@ -42,7 +44,7 @@ public: QColor bg_color = QColor(102, 86, 169, 255); bool is_msb = false; bool is_lsb = false; - QString val; + uint8_t val; QList sigs; bool valid = false; }; diff --git a/tools/cabana/cabana.cc b/tools/cabana/cabana.cc index 33403a2bff..0ccef7d3ab 100644 --- a/tools/cabana/cabana.cc +++ b/tools/cabana/cabana.cc @@ -18,6 +18,8 @@ int main(int argc, char *argv[]) { app.setWindowIcon(QIcon(":cabana-icon.png")); UnixSignalHandler signalHandler; + + settings.load(); utils::setTheme(settings.theme); QCommandLineParser cmd_parser; diff --git a/tools/cabana/chart/chart.cc b/tools/cabana/chart/chart.cc index 18a9adbe07..537aac1f28 100644 --- a/tools/cabana/chart/chart.cc +++ b/tools/cabana/chart/chart.cc @@ -14,9 +14,9 @@ #include #include #include +#include #include #include -#include #include #include "tools/cabana/chart/chartswidget.h" @@ -25,7 +25,8 @@ const int AXIS_X_TOP_MARGIN = 4; static inline bool xLessThan(const QPointF &p, float x) { return p.x() < x; } -ChartView::ChartView(const std::pair &x_range, ChartsWidget *parent) : charts_widget(parent), tip_label(this), QChartView(nullptr, parent) { +ChartView::ChartView(const std::pair &x_range, ChartsWidget *parent) + : charts_widget(parent), tip_label(this), QChartView(nullptr, parent) { series_type = (SeriesType)settings.chart_series_type; QChart *chart = new QChart(); chart->setBackgroundVisible(false); @@ -150,6 +151,11 @@ void ChartView::removeIf(std::function predicate) { void ChartView::signalUpdated(const cabana::Signal *sig) { if (std::any_of(sigs.cbegin(), sigs.cend(), [=](auto &s) { return s.sig == sig; })) { + for (const auto &s : sigs) { + if (s.sig == sig && s.series->color() != sig->color) { + setSeriesColor(s.series, sig->color); + } + } updateTitle(); updateSeries(sig); } @@ -280,15 +286,12 @@ void ChartView::updateSeries(const cabana::Signal *sig, bool clear) { s.step_vals.clear(); s.last_value_mono_time = 0; } - s.series->setColor(s.sig->color); const auto &msgs = can->events(s.msg_id); s.vals.reserve(msgs.capacity()); s.step_vals.reserve(msgs.capacity() * 2); - auto first = std::upper_bound(msgs.cbegin(), msgs.cend(), s.last_value_mono_time, [](uint64_t ts, auto e) { - return ts < e->mono_time; - }); + auto first = std::upper_bound(msgs.cbegin(), msgs.cend(), s.last_value_mono_time, CompareCanEvent()); const double route_start_time = can->routeStartTime(); for (auto end = msgs.cend(); first != end; ++first) { const CanEvent *e = *first; @@ -361,7 +364,7 @@ void ChartView::updateAxisY() { axis_y->setRange(min_y, max_y); axis_y->setTickCount(tick_count); - int n = qMax(int(-qFloor(std::log10((max_y - min_y) / (tick_count - 1)))), 0) + 1; + int n = std::max(int(-std::floor(std::log10((max_y - min_y) / (tick_count - 1)))), 0) + 1; int max_label_width = 0; QFontMetrics fm(axis_y->labelsFont()); for (int i = 0; i < tick_count; i++) { @@ -379,15 +382,15 @@ void ChartView::updateAxisY() { std::tuple ChartView::getNiceAxisNumbers(qreal min, qreal max, int tick_count) { qreal range = niceNumber((max - min), true); // range with ceiling qreal step = niceNumber(range / (tick_count - 1), false); - min = qFloor(min / step); - max = qCeil(max / step); + min = std::floor(min / step); + max = std::ceil(max / step); tick_count = int(max - min) + 1; return {min * step, max * step, tick_count}; } // nice numbers can be expressed as form of 1*10^n, 2* 10^n or 5*10^n qreal ChartView::niceNumber(qreal x, bool ceiling) { - qreal z = qPow(10, qFloor(std::log10(x))); //find corresponding number of the form of 10^n than is smaller than x + qreal z = std::pow(10, std::floor(std::log10(x))); //find corresponding number of the form of 10^n than is smaller than x qreal q = x / z; //q<10 && q>=1; if (ceiling) { if (q <= 1.0) q = 1; @@ -476,7 +479,7 @@ void ChartView::mouseReleaseEvent(QMouseEvent *event) { auto rubber = findChild(); if (event->button() == Qt::LeftButton && rubber && rubber->isVisible()) { rubber->hide(); - QRectF rect = rubber->geometry().normalized(); + auto rect = rubber->geometry().normalized(); double min = chart()->mapToValue(rect.topLeft()).x(); double max = chart()->mapToValue(rect.bottomRight()).x(); @@ -670,6 +673,7 @@ void ChartView::drawBackground(QPainter *painter, const QRectF &rect) { void ChartView::drawForeground(QPainter *painter, const QRectF &rect) { drawTimeline(painter); + drawSignalValue(painter); // draw track points painter->setPen(Qt::NoPen); qreal track_line_x = -1; @@ -700,7 +704,10 @@ void ChartView::drawForeground(QPainter *painter, const QRectF &rect) { } } - // paint zoom range + drawRubberBandTimeRange(painter); +} + +void ChartView::drawRubberBandTimeRange(QPainter *painter) { auto rubber = findChild(); if (rubber && rubber->isVisible() && rubber->width() > 1) { painter->setPen(Qt::white); @@ -717,23 +724,24 @@ void ChartView::drawForeground(QPainter *painter, const QRectF &rect) { void ChartView::drawTimeline(QPainter *painter) { const auto plot_area = chart()->plotArea(); - // draw line + // draw vertical time line qreal x = std::clamp(chart()->mapToPosition(QPointF{cur_sec, 0}).x(), plot_area.left(), plot_area.right()); painter->setPen(QPen(chart()->titleBrush().color(), 2)); painter->drawLine(QPointF{x, plot_area.top()}, QPointF{x, plot_area.bottom() + 1}); - // draw current time + // draw current time under the axis-x QString time_str = QString::number(cur_sec, 'f', 2); QSize time_str_size = QFontMetrics(axis_x->labelsFont()).size(Qt::TextSingleLine, time_str) + QSize(8, 2); - QRect time_str_rect(QPoint(x - time_str_size.width() / 2, plot_area.bottom() + AXIS_X_TOP_MARGIN), time_str_size); + QRectF time_str_rect(QPointF(x - time_str_size.width() / 2.0, plot_area.bottom() + AXIS_X_TOP_MARGIN), time_str_size); QPainterPath path; path.addRoundedRect(time_str_rect, 3, 3); painter->fillPath(path, settings.theme == DARK_THEME ? Qt::darkGray : Qt::gray); painter->setPen(palette().color(QPalette::BrightText)); painter->setFont(axis_x->labelsFont()); painter->drawText(time_str_rect, Qt::AlignCenter, time_str); +} - // draw signal value +void ChartView::drawSignalValue(QPainter *painter) { auto item_group = qgraphicsitem_cast(chart()->legend()->childItems()[0]); assert(item_group != nullptr); auto legend_markers = item_group->childItems(); @@ -785,6 +793,7 @@ QXYSeries *ChartView::createSeries(SeriesType type, QColor color) { } void ChartView::addSeries(QXYSeries *series) { + setSeriesColor(series, series->color()); chart()->addSeries(series); series->attachAxis(axis_x); series->attachAxis(axis_y); @@ -797,6 +806,21 @@ void ChartView::addSeries(QXYSeries *series) { } } +void ChartView::setSeriesColor(QXYSeries *series, QColor color) { + auto existing_series = chart()->series(); + for (auto s : existing_series) { + if (s != series && std::abs(color.hueF() - qobject_cast(s)->color().hueF()) < 0.1) { + // use different color to distinguish it from others. + auto last_color = qobject_cast(existing_series.back())->color(); + color.setHsvF(std::fmod(last_color.hueF() + 60 / 360.0, 1.0), + QRandomGenerator::global()->bounded(35, 100) / 100.0, + QRandomGenerator::global()->bounded(85, 100) / 100.0); + break; + } + } + series->setColor(color); +} + void ChartView::setSeriesType(SeriesType type) { if (type != series_type) { series_type = type; diff --git a/tools/cabana/chart/chart.h b/tools/cabana/chart/chart.h index 2740d81b66..de2a1b4510 100644 --- a/tools/cabana/chart/chart.h +++ b/tools/cabana/chart/chart.h @@ -83,10 +83,13 @@ private: void drawForeground(QPainter *painter, const QRectF &rect) override; void drawBackground(QPainter *painter, const QRectF &rect) override; void drawDropIndicator(bool draw) { if (std::exchange(can_drop, draw) != can_drop) viewport()->update(); } + void drawSignalValue(QPainter *painter); void drawTimeline(QPainter *painter); + void drawRubberBandTimeRange(QPainter *painter); std::tuple getNiceAxisNumbers(qreal min, qreal max, int tick_count); qreal niceNumber(qreal x, bool ceiling); QXYSeries *createSeries(SeriesType type, QColor color); + void setSeriesColor(QXYSeries *, QColor color); void updateSeriesPoints(); void removeIf(std::function predicate); inline void clearTrackPoints() { for (auto &s : sigs) s.track_pt = {}; } diff --git a/tools/cabana/chart/chartswidget.cc b/tools/cabana/chart/chartswidget.cc index e852e6d206..83991d914f 100644 --- a/tools/cabana/chart/chartswidget.cc +++ b/tools/cabana/chart/chartswidget.cc @@ -219,7 +219,7 @@ void ChartsWidget::updateToolBar() { undo_zoom_action->setVisible(is_zoomed); redo_zoom_action->setVisible(is_zoomed); reset_zoom_action->setVisible(is_zoomed); - reset_zoom_btn->setText(is_zoomed ? tr("%1-%2").arg(zoomed_range.first, 0, 'f', 1).arg(zoomed_range.second, 0, 'f', 1) : ""); + reset_zoom_btn->setText(is_zoomed ? tr("%1-%2").arg(zoomed_range.first, 0, 'f', 2).arg(zoomed_range.second, 0, 'f', 2) : ""); remove_all_btn->setEnabled(!charts.isEmpty()); dock_btn->setIcon(docking ? "arrow-up-right-square" : "arrow-down-left-square"); dock_btn->setToolTip(docking ? tr("Undock charts") : tr("Dock charts")); @@ -277,6 +277,10 @@ void ChartsWidget::splitChart(ChartView *src_chart) { for (auto it = src_chart->sigs.begin() + 1; it != src_chart->sigs.end(); /**/) { auto c = createChart(); src_chart->chart()->removeSeries(it->series); + + // Restore to the original color + it->series->setColor(it->sig->color); + c->addSeries(it->series); c->sigs.push_back(*it); c->updateAxisY(); @@ -285,6 +289,7 @@ void ChartsWidget::splitChart(ChartView *src_chart) { } src_chart->updateAxisY(); src_chart->updateTitle(); + QTimer::singleShot(0, src_chart, &ChartView::resetChartCache); } } diff --git a/tools/cabana/chart/chartswidget.h b/tools/cabana/chart/chartswidget.h index 88ba4226c0..3541f8d96e 100644 --- a/tools/cabana/chart/chartswidget.h +++ b/tools/cabana/chart/chartswidget.h @@ -120,7 +120,7 @@ class ZoomCommand : public QUndoCommand { public: ZoomCommand(ChartsWidget *charts, std::pair range) : charts(charts), range(range), QUndoCommand() { prev_range = charts->is_zoomed ? charts->zoomed_range : charts->display_range; - setText(QObject::tr("Zoom to %1-%2").arg(range.first, 0, 'f', 1).arg(range.second, 0, 'f', 1)); + setText(QObject::tr("Zoom to %1-%2").arg(range.first, 0, 'f', 2).arg(range.second, 0, 'f', 2)); } void undo() override { charts->setZoom(prev_range.first, prev_range.second); } void redo() override { charts->setZoom(range.first, range.second); } diff --git a/tools/cabana/chart/sparkline.cc b/tools/cabana/chart/sparkline.cc index 4692b41e4b..1ae6a1bfe0 100644 --- a/tools/cabana/chart/sparkline.cc +++ b/tools/cabana/chart/sparkline.cc @@ -9,12 +9,8 @@ void Sparkline::update(const MessageId &msg_id, const cabana::Signal *sig, doubl const auto &msgs = can->events(msg_id); uint64_t ts = (last_msg_ts + can->routeStartTime()) * 1e9; uint64_t first_ts = (ts > range * 1e9) ? ts - range * 1e9 : 0; - auto first = std::lower_bound(msgs.cbegin(), msgs.cend(), first_ts, [](auto e, uint64_t ts) { - return e->mono_time < ts; - }); - auto last = std::upper_bound(first, msgs.cend(), ts, [](uint64_t ts, auto e) { - return ts < e->mono_time; - }); + auto first = std::lower_bound(msgs.cbegin(), msgs.cend(), first_ts, CompareCanEvent()); + auto last = std::upper_bound(first, msgs.cend(), ts, CompareCanEvent()); bool update_values = last_ts != last_msg_ts || time_range != range; last_ts = last_msg_ts; diff --git a/tools/cabana/commands.cc b/tools/cabana/commands.cc index cf48abb4c9..52861723f4 100644 --- a/tools/cabana/commands.cc +++ b/tools/cabana/commands.cc @@ -4,11 +4,13 @@ // EditMsgCommand -EditMsgCommand::EditMsgCommand(const MessageId &id, const QString &name, int size, const QString &comment, QUndoCommand *parent) - : id(id), new_name(name), new_size(size), new_comment(comment), QUndoCommand(parent) { +EditMsgCommand::EditMsgCommand(const MessageId &id, const QString &name, int size, + const QString &node, const QString &comment, QUndoCommand *parent) + : id(id), new_name(name), new_size(size), new_node(node), new_comment(comment), QUndoCommand(parent) { if (auto msg = dbc()->msg(id)) { old_name = msg->name; old_size = msg->size; + old_node = msg->transmitter; old_comment = msg->comment; setText(QObject::tr("edit message %1:%2").arg(name).arg(id.address)); } else { @@ -20,11 +22,11 @@ void EditMsgCommand::undo() { if (old_name.isEmpty()) dbc()->removeMsg(id); else - dbc()->updateMsg(id, old_name, old_size, old_comment); + dbc()->updateMsg(id, old_name, old_size, old_node, old_comment); } void EditMsgCommand::redo() { - dbc()->updateMsg(id, new_name, new_size, new_comment); + dbc()->updateMsg(id, new_name, new_size, new_node, new_comment); } // RemoveMsgCommand @@ -38,7 +40,7 @@ RemoveMsgCommand::RemoveMsgCommand(const MessageId &id, QUndoCommand *parent) : void RemoveMsgCommand::undo() { if (!message.name.isEmpty()) { - dbc()->updateMsg(id, message.name, message.size, message.comment); + dbc()->updateMsg(id, message.name, message.size, message.transmitter, message.comment); for (auto s : message.getSignals()) dbc()->addSignal(id, *s); } @@ -64,7 +66,7 @@ void AddSigCommand::undo() { void AddSigCommand::redo() { if (auto msg = dbc()->msg(id); !msg) { msg_created = true; - dbc()->updateMsg(id, dbc()->newMsgName(id), can->lastMessage(id).dat.size(), ""); + dbc()->updateMsg(id, dbc()->newMsgName(id), can->lastMessage(id).dat.size(), "", ""); } signal.name = dbc()->newSignalName(id); signal.max = std::pow(2, signal.size) - 1; diff --git a/tools/cabana/commands.h b/tools/cabana/commands.h index c7f59c4c7f..0736d9b83f 100644 --- a/tools/cabana/commands.h +++ b/tools/cabana/commands.h @@ -10,13 +10,14 @@ class EditMsgCommand : public QUndoCommand { public: - EditMsgCommand(const MessageId &id, const QString &name, int size, const QString &comment, QUndoCommand *parent = nullptr); + EditMsgCommand(const MessageId &id, const QString &name, int size, const QString &node, + const QString &comment, QUndoCommand *parent = nullptr); void undo() override; void redo() override; private: const MessageId id; - QString old_name, new_name, old_comment, new_comment; + QString old_name, new_name, old_comment, new_comment, old_node, new_node; int old_size = 0, new_size = 0; }; diff --git a/tools/cabana/dbc/dbc.cc b/tools/cabana/dbc/dbc.cc index 537749e48d..a0e523d666 100644 --- a/tools/cabana/dbc/dbc.cc +++ b/tools/cabana/dbc/dbc.cc @@ -78,6 +78,9 @@ QString cabana::Msg::newSignalName() { } void cabana::Msg::update() { + if (transmitter.isEmpty()) { + transmitter = DEFAULT_NODE_NAME; + } mask.assign(size, 0x00); multiplexor = nullptr; @@ -125,6 +128,9 @@ void cabana::Msg::update() { void cabana::Signal::update() { updateMsbLsb(*this); + if (receiver_name.isEmpty()) { + receiver_name = DEFAULT_NODE_NAME; + } float h = 19 * (float)lsb / 64.0; h = fmod(h, 1.0); diff --git a/tools/cabana/dbc/dbc.h b/tools/cabana/dbc/dbc.h index e44bc41abf..bfb26c2842 100644 --- a/tools/cabana/dbc/dbc.h +++ b/tools/cabana/dbc/dbc.h @@ -13,6 +13,7 @@ #include "opendbc/can/common_dbc.h" const QString UNTITLED = "untitled"; +const QString DEFAULT_NODE_NAME = "XXX"; struct MessageId { uint8_t source = 0; diff --git a/tools/cabana/dbc/dbcfile.cc b/tools/cabana/dbc/dbcfile.cc index 2f93c1543e..063f516ead 100644 --- a/tools/cabana/dbc/dbcfile.cc +++ b/tools/cabana/dbc/dbcfile.cc @@ -60,11 +60,12 @@ bool DBCFile::writeContents(const QString &fn) { return false; } -void DBCFile::updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &comment) { +void DBCFile::updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &node, const QString &comment) { auto &m = msgs[id.address]; m.address = id.address; m.name = name; m.size = size; + m.transmitter = node.isEmpty() ? DEFAULT_NODE_NAME : node; m.comment = comment; } @@ -198,7 +199,8 @@ void DBCFile::parse(const QString &content) { QString DBCFile::generateDBC() { QString dbc_string, signal_comment, message_comment, val_desc; for (const auto &[address, m] : msgs) { - dbc_string += QString("BO_ %1 %2: %3 %4\n").arg(address).arg(m.name).arg(m.size).arg(m.transmitter.isEmpty() ? "XXX" : m.transmitter); + const QString transmitter = m.transmitter.isEmpty() ? DEFAULT_NODE_NAME : m.transmitter; + dbc_string += QString("BO_ %1 %2: %3 %4\n").arg(address).arg(m.name).arg(m.size).arg(transmitter); if (!m.comment.isEmpty()) { message_comment += QString("CM_ BO_ %1 \"%2\";\n").arg(address).arg(m.comment); } @@ -221,7 +223,7 @@ QString DBCFile::generateDBC() { .arg(doubleToString(sig->min)) .arg(doubleToString(sig->max)) .arg(sig->unit) - .arg(sig->receiver_name.isEmpty() ? "XXX" : sig->receiver_name); + .arg(sig->receiver_name.isEmpty() ? DEFAULT_NODE_NAME : sig->receiver_name); if (!sig->comment.isEmpty()) { signal_comment += QString("CM_ SG_ %1 %2 \"%3\";\n").arg(address).arg(sig->name).arg(sig->comment); } diff --git a/tools/cabana/dbc/dbcfile.h b/tools/cabana/dbc/dbcfile.h index 78a73d58e4..a3ab1cebe4 100644 --- a/tools/cabana/dbc/dbcfile.h +++ b/tools/cabana/dbc/dbcfile.h @@ -22,7 +22,7 @@ public: void cleanupAutoSaveFile(); QString generateDBC(); - void updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &comment); + void updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &node, const QString &comment); inline void removeMsg(const MessageId &id) { msgs.erase(id.address); } inline const std::map &getMessages() const { return msgs; } diff --git a/tools/cabana/dbc/dbcmanager.cc b/tools/cabana/dbc/dbcmanager.cc index 5736ac1e89..459ca0111d 100644 --- a/tools/cabana/dbc/dbcmanager.cc +++ b/tools/cabana/dbc/dbcmanager.cc @@ -82,10 +82,10 @@ void DBCManager::removeSignal(const MessageId &id, const QString &sig_name) { } } -void DBCManager::updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &comment) { +void DBCManager::updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &node, const QString &comment) { auto dbc_file = findDBCFile(id); assert(dbc_file); // This should be impossible - dbc_file->updateMsg(id, name, size, comment); + dbc_file->updateMsg(id, name, size, node, comment); emit msgUpdated(id); } diff --git a/tools/cabana/dbc/dbcmanager.h b/tools/cabana/dbc/dbcmanager.h index 3ac0829487..5f782fc930 100644 --- a/tools/cabana/dbc/dbcmanager.h +++ b/tools/cabana/dbc/dbcmanager.h @@ -28,7 +28,7 @@ public: void updateSignal(const MessageId &id, const QString &sig_name, const cabana::Signal &sig); void removeSignal(const MessageId &id, const QString &sig_name); - void updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &comment); + void updateMsg(const MessageId &id, const QString &name, uint32_t size, const QString &node, const QString &comment); void removeMsg(const MessageId &id); QString newMsgName(const MessageId &id); diff --git a/tools/cabana/detailwidget.cc b/tools/cabana/detailwidget.cc index 2f6e9cbfe8..d70aeff0c8 100644 --- a/tools/cabana/detailwidget.cc +++ b/tools/cabana/detailwidget.cc @@ -2,7 +2,6 @@ #include #include -#include #include "tools/cabana/commands.h" #include "tools/cabana/mainwin.h" @@ -135,11 +134,12 @@ void DetailWidget::refresh() { for (auto s : binary_view->getOverlappingSignals()) { warnings.push_back(tr("%1 has overlapping bits.").arg(s->name)); } + name_label->setText(QString("%1 (%2)").arg(msgName(msg_id), msg->transmitter)); } else { warnings.push_back(tr("Drag-Select in binary view to create new signal.")); + name_label->setText(msgName(msg_id)); } remove_btn->setEnabled(msg != nullptr); - name_label->setText(msgName(msg_id)); if (!warnings.isEmpty()) { warning_label->setText(warnings.join('\n')); @@ -164,8 +164,8 @@ void DetailWidget::editMsg() { int size = msg ? msg->size : can->lastMessage(msg_id).dat.size(); EditMessageDialog dlg(msg_id, msgName(msg_id), size, this); if (dlg.exec()) { - UndoStack::push(new EditMsgCommand(msg_id, dlg.name_edit->text().trimmed(), - dlg.size_spin->value(), dlg.comment_edit->toPlainText().trimmed())); + UndoStack::push(new EditMsgCommand(msg_id, dlg.name_edit->text().trimmed(), dlg.size_spin->value(), + dlg.node->text().trimmed(), dlg.comment_edit->toPlainText().trimmed())); } } @@ -182,25 +182,24 @@ EditMessageDialog::EditMessageDialog(const MessageId &msg_id, const QString &tit form_layout->addRow("", error_label = new QLabel); error_label->setVisible(false); - name_edit = new QLineEdit(title, this); + form_layout->addRow(tr("Name"), name_edit = new QLineEdit(title, this)); name_edit->setValidator(new NameValidator(name_edit)); - form_layout->addRow(tr("Name"), name_edit); - size_spin = new QSpinBox(this); + form_layout->addRow(tr("Size"), size_spin = new QSpinBox(this)); // TODO: limit the maximum? size_spin->setMinimum(1); size_spin->setValue(size); - form_layout->addRow(tr("Size"), size_spin); + form_layout->addRow(tr("Node"), node = new QLineEdit(this)); + node->setValidator(new NameValidator(name_edit)); form_layout->addRow(tr("Comment"), comment_edit = new QTextEdit(this)); + form_layout->addRow(btn_box = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel)); + if (auto msg = dbc()->msg(msg_id)) { + node->setText(msg->transmitter); comment_edit->setText(msg->comment); } - - btn_box = new QDialogButtonBox(QDialogButtonBox::Ok | QDialogButtonBox::Cancel); validateName(name_edit->text()); - form_layout->addRow(btn_box); - setFixedWidth(parent->width() * 0.9); connect(name_edit, &QLineEdit::textEdited, this, &EditMessageDialog::validateName); connect(btn_box, &QDialogButtonBox::accepted, this, &QDialog::accept); diff --git a/tools/cabana/detailwidget.h b/tools/cabana/detailwidget.h index 10c6a6c46e..ed6a865f53 100644 --- a/tools/cabana/detailwidget.h +++ b/tools/cabana/detailwidget.h @@ -21,6 +21,7 @@ public: QString original_name; QDialogButtonBox *btn_box; QLineEdit *name_edit; + QLineEdit *node; QTextEdit *comment_edit; QLabel *error_label; QSpinBox *size_spin; diff --git a/tools/cabana/historylog.cc b/tools/cabana/historylog.cc index 7381884cf3..5330549963 100644 --- a/tools/cabana/historylog.cc +++ b/tools/cabana/historylog.cc @@ -162,9 +162,7 @@ std::deque HistoryLogModel::fetchData(uint64_t from_ti return msgs; } else { assert(min_time == 0); - auto first = std::upper_bound(events.cbegin(), events.cend(), from_time, [](uint64_t ts, auto e) { - return ts < e->mono_time; - }); + auto first = std::upper_bound(events.cbegin(), events.cend(), from_time, CompareCanEvent()); auto msgs = fetchData(first, events.cend(), 0); if (update_colors) { for (auto it = msgs.begin(); it != msgs.end(); ++it) { diff --git a/tools/cabana/mainwin.cc b/tools/cabana/mainwin.cc index d3caf492a6..5cd5e70267 100644 --- a/tools/cabana/mainwin.cc +++ b/tools/cabana/mainwin.cc @@ -439,11 +439,11 @@ void MainWindow::saveFile(DBCFile *dbc_file) { if (!dbc_file->filename.isEmpty()) { dbc_file->save(); updateLoadSaveMenus(); + UndoStack::instance()->setClean(); + statusBar()->showMessage(tr("File saved"), 2000); } else if (!dbc_file->isEmpty()) { saveFileAs(dbc_file); } - UndoStack::instance()->setClean(); - statusBar()->showMessage(tr("File saved"), 2000); } void MainWindow::saveFileAs(DBCFile *dbc_file) { @@ -451,6 +451,8 @@ void MainWindow::saveFileAs(DBCFile *dbc_file) { QString fn = QFileDialog::getSaveFileName(this, title, QDir::cleanPath(settings.last_dir + "/untitled.dbc"), tr("DBC (*.dbc)")); if (!fn.isEmpty()) { dbc_file->saveAs(fn); + UndoStack::instance()->setClean(); + statusBar()->showMessage(tr("File saved as %1").arg(fn), 2000); updateRecentFiles(fn); updateLoadSaveMenus(); } @@ -606,7 +608,13 @@ void MainWindow::closeEvent(QCloseEvent *event) { settings.video_splitter_state = video_splitter->saveState(); } settings.message_header_state = messages_widget->saveHeaderState(); - settings.save(); + + auto status = settings.save(); + if (status == QSettings::AccessError) { + QString error = tr("Failed to write settings to [%1]: access denied").arg(Settings::filePath()); + qDebug() << error; + QMessageBox::warning(this, tr("Failed to write settings"), error); + } QWidget::closeEvent(event); } diff --git a/tools/cabana/settings.cc b/tools/cabana/settings.cc index d0cada680a..ee345c490c 100644 --- a/tools/cabana/settings.cc +++ b/tools/cabana/settings.cc @@ -6,19 +6,14 @@ #include #include #include -#include #include #include "tools/cabana/util.h" Settings settings; -Settings::Settings() { - load(); -} - -void Settings::save() { - QSettings s("settings", QSettings::IniFormat); +QSettings::Status Settings::save() { + QSettings s(filePath(), QSettings::IniFormat); s.setValue("fps", fps); s.setValue("max_cached_minutes", max_cached_minutes); s.setValue("chart_height", chart_height); @@ -39,10 +34,12 @@ void Settings::save() { s.setValue("log_path", log_path); s.setValue("drag_direction", drag_direction); s.setValue("suppress_defined_signals", suppress_defined_signals); + s.sync(); + return s.status(); } void Settings::load() { - QSettings s("settings", QSettings::IniFormat); + QSettings s(filePath(), QSettings::IniFormat); fps = s.value("fps", 10).toInt(); max_cached_minutes = s.value("max_cached_minutes", 30).toInt(); chart_height = s.value("chart_height", 200).toInt(); diff --git a/tools/cabana/settings.h b/tools/cabana/settings.h index b8a3797f86..42073a72de 100644 --- a/tools/cabana/settings.h +++ b/tools/cabana/settings.h @@ -1,11 +1,13 @@ #pragma once +#include #include #include #include #include #include #include +#include #include #define LIGHT_THEME 1 @@ -22,9 +24,10 @@ public: AlwaysBE, }; - Settings(); - void save(); + Settings() {} + QSettings::Status save(); void load(); + inline static QString filePath() { return QApplication::applicationDirPath() + "/settings"; } int fps = 10; int max_cached_minutes = 30; diff --git a/tools/cabana/signalview.cc b/tools/cabana/signalview.cc index 1b1c8dac22..b2798e9f3b 100644 --- a/tools/cabana/signalview.cc +++ b/tools/cabana/signalview.cc @@ -36,7 +36,7 @@ SignalModel::SignalModel(QObject *parent) : root(new Item), QAbstractItemModel(p void SignalModel::insertItem(SignalModel::Item *parent_item, int pos, const cabana::Signal *sig) { Item *item = new Item{.sig = sig, .parent = parent_item, .title = sig->name, .type = Item::Sig}; parent_item->children.insert(pos, item); - QString titles[]{"Name", "Size", "Little Endian", "Signed", "Offset", "Factor", "Type", "Multiplex Value", "Extra Info", + QString titles[]{"Name", "Size", "Node", "Little Endian", "Signed", "Offset", "Factor", "Type", "Multiplex Value", "Extra Info", "Unit", "Comment", "Minimum Value", "Maximum Value", "Value Descriptions"}; for (int i = 0; i < std::size(titles); ++i) { item->children.push_back(new Item{.sig = sig, .parent = item, .title = titles[i], .type = (Item::Type)(i + Item::Name)}); @@ -134,6 +134,7 @@ QVariant SignalModel::data(const QModelIndex &index, int role) const { case Item::Sig: return item->sig_val; case Item::Name: return item->sig->name; case Item::Size: return item->sig->size; + case Item::Node: return item->sig->receiver_name; case Item::SignalType: return signalTypeToString(item->sig->type); case Item::MultiplexValue: return item->sig->multiplex_value; case Item::Offset: return doubleToString(item->sig->offset); @@ -172,6 +173,7 @@ bool SignalModel::setData(const QModelIndex &index, const QVariant &value, int r switch (item->type) { case Item::Name: s.name = value.toString(); break; case Item::Size: s.size = value.toInt(); break; + case Item::Node: s.receiver_name = value.toString().trimmed(); break; case Item::SignalType: s.type = (cabana::Signal::Type)value.toInt(); break; case Item::MultiplexValue: s.multiplex_value = value.toInt(); break; case Item::Endian: s.is_little_endian = value.toBool(); break; @@ -195,11 +197,11 @@ void SignalModel::showExtraInfo(const QModelIndex &index) { if (item->type == Item::ExtraInfo) { if (!item->parent->extra_expanded) { item->parent->extra_expanded = true; - beginInsertRows(index.parent(), 7, 13); + beginInsertRows(index.parent(), Item::ExtraInfo - 2, Item::Desc - 2); endInsertRows(); } else { item->parent->extra_expanded = false; - beginRemoveRows(index.parent(), 7, 13); + beginRemoveRows(index.parent(), Item::ExtraInfo - 2, Item::Desc - 2); endRemoveRows(); } } @@ -267,6 +269,7 @@ void SignalModel::handleSignalRemoved(const cabana::Signal *sig) { SignalItemDelegate::SignalItemDelegate(QObject *parent) : QStyledItemDelegate(parent) { name_validator = new NameValidator(this); + node_validator = new QRegExpValidator(QRegExp("^\\w+(,\\w+)*$"), this); double_validator = new DoubleValidator(this); label_font.setPointSize(8); @@ -382,12 +385,14 @@ void SignalItemDelegate::paint(QPainter *painter, const QStyleOptionViewItem &op QWidget *SignalItemDelegate::createEditor(QWidget *parent, const QStyleOptionViewItem &option, const QModelIndex &index) const { auto item = (SignalModel::Item *)index.internalPointer(); - if (item->type == SignalModel::Item::Name || item->type == SignalModel::Item::Offset || + if (item->type == SignalModel::Item::Name || item->type == SignalModel::Item::Node || item->type == SignalModel::Item::Offset || item->type == SignalModel::Item::Factor || item->type == SignalModel::Item::MultiplexValue || item->type == SignalModel::Item::Min || item->type == SignalModel::Item::Max) { QLineEdit *e = new QLineEdit(parent); e->setFrame(false); - e->setValidator(item->type == SignalModel::Item::Name ? name_validator : double_validator); + if (item->type == SignalModel::Item::Name) e->setValidator(name_validator); + else if (item->type == SignalModel::Item::Node) e->setValidator(node_validator); + else e->setValidator(double_validator); if (item->type == SignalModel::Item::Name) { QCompleter *completer = new QCompleter(dbc()->signalNames()); @@ -395,7 +400,6 @@ QWidget *SignalItemDelegate::createEditor(QWidget *parent, const QStyleOptionVie completer->setFilterMode(Qt::MatchContains); e->setCompleter(completer); } - return e; } else if (item->type == SignalModel::Item::Size) { QSpinBox *spin = new QSpinBox(parent); diff --git a/tools/cabana/signalview.h b/tools/cabana/signalview.h index 9d8571d0b8..bcf0019bc4 100644 --- a/tools/cabana/signalview.h +++ b/tools/cabana/signalview.h @@ -17,7 +17,7 @@ class SignalModel : public QAbstractItemModel { Q_OBJECT public: struct Item { - enum Type {Root, Sig, Name, Size, Endian, Signed, Offset, Factor, SignalType, MultiplexValue, ExtraInfo, Unit, Comment, Min, Max, Desc }; + enum Type {Root, Sig, Name, Size, Node, Endian, Signed, Offset, Factor, SignalType, MultiplexValue, ExtraInfo, Unit, Comment, Min, Max, Desc }; ~Item() { qDeleteAll(children); } inline int row() { return parent->children.indexOf(this); } @@ -87,7 +87,7 @@ public: void updateEditorGeometry(QWidget *editor, const QStyleOptionViewItem &option, const QModelIndex &index) const override; void setModelData(QWidget *editor, QAbstractItemModel *model, const QModelIndex &index) const override; - QValidator *name_validator, *double_validator; + QValidator *name_validator, *double_validator, *node_validator; QFont label_font, minmax_font; const int color_label_width = 18; mutable QSize button_size; diff --git a/tools/cabana/streams/abstractstream.cc b/tools/cabana/streams/abstractstream.cc index c8945aec64..6fa479815d 100644 --- a/tools/cabana/streams/abstractstream.cc +++ b/tools/cabana/streams/abstractstream.cc @@ -27,6 +27,11 @@ AbstractStream::AbstractStream(QObject *parent) : QObject(parent) { emit StreamNotifier::instance()->changingStream(); delete can; can = this; + // TODO: add method stop() to class AbstractStream + QObject::connect(qApp, &QApplication::aboutToQuit, can, []() { + qDebug() << "stopping stream thread"; + can->pause(true); + }); emit StreamNotifier::instance()->streamStarted(); }); } @@ -157,18 +162,14 @@ void AbstractStream::mergeEvents(std::vector::const_iterator first, std } } - auto compare = [](const CanEvent *l, const CanEvent *r) { - return l->mono_time < r->mono_time; - }; - for (auto &[id, new_e] : new_events_map) { auto &e = events_[id]; - auto insert_pos = std::upper_bound(e.cbegin(), e.cend(), new_e.front(), compare); + auto insert_pos = std::upper_bound(e.cbegin(), e.cend(), new_e.front()->mono_time, CompareCanEvent()); e.insert(insert_pos, new_e.cbegin(), new_e.cend()); } if (!new_events.empty()) { - auto insert_pos = std::upper_bound(all_events_.cbegin(), all_events_.cend(), new_events.front(), compare); + auto insert_pos = std::upper_bound(all_events_.cbegin(), all_events_.cend(), new_events.front()->mono_time, CompareCanEvent()); all_events_.insert(insert_pos, new_events.cbegin(), new_events.cend()); } @@ -196,15 +197,11 @@ inline QColor blend(const QColor &a, const QColor &b) { // Calculate the frequency of the past minute. double calc_freq(const MessageId &msg_id, double current_sec) { - auto compare = [](const CanEvent *e, uint64_t mono_time) { - return e->mono_time < mono_time; - }; - const auto &events = can->events(msg_id); uint64_t cur_mono_time = (can->routeStartTime() + current_sec) * 1e9; uint64_t first_mono_time = std::max(0, cur_mono_time - 59 * 1e9); - auto first = std::lower_bound(events.begin(), events.end(), first_mono_time, compare); - auto second = std::lower_bound(first, events.end(), cur_mono_time, compare); + auto first = std::lower_bound(events.begin(), events.end(), first_mono_time, CompareCanEvent()); + auto second = std::lower_bound(first, events.end(), cur_mono_time, CompareCanEvent()); if (first != events.end() && second != events.end()) { double duration = ((*second)->mono_time - (*first)->mono_time) / 1e9; uint32_t count = std::distance(first, second); diff --git a/tools/cabana/streams/abstractstream.h b/tools/cabana/streams/abstractstream.h index 62ab3f2f4b..3a89d4e57d 100644 --- a/tools/cabana/streams/abstractstream.h +++ b/tools/cabana/streams/abstractstream.h @@ -30,7 +30,7 @@ struct CanData { std::vector> bit_change_counts; std::vector last_delta; std::vector same_delta_counter; - double last_freq_update_ts = seconds_since_boot(); + double last_freq_update_ts = 0; }; struct CanEvent { @@ -41,6 +41,11 @@ struct CanEvent { uint8_t dat[]; }; +struct CompareCanEvent { + constexpr bool operator()(const CanEvent *const e, uint64_t ts) const { return e->mono_time < ts; } + constexpr bool operator()(uint64_t ts, const CanEvent *const e) const { return ts < e->mono_time; } +}; + struct BusConfig { int can_speed_kbps = 500; int data_speed_kbps = 2000; diff --git a/tools/cabana/streams/livestream.cc b/tools/cabana/streams/livestream.cc index 3f8397f56c..17805a0b66 100644 --- a/tools/cabana/streams/livestream.cc +++ b/tools/cabana/streams/livestream.cc @@ -102,12 +102,8 @@ void LiveStream::updateEvents() { uint64_t last_ts = post_last_event && speed_ == 1.0 ? all_events_.back()->mono_time : first_event_ts + (nanos_since_boot() - first_update_ts) * speed_; - auto first = std::upper_bound(all_events_.cbegin(), all_events_.cend(), current_event_ts, [](uint64_t ts, auto e) { - return ts < e->mono_time; - }); - auto last = std::upper_bound(first, all_events_.cend(), last_ts, [](uint64_t ts, auto e) { - return ts < e->mono_time; - }); + auto first = std::upper_bound(all_events_.cbegin(), all_events_.cend(), current_event_ts, CompareCanEvent()); + auto last = std::upper_bound(first, all_events_.cend(), last_ts, CompareCanEvent()); for (auto it = first; it != last; ++it) { const CanEvent *e = *it; diff --git a/tools/cabana/tests/test_cabana.cc b/tools/cabana/tests/test_cabana.cc index a3d014dd2c..d114f72ea5 100644 --- a/tools/cabana/tests/test_cabana.cc +++ b/tools/cabana/tests/test_cabana.cc @@ -6,8 +6,7 @@ #include "tools/cabana/dbc/dbcmanager.h" #include "tools/cabana/streams/abstractstream.h" -// demo route, first segment -const std::string TEST_RLOG_URL = "https://commadata2.blob.core.windows.net/commadata2/a2a0ccea32023010/2023-07-27--13-01-19/0/rlog.bz2"; +const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2"; TEST_CASE("DBCFile::generateDBC") { QString fn = QString("%1/%2.dbc").arg(OPENDBC_FILE_PATH, "tesla_can"); diff --git a/tools/cabana/tools/findsignal.cc b/tools/cabana/tools/findsignal.cc index b19babdf88..51d86f5964 100644 --- a/tools/cabana/tools/findsignal.cc +++ b/tools/cabana/tools/findsignal.cc @@ -37,10 +37,10 @@ void FindSignalModel::search(std::function cmp) { filtered_signals.reserve(prev_sigs.size()); QtConcurrent::blockingMap(prev_sigs, [&](auto &s) { const auto &events = can->events(s.id); - auto first = std::upper_bound(events.cbegin(), events.cend(), s.mono_time, [](uint64_t ts, auto &e) { return ts < e->mono_time; }); + auto first = std::upper_bound(events.cbegin(), events.cend(), s.mono_time, CompareCanEvent()); auto last = events.cend(); if (last_time < std::numeric_limits::max()) { - last = std::upper_bound(events.cbegin(), events.cend(), last_time, [](uint64_t ts, auto &e) { return ts < e->mono_time; }); + last = std::upper_bound(events.cbegin(), events.cend(), last_time, CompareCanEvent()); } auto it = std::find_if(first, last, [&](const CanEvent *e) { return cmp(get_raw_value(e->dat, e->size, s.sig)); }); @@ -225,7 +225,7 @@ void FindSignalDlg::setInitialSignals() { for (auto it = can->last_msgs.cbegin(); it != can->last_msgs.cend(); ++it) { if (buses.isEmpty() || buses.contains(it.key().source) && (addresses.isEmpty() || addresses.contains(it.key().address))) { const auto &events = can->events(it.key()); - auto e = std::lower_bound(events.cbegin(), events.cend(), first_time, [](auto e, uint64_t ts) { return e->mono_time < ts; }); + auto e = std::lower_bound(events.cbegin(), events.cend(), first_time, CompareCanEvent()); if (e != events.cend()) { const int total_size = it.value().dat.size() * 8; for (int size = min_size->value(); size <= max_size->value(); ++size) { diff --git a/tools/cabana/util.cc b/tools/cabana/util.cc index 951856e32d..4c21530774 100644 --- a/tools/cabana/util.cc +++ b/tools/cabana/util.cc @@ -1,7 +1,6 @@ #include "tools/cabana/util.h" #include -#include #include #include #include @@ -56,6 +55,10 @@ std::pair SegmentTree::get_minmax(int n, int left, int right, in MessageBytesDelegate::MessageBytesDelegate(QObject *parent, bool multiple_lines) : multiple_lines(multiple_lines), QStyledItemDelegate(parent) { fixed_font = QFontDatabase::systemFont(QFontDatabase::FixedFont); byte_size = QFontMetrics(fixed_font).size(Qt::TextSingleLine, "00 ") + QSize(0, 2); + for (int i = 0; i < 256; ++i) { + hex_text_table[i].setText(QStringLiteral("%1").arg(i, 2, 16, QLatin1Char('0')).toUpper()); + hex_text_table[i].prepare({}, fixed_font); + } } int MessageBytesDelegate::widthForBytes(int n) const { @@ -107,7 +110,7 @@ void MessageBytesDelegate::paint(QPainter *painter, const QStyleOptionViewItem & } else if (option.state & QStyle::State_Selected) { painter->setPen(option.palette.color(QPalette::HighlightedText)); } - painter->drawText(r, Qt::AlignCenter, toHex(byte_list[i])); + utils::drawStaticText(painter, r, hex_text_table[(uint8_t)(byte_list[i])]); } painter->setFont(old_font); painter->setPen(old_pen); diff --git a/tools/cabana/util.h b/tools/cabana/util.h index 9e93e74833..2c1bc5cf7b 100644 --- a/tools/cabana/util.h +++ b/tools/cabana/util.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -10,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -75,6 +78,7 @@ public: int widthForBytes(int n) const; private: + std::array hex_text_table; QFont fixed_font; QSize byte_size = {}; bool multiple_lines = false; @@ -102,6 +106,10 @@ void setTheme(int theme); inline QString formatSeconds(int seconds) { return QDateTime::fromSecsSinceEpoch(seconds, Qt::UTC).toString(seconds > 60 * 60 ? "hh:mm:ss" : "mm:ss"); } +inline void drawStaticText(QPainter *p, const QRect &r, const QStaticText &text) { + auto size = (r.size() - text.size()) / 2; + p->drawStaticText(r.left() + size.width(), r.top() + size.height(), text); +} } class ToolButton : public QToolButton { diff --git a/tools/cabana/videowidget.cc b/tools/cabana/videowidget.cc index f535c6b729..098f26841c 100644 --- a/tools/cabana/videowidget.cc +++ b/tools/cabana/videowidget.cc @@ -159,13 +159,13 @@ Slider::Slider(QWidget *parent) : thumbnail_label(parent), QSlider(Qt::Horizonta qlog_future = std::make_unique>(QtConcurrent::run(this, &Slider::parseQLog)); } }); -} - -Slider::~Slider() { - abort_parse_qlog = true; - if (qlog_future) { - qlog_future->waitForFinished(); - } + QObject::connect(qApp, &QApplication::aboutToQuit, [this]() { + abort_parse_qlog = true; + if (qlog_future && qlog_future->isRunning()) { + qDebug() << "stopping thumbnail thread"; + qlog_future->waitForFinished(); + } + }); } AlertInfo Slider::alertInfo(double seconds) { diff --git a/tools/cabana/videowidget.h b/tools/cabana/videowidget.h index 7ddd315e48..c17dd67e2a 100644 --- a/tools/cabana/videowidget.h +++ b/tools/cabana/videowidget.h @@ -37,7 +37,6 @@ class Slider : public QSlider { public: Slider(QWidget *parent); - ~Slider(); double currentSecond() const { return value() / factor; } void setCurrentSecond(double sec) { setValue(sec * factor); } void setTimeRange(double min, double max); diff --git a/tools/gpstest/rpc_server.py b/tools/gpstest/rpc_server.py index cdedd8ea57..798237142d 100644 --- a/tools/gpstest/rpc_server.py +++ b/tools/gpstest/rpc_server.py @@ -3,6 +3,7 @@ import time import shutil from datetime import datetime from collections import defaultdict +from openpilot.system.hardware.hw import Paths import rpyc from rpyc.utils.server import ThreadedServer @@ -18,7 +19,6 @@ MATCH_NUM = 10 REPORT_STATS = 10 EPHEM_CACHE = "/data/params/d/LaikadEphemerisV3" -DOWNLOAD_CACHE = "/tmp/comma_download_cache" SERVER_LOG_FILE = "/tmp/fuzzy_server.log" server_log = open(SERVER_LOG_FILE, "w+") @@ -162,7 +162,7 @@ class RemoteCheckerService(rpyc.Service): if os.path.exists(EPHEM_CACHE): os.remove(EPHEM_CACHE) - shutil.rmtree(DOWNLOAD_CACHE, ignore_errors=True) + shutil.rmtree(Paths.download_cache_root(), ignore_errors=True) ret = self.run_checker(slat, slon, salt, sockets, procs, timeout) kill_procs(procs) diff --git a/tools/lib/auth_config.py b/tools/lib/auth_config.py index 8863dd57a2..bd76761043 100644 --- a/tools/lib/auth_config.py +++ b/tools/lib/auth_config.py @@ -13,9 +13,6 @@ if PC: else: CONFIG_DIR = "/tmp/.comma" -mkdirs_exists_ok(CONFIG_DIR) - - def get_token(): try: with open(os.path.join(CONFIG_DIR, 'auth.json')) as f: @@ -26,9 +23,13 @@ def get_token(): def set_token(token): + mkdirs_exists_ok(CONFIG_DIR) with open(os.path.join(CONFIG_DIR, 'auth.json'), 'w') as f: json.dump({'access_token': token}, f) def clear_token(): - os.unlink(os.path.join(CONFIG_DIR, 'auth.json')) + try: + os.unlink(os.path.join(CONFIG_DIR, 'auth.json')) + except FileNotFoundError: + pass diff --git a/tools/lib/cache.py b/tools/lib/cache.py index d142955e59..fd214f6bb5 100644 --- a/tools/lib/cache.py +++ b/tools/lib/cache.py @@ -2,10 +2,10 @@ import os import urllib.parse from openpilot.common.file_helpers import mkdirs_exists_ok -DEFAULT_CACHE_DIR = os.path.expanduser("~/.commacache") +DEFAULT_CACHE_DIR = os.getenv("CACHE_ROOT", os.path.expanduser("~/.commacache")) -def cache_path_for_file_path(fn, cache_prefix=None): - dir_ = os.path.join(DEFAULT_CACHE_DIR, "local") +def cache_path_for_file_path(fn, cache_dir=DEFAULT_CACHE_DIR): + dir_ = os.path.join(cache_dir, "local") mkdirs_exists_ok(dir_) fn_parsed = urllib.parse.urlparse(fn) if fn_parsed.scheme == '': diff --git a/tools/lib/framereader.py b/tools/lib/framereader.py index cbc9310790..06fee6857d 100644 --- a/tools/lib/framereader.py +++ b/tools/lib/framereader.py @@ -12,7 +12,7 @@ import numpy as np from lru import LRU import _io -from openpilot.tools.lib.cache import cache_path_for_file_path +from openpilot.tools.lib.cache import cache_path_for_file_path, DEFAULT_CACHE_DIR from openpilot.tools.lib.exceptions import DataUnreadableError from openpilot.common.file_helpers import atomic_write_in_dir @@ -106,8 +106,8 @@ def cache_fn(func): if kwargs.pop('no_cache', None): cache_path = None else: - cache_prefix = kwargs.pop('cache_prefix', None) - cache_path = cache_path_for_file_path(fn, cache_prefix) + cache_dir = kwargs.pop('cache_dir', DEFAULT_CACHE_DIR) + cache_path = cache_path_for_file_path(fn, cache_dir) if cache_path and os.path.exists(cache_path): with open(cache_path, "rb") as cache_file: @@ -140,18 +140,18 @@ def index_stream(fn, typ): } -def index_videos(camera_paths, cache_prefix=None): +def index_videos(camera_paths, cache_dir=DEFAULT_CACHE_DIR): """Requires that paths in camera_paths are contiguous and of the same type.""" if len(camera_paths) < 1: raise ValueError("must provide at least one video to index") frame_type = fingerprint_video(camera_paths[0]) for fn in camera_paths: - index_video(fn, frame_type, cache_prefix) + index_video(fn, frame_type, cache_dir) -def index_video(fn, frame_type=None, cache_prefix=None): - cache_path = cache_path_for_file_path(fn, cache_prefix) +def index_video(fn, frame_type=None, cache_dir=DEFAULT_CACHE_DIR): + cache_path = cache_path_for_file_path(fn, cache_dir) if os.path.exists(cache_path): return @@ -160,16 +160,16 @@ def index_video(fn, frame_type=None, cache_prefix=None): frame_type = fingerprint_video(fn[0]) if frame_type == FrameType.h265_stream: - index_stream(fn, "hevc", cache_prefix=cache_prefix) + index_stream(fn, "hevc", cache_dir=cache_dir) else: raise NotImplementedError("Only h265 supported") -def get_video_index(fn, frame_type, cache_prefix=None): - cache_path = cache_path_for_file_path(fn, cache_prefix) +def get_video_index(fn, frame_type, cache_dir=DEFAULT_CACHE_DIR): + cache_path = cache_path_for_file_path(fn, cache_dir) if not os.path.exists(cache_path): - index_video(fn, frame_type, cache_prefix) + index_video(fn, frame_type, cache_dir) if not os.path.exists(cache_path): return None @@ -284,13 +284,13 @@ class BaseFrameReader: raise NotImplementedError -def FrameReader(fn, cache_prefix=None, readahead=False, readbehind=False, index_data=None): +def FrameReader(fn, cache_dir=DEFAULT_CACHE_DIR, readahead=False, readbehind=False, index_data=None): frame_type = fingerprint_video(fn) if frame_type == FrameType.raw: return RawFrameReader(fn) elif frame_type in (FrameType.h265_stream,): if not index_data: - index_data = get_video_index(fn, frame_type, cache_prefix) + index_data = get_video_index(fn, frame_type, cache_dir) return StreamFrameReader(fn, frame_type, index_data, readahead=readahead, readbehind=readbehind) else: raise NotImplementedError(frame_type) diff --git a/tools/lib/tests/test_caching.py b/tools/lib/tests/test_caching.py index 7702b52e3e..b893d9815d 100755 --- a/tools/lib/tests/test_caching.py +++ b/tools/lib/tests/test_caching.py @@ -2,7 +2,6 @@ import os import unittest from openpilot.tools.lib.url_file import URLFile -from openpilot.selfdrive.test.helpers import temporary_cache_dir class TestFileDownload(unittest.TestCase): @@ -31,7 +30,6 @@ class TestFileDownload(unittest.TestCase): self.assertEqual(file_cached.get_length(), file_downloaded.get_length()) self.assertEqual(response_cached, response_downloaded) - @temporary_cache_dir def test_small_file(self): # Make sure we don't force cache os.environ["FILEREADER_CACHE"] = "0" @@ -52,7 +50,6 @@ class TestFileDownload(unittest.TestCase): for i in range(length // 100): self.compare_loads(small_file_url, 100 * i, 100) - @temporary_cache_dir def test_large_file(self): large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/qlog.bz2" # Load the end 100 bytes of both files diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index e93f8e715c..98a52cae27 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -8,12 +8,11 @@ from hashlib import sha256 from io import BytesIO from tenacity import retry, wait_random_exponential, stop_after_attempt from openpilot.common.file_helpers import mkdirs_exists_ok, atomic_write_in_dir +from openpilot.system.hardware.hw import Paths # Cache chunk size K = 1000 CHUNK_SIZE = 1000 * K -CACHE_DIR = os.environ.get("COMMA_CACHE", "/tmp/comma_download_cache/") - def hash_256(link): hsh = str(sha256((link.split("?")[0]).encode('utf-8')).hexdigest()) @@ -38,7 +37,7 @@ class URLFile: self._curl = self._tlocal.curl except AttributeError: self._curl = self._tlocal.curl = pycurl.Curl() - mkdirs_exists_ok(CACHE_DIR) + mkdirs_exists_ok(Paths.download_cache_root()) def __enter__(self): return self @@ -66,7 +65,7 @@ class URLFile: def get_length(self): if self._length is not None: return self._length - file_length_path = os.path.join(CACHE_DIR, hash_256(self._url) + "_length") + file_length_path = os.path.join(Paths.download_cache_root(), hash_256(self._url) + "_length") if os.path.exists(file_length_path) and not self._force_download: with open(file_length_path) as file_length: content = file_length.read() @@ -93,7 +92,7 @@ class URLFile: self._pos = position chunk_number = self._pos / CHUNK_SIZE file_name = hash_256(self._url) + "_" + str(chunk_number) - full_path = os.path.join(CACHE_DIR, str(file_name)) + full_path = os.path.join(Paths.download_cache_root(), str(file_name)) data = None # If we don't have a file, download it if not os.path.exists(full_path): diff --git a/tools/mac_setup.sh b/tools/mac_setup.sh index c767131f51..0c9627ca4e 100755 --- a/tools/mac_setup.sh +++ b/tools/mac_setup.sh @@ -49,7 +49,7 @@ brew "pyenv" brew "pyenv-virtualenv" brew "qt@5" brew "zeromq" -brew "gcc@12" +brew "gcc@13" cask "gcc-arm-embedded" brew "portaudio" EOS @@ -74,6 +74,27 @@ export PYCURL_SSL_LIBRARY=openssl $DIR/install_python_dependencies.sh echo "[ ] installed python dependencies t=$SECONDS" +# brew does not link qt5 by default +# check if qt5 can be linked, if not, prompt the user to link it +QT_BIN_LOCATION="$(command -v lupdate || :)" +if [ -n "$QT_BIN_LOCATION" ]; then + # if qt6 is linked, prompt the user to unlink it and link the right version + QT_BIN_VERSION="$(lupdate -version | awk '{print $NF}')" + if [[ ! "$QT_BIN_VERSION" =~ 5\.[0-9]+\.[0-9]+ ]]; then + echo + echo "lupdate/lrelease available at PATH is $QT_BIN_VERSION" + if [[ "$QT_BIN_LOCATION" == "$(brew --prefix)/"* ]]; then + echo "Run the following command to link qt5:" + echo "brew unlink qt@6 && brew link qt@5" + else + echo "Remove conflicting qt entries from PATH and run the following command to link qt5:" + echo "brew link qt@5" + fi + fi +else + brew link qt@5 +fi + echo echo "---- OPENPILOT SETUP DONE ----" echo "Open a new shell or configure your active shell env by running:" diff --git a/tools/replay/filereader.cc b/tools/replay/filereader.cc index 88879067c9..22af7f5f86 100644 --- a/tools/replay/filereader.cc +++ b/tools/replay/filereader.cc @@ -3,11 +3,12 @@ #include #include "common/util.h" +#include "system/hardware/hw.h" #include "tools/replay/util.h" std::string cacheFilePath(const std::string &url) { static std::string cache_path = [] { - const std::string comma_cache = util::getenv("COMMA_CACHE", "/tmp/comma_download_cache/"); + const std::string comma_cache = Path::download_cache_root(); util::create_directories(comma_cache, 0755); return comma_cache.back() == '/' ? comma_cache : comma_cache + "/"; }(); diff --git a/tools/replay/tests/test_replay.cc b/tools/replay/tests/test_replay.cc index 393a561c0f..05c0ca8ac4 100644 --- a/tools/replay/tests/test_replay.cc +++ b/tools/replay/tests/test_replay.cc @@ -12,7 +12,7 @@ const std::string TEST_RLOG_URL = "https://commadataci.blob.core.windows.net/openpilotci/0c94aa1e1296d7c6/2021-05-05--19-48-37/0/rlog.bz2"; const std::string TEST_RLOG_CHECKSUM = "5b966d4bb21a100a8c4e59195faeb741b975ccbe268211765efd1763d892bfb3"; -bool donload_to_file(const std::string &url, const std::string &local_file, int chunk_size = 5 * 1024 * 1024, int retries = 3) { +bool download_to_file(const std::string &url, const std::string &local_file, int chunk_size = 5 * 1024 * 1024, int retries = 3) { do { if (httpDownload(url, local_file, chunk_size)) { return true; @@ -29,7 +29,7 @@ TEST_CASE("httpMultiPartDownload") { const size_t chunk_size = 5 * 1024 * 1024; std::string content; SECTION("download to file") { - REQUIRE(donload_to_file(TEST_RLOG_URL, filename, chunk_size)); + REQUIRE(download_to_file(TEST_RLOG_URL, filename, chunk_size)); content = util::read_file(filename); } SECTION("download to buffer") { @@ -110,23 +110,36 @@ void read_segment(int n, const SegmentFile &segment_file, uint32_t flags) { loop.exec(); } -TEST_CASE("Route") { - // Create a local route from remote for testing - Route remote_route(DEMO_ROUTE); - REQUIRE(remote_route.load()); - char tmp_path[] = "/tmp/root_XXXXXX"; - const std::string data_dir = mkdtemp(tmp_path); - const std::string route_name = DEMO_ROUTE.mid(17).toStdString(); - for (int i = 0; i < 2; ++i) { - std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i); - util::create_directories(log_path, 0755); - REQUIRE(donload_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2")); - REQUIRE(donload_to_file(remote_route.at(i).road_cam.toStdString(), log_path + "fcamera.hevc")); - REQUIRE(donload_to_file(remote_route.at(i).driver_cam.toStdString(), log_path + "dcamera.hevc")); - REQUIRE(donload_to_file(remote_route.at(i).wide_road_cam.toStdString(), log_path + "ecamera.hevc")); - REQUIRE(donload_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts")); +std::string download_demo_route() { + static std::string data_dir; + + if (data_dir == "") { + char tmp_path[] = "/tmp/root_XXXXXX"; + data_dir = mkdtemp(tmp_path); + + Route remote_route(DEMO_ROUTE); + assert(remote_route.load()); + + // Create a local route from remote for testing + const std::string route_name = DEMO_ROUTE.mid(17).toStdString(); + for (int i = 0; i < 2; ++i) { + std::string log_path = util::string_format("%s/%s--%d/", data_dir.c_str(), route_name.c_str(), i); + util::create_directories(log_path, 0755); + REQUIRE(download_to_file(remote_route.at(i).rlog.toStdString(), log_path + "rlog.bz2")); + REQUIRE(download_to_file(remote_route.at(i).road_cam.toStdString(), log_path + "fcamera.hevc")); + REQUIRE(download_to_file(remote_route.at(i).driver_cam.toStdString(), log_path + "dcamera.hevc")); + REQUIRE(download_to_file(remote_route.at(i).wide_road_cam.toStdString(), log_path + "ecamera.hevc")); + REQUIRE(download_to_file(remote_route.at(i).qcamera.toStdString(), log_path + "qcamera.ts")); + } } + return data_dir; +} + + +TEST_CASE("Route") { + std::string data_dir = download_demo_route(); + SECTION("Local route") { auto flags = GENERATE(REPLAY_FLAG_DCAM | REPLAY_FLAG_ECAM, REPLAY_FLAG_QCAMERA); Route route(DEMO_ROUTE, QString::fromStdString(data_dir)); diff --git a/tools/replay/util.cc b/tools/replay/util.cc index 172d545c75..2c2de69e78 100644 --- a/tools/replay/util.cc +++ b/tools/replay/util.cc @@ -4,6 +4,7 @@ #include #include +#include #include #include #include diff --git a/tools/sim/launch_openpilot.sh b/tools/sim/launch_openpilot.sh index d5922a4819..361f7b18ea 100755 --- a/tools/sim/launch_openpilot.sh +++ b/tools/sim/launch_openpilot.sh @@ -12,5 +12,8 @@ if [[ "$CI" ]]; then export BLOCK="${BLOCK},ui" fi +SCRIPT_DIR=$(dirname "$0") +OPENPILOT_DIR=$SCRIPT_DIR/../../ + DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -cd ../../selfdrive/manager && exec ./manager.py +cd $OPENPILOT_DIR/selfdrive/manager && exec ./manager.py