move some tools back to root (#38235)

.github/workflows/docs.yaml

@@ -45,7 +45,7 @@ jobs:
       run: |
         set -x
 
-        source release/identity.sh
+        source tools/release/identity.sh
 
         cd openpilot-docs
         git checkout --orphan tmp

.github/workflows/release.yaml

@@ -26,6 +26,6 @@ jobs:
       with:
         submodules: true
         fetch-depth: 0
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Push master-ci
-      run: BRANCH=__nightly release/build_stripped.sh
+      run: BRANCH=__nightly tools/release/build_stripped.sh

.github/workflows/repo-maintenance.yaml

@@ -17,7 +17,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: uv lock
       run: uv lock --upgrade
     - name: uv pip tree

.github/workflows/tests.yaml

@@ -46,8 +46,8 @@ jobs:
         command: git lfs pull
     - name: Build devel
       timeout-minutes: 1
-      run: TARGET_DIR=$STRIPPED_DIR release/build_stripped.sh
-    - run: ./openpilot/tools/op.sh setup
+      run: TARGET_DIR=$STRIPPED_DIR tools/release/build_stripped.sh
+    - run: ./tools/op.sh setup
     - name: Build openpilot and run checks
       timeout-minutes: 30
       working-directory: ${{ env.STRIPPED_DIR }}
@@ -55,11 +55,11 @@ jobs:
     - name: Run tests
       timeout-minutes: 1
       working-directory: ${{ env.STRIPPED_DIR }}
-      run: release/check-dirty.sh
+      run: tools/release/check-dirty.sh
     - name: Check submodules
       if: github.repository == 'commaai/openpilot'
       timeout-minutes: 3
-      run: release/check-submodules.sh
+      run: tools/release/check-submodules.sh
 
   build_mac:
     name: build macOS
@@ -72,7 +72,7 @@ jobs:
       run: |
         FILTERED=$(echo "$PATH" | tr ':' '\n' | grep -v '/opt/homebrew' | tr '\n' ':')
         echo "PATH=${FILTERED}/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin" >> $GITHUB_ENV
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Building openpilot
       run: scons
 
@@ -88,7 +88,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Static analysis
       timeout-minutes: 1
       run: scripts/lint/lint.sh
@@ -105,7 +105,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Run unit tests
@@ -128,7 +128,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Run replay
@@ -201,7 +201,7 @@ jobs:
     - uses: actions/checkout@v6
       with:
         submodules: true
-    - run: ./openpilot/tools/op.sh setup
+    - run: ./tools/op.sh setup
     - name: Build openpilot
       run: scons
     - name: Driving test
@@ -222,7 +222,7 @@ jobs:
       - uses: actions/checkout@v6
         with:
           submodules: true
-      - run: ./openpilot/tools/op.sh setup
+      - run: ./tools/op.sh setup
       - name: Build openpilot
         run: scons
       - name: Create UI Report

Jenkinsfile

@@ -63,6 +63,8 @@ if [ -f /data/openpilot/launch_env.sh ]; then
   source /data/openpilot/launch_env.sh
 fi
 
+export LD_LIBRARY_PATH="\$(python -c 'import ffmpeg; print(ffmpeg.LIB_DIR)'):/usr/local/lib:\${LD_LIBRARY_PATH:-}"
+
 ln -snf ${env.TEST_DIR} /data/pythonpath
 
 cd ${env.TEST_DIR} || true
@@ -179,7 +181,7 @@ node {
   try {
     if (env.BRANCH_NAME == 'devel-staging') {
       deviceStage("build release-tizi-staging", "tizi-needs-can", [], [
-        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/release/build_release.sh"),
+        step("build release-tizi-staging", "RELEASE_BRANCH=release-tizi-staging,release-mici-staging $SOURCE_DIR/tools/release/build_release.sh"),
       ])
     }
 
@@ -187,12 +189,12 @@ node {
       parallel (
         'nightly': {
           deviceStage("build nightly", "tizi-needs-can", [], [
-            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/release/build_release.sh"),
+            step("build nightly", "RELEASE_BRANCH=nightly $SOURCE_DIR/tools/release/build_release.sh"),
           ])
         },
         'nightly-dev': {
           deviceStage("build nightly-dev", "tizi-needs-can", [], [
-            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/release/build_release.sh"),
+            step("build nightly-dev", "PANDA_DEBUG_BUILD=1 RELEASE_BRANCH=nightly-dev $SOURCE_DIR/tools/release/build_release.sh"),
           ])
         },
       )
@@ -203,7 +205,7 @@ node {
       'onroad tests': {
         deviceStage("onroad", "tizi-needs-can", ["UNSAFE=1"], [
           step("build openpilot", "cd openpilot/system/manager && ./build.py"),
-          step("check dirty", "release/check-dirty.sh"),
+          step("check dirty", "tools/release/check-dirty.sh"),
           step("onroad tests", "pytest openpilot/selfdrive/test/test_onroad.py -s", [timeout: 60]),
         ])
       },
@@ -211,7 +213,7 @@ node {
         deviceStage("tizi-hardware", "tizi-common", ["UNSAFE=1"], [
           step("build", "cd openpilot/system/manager && ./build.py"),
           step("test power draw", "pytest -s openpilot/selfdrive/test//test_power_draw.py"),
-          step("test encoder", "LD_LIBRARY_PATH=/usr/local/lib pytest openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
+          step("test encoder", "pytest openpilot/system/loggerd/tests/test_encoder.py", [diffPaths: ["openpilot/system/loggerd/"]]),
           step("test manager", "pytest openpilot/system/manager/test/test_manager.py"),
         ])
       },

openpilot/selfdrive/test/mem_usage.py

@@ -1,12 +1,9 @@
-#!/usr/bin/env python3
-import argparse
 import os
 from collections import defaultdict
 
 import numpy as np
 
 from openpilot.common.utils import tabulate
-from openpilot.tools.lib.logreader import LogReader
 
 DEMO_ROUTE = "5beb9b58bd12b691/0000010a--a51155e496"
 MB = 1024 * 1024
@@ -210,29 +207,3 @@ def print_report(proc_logs, device_states=None):
 
   print_process_tables(op_procs, other_procs, total_mb, use_pss)
   print_memory_accounting(proc_logs, op_procs, other_procs, total_mb, use_pss)
-
-
-if __name__ == "__main__":
-  parser = argparse.ArgumentParser(description="Analyze memory usage from route logs")
-  parser.add_argument("route", nargs="?", default=None, help="route ID or local rlog path")
-  parser.add_argument("--demo", action="store_true", help=f"use demo route ({DEMO_ROUTE})")
-  args = parser.parse_args()
-
-  if args.demo:
-    route = DEMO_ROUTE
-  elif args.route:
-    route = args.route
-  else:
-    parser.error("provide a route or use --demo")
-
-  print(f"Reading logs from: {route}")
-
-  proc_logs = []
-  device_states = []
-  for msg in LogReader(route):
-    if msg.which() == 'procLog':
-      proc_logs.append(msg)
-    elif msg.which() == 'deviceState':
-      device_states.append(msg)
-
-  print_report(proc_logs, device_states)

openpilot/selfdrive/test/test_onroad.py

@@ -283,7 +283,7 @@ class TestOnroad:
     print("--------------- Memory Usage -------------------")
     print("------------------------------------------------")
 
-    from openpilot.tools.scripts.mem_usage import print_report
+    from openpilot.selfdrive.test.mem_usage import print_report
     print_report(self.msgs['procLog'], self.msgs['deviceState'])
 
     offset = int(SERVICE_LIST['deviceState'].frequency * LOG_OFFSET)

scripts/post-commit

@@ -3,5 +3,5 @@ set -e
 if [[ -f .git/hooks/post-commit.d/post-commit ]]; then
   .git/hooks/post-commit.d/post-commit
 fi
-openpilot/tools/op.sh lint --fast
+tools/op.sh lint --fast
 echo ""

tools/README.md

@@ -18,7 +18,7 @@ git clone https://github.com/commaai/openpilot.git
 **2. Run the setup script**
 ``` bash
 cd openpilot
-openpilot/tools/op.sh setup
+tools/op.sh setup
 ```
 
 **3. Activate a Python shell**
@@ -41,7 +41,7 @@ Follow [these instructions](https://docs.microsoft.com/en-us/windows/wsl/install
 **NOTE**: If you are running WSL 2 and experiencing performance issues with the UI or simulator, you may need to explicitly enable hardware acceleration by setting `GALLIUM_DRIVER=d3d12` before commands. Add `export GALLIUM_DRIVER=d3d12` to your `~/.bashrc` file to make it automatic for future sessions.
 
 ## CTF
-Learn about the openpilot ecosystem and tools by playing our [CTF](/openpilot/tools/CTF.md).
+Learn about the openpilot ecosystem and tools by playing our [CTF](/tools/CTF.md).
 
 ## Directory Structure
 

tools/car_porting/README.md

@@ -1,4 +1,4 @@
-# openpilot/tools/car_porting
+# tools/car_porting
 
 Check out [this blog post](https://blog.comma.ai/how-to-write-a-car-port-for-openpilot/) for a high-level overview of porting a car.
 
@@ -15,13 +15,13 @@ Example:
 > openpilot/tools/cabana/cabana '1bbe6bf2d62f58a8|2022-07-14--17-11-43'
 ```
 
-### [openpilot/tools/car_porting/auto_fingerprint.py](/openpilot/tools/car_porting/auto_fingerprint.py)
+### [tools/car_porting/auto_fingerprint.py](/tools/car_porting/auto_fingerprint.py)
 
 Given a route and platform, automatically inserts FW fingerprints from the platform into the correct place in fingerprints.py
 
 Example:
 ```bash
-> python3 openpilot/tools/car_porting/auto_fingerprint.py '1bbe6bf2d62f58a8|2022-07-14--17-11-43' 'OUTBACK'
+> python3 tools/car_porting/auto_fingerprint.py '1bbe6bf2d62f58a8|2022-07-14--17-11-43' 'OUTBACK'
 Attempting to add fw version for:  OUTBACK
 ```
 
@@ -39,13 +39,13 @@ FAILED openpilot/selfdrive/car/tests/test_car_interfaces.py::TestCarInterfaces::
 
 ```
 
-### [openpilot/tools/car_porting/test_car_model.py](/openpilot/tools/car_porting/test_car_model.py)
+### [tools/car_porting/test_car_model.py](/tools/car_porting/test_car_model.py)
 
 Given a route, runs most of the car interface to check for common errors like missing signals, blocked panda messages, and safety mismatches.
 
 #### Example: panda safety mismatch for gasPressed
 ```bash
-> python3 openpilot/tools/car_porting/test_car_model.py '4822a427b188122a|2023-08-14--16-22-21'
+> python3 tools/car_porting/test_car_model.py '4822a427b188122a|2023-08-14--16-22-21'
 
 =====================================================================
 FAIL: test_panda_safety_carstate (__main__.CarModelTestCase.test_panda_safety_carstate)
@@ -59,7 +59,7 @@ AssertionError: 1 is not false : panda safety doesn't agree with openpilot: {'ga
 
 ## Jupyter notebooks
 
-To use these notebooks, install Jupyter within your [openpilot virtual environment](/openpilot/tools/README.md).
+To use these notebooks, install Jupyter within your [openpilot virtual environment](/tools/README.md).
 
 ```bash
 uv pip install jupyter ipykernel
@@ -71,7 +71,7 @@ Launching:
 jupyter notebook
 ```
 
-### [examples/subaru_steer_temp_fault.ipynb](/openpilot/tools/car_porting/examples/subaru_steer_temp_fault.ipynb)
+### [examples/subaru_steer_temp_fault.ipynb](/tools/car_porting/examples/subaru_steer_temp_fault.ipynb)
 
 An example of searching through a database of segments for a specific condition, and plotting the results.
 
@@ -79,7 +79,7 @@ An example of searching through a database of segments for a specific condition,
 
 *a plot of the steer_warning vs steering angle, where we can see it is clearly caused by a large steering angle change*
 
-### [examples/subaru_long_accel.ipynb](/openpilot/tools/car_porting/examples/subaru_long_accel.ipynb)
+### [examples/subaru_long_accel.ipynb](/tools/car_porting/examples/subaru_long_accel.ipynb)
 
 An example of plotting the response of an actuator when it is active.
 
@@ -87,7 +87,7 @@ An example of plotting the response of an actuator when it is active.
 
 *a plot of the brake_pressure vs acceleration, where we can see it is a fairly linear response.*
 
-### [examples/ford_vin_fingerprint.ipynb](/openpilot/tools/car_porting/examples/ford_vin_fingerprint.ipynb)
+### [examples/ford_vin_fingerprint.ipynb](/tools/car_porting/examples/ford_vin_fingerprint.ipynb)
 
 In this example, we use the public comma car segments database to check if vin fingerprinting is feasible for ford.
 
@@ -109,7 +109,7 @@ vin: 3FTTW8E31PRXXXXXX real platform: FORD MAVERICK 1ST GEN              determi
 vin: 3FTTW8E99NRXXXXXX real platform: FORD MAVERICK 1ST GEN              determined platform: mock                              correct: False
 ```
 
-### [examples/find_segments_with_message.ipynb](/openpilot/tools/car_porting/examples/find_segments_with_message.ipynb)
+### [examples/find_segments_with_message.ipynb](/tools/car_porting/examples/find_segments_with_message.ipynb)
 
 Searches for segments where a set of given CAN message IDs are present. In the example, we search for all messages
 used for CAN-based ignition detection.

tools/op.sh

@@ -201,7 +201,7 @@ function op_setup() {
 
   echo "Installing dependencies..."
   st="$(date +%s)"
-  SETUP_SCRIPT="openpilot/tools/setup_dependencies.sh"
+  SETUP_SCRIPT="tools/setup_dependencies.sh"
   if ! $OPENPILOT_ROOT/$SETUP_SCRIPT; then
     echo -e " ↳ [${RED}✗${NC}] Dependencies installation failed!"
     return 1
@@ -269,12 +269,12 @@ function op_venv() {
 
 function op_adb() {
   op_before_cmd
-  op_run_command openpilot/tools/scripts/adb_ssh.sh "$@"
+  op_run_command tools/scripts/adb_ssh.sh "$@"
 }
 
 function op_ssh() {
   op_before_cmd
-  op_run_command openpilot/tools/scripts/ssh.py "$@"
+  op_run_command tools/scripts/ssh.py "$@"
 }
 
 function op_script() {

tools/release/README.md

@@ -11,7 +11,7 @@
   - [ ] push the new branch
 - [ ] push to staging:
   - [ ] make sure you are on the newly created release master branch (`zerotentwo`)
-  - [ ] run `BRANCH=devel-staging release/build_stripped.sh`. Jenkins will then automatically build staging on device, run `test_onroad` and update the staging branch
+  - [ ] run `BRANCH=devel-staging tools/release/build_stripped.sh`. Jenkins will then automatically build staging on device, run `test_onroad` and update the staging branch
 - [ ] bump version on master: `openpilot/common/version.h` and `RELEASES.md`
 - [ ] post on Discord, tag `@release crew`
 

tools/release/build_release.sh

@@ -33,7 +33,7 @@ git checkout --orphan $BUILD_BRANCH
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-cp -pR --parents $(./release/release_files.py) $BUILD_DIR/
+cp -pR --parents $(./tools/release/release_files.py) $BUILD_DIR/
 
 # in the directory
 cd $BUILD_DIR
@@ -73,7 +73,7 @@ find . -name '*.os' -delete
 find . -name '*.pyc' -delete
 find . -name 'moc_*' -delete
 find . -name '__pycache__' -delete
-rm -rf .sconsign.dblite Jenkinsfile release/
+rm -rf .sconsign.dblite Jenkinsfile tools/release/
 rm -f openpilot/selfdrive/modeld/models/*.onnx*
 
 # Mark as prebuilt release

tools/release/build_stripped.sh

@@ -38,7 +38,7 @@ git submodule foreach --recursive git clean -xdff
 # do the files copy
 echo "[-] copying files T=$SECONDS"
 cd $SOURCE_DIR
-cp -pR --parents $(./release/release_files.py) $TARGET_DIR/
+cp -pR --parents $(./tools/release/release_files.py) $TARGET_DIR/
 
 # in the directory
 cd $TARGET_DIR

tools/release/release_files.py

@@ -4,7 +4,7 @@ import re
 from pathlib import Path
 
 HERE = os.path.abspath(os.path.dirname(__file__))
-ROOT = HERE + "/.."
+ROOT = os.path.abspath(os.path.join(HERE, "../.."))
 
 blacklist = [
   ".git/",

tools/scripts/car/can_print_changes.py

@@ -5,7 +5,7 @@ import time
 from collections import defaultdict
 
 import openpilot.cereal.messaging as messaging
-from openpilot.tools.scripts.can_table import can_table
+from tools.scripts.car.can_table import can_table
 from openpilot.tools.lib.logreader import LogIterable, LogReader
 
 RED = '\033[91m'

tools/scripts/mem_usage.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import argparse
+import os
+import sys
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+
+from openpilot.selfdrive.test.mem_usage import DEMO_ROUTE, print_report
+from openpilot.tools.lib.logreader import LogReader
+
+
+if __name__ == "__main__":
+  parser = argparse.ArgumentParser(description="Analyze memory usage from route logs")
+  parser.add_argument("route", nargs="?", default=None, help="route ID or local rlog path")
+  parser.add_argument("--demo", action="store_true", help=f"use demo route ({DEMO_ROUTE})")
+  args = parser.parse_args()
+
+  if args.demo:
+    route = DEMO_ROUTE
+  elif args.route:
+    route = args.route
+  else:
+    parser.error("provide a route or use --demo")
+
+  print(f"Reading logs from: {route}")
+
+  proc_logs = []
+  device_states = []
+  for msg in LogReader(route):
+    if msg.which() == 'procLog':
+      proc_logs.append(msg)
+    elif msg.which() == 'deviceState':
+      device_states.append(msg)
+
+  print_report(proc_logs, device_states)

tools/setup.sh

@@ -121,10 +121,10 @@ function git_clone() {
 
 function install_with_op() {
   cd $OPENPILOT_ROOT
-  $OPENPILOT_ROOT/openpilot/tools/op.sh install
-  $OPENPILOT_ROOT/openpilot/tools/op.sh post-commit
+  $OPENPILOT_ROOT/tools/op.sh install
+  $OPENPILOT_ROOT/tools/op.sh post-commit
 
-  if ! $OPENPILOT_ROOT/openpilot/tools/op.sh setup; then
+  if ! $OPENPILOT_ROOT/tools/op.sh setup; then
     echo -e "\n[${RED}✗${NC}] failed to install openpilot!"
     return 1
   fi