From 0f4bae1c771ba3f94fc96591f944cd3d32c25d76 Mon Sep 17 00:00:00 2001 From: firestar5683 <168790843+firestar5683@users.noreply.github.com> Date: Fri, 27 Mar 2026 00:48:53 -0500 Subject: [PATCH] TOOLS --- .gitignore | 1 + SConstruct | 6 + c3 | 2 +- c4 | 2 +- dev | 6 + docs/how-to/laptop-device-build.md | 27 +- raybig | 2 +- scripts/host_tool_runner.sh | 531 +++++++++++++++++++++ selfdrive/test/process_replay/__init__.py | 30 +- selfdrive/test/process_replay/migration.py | 3 +- tool | 6 + tools/README.md | 3 + tools/STARPILOT_DEVELOPMENT.md | 209 ++++++++ tools/host | 6 + 14 files changed, 818 insertions(+), 16 deletions(-) create mode 100755 dev create mode 100755 scripts/host_tool_runner.sh create mode 100755 tool create mode 100644 tools/STARPILOT_DEVELOPMENT.md create mode 100755 tools/host diff --git a/.gitignore b/.gitignore index a1c7bcc6..5fb63fc5 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ venv/ a.out .hypothesis .cache/ +.host_runtime/ .comma_sysroot/ .venv-linux-arm64/ compiledmodels/ diff --git a/SConstruct b/SConstruct index 20e10299..dff2cd40 100644 --- a/SConstruct +++ b/SConstruct @@ -63,6 +63,12 @@ AddOption('--minimal', default=os.path.exists(File('#.lfsconfig').abspath), # minimal by default on release branch (where there's no LFS) help='the minimum build to run openpilot. no tests, tools, etc.') +AddOption('--extras', + action='store_true', + dest='extras', + default=os.path.exists(File('#.lfsconfig').abspath), + help='build optional tools/tests even when minimal is the default') + def maybe_delegate_to_laptop_device_builder() -> None: if platform.system() != "Darwin": return diff --git a/c3 b/c3 index b82d7624..c1250b67 100755 --- a/c3 +++ b/c3 @@ -3,4 +3,4 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec "${ROOT_DIR}/scripts/launch_ui_desktop.sh" "$@" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" c3 "$@" diff --git a/c4 b/c4 index 0023721d..0e9a7f3d 100755 --- a/c4 +++ b/c4 @@ -3,4 +3,4 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec "${ROOT_DIR}/scripts/launch_ui_c4_desktop.sh" "$@" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" c4 "$@" diff --git a/dev b/dev new file mode 100755 index 00000000..00b24988 --- /dev/null +++ b/dev @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@" diff --git a/docs/how-to/laptop-device-build.md b/docs/how-to/laptop-device-build.md index d6381b65..234223a8 100644 --- a/docs/how-to/laptop-device-build.md +++ b/docs/how-to/laptop-device-build.md @@ -2,6 +2,8 @@ This flow builds **device-target (`larch64`) binaries on your laptop** using a Linux/aarch64 container and a synced comma sysroot. +For the full StarPilot branch workflow, including host-native shorthand tools such as `./dev`, `./c3`, `./c4`, and `./raybig`, see [tools/STARPILOT_DEVELOPMENT.md](../../tools/STARPILOT_DEVELOPMENT.md). + ## Prerequisites - Docker Desktop (or Podman) with Linux/aarch64 support. @@ -105,18 +107,23 @@ Note: - `manager` mode requires the container runtime machine to be `aarch64`. - If your built image runs as `x86_64`, build mode still works for artifact generation, but manager launch in-container is not supported. -Desktop UI workaround (keep device build + still inspect UI on macOS): +Preferred host-side shorthand commands on this branch: + +```bash +./c3 +./c4 +./raybig +./dev replay +./dev cabana +./dev plotjuggler +``` + +These commands use the isolated `.host_runtime` cache so host-native artifacts do not churn the main tree. + +Legacy direct script entrypoints still exist if needed: ```bash scripts/launch_ui_desktop.sh -``` - -This builds a native macOS `selfdrive/ui/ui.macos`, restores device `selfdrive/ui/ui`, and launches the mac UI binary. - -Desktop C4/raylib UI launcher: - -```bash scripts/launch_ui_c4_desktop.sh +scripts/launch_ui_raybig_desktop.sh ``` - -This launches `selfdrive/ui/ui.py` in small-UI mode (`BIG=0`) with onboarding pre-accepted params. diff --git a/raybig b/raybig index fec34670..618a8ef2 100755 --- a/raybig +++ b/raybig @@ -3,4 +3,4 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -exec "${ROOT_DIR}/scripts/launch_ui_raybig_desktop.sh" "$@" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" raybig "$@" diff --git a/scripts/host_tool_runner.sh b/scripts/host_tool_runner.sh new file mode 100755 index 00000000..d6c8d16f --- /dev/null +++ b/scripts/host_tool_runner.sh @@ -0,0 +1,531 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "${ROOT_DIR}" + +PLATFORM_KEY="$(uname -s | tr '[:upper:]' '[:lower:]')" +HOST_PLATFORM_ROOT="${ROOT_DIR}/.host_runtime/${PLATFORM_KEY}" +HOST_ROOT="" +WORK_DIR="" +HOST_VENV="" +HOST_LOCK_DIR="" +HOST_LOCK_PID_FILE="" +HOST_LOCK_CMD_FILE="" +HOST_LOCK_HELD=0 +HOST_BUCKETS=(shared cabana plotjuggler) + +usage() { + cat <<'EOF' +Usage: + ./dev [args...] + ./tool [args...] + ./tools/host [args...] + +Commands: + c3 Launch the desktop Qt UI from the isolated host cache. + c4 Launch the small raylib UI from the isolated host cache. + raybig Launch the large raylib UI from the isolated host cache. + replay Build and run replay from the isolated host cache. + cabana Build and run cabana from the isolated host cache. + plotjuggler Run PlotJuggler helper from the isolated host cache. + juggle Alias for plotjuggler. + sync Refresh the isolated host cache only. + shell Open a shell inside the isolated host cache. + help Show this help text. + +Notes: + - Host-tool builds happen under ./.host_runtime/ and do not touch the main tree. + - `cabana` and `plotjuggler` use their own host-runtime buckets and can run together. + - Other commands that share a bucket still wait on that bucket's lock. + - `./build` remains the device-target flow. + - For c3/c4/raybig, pass the jobs count first to preserve existing shorthand: + ./dev c3 8 + ./dev raybig 12 + - `./dev sync` refreshes all host buckets. Use `./dev sync cabana` to sync one. +EOF +} + +default_jobs() { + if command -v nproc >/dev/null 2>&1; then + nproc + elif command -v sysctl >/dev/null 2>&1; then + sysctl -n hw.ncpu + else + echo 8 + fi +} + +resolve_host_bucket() { + local name="${1:-shared}" + + case "${name}" in + shared|default|ui|c3|c4|raybig|replay|shell) + echo "shared" + ;; + cabana) + echo "cabana" + ;; + plotjuggler|juggle) + echo "plotjuggler" + ;; + *) + return 1 + ;; + esac +} + +set_host_bucket() { + local bucket="$1" + + if [[ "${bucket}" == "shared" ]]; then + HOST_ROOT="${HOST_PLATFORM_ROOT}" + else + HOST_ROOT="${HOST_PLATFORM_ROOT}/${bucket}" + fi + WORK_DIR="${HOST_ROOT}/worktree" + HOST_VENV="${HOST_ROOT}/venv" + HOST_LOCK_DIR="${HOST_ROOT}/lock" + HOST_LOCK_PID_FILE="${HOST_LOCK_DIR}/pid" + HOST_LOCK_CMD_FILE="${HOST_LOCK_DIR}/command" +} + +lock_owner_summary() { + local lock_pid="" + local lock_cmd="unknown" + + if [[ -f "${HOST_LOCK_PID_FILE}" ]]; then + lock_pid="$(<"${HOST_LOCK_PID_FILE}")" + fi + if [[ -f "${HOST_LOCK_CMD_FILE}" ]]; then + lock_cmd="$(<"${HOST_LOCK_CMD_FILE}")" + fi + + if [[ -n "${lock_pid}" ]]; then + echo "pid ${lock_pid} (${lock_cmd})" + else + echo "${lock_cmd}" + fi +} + +lock_is_stale() { + local lock_pid="" + + if [[ ! -f "${HOST_LOCK_PID_FILE}" ]]; then + return 1 + fi + + lock_pid="$(<"${HOST_LOCK_PID_FILE}")" + if [[ ! "${lock_pid}" =~ ^[0-9]+$ ]]; then + return 0 + fi + + if kill -0 "${lock_pid}" 2>/dev/null; then + return 1 + fi + + return 0 +} + +release_host_lock() { + if [[ "${HOST_LOCK_HELD}" != "1" ]]; then + return + fi + + rm -f "${HOST_LOCK_PID_FILE}" "${HOST_LOCK_CMD_FILE}" + rmdir "${HOST_LOCK_DIR}" 2>/dev/null || rm -rf "${HOST_LOCK_DIR}" + HOST_LOCK_HELD=0 +} + +acquire_host_lock() { + local lock_label="$1" + local notified=0 + + mkdir -p "${HOST_ROOT}" + + while true; do + if mkdir "${HOST_LOCK_DIR}" 2>/dev/null; then + printf '%s\n' "$$" > "${HOST_LOCK_PID_FILE}" + printf '%s\n' "${lock_label}" > "${HOST_LOCK_CMD_FILE}" + HOST_LOCK_HELD=1 + trap release_host_lock EXIT + return + fi + + if lock_is_stale; then + echo "Removing stale host runtime lock: $(lock_owner_summary)" >&2 + rm -rf "${HOST_LOCK_DIR}" + continue + fi + + if [[ "${notified}" == "0" ]]; then + echo "Waiting for host runtime lock in ${HOST_ROOT}. Another shorthand command is using it: $(lock_owner_summary)." >&2 + notified=1 + fi + sleep 1 + done +} + +ensure_venv() { + if [[ ! -x "${ROOT_DIR}/.venv/bin/python3" ]]; then + echo "Missing .venv. Run tools/install_python_dependencies.sh first." + exit 1 + fi + + if ! command -v uv >/dev/null 2>&1; then + echo "Missing uv. Run tools/install_python_dependencies.sh first." + exit 1 + fi +} + +sync_selected_bucket() { + local bucket="$1" + + set_host_bucket "${bucket}" + acquire_host_lock "sync ${bucket}" + sync_worktree + echo "Host tool cache synced (${bucket}): ${WORK_DIR}" + release_host_lock +} + +sync_all_buckets() { + local bucket="" + for bucket in "${HOST_BUCKETS[@]}"; do + sync_selected_bucket "${bucket}" + done +} + +purge_host_python_artifacts() { + rm -f \ + "${WORK_DIR}/common/params_pyx.so" \ + "${WORK_DIR}/common/params_pyx.o" \ + "${WORK_DIR}/common/params_pyx.cpp" \ + "${WORK_DIR}/common/transformations/transformations.so" \ + "${WORK_DIR}/common/transformations/transformations.o" \ + "${WORK_DIR}/common/transformations/transformations.cpp" \ + "${WORK_DIR}/msgq_repo/msgq/ipc_pyx.so" \ + "${WORK_DIR}/msgq_repo/msgq/ipc_pyx.o" \ + "${WORK_DIR}/msgq_repo/msgq/ipc_pyx.cpp" \ + "${WORK_DIR}/msgq_repo/msgq/visionipc/visionipc_pyx.so" \ + "${WORK_DIR}/msgq_repo/msgq/visionipc/visionipc_pyx.o" \ + "${WORK_DIR}/msgq_repo/msgq/visionipc/visionipc_pyx.cpp" +} + +ensure_host_python_tools() { + ensure_venv + + local root_py_minor + root_py_minor="$("${ROOT_DIR}/.venv/bin/python3" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + + if [[ -x "${HOST_VENV}/bin/python3" ]]; then + local host_py_minor + host_py_minor="$("${HOST_VENV}/bin/python3" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")')" + if [[ "${host_py_minor}" != "${root_py_minor}" ]]; then + rm -rf "${HOST_VENV}" + purge_host_python_artifacts + fi + fi + + if [[ -x "${HOST_VENV}/bin/scons" && -x "${HOST_VENV}/bin/cythonize" && -x "${HOST_VENV}/bin/python3" ]]; then + return + fi + + ( + cd "${WORK_DIR}" + UV_PROJECT_ENVIRONMENT="${HOST_VENV}" UV_PYTHON="${ROOT_DIR}/.venv/bin/python3" uv sync --frozen --all-extras + ) +} + +sync_worktree() { + ensure_venv + + mkdir -p "${HOST_ROOT}" + + local excludes=( + ".git/" + ".venv/" + ".venv-linux-arm64/" + ".cache/" + ".comma_sysroot/" + ".host_runtime/" + "__pycache__/" + "*.pyc" + "*.pyo" + "*.o" + "*.os" + ".sconsign.dblite" + "compile_commands.json" + "tools/plotjuggler/bin/" + "tools/replay/replay" + "tools/replay/tests/test_replay" + "tools/cabana/cabana" + "tools/cabana/tests/test_cabana" + "selfdrive/ui/ui" + "selfdrive/ui/ui.macos" + "selfdrive/ui/ui.larch64" + "cereal/libcereal.a" + "cereal/libsocketmaster.a" + "cereal/messaging/bridge" + "common/libcommon.a" + "common/params_pyx.so" + "common/params_pyx.cpp" + "common/transformations/libtransformations.a" + "common/transformations/transformations.so" + "msgq_repo/libmsgq.a" + "msgq_repo/libvisionipc.a" + "msgq_repo/msgq/ipc_pyx.so" + "msgq_repo/msgq/visionipc/visionipc_pyx.so" + "rednose_repo/rednose/helpers/ekf_sym_pyx.so" + "selfdrive/modeld/models/commonmodel_pyx.so" + "selfdrive/pandad/pandad_api_impl.so" + "selfdrive/controls/lib/lateral_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so" + "selfdrive/controls/lib/lateral_mpc_lib/c_generated_code/libacados_ocp_solver_lat.so" + "selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so" + "selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libacados_ocp_solver_long.so" + "third_party/libjson11.a" + "third_party/libkaitai.a" + ) + + local rsync_args=(-a --delete) + local pattern="" + for pattern in "${excludes[@]}"; do + rsync_args+=(--exclude "${pattern}") + done + + rsync "${rsync_args[@]}" "${ROOT_DIR}/" "${WORK_DIR}/" + rm -f "${WORK_DIR}/third_party/libjson11.a" "${WORK_DIR}/third_party/libkaitai.a" + ensure_host_python_tools + rm -rf "${WORK_DIR}/.venv" + ln -s "${HOST_VENV}" "${WORK_DIR}/.venv" +} + +setup_build_env() { + if [[ -d /opt/homebrew/bin ]]; then + export PATH="/opt/homebrew/bin:${PATH}" + fi + + if [[ "$(uname -s)" == "Darwin" ]]; then + export CC="/usr/bin/clang" + export CXX="/usr/bin/clang++" + fi + + unset CPATH C_INCLUDE_PATH CPLUS_INCLUDE_PATH CPPFLAGS CFLAGS CXXFLAGS LDFLAGS +} + +export_workdir_pythonpath() { + export PYTHONPATH="${WORK_DIR}:${WORK_DIR}/frogpilot/third_party" + local repo_dir="" + for repo_dir in "${WORK_DIR}"/*_repo; do + [[ -d "${repo_dir}" ]] && export PYTHONPATH="${PYTHONPATH}:${repo_dir}" + done + [[ -d "${WORK_DIR}/third_party/acados" ]] && export PYTHONPATH="${PYTHONPATH}:${WORK_DIR}/third_party/acados" +} + +run_host_scons() { + local jobs="$1" + shift || true + + ( + cd "${WORK_DIR}" + setup_build_env + source .venv/bin/activate + SP_DISABLE_AUTO_DEVICE_SCONS=1 "${WORK_DIR}/.venv/bin/scons" --extras -j"${jobs}" "$@" + ) +} + +run_in_worktree() { + ( + cd "${WORK_DIR}" + setup_build_env + "$@" + ) +} + +run_python_tool() { + local script_path="$1" + shift || true + + ( + cd "${WORK_DIR}" + setup_build_env + export_workdir_pythonpath + source .venv/bin/activate + exec "${WORK_DIR}/.venv/bin/python3" "${WORK_DIR}/${script_path}" "$@" + ) +} + +launch_c3() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_in_worktree "${WORK_DIR}/scripts/launch_ui_desktop.sh" "${jobs}" "$@" +} + +launch_c4() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_in_worktree "${WORK_DIR}/scripts/launch_ui_c4_desktop.sh" "${jobs}" "$@" +} + +launch_raybig() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_in_worktree "${WORK_DIR}/scripts/launch_ui_raybig_desktop.sh" "${jobs}" "$@" +} + +launch_replay() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_host_scons "${jobs}" tools/replay/replay + + ( + cd "${WORK_DIR}" + setup_build_env + export BASEDIR="${WORK_DIR}" + exec "${WORK_DIR}/tools/replay/replay" "$@" + ) +} + +launch_cabana() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_host_scons "${jobs}" tools/cabana/cabana + + ( + cd "${WORK_DIR}" + setup_build_env + export BASEDIR="${WORK_DIR}" + exec "${WORK_DIR}/tools/cabana/cabana" "$@" + ) +} + +launch_plotjuggler() { + local jobs + jobs="$(default_jobs)" + if [[ "${1:-}" =~ ^[0-9]+$ ]]; then + jobs="$1" + shift || true + fi + + sync_worktree + run_host_scons "${jobs}" \ + common/params_pyx.so \ + common/transformations/transformations.so \ + msgq_repo/msgq/ipc_pyx.so \ + msgq_repo/msgq/visionipc/visionipc_pyx.so + run_python_tool "tools/plotjuggler/juggle.py" "$@" +} + +open_shell() { + sync_worktree + ( + cd "${WORK_DIR}" + setup_build_env + export_workdir_pythonpath + source .venv/bin/activate + exec "${SHELL:-/bin/bash}" + ) +} + +main() { + local command="${1:-help}" + local bucket="" + if [[ $# -gt 0 ]]; then + shift || true + fi + + case "${command}" in + help|-h|--help) + usage + ;; + c3|c4|raybig|replay|shell) + set_host_bucket "shared" + acquire_host_lock "${command} $*" + ;; + cabana) + set_host_bucket "cabana" + acquire_host_lock "${command} $*" + ;; + plotjuggler|juggle) + set_host_bucket "plotjuggler" + acquire_host_lock "${command} $*" + ;; + sync) + if [[ $# -gt 0 ]]; then + bucket="$(resolve_host_bucket "${1}")" || { + echo "Unknown host bucket for sync: ${1}" >&2 + echo "Valid sync buckets: shared, cabana, plotjuggler" >&2 + exit 1 + } + shift || true + sync_selected_bucket "${bucket}" + else + sync_all_buckets + fi + exit 0 + ;; + *) + echo "Unknown command: ${command}" >&2 + echo >&2 + usage >&2 + exit 1 + ;; + esac + + case "${command}" in + c3) + launch_c3 "$@" + ;; + c4) + launch_c4 "$@" + ;; + raybig) + launch_raybig "$@" + ;; + replay) + launch_replay "$@" + ;; + cabana) + launch_cabana "$@" + ;; + plotjuggler|juggle) + launch_plotjuggler "$@" + ;; + shell) + open_shell + ;; + esac +} + +set_host_bucket "shared" +main "$@" diff --git a/selfdrive/test/process_replay/__init__.py b/selfdrive/test/process_replay/__init__.py index b9942771..3bc0880b 100644 --- a/selfdrive/test/process_replay/__init__.py +++ b/selfdrive/test/process_replay/__init__.py @@ -1,2 +1,28 @@ -from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, get_process_config, get_custom_params_from_lr, \ - replay_process, replay_process_with_name # noqa: F401 +from typing import TYPE_CHECKING + +__all__ = [ + "CONFIGS", + "get_process_config", + "get_custom_params_from_lr", + "replay_process", + "replay_process_with_name", +] + +if TYPE_CHECKING: + from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, get_custom_params_from_lr, get_process_config, replay_process, replay_process_with_name + + +def __getattr__(name: str): + if name in __all__: + from openpilot.selfdrive.test.process_replay.process_replay import CONFIGS, get_custom_params_from_lr, get_process_config, replay_process, replay_process_with_name + + exports = { + "CONFIGS": CONFIGS, + "get_process_config": get_process_config, + "get_custom_params_from_lr": get_custom_params_from_lr, + "replay_process": replay_process, + "replay_process_with_name": replay_process_with_name, + } + return exports[name] + + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/selfdrive/test/process_replay/migration.py b/selfdrive/test/process_replay/migration.py index 33b363cf..aaf8d101 100644 --- a/selfdrive/test/process_replay/migration.py +++ b/selfdrive/test/process_replay/migration.py @@ -13,7 +13,6 @@ from opendbc.car.gm.values import GMSafetyFlags from openpilot.selfdrive.modeld.constants import ModelConstants from openpilot.selfdrive.modeld.fill_model_msg import fill_xyz_poly, fill_lane_line_meta from openpilot.selfdrive.test.process_replay.vision_meta import meta_from_encode_index -from openpilot.selfdrive.controls.lib.longitudinal_planner import get_accel_from_plan, CONTROL_N_T_IDX from openpilot.system.manager.process_config import managed_processes from openpilot.tools.lib.logreader import LogIterable @@ -98,6 +97,8 @@ def migration(inputs: list[str], product: str|None=None): @migration(inputs=["longitudinalPlan", "carParams"]) def migrate_longitudinalPlan(msgs): + from openpilot.selfdrive.controls.lib.longitudinal_planner import CONTROL_N_T_IDX, get_accel_from_plan + ops = [] needs_migration = all(msg.longitudinalPlan.aTarget == 0.0 for _, msg in msgs if msg.which() == 'longitudinalPlan') diff --git a/tool b/tool new file mode 100755 index 00000000..00b24988 --- /dev/null +++ b/tool @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@" diff --git a/tools/README.md b/tools/README.md index d52c8f45..cdd5b303 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,5 +1,8 @@ # openpilot tools +StarPilot branch-specific shorthand and split-build workflow: +[STARPILOT_DEVELOPMENT.md](STARPILOT_DEVELOPMENT.md) + ## System Requirements openpilot is developed and tested on **Ubuntu 24.04**, which is the primary development target aside from the [supported embedded hardware](https://github.com/commaai/openpilot#running-on-a-dedicated-device-in-a-car). diff --git a/tools/STARPILOT_DEVELOPMENT.md b/tools/STARPILOT_DEVELOPMENT.md new file mode 100644 index 00000000..f9cf71be --- /dev/null +++ b/tools/STARPILOT_DEVELOPMENT.md @@ -0,0 +1,209 @@ +# StarPilot Development + +This branch uses a split development flow: + +- `./build` keeps producing device-target (`larch64`) artifacts for comma runtime compatibility. +- `./dev`, `./tool`, `./tools/host`, `./c3`, `./c4`, and `./raybig` use an isolated host cache under `.host_runtime/` for native macOS/Linux tooling. + +The goal is simple: keep the device build intact, keep host tools consistent with runtime behavior, and stop polluting the repo with host-only `.so`, `.o`, and desktop UI artifacts. + +## Prerequisites + +- Run `tools/install_python_dependencies.sh` +- Have `uv` available in your shell +- For device-target builds, install Docker Desktop or Podman with Linux/aarch64 support +- For macOS Qt UI (`./c3`), install the Qt 5/Homebrew dependencies already expected by the repo + +## One-time device build setup + +Fast path: + +```bash +scripts/laptop_device_build.sh setup +``` + +Equivalent wrapper: + +```bash +scripts/starpilot_build_flow.sh laptop-setup +``` + +If you need the manual sysroot flow instead: + +```bash +scripts/laptop_device_build.sh build-image +scripts/laptop_device_build.sh setup-sysroot-agnos +``` + +Or from a physical comma: + +```bash +scripts/laptop_device_build.sh setup-sysroot comma 22 +scripts/laptop_device_build.sh build-image +``` + +## Device-target build flow + +This stays the same: + +```bash +./build +``` + +Equivalent long form: + +```bash +scripts/laptop_device_build.sh build +``` + +This is the path you use for the actual comma/device-compatible build. It writes the normal device artifacts and does not depend on `.host_runtime`. + +## Host tooling flow + +Use the shorthand launchers for host-native tools: + +```bash +./dev [args...] +./tool [args...] +./tools/host [args...] +``` + +All three entrypoints do the same thing. `./dev` is the shortest general-purpose form. + +### Available host commands + +- `./dev replay [args...]` +- `./dev cabana [args...]` +- `./dev plotjuggler [args...]` +- `./dev juggle [args...]` +- `./dev sync` +- `./dev shell` + +### Desktop UI shorthands + +- `./c3 [jobs] [args...]` +- `./c4 [jobs] [args...]` +- `./raybig [jobs] [args...]` + +These are wrappers around the same isolated host runner used by `./dev`. + +Examples: + +```bash +./dev replay +./dev plotjuggler --help +./dev cabana +./c3 8 +./c4 8 +./raybig 8 +./dev shell +``` + +## What gets isolated + +Host-native artifacts live under: + +```bash +.host_runtime// +``` + +That host area contains: + +- `worktree/` and `venv/` for the shared bucket used by UI/replay commands +- `cabana/worktree/` and `cabana/venv/` for Cabana +- `plotjuggler/worktree/` and `plotjuggler/venv/` for PlotJuggler +- host-built binaries, static libs, objects, and Python extensions for each bucket + +Because `.host_runtime/` is git-ignored, running host tools no longer churns tracked files in the main repo. + +## Build and rebuild behavior + +Expected behavior: + +- First run builds whatever the host tool needs +- Later runs reuse the cached host artifacts +- Source changes or branch changes trigger a resync into `.host_runtime/.../worktree` +- SCons rebuilds only the host artifacts that are out of date +- Deleting `.host_runtime/` forces a clean rebuild of the host cache + +In other words, the host-side shorthand commands should not fully recompile every time unless something actually changed. + +## Concurrency rule + +Host shorthand commands are isolated by bucket. + +That means: + +- `./dev cabana` and `./dev plotjuggler` can run at the same time +- `./build` is separate from all host buckets and can run at the same time as host tools +- commands that share the same bucket still wait on that bucket's lock +- long-running UI sessions hold the shared bucket lock until they exit + +Current bucket split: + +- shared bucket: `./c3`, `./c4`, `./raybig`, `./dev replay`, `./dev shell` +- cabana bucket: `./dev cabana` +- plotjuggler bucket: `./dev plotjuggler`, `./dev juggle` + +This prevents one command from syncing or rebuilding over another live host session while still allowing the common Cabana + PlotJuggler pairing. + +## Choosing the right command + +Use `./build` when: + +- you want device-target artifacts +- you care about matching the comma runtime build path +- you are preparing binaries for deployment/runtime validation + +Use `./dev ...` when: + +- you want host-native tools like replay, cabana, or PlotJuggler +- you want host-native `.so` files separated from the main repo +- you do not want AI tools or git status confused by temporary build churn + +Use `./c3`, `./c4`, or `./raybig` when: + +- you want the desktop UI variants +- you want them to build/run from the isolated host cache instead of touching tracked files + +## Troubleshooting + +If a shorthand command fails immediately: + +- make sure `.venv` exists +- run `tools/install_python_dependencies.sh` +- confirm `uv` is installed + +If device builds fail: + +- run `scripts/laptop_device_build.sh doctor` +- finish the sysroot/container setup before retrying `./build` + +If you want to reset the host cache: + +```bash +rm -rf .host_runtime +``` + +Then rerun the shorthand command you need. + +To refresh all buckets without deleting them: + +```bash +./dev sync +``` + +To refresh one bucket only: + +```bash +./dev sync cabana +./dev sync plotjuggler +./dev sync shared +``` + +## Recommended day-to-day workflow + +1. Use `./build` when you need the real device-target build. +2. Use `./dev replay`, `./dev cabana`, or `./dev plotjuggler` for host-side tooling. +3. Use `./c3`, `./c4`, or `./raybig` for desktop UI work. +4. Let `.host_runtime` keep host artifacts out of the repo. diff --git a/tools/host b/tools/host new file mode 100755 index 00000000..d46026db --- /dev/null +++ b/tools/host @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +exec "${ROOT_DIR}/scripts/host_tool_runner.sh" "$@"