diff --git a/launch_chffrplus.sh b/launch_chffrplus.sh index f30e03ca62..240bc3b648 100755 --- a/launch_chffrplus.sh +++ b/launch_chffrplus.sh @@ -18,21 +18,21 @@ function agnos_init { sudo chmod 660 /dev/adsprpc-smd /dev/ion /dev/kgsl-3d0 # Check if AGNOS update is required - if [ $(< /VERSION) != "$AGNOS_VERSION" ]; then + if [ "$(< /VERSION)" != "$AGNOS_VERSION" ]; then AGNOS_PY="$DIR/openpilot/common/hardware/comma/agnos.py" MANIFEST="$DIR/openpilot/system/hardware/comma/agnos.json" - if $AGNOS_PY --verify $MANIFEST; then + if "$AGNOS_PY" --verify "$MANIFEST"; then sudo reboot fi while true; do - $DIR/openpilot/common/hardware/comma/updater $AGNOS_PY $MANIFEST + "$DIR/openpilot/common/hardware/comma/updater" "$AGNOS_PY" "$MANIFEST" done fi } function launch { # Remove orphaned git lock if it exists on boot - [ -f "$DIR/.git/index.lock" ] && rm -f $DIR/.git/index.lock + [ -f "$DIR/.git/index.lock" ] && rm -f "$DIR/.git/index.lock" # Check to see if there's a valid overlay-based update available. Conditions # are as follows: @@ -44,7 +44,7 @@ function launch { # that completed successfully and synced to disk. if [ -f "${DIR}/.overlay_init" ]; then - find ${DIR}/.git -newer ${DIR}/.overlay_init | grep -q '.' 2> /dev/null + find "${DIR}/.git" -newer "${DIR}/.overlay_init" | grep -q '.' 2> /dev/null if [ $? -eq 0 ]; then echo "${DIR} has been modified, skipping overlay update installation" else @@ -53,9 +53,9 @@ function launch { echo "Valid overlay update found, installing" LAUNCHER_LOCATION="${BASH_SOURCE[0]}" - mv $DIR /data/safe_staging/old_openpilot - mv "${STAGING_ROOT}/finalized" $DIR - cd $DIR + mv "$DIR" /data/safe_staging/old_openpilot + mv "${STAGING_ROOT}/finalized" "$DIR" + cd "$DIR" echo "Restarting launch script ${LAUNCHER_LOCATION}" unset AGNOS_VERSION @@ -69,7 +69,7 @@ function launch { fi # handle pythonpath - ln -sfn $(pwd) /data/pythonpath + ln -sfn "$(pwd)" /data/pythonpath export PYTHONPATH="$PWD" # submodule package symlinks for PYTHONPATH imports on device. @@ -90,7 +90,7 @@ function launch { # start manager cd openpilot/system/manager - if [ ! -f $DIR/prebuilt ]; then + if [ ! -f "$DIR/prebuilt" ]; then ./build.py fi ./manager.py diff --git a/openpilot/selfdrive/assets/prep-svg.sh b/openpilot/selfdrive/assets/prep-svg.sh index 567c17da32..ee6ba0a3ac 100755 --- a/openpilot/selfdrive/assets/prep-svg.sh +++ b/openpilot/selfdrive/assets/prep-svg.sh @@ -23,8 +23,8 @@ done # sudo apt install inkscape -for svg in $(find $DIR -type f | grep svg$); do - bunx svgo $svg --multipass --pretty --indent 2 +for svg in $(find "$DIR" -type f | grep svg$); do + bunx svgo "$svg" --multipass --pretty --indent 2 # convert to PNG png="${svg%.svg}.png" diff --git a/openpilot/selfdrive/test/scons_build_test.sh b/openpilot/selfdrive/test/scons_build_test.sh index 6f6eafcad1..dfdaf6189f 100755 --- a/openpilot/selfdrive/test/scons_build_test.sh +++ b/openpilot/selfdrive/test/scons_build_test.sh @@ -3,7 +3,7 @@ set -e SCRIPT_DIR=$(dirname "$0") BASEDIR=$(realpath "$SCRIPT_DIR/../../../") -cd $BASEDIR +cd "$BASEDIR" # tests that our build system's dependencies are configured properly, # needs a machine with lots of cores @@ -11,7 +11,7 @@ cd $BASEDIR # helpful commands: # scons -Q --tree=derived -cd $BASEDIR/opendbc_repo/ +cd "$BASEDIR/opendbc_repo/" scons --clean scons --no-cache --random if ! scons -q; then diff --git a/openpilot/selfdrive/test/setup_device_ci.sh b/openpilot/selfdrive/test/setup_device_ci.sh index 8558a38bc9..a1dd88dcf4 100755 --- a/openpilot/selfdrive/test/setup_device_ci.sh +++ b/openpilot/selfdrive/test/setup_device_ci.sh @@ -29,7 +29,7 @@ if [ -d /data/safe_staging/ ]; then fi CONTINUE_PATH="/data/continue.sh" -tee $CONTINUE_PATH << EOF +tee "$CONTINUE_PATH" << EOF #!/usr/bin/env bash sudo abctl --set_success @@ -54,7 +54,7 @@ done sleep infinity EOF -chmod +x $CONTINUE_PATH +chmod +x "$CONTINUE_PATH" export GIT_LFS_SKIP_SMUDGE=1 pull_lfs() { @@ -87,16 +87,16 @@ pull_lfs() { safe_checkout() { # completely clean TEST_DIR - cd $SOURCE_DIR + cd "$SOURCE_DIR" # cleanup orphaned locks find .git -type f -name "*.lock" -exec rm {} + git reset --hard - git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin $GIT_COMMIT + git fetch --no-tags --no-recurse-submodules -j4 --verbose --depth 1 origin "$GIT_COMMIT" find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \; - git reset --hard $GIT_COMMIT - git checkout $GIT_COMMIT + git reset --hard "$GIT_COMMIT" + git checkout "$GIT_COMMIT" git clean -xdff git submodule sync git submodule foreach --recursive "git reset --hard && git clean -xdff" @@ -106,22 +106,22 @@ safe_checkout() { pull_lfs echo "git checkout done, t=$SECONDS" - du -hs $SOURCE_DIR $SOURCE_DIR/.git + du -hs "$SOURCE_DIR" "$SOURCE_DIR/.git" - rsync -a --delete $SOURCE_DIR $TEST_DIR + rsync -a --delete "$SOURCE_DIR" "$TEST_DIR" } unsafe_checkout() {( set -e # checkout directly in test dir, leave old build products - cd $TEST_DIR + cd "$TEST_DIR" # cleanup orphaned locks find .git -type f -name "*.lock" -exec rm {} + - git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin $GIT_COMMIT - git checkout --force --no-recurse-submodules $GIT_COMMIT - git reset --hard $GIT_COMMIT + git fetch --no-tags --no-recurse-submodules -j8 --verbose --depth 1 origin "$GIT_COMMIT" + git checkout --force --no-recurse-submodules "$GIT_COMMIT" + git reset --hard "$GIT_COMMIT" git clean -dff git submodule sync git submodule foreach --recursive "git reset --hard && git clean -df" @@ -135,7 +135,7 @@ export GIT_PACK_THREADS=8 # set up environment if [ ! -d "$SOURCE_DIR" ]; then - git clone https://github.com/commaai/openpilot.git $SOURCE_DIR + git clone https://github.com/commaai/openpilot.git "$SOURCE_DIR" fi if [ ! -z "$UNSAFE" ]; then @@ -152,7 +152,7 @@ else fi # submodule package symlinks for PYTHONPATH imports on device (same as launch_chffrplus.sh) -cd $TEST_DIR +cd "$TEST_DIR" ln -sfn msgq_repo/msgq msgq ln -sfn opendbc_repo/opendbc opendbc ln -sfn rednose_repo/rednose rednose diff --git a/openpilot/selfdrive/ui/translations/auto_translate.sh b/openpilot/selfdrive/ui/translations/auto_translate.sh index 7238426c75..858e60eb26 100755 --- a/openpilot/selfdrive/ui/translations/auto_translate.sh +++ b/openpilot/selfdrive/ui/translations/auto_translate.sh @@ -4,7 +4,7 @@ set -euo pipefail DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd)" ROOT="$DIR/../../../" -cd $DIR +cd "$DIR" ./update_translations.py command -v codex >/dev/null || { diff --git a/openpilot/system/camerad/test/stress_restart.sh b/openpilot/system/camerad/test/stress_restart.sh index 0445dcba79..5f0f2dd2f9 100755 --- a/openpilot/system/camerad/test/stress_restart.sh +++ b/openpilot/system/camerad/test/stress_restart.sh @@ -4,6 +4,6 @@ while :; do ./camerad & pid="$!" sleep 2 - kill -2 $pid - wait $pid + kill -2 "$pid" + wait "$pid" done diff --git a/openpilot/tools/sim/launch_openpilot.sh b/openpilot/tools/sim/launch_openpilot.sh index 392f365d03..813711dd79 100755 --- a/openpilot/tools/sim/launch_openpilot.sh +++ b/openpilot/tools/sim/launch_openpilot.sh @@ -18,4 +18,4 @@ SCRIPT_DIR=$(dirname "$0") OPENPILOT_DIR=$SCRIPT_DIR/../../ DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" -cd $OPENPILOT_DIR/system/manager && exec ./manager.py +cd "$OPENPILOT_DIR/system/manager" && exec ./manager.py diff --git a/scripts/apply-pr.sh b/scripts/apply-pr.sh index ad0af46b49..f4a29ec087 100755 --- a/scripts/apply-pr.sh +++ b/scripts/apply-pr.sh @@ -6,6 +6,6 @@ if [ $# -eq 0 ]; then fi BASE="https://github.com/commaai/openpilot/pull/" -PR_NUM="$(echo $1 | grep -o -E '[0-9]+')" +PR_NUM="$(echo "$1" | grep -o -E '[0-9]+')" -curl -L $BASE/$PR_NUM.patch | git apply -3 +curl -L "$BASE/$PR_NUM.patch" | git apply -3 diff --git a/scripts/checkout-pr.sh b/scripts/checkout-pr.sh index eeba816d88..5f5dc8b1e3 100755 --- a/scripts/checkout-pr.sh +++ b/scripts/checkout-pr.sh @@ -7,10 +7,10 @@ if [ $# -eq 0 ]; then fi BASE="https://github.com/commaai/openpilot/pull/" -PR_NUM="$(echo $1 | grep -o -E '[0-9]+')" +PR_NUM="$(echo "$1" | grep -o -E '[0-9]+')" BRANCH=tmp-pr${PR_NUM} -git branch -D -f $BRANCH || true -git fetch -u -f origin pull/$PR_NUM/head:$BRANCH -git switch $BRANCH +git branch -D -f "$BRANCH" || true +git fetch -u -f origin "pull/$PR_NUM/head:$BRANCH" +git switch "$BRANCH" git reset --hard FETCH_HEAD diff --git a/scripts/jenkins_loop_test.sh b/scripts/jenkins_loop_test.sh index 8073f4668c..6cbdafd355 100755 --- a/scripts/jenkins_loop_test.sh +++ b/scripts/jenkins_loop_test.sh @@ -11,7 +11,7 @@ BRANCH="master" RUNS="20" COOKIE_JAR=/tmp/cookies -CRUMB=$(curl -s --cookie-jar $COOKIE_JAR 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') +CRUMB=$(curl -s --cookie-jar "$COOKIE_JAR" 'https://jenkins.comma.life/crumbIssuer/api/xml?xpath=concat(//crumbRequestField,":",//crumb)') FIRST_LOOP=1 @@ -25,13 +25,13 @@ function loop() { if [[ $FIRST_LOOP ]]; then TEMP_DIR=$(mktemp -d) - GIT_LFS_SKIP_SMUDGE=1 git clone --quiet -b $BRANCH --depth=1 --no-tags git@github.com:commaai/openpilot $TEMP_DIR - git -C $TEMP_DIR checkout --quiet -b $JENKINS_BRANCH - echo "TESTING: $(date)" >> $TEMP_DIR/testing_jenkins - git -C $TEMP_DIR add testing_jenkins - git -C $TEMP_DIR commit --quiet -m "testing" - git -C $TEMP_DIR push --quiet -f origin $JENKINS_BRANCH - rm -rf $TEMP_DIR + GIT_LFS_SKIP_SMUDGE=1 git clone --quiet -b "$BRANCH" --depth=1 --no-tags git@github.com:commaai/openpilot "$TEMP_DIR" + git -C "$TEMP_DIR" checkout --quiet -b "$JENKINS_BRANCH" + echo "TESTING: $(date)" >> "$TEMP_DIR/testing_jenkins" + git -C "$TEMP_DIR" add testing_jenkins + git -C "$TEMP_DIR" commit --quiet -m "testing" + git -C "$TEMP_DIR" push --quiet -f origin "$JENKINS_BRANCH" + rm -rf "$TEMP_DIR" FIRST_BUILD=1 echo '' echo 'waiting on Jenkins...' @@ -40,15 +40,15 @@ function loop() { FIRST_LOOP="" fi - FIRST_BUILD=$(curl -s $API_ROUTE/api/json | jq .nextBuildNumber) + FIRST_BUILD=$(curl -s "$API_ROUTE/api/json" | jq .nextBuildNumber) LAST_BUILD=$((FIRST_BUILD+N-1)) - TEST_BUILDS=( $(seq $FIRST_BUILD $LAST_BUILD) ) + read -r -a TEST_BUILDS <<< "$(seq -s ' ' "$FIRST_BUILD" "$LAST_BUILD")" # Start N new builds - for i in ${TEST_BUILDS[@]}; + for i in "${TEST_BUILDS[@]}"; do echo "Starting build $i" - curl -s --output /dev/null --cookie $COOKIE_JAR -H "$CRUMB" -X POST $API_ROUTE/build?delay=0sec + curl -s --output /dev/null --cookie "$COOKIE_JAR" -H "$CRUMB" -X POST "$API_ROUTE/build?delay=0sec" sleep 5 done echo "" @@ -58,14 +58,14 @@ function loop() { sleep 30 count=0 - for i in ${TEST_BUILDS[@]}; + for i in "${TEST_BUILDS[@]}"; do - RES=$(curl -s -w "\n%{http_code}" --cookie $COOKIE_JAR -H "$CRUMB" $API_ROUTE/$i/api/json) + RES=$(curl -s -w "\n%{http_code}" --cookie "$COOKIE_JAR" -H "$CRUMB" "$API_ROUTE/$i/api/json") HTTP_CODE=$(tail -n1 <<< "$RES") JSON=$(sed '$ d' <<< "$RES") if [[ $HTTP_CODE == "200" ]]; then - STILL_RUNNING=$(echo $JSON | jq .inProgress) + STILL_RUNNING=$(echo "$JSON" | jq .inProgress) if [[ $STILL_RUNNING == "true" ]]; then echo -e "Build $i: ${YELLOW}still running${NC}" continue @@ -119,11 +119,11 @@ function _looper() { echo -e "You are about to start $RUNS Jenkins builds against the $BRANCH branch." echo -e "If you expect this to run overnight, ${UNDERLINE}${BOLD}unplug the cold reboot power switch${NC} from the testing closet before." echo "" - read -p "Press (y/Y) to confirm: " choice + read -r -p "Press (y/Y) to confirm: " choice if [[ "$choice" == "y" || "$choice" == "Y" ]]; then loop fi } -_looper $@ +_looper "$@" diff --git a/scripts/launch_corolla.sh b/scripts/launch_corolla.sh index 926569a1e0..8894caaf74 100755 --- a/scripts/launch_corolla.sh +++ b/scripts/launch_corolla.sh @@ -4,4 +4,4 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" export FINGERPRINT="TOYOTA_COROLLA_TSS2" export SKIP_FW_QUERY="1" -$DIR/../launch_openpilot.sh +"$DIR/../launch_openpilot.sh" diff --git a/scripts/lint/check_nomerge_comments.sh b/scripts/lint/check_nomerge_comments.sh index 6737d62a20..0dc3a7bd73 100755 --- a/scripts/lint/check_nomerge_comments.sh +++ b/scripts/lint/check_nomerge_comments.sh @@ -2,9 +2,9 @@ FAIL=0 -if grep -n '\(#\|//\)\([[:space:]]*\)NOMERGE' $@; then +if grep -n '\(#\|//\)\([[:space:]]*\)NOMERGE' "$@"; then echo -e "NOMERGE comments found! Remove them before merging\n" FAIL=1 fi -exit $FAIL +exit "$FAIL" diff --git a/scripts/lint/check_shebang_format.sh b/scripts/lint/check_shebang_format.sh index 89b95d5929..6a3657a10b 100755 --- a/scripts/lint/check_shebang_format.sh +++ b/scripts/lint/check_shebang_format.sh @@ -2,14 +2,14 @@ FAIL=0 -if grep '^#!.*python' $@ | grep -v '#!/usr/bin/env python3$'; then +if grep '^#!.*python' "$@" | grep -v '#!/usr/bin/env python3$'; then echo -e "Invalid shebang! Must use '#!/usr/bin/env python3'\n" FAIL=1 fi -if grep '^#!.*bash' $@ | grep -v '#!/usr/bin/env bash$'; then +if grep '^#!.*bash' "$@" | grep -v '#!/usr/bin/env bash$'; then echo -e "Invalid shebang! Must use '#!/usr/bin/env bash'" FAIL=1 fi -exit $FAIL +exit "$FAIL" diff --git a/scripts/lint/check_shell.py b/scripts/lint/check_shell.py new file mode 100755 index 0000000000..d34bd6e454 --- /dev/null +++ b/scripts/lint/check_shell.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""A minimal shellcheck-like static analysis tool for shell scripts. + +Covers syntax, unquoted expansions, scalar $@ assignments, and read -r. +""" +import re +import argparse +import subprocess +from pathlib import Path + +VARIABLE = re.compile(r"\$(?:\{[^}\n]*\}|[A-Za-z_]\w*|[@*0-9])") +ASSIGNMENT = re.compile(r"[A-Za-z_]\w*(?:\[[^]]*\])?\+?=") +OPAQUE = re.compile(r"\$?\(\([^\n]*?\)\)|\[\[.*?\]\]", re.DOTALL) +HEREDOC = re.compile(r"<<(-?)\s*('[^']+'|\"[^\"]+\"|\\?[A-Za-z_]\w*)") + + +def commands(text): + def scan(i=0, end="", pattern_group=False): + start, quote, words, expansions, documents = i, False, [], [], [] + cases = [] + while i < len(text): + c = text[i] + if c == "\\": + i += 2 + continue + if c == "'" and not quote: + i = text.find("'", i + 1) + 1 or len(text) + continue + if c == '"': + quote = not quote + elif (not quote or text.startswith("$((", i)) and (match := OPAQUE.match(text, i)): + i = match.end() + continue + elif text.startswith("$(", i): + if not quote: + expansions.append(i) + i = yield from scan(i + 2, ")") + continue + elif match := VARIABLE.match(text, i): + if not quote and not match[0].startswith("${#"): + expansions.append(i) + i = match.end() + continue + elif not quote: + if text.startswith("<<<", i): + i += 3 + continue + if c == "#" and i == start: + i = text.find("\n", i) + i = len(text) if i < 0 else i + start = i + continue + if not text.startswith("<<<", i) and (match := HEREDOC.match(text, i)): + documents.append((match[2].strip("'\"").lstrip("\\"), bool(match[1]))) + i = match.end() + continue + if c in " \t\r\n;|&()": + if start < i: + words.append((text[start:i], start, expansions)) + if len(words) >= 3 and words[0][0] == "case" and words[-1][0] == "in": + cases.append(True) + words = [] + if words and words[0][0] == "esac" and cases: + cases.pop() + words = [] + pattern = bool(cases and cases[-1]) + expansions = [] + if c in "\n;|&()": + if words and not (pattern or pattern_group) and (c != ")" or end): + yield words + words = [] + if c == end and not pattern: + return i + 1 + if c == ")" and pattern: + cases[-1] = False + if cases and (terminator := re.match(r";(?:;&|;|&)", text[i:])): + cases[-1] = True + i += len(terminator[0]) - 1 + if c == "(": + if pattern and i == start: + cases[-1] = False + i = yield from scan(i + 1, ")", pattern or pattern_group) + start = i + continue + if c == "\n": + for delimiter, strip_tabs in documents: + while i < len(text): + stop = text.find("\n", i + 1) + stop = len(text) if stop < 0 else stop + line, i = text[i + 1:stop], stop + if (line.lstrip("\t") if strip_tabs else line) == delimiter: + break + documents = [] + start = i + 1 + i += 1 + if start < i: + words.append((text[start:i], start, expansions)) + if words: + yield words + return i + yield from scan() + + +def check_text(text): + for words in commands(text): + command = next((w for w, _, _ in words if not ASSIGNMENT.match(w) and w not in {"if", "then", "elif", "while", "until", "do", "!"}), "") + prefix = True + for index, (word, start, expansions) in enumerate(words): + assignment = ASSIGNMENT.match(word) and (prefix or command in {"export", "local", "declare", "readonly", "typeset"}) + prefix = prefix and (bool(assignment) or word in {"if", "then", "elif", "while", "until", "do", "!"}) + for offset in expansions: + array = word in {"$@", "$*"} or "[@]" in word or "[*]" in word + if not assignment and (command not in {"case", "for", "select"} or array) and not (index and words[index - 1][0] == "<<<"): + yield text.count("\n", 0, offset) + 1, "Quote this expansion to prevent word splitting and globbing" + if assignment and re.fullmatch(r'"[^"\n]*\$@[^"\n]*"', ASSIGNMENT.sub("", word, count=1)): + yield text.count("\n", 0, start) + 1, 'Use an array for "$@", or "$*" to join arguments' + if command == "read" and not any(re.fullmatch(r"-[A-Za-z]*r[A-Za-z0-9]*", w) for w, _, _ in words): + yield text.count("\n", 0, next(start for w, start, _ in words if w == "read")) + 1, "Use read -r to preserve backslashes" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("filenames", nargs="+") + failed = False + for filename in parser.parse_args().filenames: + syntax = subprocess.run(["bash", "-n", "--", filename], check=False) + failed |= syntax.returncode != 0 + if syntax.returncode == 0: + for line, message in check_text(Path(filename).read_text()): + print(f"{filename}:{line}: {message}") + failed = True + raise SystemExit(failed) diff --git a/scripts/lint/lint.sh b/scripts/lint/lint.sh index fe3644d670..b05b2475cb 100755 --- a/scripts/lint/lint.sh +++ b/scripts/lint/lint.sh @@ -9,7 +9,7 @@ NC='\033[0m' DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" ROOT="$DIR/../../" -cd $ROOT +cd "$ROOT" FAILED=0 @@ -26,10 +26,10 @@ function run() { done shift 1; - CMD="$@" + CMD=("$@") set +e - log="$((eval "$CMD" ) 2>&1)" + log="$("${CMD[@]}" 2>&1)" if [[ $? -eq 0 ]]; then echo -e "[${GREEN}✔${NC}]" @@ -42,23 +42,21 @@ function run() { } function run_tests() { - ALL_FILES=$1 - PYTHON_FILES=$2 - run "ruff" ruff check openpilot --quiet - run "check_dependencies" python3 $DIR/check_dependencies.py - run "check_indentation" $DIR/check_indentation.py $PYTHON_FILES - run "check_added_large_files" $DIR/check_added_large_files.py --maxkb=120 $ALL_FILES - run "check_shebang_scripts_are_executable" $DIR/check_shebang_scripts_are_executable.py $ALL_FILES - run "check_shebang_format" $DIR/check_shebang_format.sh $ALL_FILES - run "check_nomerge_comments" $DIR/check_nomerge_comments.sh $ALL_FILES + run "check_shell" python3 "$DIR/check_shell.py" "${SHELL_FILES[@]}" + run "check_dependencies" python3 "$DIR/check_dependencies.py" + run "check_indentation" "$DIR/check_indentation.py" "${PYTHON_FILES[@]}" + run "check_added_large_files" "$DIR/check_added_large_files.py" --maxkb=120 "${ALL_FILES[@]}" + run "check_shebang_scripts_are_executable" "$DIR/check_shebang_scripts_are_executable.py" "${ALL_FILES[@]}" + run "check_shebang_format" "$DIR/check_shebang_format.sh" "${ALL_FILES[@]}" + run "check_nomerge_comments" "$DIR/check_nomerge_comments.sh" "${ALL_FILES[@]}" if [[ -z "$FAST" ]]; then run "ty" ty check openpilot - run "codespell" codespell $ALL_FILES + run "codespell" codespell "${ALL_FILES[@]}" fi - return $FAILED + return "$FAILED" } function help() { @@ -68,6 +66,7 @@ function help() { echo "" echo -e "${BOLD}${UNDERLINE}Tests:${NC}" echo -e " ${BOLD}ruff${NC}" + echo -e " ${BOLD}check_shell${NC}" echo -e " ${BOLD}check_dependencies${NC}" echo -e " ${BOLD}check_indentation${NC}" echo -e " ${BOLD}ty${NC}" @@ -103,16 +102,26 @@ while [[ $# -gt 0 ]]; do esac done -RUN=$([ -z "$RUN" ] && echo "" || echo "!($(echo $RUN | sed 's/ /|/g'))") -SKIP="@($(echo $SKIP | sed 's/ /|/g'))" +RUN=$([ -z "$RUN" ] && echo "" || echo "!($(echo "$RUN" | sed 's/ /|/g'))") +SKIP="@($(echo "$SKIP" | sed 's/ /|/g'))" -GIT_FILES="$(git ls-files openpilot)" -ALL_FILES="" -for f in $GIT_FILES; do +ALL_FILES=() +PYTHON_FILES=() +while IFS= read -r -d '' f; do if [[ -f $f ]]; then - ALL_FILES+="$f"$'\n' + ALL_FILES+=("$f") + if [[ $f == *.py ]]; then + PYTHON_FILES+=("$f") + fi fi -done -PYTHON_FILES=$(echo "$ALL_FILES" | grep --color=never '.py$' || true) +done < <(git ls-files -z openpilot) -run_tests "$ALL_FILES" "$PYTHON_FILES" +# Include tooling, launchers, and the extensionless Git hook. +SHELL_FILES=() +while IFS= read -r -d '' f; do + if [[ -f $f ]]; then + SHELL_FILES+=("$f") + fi +done < <(git ls-files -z '*.sh' '*.bash' scripts/post-commit) + +run_tests diff --git a/scripts/retry.sh b/scripts/retry.sh index 23501d7559..382190fe10 100755 --- a/scripts/retry.sh +++ b/scripts/retry.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash function fail { - echo $1 >&2 + echo "$1" >&2 exit 1 } @@ -14,7 +14,7 @@ function retry { "$@" && break || { if [[ $n -lt $max ]]; then ((n++)) - sleep $delay; + sleep "$delay"; else fail "The command has failed after $n attempts." fi diff --git a/tools/op.sh b/tools/op.sh index 183b4b1012..d1a7dcebbb 100755 --- a/tools/op.sh +++ b/tools/op.sh @@ -14,9 +14,9 @@ UNDERLINE='\033[4m' BOLD='\033[1m' NC='\033[0m' -SHELL_NAME="$(basename ${SHELL})" -RC_FILE="${HOME}/.$(basename ${SHELL})rc" -if [ "$(uname)" == "Darwin" ] && [ $SHELL == "/bin/bash" ]; then +SHELL_NAME="$(basename "${SHELL}")" +RC_FILE="${HOME}/.$(basename "${SHELL}")rc" +if [ "$(uname)" == "Darwin" ] && [ "$SHELL" == "/bin/bash" ]; then RC_FILE="$HOME/.bash_profile" fi @@ -250,7 +250,7 @@ function op_venv() { case $SHELL_NAME in "zsh") ZSHRC_DIR=$(mktemp -d 2>/dev/null || mktemp -d -t 'tmp_zsh') - echo "source \"$RC_FILE\"; source \"$OPENPILOT_ROOT/.venv/bin/activate\"" >> $ZSHRC_DIR/.zshrc + echo "source \"$RC_FILE\"; source \"$OPENPILOT_ROOT/.venv/bin/activate\"" >> "$ZSHRC_DIR/.zshrc" ZDOTDIR=$ZSHRC_DIR zsh ;; *) bash --rcfile <(echo "source \"$RC_FILE\"; source \"$OPENPILOT_ROOT/.venv/bin/activate\"") ;; @@ -405,14 +405,14 @@ function op_start() { if [[ -f "/AGNOS" ]]; then op_before_cmd op_check_agnos_update - op_run_command sudo systemctl restart comma $@ + op_run_command sudo systemctl restart comma "$@" fi } function op_stop() { if [[ -f "/AGNOS" ]]; then op_before_cmd - op_run_command sudo systemctl stop comma $@ + op_run_command sudo systemctl stop comma "$@" fi } diff --git a/tools/release/build_release.sh b/tools/release/build_release.sh index cced9f7a54..6dcf99957f 100755 --- a/tools/release/build_release.sh +++ b/tools/release/build_release.sh @@ -3,7 +3,7 @@ set -e set -x DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR +cd "$DIR" BUILD_DIR=/data/openpilot SOURCE_DIR="$(git rev-parse --show-toplevel)" @@ -19,26 +19,26 @@ BUILD_BRANCH=release-mici-staging # set git identity -source $DIR/identity.sh +source "$DIR/identity.sh" echo "[-] Setting up repo T=$SECONDS" if ! git -C "$SOURCE_DIR" worktree remove --force "$BUILD_DIR" 2>/dev/null; then - rm -rf $BUILD_DIR + rm -rf "$BUILD_DIR" fi git -C "$SOURCE_DIR" worktree prune git -C "$SOURCE_DIR" worktree add --detach --no-checkout "$BUILD_DIR" -cd $BUILD_DIR +cd "$BUILD_DIR" git update-ref -d "refs/heads/$BUILD_BRANCH" git symbolic-ref HEAD "refs/heads/$BUILD_BRANCH" git read-tree --empty # do the files copy echo "[-] copying files T=$SECONDS" -cd $SOURCE_DIR +cd "$SOURCE_DIR" ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$BUILD_DIR" -- # in the directory -cd $BUILD_DIR +cd "$BUILD_DIR" # use the full CPU available for speeding up the build. # openpilot resets the CPU frequencies when test_onroad.py runs below. @@ -88,7 +88,7 @@ git -c core.compression=0 add -f . git -c core.compression=0 -c gc.auto=0 commit -m "openpilot v$VERSION" # Run tests -cd $BUILD_DIR +cd "$BUILD_DIR" RELEASE=1 ./openpilot/selfdrive/test/test_onroad.py "$@" #tools/test_runner.py openpilot/selfdrive/car/tests/test_car_interfaces.py diff --git a/tools/release/build_stripped.sh b/tools/release/build_stripped.sh index a216dfe11c..239add2518 100755 --- a/tools/release/build_stripped.sh +++ b/tools/release/build_stripped.sh @@ -3,23 +3,23 @@ set -ex DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)" +SOURCE_DIR="$(git -C "$DIR" rev-parse --show-toplevel)" if [ -z "$TARGET_DIR" ]; then TARGET_DIR="$(mktemp -d)" fi # set git identity -source $DIR/identity.sh +source "$DIR/identity.sh" echo "[-] Setting up target repo T=$SECONDS" -rm -rf $TARGET_DIR -mkdir -p $TARGET_DIR -cd $TARGET_DIR -cp -r $SOURCE_DIR/.git $TARGET_DIR +rm -rf "$TARGET_DIR" +mkdir -p "$TARGET_DIR" +cd "$TARGET_DIR" +cp -r "$SOURCE_DIR/.git" "$TARGET_DIR" echo "[-] setting up stripped branch sync T=$SECONDS" -cd $TARGET_DIR +cd "$TARGET_DIR" # tmp branch git checkout --orphan tmp @@ -32,20 +32,20 @@ find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm - # do the files copy echo "[-] copying files T=$SECONDS" -cd $SOURCE_DIR +cd "$SOURCE_DIR" ./tools/release/release_files.py | xargs -0 cp -pR --parents -t "$TARGET_DIR" -- # in the directory -cd $TARGET_DIR +cd "$TARGET_DIR" rm -rf .git/modules/ find openpilot/selfdrive/modeld/models -name '*.onnx' -size +95M -exec ./openpilot/common/file_chunker.py {} \; # include source commit hash and build date in commit -GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD) -GIT_COMMIT_DATE=$(git --git-dir=$SOURCE_DIR/.git show --no-patch --format='%ct %ci' HEAD) +GIT_HASH=$(git --git-dir="$SOURCE_DIR/.git" rev-parse HEAD) +GIT_COMMIT_DATE=$(git --git-dir="$SOURCE_DIR/.git" show --no-patch --format='%ct %ci' HEAD) DATETIME=$(date '+%Y-%m-%dT%H:%M:%S') -VERSION=$(cat $SOURCE_DIR/openpilot/common/version.h | awk -F\" '{print $2}') +VERSION=$(cat "$SOURCE_DIR/openpilot/common/version.h" | awk -F\" '{print $2}') echo -n "$GIT_HASH" > git_src_commit echo -n "$GIT_COMMIT_DATE" > git_src_commit_date @@ -82,7 +82,7 @@ if [ ! -z "$BRANCH" ]; then git config --local core.hooksPath .git/hooks git lfs update --force # uploading the larger pack is faster than spending CPU to optimize it - git -c pack.window=0 -c pack.depth=0 -c pack.compression=0 push -f origin tmp:$BRANCH + git -c pack.window=0 -c pack.depth=0 -c pack.compression=0 push -f origin "tmp:$BRANCH" fi echo "[-] done T=$SECONDS, ready at $TARGET_DIR" diff --git a/tools/release/check-dirty.sh b/tools/release/check-dirty.sh index ac049970cf..73d393f3c2 100755 --- a/tools/release/check-dirty.sh +++ b/tools/release/check-dirty.sh @@ -2,7 +2,7 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" -cd $DIR +cd "$DIR" if [ ! -z "$(git status --porcelain)" ]; then echo "Dirty working tree after build:" diff --git a/tools/release/check-submodules.sh b/tools/release/check-submodules.sh index 93869a7403..c93bb58752 100755 --- a/tools/release/check-submodules.sh +++ b/tools/release/check-submodules.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash -while read hash submodule ref; do +while read -r hash submodule ref; do if [ "$submodule" = "tinygrad_repo" ]; then echo "Skipping $submodule" continue fi - git -C $submodule fetch --depth 100 origin master - git -C $submodule branch -r --contains $hash | grep "origin/master" + git -C "$submodule" fetch --depth 100 origin master + git -C "$submodule" branch -r --contains "$hash" | grep "origin/master" if [ "$?" -eq 0 ]; then echo "$submodule ok" else diff --git a/tools/scripts/adb_ssh.sh b/tools/scripts/adb_ssh.sh index c584a72f2b..b808243d36 100755 --- a/tools/scripts/adb_ssh.sh +++ b/tools/scripts/adb_ssh.sh @@ -36,7 +36,7 @@ SSH_PORT=2222 while ss -tln | grep -q ":${SSH_PORT} "; do SSH_PORT=$((SSH_PORT + 1)) done -adb forward tcp:${SSH_PORT} tcp:22 +adb forward tcp:"${SSH_PORT}" tcp:22 # SSH! -ssh comma@localhost -p ${SSH_PORT} "$@" +ssh comma@localhost -p "${SSH_PORT}" "$@" diff --git a/tools/setup.sh b/tools/setup.sh index ced451ab11..81eb543114 100755 --- a/tools/setup.sh +++ b/tools/setup.sh @@ -59,10 +59,10 @@ function ask_dir() { return 0 fi - read + read -r if [[ ! -z "$REPLY" ]]; then - mkdir -p $REPLY - OPENPILOT_ROOT="$(realpath $REPLY)/openpilot" + mkdir -p "$REPLY" + OPENPILOT_ROOT="$(realpath "$REPLY")/openpilot" fi } @@ -114,7 +114,7 @@ function check_git() { function git_clone() { st="$(date +%s)" echo "Cloning openpilot..." - if $(git clone --filter=blob:none https://github.com/commaai/openpilot.git "$OPENPILOT_ROOT"); then + if git clone --filter=blob:none https://github.com/commaai/openpilot.git "$OPENPILOT_ROOT"; then if [[ -f $OPENPILOT_ROOT/launch_openpilot.sh ]]; then et="$(date +%s)" echo -e " ↳ [${GREEN}✔${NC}] Successfully cloned openpilot in $((et - st)) seconds.\n" @@ -127,10 +127,10 @@ function git_clone() { } function install_with_op() { - cd $OPENPILOT_ROOT - $OPENPILOT_ROOT/tools/op.sh post-commit + cd "$OPENPILOT_ROOT" + "$OPENPILOT_ROOT/tools/op.sh" post-commit - if ! $OPENPILOT_ROOT/tools/op.sh setup; then + if ! "$OPENPILOT_ROOT/tools/op.sh" setup; then echo -e "\n[${RED}✗${NC}] failed to install openpilot!" return 1 fi @@ -147,5 +147,5 @@ check_stdin ask_dir check_dir check_git -[ -z $SKIP_GIT_CLONE ] && git_clone +[ -z "$SKIP_GIT_CLONE" ] && git_clone install_with_op diff --git a/tools/setup_dependencies.sh b/tools/setup_dependencies.sh index 7af2180686..c9e635836b 100755 --- a/tools/setup_dependencies.sh +++ b/tools/setup_dependencies.sh @@ -20,14 +20,14 @@ function retry() { } function install_linux_deps() { - SUDO="" + SUDO=() if [[ ! $(id -u) -eq 0 ]]; then if [[ -z $(which sudo) ]]; then echo "Please install sudo or run as root" exit 1 fi - SUDO="sudo" + SUDO=(sudo) fi local missing_linux_deps=0 @@ -51,28 +51,28 @@ function install_linux_deps() { # the native package managers are slow, so skip if we can echo "[ ] system packages already installed t=$SECONDS" elif command -v apt-get > /dev/null 2>&1; then - $SUDO apt-get update - $SUDO apt-get install -y --no-install-recommends ca-certificates build-essential curl libcurl4-openssl-dev locales git xclip wl-clipboard + "${SUDO[@]}" apt-get update + "${SUDO[@]}" apt-get install -y --no-install-recommends ca-certificates build-essential curl libcurl4-openssl-dev locales git xclip wl-clipboard elif command -v dnf > /dev/null 2>&1; then - $SUDO dnf install -y ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-langpack-en git + "${SUDO[@]}" dnf install -y ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-langpack-en git elif command -v yum > /dev/null 2>&1; then - $SUDO yum install -y ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-langpack-en git + "${SUDO[@]}" yum install -y ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-langpack-en git elif command -v pacman > /dev/null 2>&1; then - $SUDO pacman -Syu --noconfirm --needed base-devel ca-certificates curl git + "${SUDO[@]}" pacman -Syu --noconfirm --needed base-devel ca-certificates curl git elif command -v zypper > /dev/null 2>&1; then - $SUDO zypper --non-interactive refresh - $SUDO zypper --non-interactive install ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-locale git + "${SUDO[@]}" zypper --non-interactive refresh + "${SUDO[@]}" zypper --non-interactive install ca-certificates gcc gcc-c++ make curl libcurl-devel glibc-locale git elif command -v apk > /dev/null 2>&1; then - $SUDO apk add --no-cache ca-certificates build-base curl curl-dev musl-locales git + "${SUDO[@]}" apk add --no-cache ca-certificates build-base curl curl-dev musl-locales git elif command -v xbps-install > /dev/null 2>&1; then - $SUDO xbps-install -Syu base-devel ca-certificates curl git libcurl-devel glibc-locales + "${SUDO[@]}" xbps-install -Syu base-devel ca-certificates curl git libcurl-devel glibc-locales else echo "Unsupported Linux distribution. Supported package managers: apt-get, dnf, yum, pacman, zypper, apk, xbps-install." exit 1 fi if [[ -d "/etc/udev/rules.d/" ]]; then - $SUDO tee /etc/udev/rules.d/11-openpilot.rules > /dev/null <<-EOF + "${SUDO[@]}" tee /etc/udev/rules.d/11-openpilot.rules > /dev/null <<-EOF # Panda Jungle devices SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddcf", MODE="0666" SUBSYSTEM=="usb", ATTRS{idVendor}=="3801", ATTRS{idProduct}=="ddef", MODE="0666" @@ -91,9 +91,9 @@ function install_linux_deps() { EOF # delete the old ones - $SUDO rm -f /etc/udev/rules.d/11-panda.rules /etc/udev/rules.d/12-panda_jungle.rules /etc/udev/rules.d/50-comma-adb.rules + "${SUDO[@]}" rm -f /etc/udev/rules.d/11-panda.rules /etc/udev/rules.d/12-panda_jungle.rules /etc/udev/rules.d/50-comma-adb.rules - $SUDO udevadm control --reload-rules && $SUDO udevadm trigger || true + "${SUDO[@]}" udevadm control --reload-rules && "${SUDO[@]}" udevadm trigger || true fi }