openpilot v0.9.7 release

date: 2024-06-11T01:36:39
master commit: f8cb04e4a8b032b72a909f68b808a50936184bee

release/build_devel.sh

@@ -0,0 +1,87 @@
+#!/usr/bin/bash
+set -ex
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+SOURCE_DIR="$(git -C $DIR rev-parse --show-toplevel)"
+if [ -z "$TARGET_DIR" ]; then
+  TARGET_DIR="$(mktemp -d)"
+fi
+
+# set git identity
+source $DIR/identity.sh
+
+echo "[-] Setting up target repo T=$SECONDS"
+
+rm -rf $TARGET_DIR
+mkdir -p $TARGET_DIR
+cd $TARGET_DIR
+cp -r $SOURCE_DIR/.git $TARGET_DIR
+pre-commit uninstall || true
+
+echo "[-] bringing master-ci and devel in sync T=$SECONDS"
+cd $TARGET_DIR
+
+git fetch --depth 1 origin master-ci
+git fetch --depth 1 origin devel
+
+git checkout -f --track origin/master-ci
+git reset --hard master-ci
+git checkout master-ci
+git reset --hard origin/devel
+git clean -xdff
+git lfs uninstall
+
+# remove everything except .git
+echo "[-] erasing old openpilot T=$SECONDS"
+find . -maxdepth 1 -not -path './.git' -not -name '.' -not -name '..' -exec rm -rf '{}' \;
+
+# reset source tree
+cd $SOURCE_DIR
+git clean -xdff
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(./release/release_files.py) $TARGET_DIR/
+
+# in the directory
+cd $TARGET_DIR
+rm -f panda/board/obj/panda.bin.signed
+
+# include source commit hash and build date in commit
+GIT_HASH=$(git --git-dir=$SOURCE_DIR/.git rev-parse HEAD)
+DATETIME=$(date '+%Y-%m-%dT%H:%M:%S')
+VERSION=$(cat $SOURCE_DIR/common/version.h | awk -F\" '{print $2}')
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git status
+git commit -a -m "openpilot v$VERSION release
+
+date: $DATETIME
+master commit: $GIT_HASH
+"
+
+# should be no submodules or LFS files
+git submodule status
+if [ ! -z "$(git lfs ls-files)" ]; then
+  echo "LFS files detected!"
+  exit 1
+fi
+
+# ensure files are within GitHub's limit
+BIG_FILES="$(find . -type f -not -path './.git/*' -size +95M)"
+if [ ! -z "$BIG_FILES" ]; then
+  printf '\n\n\n'
+  echo "Found files exceeding GitHub's 100MB limit:"
+  echo "$BIG_FILES"
+  exit 1
+fi
+
+if [ ! -z "$BRANCH" ]; then
+  echo "[-] Pushing to $BRANCH T=$SECONDS"
+  git push -f origin master-ci:$BRANCH
+fi
+
+echo "[-] done T=$SECONDS, ready at $TARGET_DIR"

release/build_release.sh

@@ -0,0 +1,103 @@
+#!/usr/bin/bash
+set -e
+set -x
+
+# git diff --name-status origin/release3-staging | grep "^A" | less
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+
+cd $DIR
+
+BUILD_DIR=/data/openpilot
+SOURCE_DIR="$(git rev-parse --show-toplevel)"
+
+if [ -z "$RELEASE_BRANCH" ]; then
+  echo "RELEASE_BRANCH is not set"
+  exit 1
+fi
+
+
+# set git identity
+source $DIR/identity.sh
+
+echo "[-] Setting up repo T=$SECONDS"
+rm -rf $BUILD_DIR
+mkdir -p $BUILD_DIR
+cd $BUILD_DIR
+git init
+git remote add origin git@github.com:commaai/openpilot.git
+git checkout --orphan $RELEASE_BRANCH
+
+# do the files copy
+echo "[-] copying files T=$SECONDS"
+cd $SOURCE_DIR
+cp -pR --parents $(./release/release_files.py) $BUILD_DIR/
+
+# in the directory
+cd $BUILD_DIR
+
+rm -f panda/board/obj/panda.bin.signed
+rm -f panda/board/obj/panda_h7.bin.signed
+
+VERSION=$(cat common/version.h | awk -F[\"-]  '{print $2}')
+echo "#define COMMA_VERSION \"$VERSION-release\"" > common/version.h
+
+echo "[-] committing version $VERSION T=$SECONDS"
+git add -f .
+git commit -a -m "openpilot v$VERSION release"
+
+# Build
+export PYTHONPATH="$BUILD_DIR"
+scons -j$(nproc) --minimal
+
+# release panda fw
+CERT=/data/pandaextra/certs/release RELEASE=1 scons -j$(nproc) panda/
+
+# Ensure no submodules in release
+if test "$(git submodule--helper list | wc -l)" -gt "0"; then
+  echo "submodules found:"
+  git submodule--helper list
+  exit 1
+fi
+git submodule status
+
+# Cleanup
+find . -name '*.a' -delete
+find . -name '*.o' -delete
+find . -name '*.os' -delete
+find . -name '*.pyc' -delete
+find . -name 'moc_*' -delete
+find . -name '__pycache__' -delete
+rm -rf .sconsign.dblite Jenkinsfile release/
+rm selfdrive/modeld/models/supercombo.onnx
+
+find third_party/ -name '*x86*' -exec rm -r {} +
+find third_party/ -name '*Darwin*' -exec rm -r {} +
+
+
+# Restore third_party
+git checkout third_party/
+
+# Mark as prebuilt release
+touch prebuilt
+
+# Add built files to git
+git add -f .
+git commit --amend -m "openpilot v$VERSION"
+
+# Run tests
+TEST_FILES="tools/"
+cd $SOURCE_DIR
+cp -pR -n --parents $TEST_FILES $BUILD_DIR/
+cd $BUILD_DIR
+RELEASE=1 pytest -n0 -s selfdrive/test/test_onroad.py
+#system/manager/test/test_manager.py
+#pytest selfdrive/car/tests/test_car_interfaces.py
+rm -rf $TEST_FILES
+
+if [ ! -z "$RELEASE_BRANCH" ]; then
+  echo "[-] pushing release T=$SECONDS"
+  git push -f origin $RELEASE_BRANCH:$RELEASE_BRANCH
+fi
+
+echo "[-] done T=$SECONDS"

release/check-dirty.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/bash
+set -e
+
+DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
+cd $DIR
+
+if [ ! -z "$(git status --porcelain)" ]; then
+  echo "Dirty working tree after build:"
+  git status --porcelain
+  exit 1
+fi

release/check-submodules.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+while read hash submodule ref; do
+  git -C $submodule fetch --depth 2000 origin master
+  git -C $submodule branch -r --contains $hash | grep "origin/master"
+  if [ "$?" -eq 0 ]; then
+    echo "$submodule ok"
+  else
+    echo "$submodule: $hash is not on master"
+    exit 1
+  fi
+done <<< $(git submodule status --recursive)

release/identity.sh

@@ -0,0 +1,4 @@
+export GIT_COMMITTER_NAME="Vehicle Researcher"
+export GIT_COMMITTER_EMAIL="user@comma.ai"
+export GIT_AUTHOR_NAME="Vehicle Researcher"
+export GIT_AUTHOR_EMAIL="user@comma.ai"

release/release_files.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+import os
+import re
+from pathlib import Path
+
+HERE = os.path.abspath(os.path.dirname(__file__))
+ROOT = HERE + "/.."
+
+# blacklisting is for two purposes:
+# - minimizing release download size
+# - keeping the diff readable
+blacklist = [
+  "body/STL/",
+
+  "panda/drivers/",
+  "panda/examples/",
+  "panda/tests/safety/",
+
+  "opendbc/.*.dbc$",
+  "opendbc/generator/",
+
+  "cereal/.*test.*",
+  "^common/tests/",
+
+  # particularly large text files
+  "poetry.lock",
+  "third_party/catch2",
+  "selfdrive/car/tests/test_models.*",
+
+  "^tools/",
+  "^scripts/",
+  "^tinygrad_repo/",
+
+  "matlab.*.md",
+
+  ".git/",
+  ".github/",
+  ".devcontainer/",
+  "Darwin/",
+  ".vscode",
+
+  # common things
+  "LICENSE",
+  "Dockerfile",
+  ".pre-commit",
+
+  # no LFS or submodules in release
+  ".lfsconfig",
+  ".gitattributes",
+  ".git$",
+  ".gitmodules",
+]
+
+# gets you through the blacklist
+whitelist = [
+  "tools/lib/",
+  "tools/bodyteleop/",
+
+  "tinygrad_repo/openpilot/compile2.py",
+  "tinygrad_repo/extra/onnx.py",
+  "tinygrad_repo/extra/onnx_ops.py",
+  "tinygrad_repo/extra/thneed.py",
+  "tinygrad_repo/extra/utils.py",
+  "tinygrad_repo/tinygrad/codegen/kernel.py",
+  "tinygrad_repo/tinygrad/codegen/linearizer.py",
+  "tinygrad_repo/tinygrad/features/image.py",
+  "tinygrad_repo/tinygrad/features/search.py",
+  "tinygrad_repo/tinygrad/nn/*",
+  "tinygrad_repo/tinygrad/renderer/cstyle.py",
+  "tinygrad_repo/tinygrad/renderer/opencl.py",
+  "tinygrad_repo/tinygrad/runtime/lib.py",
+  "tinygrad_repo/tinygrad/runtime/ops_cpu.py",
+  "tinygrad_repo/tinygrad/runtime/ops_disk.py",
+  "tinygrad_repo/tinygrad/runtime/ops_gpu.py",
+  "tinygrad_repo/tinygrad/shape/*",
+  "tinygrad_repo/tinygrad/.*.py",
+
+  # TODO: do this automatically
+  "opendbc/comma_body.dbc",
+  "opendbc/chrysler_ram_hd_generated.dbc",
+  "opendbc/chrysler_ram_dt_generated.dbc",
+  "opendbc/chrysler_pacifica_2017_hybrid_generated.dbc",
+  "opendbc/chrysler_pacifica_2017_hybrid_private_fusion.dbc",
+  "opendbc/gm_global_a_powertrain_generated.dbc",
+  "opendbc/gm_global_a_object.dbc",
+  "opendbc/gm_global_a_chassis.dbc",
+  "opendbc/FORD_CADS.dbc",
+  "opendbc/ford_fusion_2018_adas.dbc",
+  "opendbc/ford_lincoln_base_pt.dbc",
+  "opendbc/honda_accord_2018_can_generated.dbc",
+  "opendbc/acura_ilx_2016_can_generated.dbc",
+  "opendbc/acura_rdx_2018_can_generated.dbc",
+  "opendbc/acura_rdx_2020_can_generated.dbc",
+  "opendbc/honda_civic_touring_2016_can_generated.dbc",
+  "opendbc/honda_civic_hatchback_ex_2017_can_generated.dbc",
+  "opendbc/honda_crv_touring_2016_can_generated.dbc",
+  "opendbc/honda_crv_ex_2017_can_generated.dbc",
+  "opendbc/honda_crv_ex_2017_body_generated.dbc",
+  "opendbc/honda_crv_executive_2016_can_generated.dbc",
+  "opendbc/honda_fit_ex_2018_can_generated.dbc",
+  "opendbc/honda_odyssey_exl_2018_generated.dbc",
+  "opendbc/honda_odyssey_extreme_edition_2018_china_can_generated.dbc",
+  "opendbc/honda_insight_ex_2019_can_generated.dbc",
+  "opendbc/acura_ilx_2016_nidec.dbc",
+  "opendbc/honda_civic_ex_2022_can_generated.dbc",
+  "opendbc/hyundai_canfd.dbc",
+  "opendbc/hyundai_kia_generic.dbc",
+  "opendbc/hyundai_kia_mando_front_radar_generated.dbc",
+  "opendbc/mazda_2017.dbc",
+  "opendbc/nissan_x_trail_2017_generated.dbc",
+  "opendbc/nissan_leaf_2018_generated.dbc",
+  "opendbc/subaru_global_2017_generated.dbc",
+  "opendbc/subaru_global_2020_hybrid_generated.dbc",
+  "opendbc/subaru_outback_2015_generated.dbc",
+  "opendbc/subaru_outback_2019_generated.dbc",
+  "opendbc/subaru_forester_2017_generated.dbc",
+  "opendbc/toyota_tnga_k_pt_generated.dbc",
+  "opendbc/toyota_new_mc_pt_generated.dbc",
+  "opendbc/toyota_nodsu_pt_generated.dbc",
+  "opendbc/toyota_adas.dbc",
+  "opendbc/toyota_tss2_adas.dbc",
+  "opendbc/vw_golf_mk4.dbc",
+  "opendbc/vw_mqb_2010.dbc",
+  "opendbc/tesla_can.dbc",
+  "opendbc/tesla_radar_bosch_generated.dbc",
+  "opendbc/tesla_radar_continental_generated.dbc",
+  "opendbc/tesla_powertrain.dbc",
+]
+
+
+if __name__ == "__main__":
+  for f in Path(ROOT).rglob("**/*"):
+    if not (f.is_file() or f.is_symlink()):
+      continue
+
+    rf = str(f.relative_to(ROOT))
+    blacklisted = any(re.search(p, rf) for p in blacklist)
+    whitelisted = any(re.search(p, rf) for p in whitelist)
+    if blacklisted and not whitelisted:
+      continue
+
+    print(rf)