From 9413bb59529a6571a0e54acb9d68d73150790700 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Tue, 13 Feb 2024 08:47:10 -0500 Subject: [PATCH 01/10] Update CHANGELOGS.md --- CHANGELOGS.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index 76cbb73382..348bf3467c 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -4,6 +4,7 @@ sunnypilot - 0.9.6.0 (2024-xx-xx) * NEW❗: Default Driving Model: Los Angeles v2 (January 24, 2024) * UPDATED: Driving Model Selector v3 * NEW❗: Driving Model additions + * Certified Herbalist v2 (February 13, 2024) - CHv2 * Certified Herbalist (February 5, 2024) - CH * Los Angeles v2 (January 24, 2024) - LAv2 * Los Angeles (January 22, 2024) - LAv1 @@ -21,10 +22,10 @@ sunnypilot - 0.9.6.0 (2024-xx-xx) * Custom Offsets * UPDATED: Dynamic Lane Profile (DLP) * Continued support for Legacy Driving Models (e.g., ND, BDv2, BDv1, FV, NS) - * Deprecate support for newer Driving Models (e.g., CH, LAv2, LAv1) + * Deprecate support for newer Driving Models (e.g., CHv2, CH, LAv2, LAv1) * UPDATED: Custom Offsets * Continued support for Legacy Driving Models (e.g., ND, BDv2, BDv1, FV, NS) - * Deprecate support for newer Driving Models (e.g., CH, LAv2, LAv1) + * Deprecate support for newer Driving Models (e.g., CHv2, CH, LAv2, LAv1) * FIXED: New comma 3X support * FIXED: New comma eSIM support * Bug fixes and performance improvements From 0e90bf751bb717a09d3454f79507ef740570ff3f Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Tue, 13 Feb 2024 13:36:17 -0500 Subject: [PATCH 02/10] Update CHANGELOGS.md --- CHANGELOGS.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOGS.md b/CHANGELOGS.md index 348bf3467c..7f729fc9cf 100644 --- a/CHANGELOGS.md +++ b/CHANGELOGS.md @@ -17,7 +17,7 @@ sunnypilot - 0.9.6.0 (2024-xx-xx) * Blue Diamond (November 18, 2023) - BDv1 * Farmville (November 7, 2023) - FV * Night Strike (October 3, 2023) - NS - * Certain features are deprecated with Legacy Driving Models + * Certain features are deprecated with newer Driving Models * Dynamic Lane Profile * Custom Offsets * UPDATED: Dynamic Lane Profile (DLP) From d2648d42f9b11dd4e4f92fa4260dd043aca0b043 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Tue, 13 Feb 2024 15:34:04 -0500 Subject: [PATCH 03/10] Driving Model Selector: Handle custom navmodel --- selfdrive/modeld/navmodeld.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/selfdrive/modeld/navmodeld.py b/selfdrive/modeld/navmodeld.py index bfe9c8fbed..47439dbab6 100755 --- a/selfdrive/modeld/navmodeld.py +++ b/selfdrive/modeld/navmodeld.py @@ -15,6 +15,7 @@ from openpilot.common.params import Params from openpilot.common.realtime import set_realtime_priority from openpilot.selfdrive.modeld.constants import ModelConstants from openpilot.selfdrive.modeld.runners import ModelRunner, Runtime +from openpilot.selfdrive.sunnypilot import get_model_generation NAV_INPUT_SIZE = 256*256 NAV_FEATURE_LEN = 256 @@ -51,7 +52,8 @@ class ModelState: assert ctypes.sizeof(NavModelResult) == NAV_OUTPUT_SIZE * ctypes.sizeof(ctypes.c_float) self.output = np.zeros(NAV_OUTPUT_SIZE, dtype=np.float32) self.param_s = Params() - if self.param_s.get_bool("CustomDrivingModel"): + self.custom_model, self.model_gen = get_model_generation(self.param_s) + if self.custom_model and self.model_gen != 0: _model_name = self.param_s.get("NavModelText", encoding="utf8") _model_paths = { ModelRunner.SNPE: f"{CUSTOM_MODEL_PATH}/navmodel_q_{_model_name}.dlc"} From 896bd18c0da0e2731aad4c1b818b69bee914ea1f Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Tue, 13 Feb 2024 15:39:03 -0500 Subject: [PATCH 04/10] Driving Model Selector: Handle default model name display --- selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.cc b/selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.cc index 616bee253d..17e219ec7c 100644 --- a/selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.cc +++ b/selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.cc @@ -68,7 +68,7 @@ QString SoftwarePanelSP::GetModelName() { return selectedModelToDownload->displayName; } - if (params.getBool("CustomDrivingModel")) { + if (params.getBool("CustomDrivingModel") && QString::fromStdString(params.get("DrivingModelGeneration")) != "0") { return QString::fromStdString(params.get("DrivingModelName")); } From 17acb0dc494e003789028a9655f93642e666029d Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Tue, 13 Feb 2024 17:57:24 -0500 Subject: [PATCH 05/10] Driving Model Selector: Create directory if does not exist --- selfdrive/manager/manager.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/selfdrive/manager/manager.py b/selfdrive/manager/manager.py index cd7e14a8e3..e39892a13b 100755 --- a/selfdrive/manager/manager.py +++ b/selfdrive/manager/manager.py @@ -166,6 +166,10 @@ def manager_init() -> None: if os.path.isfile(os.path.join(sentry.CRASHES_DIR, 'error.txt')): os.remove(os.path.join(sentry.CRASHES_DIR, 'error.txt')) + models_dir = "/data/media/0/models" + if not os.path.exists(models_dir): + os.makedirs(models_dir) + # preimport all processes for p in managed_processes.values(): p.prepare() From a0dd4d73590a317eebf46038430b1c89f869fe07 Mon Sep 17 00:00:00 2001 From: DevTekVE Date: Wed, 14 Feb 2024 17:51:26 +0000 Subject: [PATCH 06/10] Adding sanity checking and using a different method for the actual artifact being built --- .gitlab-ci.yml | 88 ++++++++++++++++++++------- release/ci/install_gitlab_runner.sh | 1 + release/ci/sanity_check.sh | 75 +++++++++++++++++++++++ release/ci/uninstall_gitlab_runner.sh | 0 4 files changed, 143 insertions(+), 21 deletions(-) create mode 100755 release/ci/sanity_check.sh mode change 100644 => 100755 release/ci/uninstall_gitlab_runner.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5f0cdc3cef..b437df7c90 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -12,6 +12,7 @@ variables: stages: - build + - sanity - publish - notify @@ -70,6 +71,7 @@ build: paths: - "${CI_DIR}/scons_cache" before_script: + - find $BUILD_DIR/ -mindepth 1 -delete - *default_before_script - "echo Starting build stage..." - "echo BUILD_DIR: ${BUILD_DIR}" @@ -79,7 +81,6 @@ build: - git config --global --add safe.directory ${CI_PROJECT_DIR} script: - export PYTHONPATH="$BUILD_DIR" - - "echo Calling to build [${CI_DIR}/build.sh ${CI_PROJECT_DIR} ${BUILD_DIR} ${VERSION}]" - "echo Building Panda..." - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache ${CI_PROJECT_DIR}/panda - mkdir -p ${BUILD_DIR} @@ -89,28 +90,49 @@ build: - cd $BUILD_DIR - sed -i '/from .board.jungle import PandaJungle, PandaJungleDFU/s/^/#/' panda/__init__.py # comment panda jungle when prebuilt - scons -j$(nproc) cache_dir=${CI_DIR}/scons_cache - - # Cleanup - - echo "Doing cleanup for prebuilt\n\n" - - echo "Deleting specified files..." - - find ${BUILD_DIR} \( -name '*.a' -o -name '*.o' -o -name '*.os' -o -name '*.pyc' -o -name 'moc_*' -o -name '*.cc' -o -name "Jenkinsfile" -o -name "supercombo.onnx" -o -name ".sconsign.dblite" \) -print -delete - - - echo "Deleting Headers from selfdrive/ui path..." - - find ${BUILD_DIR}/selfdrive/ui -type f -name '*.h' -print -delete - - - echo "Deleting the following folders and subdirectories..." - - find "${BUILD_DIR}" -type d \( -path "*panda/board" -o -path "*panda/certs" -o -path "*panda/crypto" -o -path "*release" -o -path "*.github" -o -path "*selfdrive/ui/replay" -o -path "*__pycache__" \) -print -exec rm -rf {} + - - # Move back signed panda fw - - mkdir -p ${BUILD_DIR}/panda/board/obj - - cp ${CI_PROJECT_DIR}/panda/board/obj/panda.bin.signed ${BUILD_DIR}/panda/board/obj/panda.bin.signed - - cp ${CI_PROJECT_DIR}/panda/board/obj/panda_h7.bin.signed ${BUILD_DIR}/panda/board/obj/panda_h7.bin.signed - - cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda.bin - - cp ${CI_PROJECT_DIR}/panda/board/obj/bootstub.panda_h7.bin ${BUILD_DIR}/panda/board/obj/bootstub.panda_h7.bin - touch ${BUILD_DIR}/prebuilt + - sudo rm -rf ${OUTPUT_DIR} - mkdir -p ${OUTPUT_DIR} - - shopt -s dotglob && mv ${BUILD_DIR}/* ${OUTPUT_DIR} - - sudo chown -R comma:comma ${OUTPUT_DIR} + # We first include the paths we want to keep, even if we later will be excluding the other things on those paths + - rsync -av + --include='**/panda/board/' + --include='**/panda/board/obj' + --include='**/panda/board/obj/panda.bin.signed' + --include='**/panda/board/obj/panda_h7.bin.signed' + --include='**/panda/board/obj/bootstub.panda.bin' + --include='**/panda/board/obj/bootstub.panda_h7.bin' + --exclude='.sconsign.dblite' + --exclude='*.a' + --exclude='*.A' + --exclude='*.o' + --exclude='*.O' + --exclude='*.os' + --exclude='*.OS' + --exclude='*.pyc' + --exclude='*.PYC' + --exclude='moc_*' + --exclude='MOC_*' + --exclude='*.cc' + --exclude='*.CC' + --exclude='Jenkinsfile' + --exclude='supercombo.onnx' + --exclude='**/panda/board/*' + --exclude='**/panda/board/obj/**' + --exclude='**/panda/certs/' + --exclude='**/panda/crypto/' + --exclude='**/release/' + --exclude='**/.github/' + --exclude='**/selfdrive/ui/replay/' + --exclude='**/__pycache__/' + --exclude='**/selfdrive/ui/**/*.h' + --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' + --exclude='**/.git/' + --delete-excluded + --chown=comma:comma + ${BUILD_DIR}/ ${OUTPUT_DIR}/ + after_script: + # cleanup build dir after doing work + - find $BUILD_DIR/ -mindepth 1 -delete artifacts: paths: - ${OUTPUT_DIR}/ @@ -120,6 +142,27 @@ build: when: manual - if: $NEW_BRANCH when: always + +check no source code sent: + stage: sanity + variables: + FORBIDDEN_FILE_EXTENSIONS: "*.a,*.o,*.os,*.pyc,moc_*,*.cc,Jenkinsfile,supercombo.onnx,.sconsign.dblite" + FORBIDDEN_DIR_PATTERNS: "*panda/certs,*panda/crypto,*release,*.github,*selfdrive/ui/replay,*__pycache__" + REQUIRED_FILE_EXTENSIONS: "*.py,*.json" + REQUIRED_DIR_PATTERNS: "*selfdrive/ui,*openpilot" + before_script: + - apk update && apk upgrade + - apk add bash findutils + script: + - cd ${OUTPUT_DIR} + - echo "Checking that we have properly cleaned up" + - ${CI_DIR}/sanity_check.sh "$FORBIDDEN_FILE_EXTENSIONS" "$FORBIDDEN_DIR_PATTERNS" true + - echo "Checking that our sanity check works and also checking that some required files are indeed found" + - ${CI_DIR}/sanity_check.sh "$REQUIRED_FILE_EXTENSIONS" "$REQUIRED_DIR_PATTERNS" false + rules: + - if: $NEW_BRANCH + when: on_success + - when: never .publish_base: &publish_base variables: @@ -128,6 +171,9 @@ build: needs: - job: build artifacts: true + - job: "check no source code sent" + artifacts: false + optional: false before_script: - 'apk update && apk upgrade' - 'apk add git bash openssh' diff --git a/release/ci/install_gitlab_runner.sh b/release/ci/install_gitlab_runner.sh index a44c13cc92..0dccc95154 100755 --- a/release/ci/install_gitlab_runner.sh +++ b/release/ci/install_gitlab_runner.sh @@ -64,6 +64,7 @@ set_gitlab_directory_permissions() { sudo chmod g+s "$GITLAB_BASE_DIR" } +# Please note that when the gitlab runner is started, linux implicitly executes "source /etc/profile" for every logged shell. This will make scons available create_gitlab_runner_service() { cat < Date: Wed, 14 Feb 2024 13:08:54 -0500 Subject: [PATCH 07/10] NNLC: Slight fixes --- release/files_common | 1 + selfdrive/controls/lib/latcontrol_torque.py | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/release/files_common b/release/files_common index cbce7906f4..a12ed62db6 100644 --- a/release/files_common +++ b/release/files_common @@ -102,6 +102,7 @@ selfdrive/car/tests/__init__.py selfdrive/car/tests/test_car_interfaces.py selfdrive/car/torque_data/*.toml selfdrive/car/torque_data/lat_models/* +selfdrive/car/torque_data/neural_ff_weights.json selfdrive/car/body/*.py selfdrive/car/chrysler/*.py diff --git a/selfdrive/controls/lib/latcontrol_torque.py b/selfdrive/controls/lib/latcontrol_torque.py index 4e0d69a6ea..c5d0e7ef8f 100644 --- a/selfdrive/controls/lib/latcontrol_torque.py +++ b/selfdrive/controls/lib/latcontrol_torque.py @@ -189,12 +189,13 @@ class LatControlTorque(LatControl): lateral_jerk_measurement = 0 lookahead_lateral_jerk = 0 - if self.use_nn or self.use_lateral_jerk: + model_good = model_data is not None and len(model_data.orientation.x) >= CONTROL_N + if model_good and (self.use_nn or self.use_lateral_jerk): # prepare "look-ahead" desired lateral jerk lookahead = interp(CS.vEgo, self.friction_look_ahead_bp, self.friction_look_ahead_v) friction_upper_idx = next((i for i, val in enumerate(ModelConstants.T_IDXS) if val > lookahead), 16) predicted_lateral_jerk = get_predicted_lateral_jerk(model_data.acceleration.y, self.t_diffs) - desired_lateral_jerk = (interp(self.desired_lat_jerk_time, ModelConstants.T_IDXS, model_data.acceleration.y) - actual_lateral_accel) / self.desired_lat_jerk_time + desired_lateral_jerk = (interp(self.desired_lat_jerk_time, ModelConstants.T_IDXS, model_data.acceleration.y) - desired_lateral_accel) / self.desired_lat_jerk_time lookahead_lateral_jerk = get_lookahead_value(predicted_lateral_jerk[LAT_PLAN_MIN_IDX:friction_upper_idx], desired_lateral_jerk) if self.use_steering_angle or lookahead_lateral_jerk == 0.0: lookahead_lateral_jerk = 0.0 @@ -203,12 +204,12 @@ class LatControlTorque(LatControl): lateral_jerk_setpoint = self.lat_jerk_friction_factor * lookahead_lateral_jerk lateral_jerk_measurement = self.lat_jerk_friction_factor * actual_lateral_jerk - model_good = model_data is not None and len(model_data.orientation.x) >= CONTROL_N if self.use_nn and model_good: # update past data roll = params.roll - pitch = self.pitch.update(llk.calibratedOrientationNED.value[1]) - roll = roll_pitch_adjust(roll, pitch) + if len(llk.calibratedOrientationNED.value) > 1: + pitch = self.pitch.update(llk.calibratedOrientationNED.value[1]) + roll = roll_pitch_adjust(roll, pitch) self.roll_deque.append(roll) self.lateral_accel_desired_deque.append(desired_lateral_accel) From 57be1f4c5f0bede715fddf0b19d8a5039445bd81 Mon Sep 17 00:00:00 2001 From: Miguel Fernandez Date: Wed, 14 Feb 2024 19:40:40 +0100 Subject: [PATCH 08/10] PIPELINE: hardening pipeline - excluding ui.h from prebuilts --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b437df7c90..02fbd9c892 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -124,6 +124,7 @@ build: --exclude='**/.github/' --exclude='**/selfdrive/ui/replay/' --exclude='**/__pycache__/' + --exclude='**/selfdrive/ui/*.h' --exclude='**/selfdrive/ui/**/*.h' --exclude='**/selfdrive/ui/qt/offroad/sunnypilot/' --exclude='**/.git/' From f17ebeeef6179a7e5ce92dd531ff54c75aced500 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Wed, 14 Feb 2024 14:02:55 -0500 Subject: [PATCH 09/10] Torque: Remove additional files --- selfdrive/car/torque_data/override.yaml | 0 selfdrive/car/torque_data/params.yaml | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 selfdrive/car/torque_data/override.yaml delete mode 100644 selfdrive/car/torque_data/params.yaml diff --git a/selfdrive/car/torque_data/override.yaml b/selfdrive/car/torque_data/override.yaml deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/selfdrive/car/torque_data/params.yaml b/selfdrive/car/torque_data/params.yaml deleted file mode 100644 index e69de29bb2..0000000000 From 170c63fbc5eba3bef140b425890506b26aff8ea8 Mon Sep 17 00:00:00 2001 From: Jason Wen Date: Thu, 15 Feb 2024 11:40:23 -0500 Subject: [PATCH 10/10] FCR: Add description to generate LEAF_IC in Nissan FCR --- selfdrive/car/nissan/values.py | 2 +- selfdrive/car/sunnypilot_carname.json | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/selfdrive/car/nissan/values.py b/selfdrive/car/nissan/values.py index d064ce894d..b0a3d74e1a 100644 --- a/selfdrive/car/nissan/values.py +++ b/selfdrive/car/nissan/values.py @@ -40,7 +40,7 @@ class NissanCarInfo(CarInfo): CAR_INFO: Dict[str, Optional[Union[NissanCarInfo, List[NissanCarInfo]]]] = { CAR.XTRAIL: NissanCarInfo("Nissan X-Trail 2017"), CAR.LEAF: NissanCarInfo("Nissan Leaf 2018-23", video_link="https://youtu.be/vaMbtAh_0cY"), - CAR.LEAF_IC: None, # same platforms + CAR.LEAF_IC: NissanCarInfo("Nissan Leaf Instrument Cluster 2018-23", video_link="https://youtu.be/vaMbtAh_0cY"), # same platforms CAR.ROGUE: NissanCarInfo("Nissan Rogue 2018-20"), CAR.ALTIMA: NissanCarInfo("Nissan Altima 2019-20", car_parts=CarParts.common([CarHarness.nissan_b])), } diff --git a/selfdrive/car/sunnypilot_carname.json b/selfdrive/car/sunnypilot_carname.json index c8090c8b3b..79b3176d59 100644 --- a/selfdrive/car/sunnypilot_carname.json +++ b/selfdrive/car/sunnypilot_carname.json @@ -180,6 +180,7 @@ "Mazda CX-9 2021-23": "MAZDA CX-9 2021", "Nissan Altima 2019-20": "NISSAN ALTIMA 2020", "Nissan Leaf 2018-23": "NISSAN LEAF 2018", + "Nissan Leaf Instrument Cluster 2018-23": "NISSAN LEAF 2018 Instrument Cluster", "Nissan Rogue 2018-20": "NISSAN ROGUE 2019", "Nissan X-Trail 2017": "NISSAN X-TRAIL 2017", "Ram 1500 2019-24": "RAM 1500 5TH GEN",