diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index 14b4d904e3..49dae93747 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -43,6 +43,7 @@ jobs: with: submodules: true - name: uv lock + if: github.repository == 'commaai/openpilot' run: | python3 -m ensurepip --upgrade pip3 install uv diff --git a/common/params_keys.h b/common/params_keys.h index e0af53b299..afb6b348eb 100644 --- a/common/params_keys.h +++ b/common/params_keys.h @@ -154,6 +154,7 @@ inline static std::unordered_map keys = { {"MaxTimeOffroad", {PERSISTENT | BACKUP, INT, "1800"}}, {"ModelRunnerTypeCache", {CLEAR_ON_ONROAD_TRANSITION, INT}}, {"OffroadMode", {CLEAR_ON_MANAGER_START, BOOL}}, + {"Offroad_TiciSupport", {CLEAR_ON_MANAGER_START, JSON}}, {"QuickBootToggle", {PERSISTENT | BACKUP, BOOL, "0"}}, {"QuietMode", {PERSISTENT | BACKUP, BOOL, "0"}}, {"RainbowMode", {PERSISTENT | BACKUP, BOOL, "0"}}, diff --git a/panda b/panda index f10ddc6a89..7eab6fd61b 160000 --- a/panda +++ b/panda @@ -1 +1 @@ -Subproject commit f10ddc6a89953440a15deec6352fff1d406a627a +Subproject commit 7eab6fd61bae085e0fd44cccb79dc6451163029e diff --git a/selfdrive/selfdrived/alerts_offroad.json b/selfdrive/selfdrived/alerts_offroad.json index 183c1f8547..87a007211f 100644 --- a/selfdrive/selfdrived/alerts_offroad.json +++ b/selfdrive/selfdrived/alerts_offroad.json @@ -49,5 +49,10 @@ "text": "openpilot detected excessive %1 actuation on your last drive. Please contact support at https://comma.ai/support and share your device's Dongle ID for troubleshooting.", "severity": 1, "_comment": "Set extra field to lateral or longitudinal." + }, + "Offroad_TiciSupport": { + "text": "Unsupported branch detected - The current version of %1 branch is no longer supported on the comma three. Please go to [Device > Software] and install a supported branch with -tici in the branch name for the comma three.", + "severity": 1, + "_comment": "Set extra field to the current branch name." } } diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.cc index 02a01a4b63..c3f795e18c 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.cc +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.cc @@ -48,25 +48,6 @@ static const QString progressStyleError = progressStyleActive + " background-color: transparent;" "}"; -std::optional safeParamEventLoad(Params& params, const std::string& paramName) { - std::string raw = params.get(paramName); - if (raw.empty()) { - return std::nullopt; - } - - try { - AlignedBuffer alignedBuf; - auto buf = alignedBuf.align(raw.data(), raw.size()); - - capnp::FlatArrayMessageReader msg(kj::ArrayPtr(buf.begin(), buf.size())); - return msg.getRoot(); - } - catch (const kj::Exception& e) { - qInfo() << "Invalid param" << QString::fromStdString(paramName) << ":" << e.getDescription().cStr(); - return std::nullopt; - } -} - ModelsPanel::ModelsPanel(QWidget *parent) : QWidget(parent) { QVBoxLayout *main_layout = new QVBoxLayout(this); main_layout->setContentsMargins(50, 20, 50, 20); @@ -152,16 +133,10 @@ ModelsPanel::ModelsPanel(QWidget *parent) : QWidget(parent) { list->addItem(lagd_toggle_control); // Software delay control - int liveDelayMaxInt = 30; - if (const auto event = safeParamEventLoad(params, "LiveDelay"); event && event->hasLiveDelay()) { - auto liveDelay = event->getLiveDelay(); - float lateralDelay = liveDelay.getLateralDelay(); - liveDelayMaxInt = static_cast(lateralDelay * 100.0f) + 20; - } delay_control = new OptionControlSP("LagdToggleDelay", tr("Adjust Software Delay"), - tr("Adjust the software delay when Live Learning Steer Delay is toggled off." - "\nThe default software delay value is 0.2"), - "", {5, liveDelayMaxInt}, 1, false, nullptr, true, true); + tr("Adjust the software delay when Live Learning Steer Delay is toggled off." + "\nThe default software delay value is 0.2"), + "", {5, 50}, 1, false, nullptr, true, true); connect(delay_control, &OptionControlSP::updateLabels, [=]() { float value = QString::fromStdString(params.get("LagdToggleDelay")).toFloat(); @@ -449,27 +424,28 @@ void ModelsPanel::updateLabels() { "Disable to use a fixed steering response time. Keeping this on provides the stock openpilot experience."); bool lagdEnabled = params.getBool("LagdToggle"); if (lagdEnabled) { - if (const auto event = safeParamEventLoad(params, "LiveDelay"); event && event->hasLiveDelay()) { - auto liveDelay = event->getLiveDelay(); - float lateralDelay = liveDelay.getLateralDelay(); + auto liveDelayBytes = params.get("LiveDelay"); + if (!liveDelayBytes.empty()) { + auto LD = loadCerealEvent(params, "LiveDelay"); + float lateralDelay = LD->getLiveDelay().getLateralDelay(); desc += QString("

%1 %2 s") - .arg(tr("Live Steer Delay:")).arg(QString::number(lateralDelay, 'f', 3)); + .arg(tr("Live Steer Delay:")).arg(QString::number(lateralDelay, 'f', 3)); } } else { - std::string carParamsBytes = params.get("CarParamsPersistent"); + auto carParamsBytes = params.get("CarParamsPersistent"); if (!carParamsBytes.empty()) { - capnp::FlatArrayMessageReader msg(kj::ArrayPtr( - reinterpret_cast(carParamsBytes.data()), - carParamsBytes.size() / sizeof(capnp::word))); - auto carParams = msg.getRoot(); - float steerDelay = carParams.getSteerActuatorDelay(); + AlignedBuffer aligned_buf_cp; + capnp::FlatArrayMessageReader cmsg(aligned_buf_cp.align(carParamsBytes.data(), carParamsBytes.size())); + cereal::CarParams::Reader CP = cmsg.getRoot(); + + float steerDelay = CP.getSteerActuatorDelay(); float softwareDelay = QString::fromStdString(params.get("LagdToggleDelay")).toFloat(); float totalLag = steerDelay + softwareDelay; desc += QString("

" "%1 %2 s + %3 %4 s = %5 %6 s") - .arg(tr("Actuator Delay:"), QString::number(steerDelay, 'f', 2), - tr("Software Delay:"), QString::number(softwareDelay, 'f', 2), - tr("Total Delay:"), QString::number(totalLag, 'f', 2)); + .arg(tr("Actuator Delay:"), QString::number(steerDelay, 'f', 2), + tr("Software Delay:"), QString::number(softwareDelay, 'f', 2), + tr("Total Delay:"), QString::number(totalLag, 'f', 2)); } } lagd_toggle_control->setDescription(desc); diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.h b/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.h index 1906ebd2a0..1a39800dde 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.h +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/models_panel.h @@ -9,6 +9,7 @@ #include +#include "selfdrive/ui/sunnypilot/qt/util.h" #include "selfdrive/ui/sunnypilot/qt/offroad/settings/settings.h" class ModelsPanel : public QWidget { diff --git a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc index a1961cb1ea..8bdb7703f2 100644 --- a/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc +++ b/selfdrive/ui/sunnypilot/qt/offroad/settings/software_panel.cc @@ -11,12 +11,39 @@ SoftwarePanelSP::SoftwarePanelSP(QWidget *parent) : SoftwarePanel(parent) { // branch selector QObject::disconnect(targetBranchBtn, nullptr, nullptr, nullptr); connect(targetBranchBtn, &ButtonControlSP::clicked, [=]() { - InputDialog d(tr("Search Branch"), this, tr("Enter search keywords, or leave blank to list all branches."), false); + if (Hardware::get_device_type() == cereal::InitData::DeviceType::TICI) { + auto current = params.get("GitBranch"); + QStringList allBranches = QString::fromStdString(params.get("UpdaterAvailableBranches")).split(","); + QStringList branches; + for (const QString &b : allBranches) { + if (b.endsWith("-tici")) { + branches.append(b); + } + } + + for (QString b : {current.c_str(), "master-tici", "staging-tici", "release-tici"}) { + auto i = branches.indexOf(b); + if (i >= 0) { + branches.removeAt(i); + branches.insert(0, b); + } + } + + QString cur = QString::fromStdString(params.get("UpdaterTargetBranch")); + QString selection = MultiOptionDialog::getSelection(tr("Select a branch"), branches, cur, this); + if (!selection.isEmpty()) { + params.put("UpdaterTargetBranch", selection.toStdString()); + targetBranchBtn->setValue(QString::fromStdString(params.get("UpdaterTargetBranch"))); + checkForUpdates(); + } + } else { + InputDialog d(tr("Search Branch"), this, tr("Enter search keywords, or leave blank to list all branches."), false); d.setMinLength(0); const int ret = d.exec(); if (ret) { searchBranches(d.text()); } + } }); // Disable Updates toggle diff --git a/selfdrive/ui/sunnypilot/qt/util.cc b/selfdrive/ui/sunnypilot/qt/util.cc index ca85935d0b..2e066e88b5 100644 --- a/selfdrive/ui/sunnypilot/qt/util.cc +++ b/selfdrive/ui/sunnypilot/qt/util.cc @@ -110,3 +110,16 @@ QStringList searchFromList(const QString &query, const QStringList &list) { } return search_results; } + +std::optional loadCerealEvent(Params& params, const std::string& _param) { + std::string bytes = params.get(_param); + + try { + AlignedBuffer aligned_buf; + capnp::FlatArrayMessageReader cmsg(aligned_buf.align(bytes.data(), bytes.size())); + return cmsg.getRoot(); + } catch (kj::Exception& e) { + qInfo() << "invalid " << QString::fromStdString(_param) << ":" << e.getDescription().cStr(); + return std::nullopt; + } +} diff --git a/selfdrive/ui/sunnypilot/qt/util.h b/selfdrive/ui/sunnypilot/qt/util.h index 089b5370cc..4b9d615ce5 100644 --- a/selfdrive/ui/sunnypilot/qt/util.h +++ b/selfdrive/ui/sunnypilot/qt/util.h @@ -15,8 +15,11 @@ #include #include +#include "selfdrive/ui/sunnypilot/ui.h" + QString getUserAgent(bool sunnylink = false); std::optional getSunnylinkDongleId(); std::optional getParamIgnoringDefault(const std::string ¶m_name, const std::string &default_value); QMap loadPlatformList(); QStringList searchFromList(const QString &query, const QStringList &list); +std::optional loadCerealEvent(Params& params, const std::string& _param); diff --git a/sunnypilot/modeld_v2/modeld.py b/sunnypilot/modeld_v2/modeld.py index bf89bc98d6..0fd45940d8 100755 --- a/sunnypilot/modeld_v2/modeld.py +++ b/sunnypilot/modeld_v2/modeld.py @@ -27,7 +27,7 @@ from openpilot.sunnypilot.modeld.modeld_base import ModelStateBase from openpilot.sunnypilot.models.helpers import get_active_bundle from openpilot.sunnypilot.models.runners.helpers import get_model_runner -PROCESS_NAME = "selfdrive.modeld.modeld" +PROCESS_NAME = "selfdrive.modeld.modeld_tinygrad" class FrameMeta: @@ -77,42 +77,47 @@ class ModelState(ModelStateBase): self.numpy_inputs[key] = np.zeros(shape, dtype=np.float32) # Temporal input: shape is [batch, history, features] if len(shape) == 3 and shape[1] > 1: - buffer_history_len = max(100, (shape[1] * 4 if shape[1] < 100 else shape[1])) # Allow for higher history buffers in the future + buffer_history_len = shape[1] * 4 if shape[1] < 99 else shape[1] # Allow for higher history buffers in the future feature_len = shape[2] - self.temporal_buffers[key] = np.zeros((1, buffer_history_len, feature_len), dtype=np.float32) features_buffer_shape = self.model_runner.input_shapes.get('features_buffer') if shape[1] in (24, 25) and features_buffer_shape is not None and features_buffer_shape[1] == 24: # 20Hz + buffer_history_len = (features_buffer_shape[1] + 1) * 4 step = int(-buffer_history_len / shape[1]) self.temporal_idxs_map[key] = np.arange(step, step * (shape[1] + 1), step)[::-1] elif shape[1] == 25: # Split skip = buffer_history_len // shape[1] self.temporal_idxs_map[key] = np.arange(buffer_history_len)[-1 - (skip * (shape[1] - 1))::skip] - elif shape[1] == buffer_history_len: # non20hz - self.temporal_idxs_map[key] = np.arange(buffer_history_len) + elif shape[1] >= 99: # non20hz + self.temporal_idxs_map[key] = np.arange(shape[1]) + self.temporal_buffers[key] = np.zeros((1, buffer_history_len, feature_len), dtype=np.float32) @property def mlsim(self) -> bool: return bool(self.generation is not None and self.generation >= 11) + @property + def desire_key(self) -> str: + return next(key for key in self.numpy_inputs if key.startswith('desire')) + def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray], inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None: # Model decides when action is completed, so desire input is just a pulse triggered on rising edge - inputs['desire'][0] = 0 - new_desire = np.where(inputs['desire'] - self.prev_desire > .99, inputs['desire'], 0) - self.prev_desire[:] = inputs['desire'] - self.temporal_buffers['desire'][0,:-1] = self.temporal_buffers['desire'][0,1:] - self.temporal_buffers['desire'][0,-1] = new_desire + inputs[self.desire_key][0] = 0 + new_desire = np.where(inputs[self.desire_key] - self.prev_desire > .99, inputs[self.desire_key], 0) + self.prev_desire[:] = inputs[self.desire_key] + self.temporal_buffers[self.desire_key][0,:-1] = self.temporal_buffers[self.desire_key][0,1:] + self.temporal_buffers[self.desire_key][0,-1] = new_desire # Roll buffer and assign based on desire.shape[1] value - if self.temporal_buffers['desire'].shape[1] > self.numpy_inputs['desire'].shape[1]: - skip = self.temporal_buffers['desire'].shape[1] // self.numpy_inputs['desire'].shape[1] - self.numpy_inputs['desire'][:] = ( - self.temporal_buffers['desire'][0].reshape(self.numpy_inputs['desire'].shape[0], self.numpy_inputs['desire'].shape[1], skip, -1).max(axis=2)) + if self.temporal_buffers[self.desire_key].shape[1] > self.numpy_inputs[self.desire_key].shape[1]: + skip = self.temporal_buffers[self.desire_key].shape[1] // self.numpy_inputs[self.desire_key].shape[1] + self.numpy_inputs[self.desire_key][:] = (self.temporal_buffers[self.desire_key][0].reshape( + self.numpy_inputs[self.desire_key].shape[0], self.numpy_inputs[self.desire_key].shape[1], skip, -1).max(axis=2)) else: - self.numpy_inputs['desire'][:] = self.temporal_buffers['desire'][0, self.temporal_idxs_map['desire']] + self.numpy_inputs[self.desire_key][:] = self.temporal_buffers[self.desire_key][0, self.temporal_idxs_map[self.desire_key]] for key in self.numpy_inputs: - if key in inputs and key not in ['desire']: + if key in inputs and key not in [self.desire_key]: self.numpy_inputs[key][:] = inputs[key] imgs_cl = {name: self.frames[name].prepare(bufs[name], transforms[name].flatten()) for name in self.model_runner.vision_input_names} @@ -156,10 +161,11 @@ class ModelState(ModelStateBase): desired_accel = smooth_value(desired_accel, prev_action.desiredAcceleration, self.LONG_SMOOTH_SECONDS) desired_curvature = get_curvature_from_output(model_output, v_ego, lat_action_t, self.mlsim) - if v_ego > self.MIN_LAT_CONTROL_SPEED: - desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, self.LAT_SMOOTH_SECONDS) - else: - desired_curvature = prev_action.desiredCurvature + if self.generation is not None and self.generation >= 10: # smooth curvature for post FOF models + if v_ego > self.MIN_LAT_CONTROL_SPEED: + desired_curvature = smooth_value(desired_curvature, prev_action.desiredCurvature, self.LAT_SMOOTH_SECONDS) + else: + desired_curvature = prev_action.desiredCurvature return log.ModelDataV2.Action(desiredCurvature=float(desired_curvature),desiredAcceleration=float(desired_accel), shouldStop=bool(should_stop)) @@ -306,7 +312,7 @@ def main(demo=False): bufs = {name: buf_extra if 'big' in name else buf_main for name in model.model_runner.vision_input_names} transforms = {name: model_transform_extra if 'big' in name else model_transform_main for name in model.model_runner.vision_input_names} inputs:dict[str, np.ndarray] = { - 'desire': vec_desire, + model.desire_key: vec_desire, 'traffic_convention': traffic_convention, } diff --git a/sunnypilot/modeld_v2/tests/test_buffer_logic_inspect.py b/sunnypilot/modeld_v2/tests/test_buffer_logic_inspect.py index 8a0cfd97c8..f664db31b3 100644 --- a/sunnypilot/modeld_v2/tests/test_buffer_logic_inspect.py +++ b/sunnypilot/modeld_v2/tests/test_buffer_logic_inspect.py @@ -8,12 +8,16 @@ import openpilot.sunnypilot.modeld_v2.modeld as modeld_module ModelState = modeld_module.ModelState - # These are the shapes extracted/loaded from the model onnx SHAPE_MODE_PARAMS = [ - ({'desire': (1, 25, 8), 'features_buffer': (1, 25, 512), 'prev_desired_curv': (1, 25, 1)}, 'split'), - ({'desire': (1, 25, 8), 'features_buffer': (1, 24, 512), 'prev_desired_curv': (1, 25, 1)}, '20hz'), - ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), 'prev_desired_curv': (1, 100, 1)}, 'non20hz'), + ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), "nav_features": (1, 256), "nav_instructions": (1, 150)}, 'non20hz'), # Optimus Prime + ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), "lat_planner_state": (1, 4),}, 'non20hz'), # farmville + ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), "lateral_control_params": (1, 2), "prev_desired_curv": (1, 100, 1)}, 'non20hz'), # wd40 + ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), 'prev_desired_curv': (1, 100, 1), "lateral_control_params": (1, 2),}, 'non20hz'), # NTS + ({'desire': (1, 25, 8), 'features_buffer': (1, 24, 512)}, '20hz'), # NPR + ({'desire': (1, 100, 8), 'features_buffer': (1, 99, 512), 'prev_desired_curv': (1, 100, 1), "lateral_control_params": (1, 2),}, 'non20hz'), # NTS + ({'desire': (1, 25, 8), 'features_buffer': (1, 25, 512)}, 'split'), # Steam Powered v2 + ({'desire_pulse': (1, 25, 8), 'features_buffer': (1, 25, 512)}, 'split'), # desire rename ] @@ -95,9 +99,7 @@ def get_expected_indices(shape, constants, mode, key=None): idxs = np.arange(step_size, step_size * (num_elements + 1), step_size)[::-1] return idxs elif mode == 'non20hz': - if key and shape[1] == constants.FULL_HISTORY_BUFFER_LEN: - return np.arange(constants.FULL_HISTORY_BUFFER_LEN) - return None + return np.arange(shape[1]) return None @@ -108,6 +110,8 @@ def test_buffer_shapes_and_indices(shapes, mode, apply_patches): for key in shapes: buf = state.temporal_buffers.get(key, None) idxs = state.temporal_idxs_map.get(key, None) + if buf is None: + continue # not all shapes are 3D, and the non-3D ones are not buffered # Buffer shape logic if mode == 'split': expected_shape = (1, constants.FULL_HISTORY_BUFFER_LEN, shapes[key][2]) @@ -116,10 +120,7 @@ def test_buffer_shapes_and_indices(shapes, mode, apply_patches): expected_shape = (1, constants.FULL_HISTORY_BUFFER_LEN, shapes[key][2]) expected_idxs = get_expected_indices(shapes[key], constants, '20hz', key) elif mode == 'non20hz': - if key == 'features_buffer': - expected_shape = (1, shapes[key][1]*4, shapes[key][2]) - else: - expected_shape = (1, shapes[key][1], shapes[key][2]) + expected_shape = (1, shapes[key][1], shapes[key][2]) expected_idxs = get_expected_indices(shapes[key], constants, 'non20hz', key) assert buf is not None, f"{key}: buffer not found" @@ -130,10 +131,10 @@ def test_buffer_shapes_and_indices(shapes, mode, apply_patches): assert idxs is None or idxs.size == 0, f"{key}: buffer idxs should be None or empty" -def legacy_buffer_update(buf, new_val, mode, key, constants, idxs): +def legacy_buffer_update(buf, new_val, mode, key, constants, idxs, input_shape, prev_desire=None): # This is what we compare the new dynamic logic to, to ensure it does the same thing if mode == 'split': - if key == 'desire': + if key == 'desire' or key.startswith('desire'): buf[0,:-1] = buf[0,1:] buf[0,-1] = new_val return buf.reshape((1, constants.INPUT_HISTORY_BUFFER_LEN, constants.TEMPORAL_SKIP, -1)).max(axis=2) @@ -173,15 +174,22 @@ def legacy_buffer_update(buf, new_val, mode, key, constants, idxs): return legacy_buf[idxs] elif mode == 'non20hz': if key == 'desire': - length = new_val.shape[0] - buf[0,:-1,:length] = buf[0,1:,:length] - buf[0,-1,:length] = new_val[:length] + desire_len = constants.DESIRE_LEN + if prev_desire is None: + prev_desire = np.zeros(desire_len, dtype=np.float32) + # Set first element to zero + new_val = new_val.copy() + new_val[0] = 0 + # Shift buffer by desire len + buf[0][:-desire_len] = buf[0][desire_len:] + # Only insert new desire if rising edge + buf[0][-desire_len:] = np.where(new_val - prev_desire > 0.99, new_val, 0) + prev_desire[:] = new_val return buf[0] elif key == 'features_buffer': - feature_len = new_val.shape[0] - buf[0,:-1,:feature_len] = buf[0,1:,:feature_len] - buf[0,-1,:feature_len] = new_val[:feature_len] - return buf[0] + buf[0, :-1] = buf[0, 1:] + buf[0, -1] = new_val + return buf[0, -input_shape[1]:] # (99, 512) elif key == 'prev_desired_curv': length = new_val.shape[0] buf[0,:-length,0] = buf[0,length:,0] @@ -191,32 +199,18 @@ def legacy_buffer_update(buf, new_val, mode, key, constants, idxs): def dynamic_buffer_update(state, key, new_val, mode): - if key == 'desire': - state.temporal_buffers['desire'][0,:-1] = state.temporal_buffers['desire'][0,1:] - state.temporal_buffers['desire'][0,-1] = new_val - if state.temporal_buffers['desire'].shape[1] > state.numpy_inputs['desire'].shape[1]: - skip = state.temporal_buffers['desire'].shape[1] // state.numpy_inputs['desire'].shape[1] - return state.temporal_buffers['desire'][0].reshape( - state.numpy_inputs['desire'].shape[0], state.numpy_inputs['desire'].shape[1], skip, -1 - ).max(axis=2) - else: - return state.temporal_buffers['desire'][0, state.temporal_idxs_map['desire']] - - inputs = {'desire': np.zeros((1, state.constants.DESIRE_LEN), dtype=np.float32)} - for k, tb in state.temporal_buffers.items(): - if k in state.temporal_idxs_map: - continue - buf_len = tb.shape[1] - if k in state.numpy_inputs: - out_len = state.numpy_inputs[k].shape[1] - if out_len <= buf_len: - state.temporal_idxs_map[k] = np.arange(buf_len)[-out_len:] - else: - state.temporal_idxs_map[k] = np.arange(buf_len) - else: - state.temporal_idxs_map[k] = np.arange(buf_len) + if key == 'desire' or key.startswith('desire'): + inputs = {k: np.zeros(v[2], dtype=np.float32) if len(v) == 3 else np.zeros(v[1], dtype=np.float32) + for k, v in state.model_runner.input_shapes.items() if k != key} + inputs[key] = new_val.copy() + # ModelState.run expects desire as a pulse, so we zero the first element. + inputs[key][0] = 0 + state.run({}, {}, inputs, prepare_only=False) + return state.numpy_inputs[key] if key == 'features_buffer': + inputs = {k: np.zeros(v[2], dtype=np.float32) if len(v) == 3 else np.zeros(v[1], dtype=np.float32) + for k, v in state.model_runner.input_shapes.items() if k != 'features_buffer'} def run_model_stub(): return { 'hidden_state': np.asarray(new_val, dtype=np.float32).reshape(1, -1), @@ -226,6 +220,8 @@ def dynamic_buffer_update(state, key, new_val, mode): return state.numpy_inputs['features_buffer'][0] if key == 'prev_desired_curv': + inputs = {k: np.zeros(v[2], dtype=np.float32) if len(v) == 3 else np.zeros(v[1], dtype=np.float32) + for k, v in state.model_runner.input_shapes.items() if k != 'prev_desired_curv'} def run_model_stub(): return { 'hidden_state': np.zeros((1, state.constants.FEATURE_LEN), dtype=np.float32), @@ -241,16 +237,27 @@ def dynamic_buffer_update(state, key, new_val, mode): @pytest.mark.parametrize("key", ["desire", "features_buffer", "prev_desired_curv"]) def test_buffer_update_equivalence(shapes, mode, key, apply_patches): state = ModelState(None) + if key == "desire": + desire_keys = [k for k in shapes.keys() if k.startswith('desire')] + if desire_keys: + actual_key = desire_keys[0] # Use the first (and likely only) desire key + else: + actual_key = key + + if actual_key not in state.numpy_inputs: + pytest.skip() + constants = DummyModelRunner(shapes).constants - buf = state.temporal_buffers.get(key, None) - idxs = state.temporal_idxs_map.get(key, None) - input_shape = shapes[key] + buf = state.temporal_buffers.get(actual_key, None) + idxs = state.temporal_idxs_map.get(actual_key, None) + input_shape = shapes[actual_key] + prev_desire = np.zeros(constants.DESIRE_LEN, dtype=np.float32) if key == 'desire' else None + for step in range(20): # multiple steps to ensure history is built up new_val = np.full((input_shape[2],), step, dtype=np.float32) - expected = legacy_buffer_update(buf, new_val, mode, key, constants, idxs) - actual = dynamic_buffer_update(state, key, new_val, mode) - # Model returns the reduced numpy_inputs history, compare the last n entries so the test is checking the same slices. + expected = legacy_buffer_update(buf, new_val, mode, actual_key, constants, idxs, input_shape, prev_desire) + actual = dynamic_buffer_update(state, actual_key, new_val, mode) if expected is not None and actual is not None and expected.shape != actual.shape: if expected.ndim == 2 and actual.ndim == 2 and expected.shape[1] == actual.shape[1]: expected = expected[-actual.shape[0]:] - assert np.allclose(actual, expected), f"{mode} {key}: dynamic buffer update does not match legacy logic" + assert np.allclose(actual, expected), f"{mode} {actual_key}: dynamic buffer update does not match legacy logic" diff --git a/sunnypilot/models/fetcher.py b/sunnypilot/models/fetcher.py index 358c65fe34..3be6e0b46c 100644 --- a/sunnypilot/models/fetcher.py +++ b/sunnypilot/models/fetcher.py @@ -8,6 +8,7 @@ See the LICENSE.md file in the root directory for more details. import time import requests +from requests.exceptions import (SSLError, RequestException, HTTPError) from openpilot.common.params import Params from openpilot.common.swaglog import cloudlog from sunnypilot.models.helpers import is_bundle_version_compatible @@ -122,19 +123,36 @@ class ModelFetcher: self.model_cache = ModelCache(params) self.model_parser = ModelParser() - def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle]: - """Fetches fresh model data from remote and updates cache""" + def _fetch_and_cache_models(self) -> list[custom.ModelManagerSP.ModelBundle] | None: + """Fetches fresh model data from remote and updates cache. + Returns None on transport errors. Raises on 404 and other fatal HTTP errors. + """ try: response = requests.get(self.MODEL_URL, timeout=10) - response.raise_for_status() - json_data = response.json() + # Explicitly handle 404 differently + if response.status_code == 404: + cloudlog.error(f"Models URL returned 404 Not Found: {self.MODEL_URL}") + raise HTTPError(f"404 Not Found: {self.MODEL_URL}", response=response) + + # Raise for any other 4xx/5xx + response.raise_for_status() + + json_data = response.json() self.model_cache.set(json_data) cloudlog.debug("Successfully updated models cache") return self.model_parser.parse_models(json_data) - except Exception: - cloudlog.exception("Error fetching models") - raise + + except ConnectionError as e: + cloudlog.warning(f"DNS/connection error while fetching models: {e}") + except SSLError as e: + cloudlog.warning(f"SSL error while fetching models: {e}") + except RequestException as e: + cloudlog.warning(f"Request transport error while fetching models: {e}") + except Exception as e: + cloudlog.exception(f"Unexpected error fetching models: {e}") + + return None def get_available_bundles(self) -> list[custom.ModelManagerSP.ModelBundle]: """Gets the list of available models, with smart cache handling""" @@ -144,12 +162,12 @@ class ModelFetcher: cloudlog.debug("Using valid cached models data") return self.model_parser.parse_models(cached_data) - try: - return self._fetch_and_cache_models() - except Exception: - if not cached_data: - cloudlog.exception("Failed to fetch fresh data and no cache available") - raise + fetched_bundles = self._fetch_and_cache_models() + if fetched_bundles is not None: + return fetched_bundles + + if not cached_data: + cloudlog.warning("Failed to fetch fresh data and no cache available") cloudlog.warning("Failed to fetch fresh data. Using expired cache as fallback") return self.model_parser.parse_models(cached_data) diff --git a/sunnypilot/models/helpers.py b/sunnypilot/models/helpers.py index 79241cd831..ecf0a39b72 100644 --- a/sunnypilot/models/helpers.py +++ b/sunnypilot/models/helpers.py @@ -19,7 +19,7 @@ from openpilot.system.hardware.hw import Paths from pathlib import Path # see the README.md for more details on the model selector versioning -CURRENT_SELECTOR_VERSION = 9 +CURRENT_SELECTOR_VERSION = 10 REQUIRED_MIN_SELECTOR_VERSION = 9 USE_ONNX = os.getenv('USE_ONNX', PC) diff --git a/sunnypilot/sunnylink/athena/sunnylinkd.py b/sunnypilot/sunnylink/athena/sunnylinkd.py index 90eae1dfe8..363fa1defc 100755 --- a/sunnypilot/sunnylink/athena/sunnylinkd.py +++ b/sunnypilot/sunnylink/athena/sunnylinkd.py @@ -3,7 +3,9 @@ from __future__ import annotations import base64 +import errno import gzip +import json import os import ssl import threading @@ -17,11 +19,11 @@ from openpilot.common.swaglog import cloudlog from openpilot.system.athena.athenad import ws_send, jsonrpc_handler, \ recv_queue, UploadQueueCache, upload_queue, cur_upload_items, backoff, ws_manage, log_handler, start_local_proxy_shim, upload_handler from websocket import (ABNF, WebSocket, WebSocketException, WebSocketTimeoutException, - create_connection) + create_connection, WebSocketConnectionClosedException) import cereal.messaging as messaging from sunnypilot.sunnylink.api import SunnylinkApi -from sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready +from sunnypilot.sunnylink.utils import sunnylink_need_register, sunnylink_ready, get_param_as_byte SUNNYLINK_ATHENA_HOST = os.getenv('SUNNYLINK_ATHENA_HOST', 'wss://ws.stg.api.sunnypilot.ai') HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4")) @@ -107,10 +109,13 @@ def ws_recv(ws: WebSocket, end_event: threading.Event) -> None: except WebSocketTimeoutException: ns_since_last_ping = int(time.monotonic() * 1e9) - last_ping if ns_since_last_ping > SUNNYLINK_RECONNECT_TIMEOUT_S * 1e9: - cloudlog.exception("sunnylinkd.ws_recv.timeout") + cloudlog.warning("sunnylinkd.ws_recv.timeout") end_event.set() - except Exception: - cloudlog.exception("sunnylinkd.ws_recv.exception") + except Exception as e: + if isinstance(e, WebSocketConnectionClosedException): + cloudlog.warning(f"sunnylinkd.ws_recv.{type(e).__name__}") + else: + cloudlog.exception("sunnylinkd.ws_recv.exception") end_event.set() @@ -137,11 +142,15 @@ def ws_queue(end_event: threading.Event) -> None: sunnylink_api.resume_queued(timeout=29) resume_requested = True tries = 0 - except Exception: - cloudlog.exception("sunnylinkd.ws_queue.resume_queued.exception") + except Exception as e: + if isinstance(e, (ConnectionError, TimeoutError)): + cloudlog.warning(f"sunnylinkd.ws_queue.resume_queued.{type(e).__name__}") + else: + cloudlog.exception("sunnylinkd.ws_queue.resume_queued.exception") + resume_requested = False tries += 1 - time.sleep(backoff(tries)) # Wait for the backoff time before the next attempt + time.sleep(backoff(tries)) if end_event.is_set(): cloudlog.debug("end_event is set, exiting ws_queue thread") @@ -171,16 +180,22 @@ def getParamsAllKeys() -> list[str]: @dispatcher.add_method def getParams(params_keys: list[str], compression: bool = False) -> str | dict[str, str]: + params = Params() + try: - params = Params() - params_dict: dict[str, bytes] = {key: params.get(key) or b'' for key in params_keys} + param_keys_validated = [key for key in params_keys if key in getParamsAllKeys()] + params_dict: dict[str, list[dict[str, str | bool | int ]]] = {"params": [ + { + "key": key, + "value": base64.b64encode(gzip.compress(get_param_as_byte(key)) if compression else get_param_as_byte(key)).decode('utf-8'), + "type": int(params.get_type(key).value), + "is_compressed": compression + } for key in param_keys_validated + ]} - # Compress the values before encoding to base64 as output from params.get is bytes and same for compression - if compression: - params_dict = {key: gzip.compress(value) for key, value in params_dict.items()} - - # Last step is to encode the values to base64 and decode to utf-8 for JSON serialization - return {key: base64.b64encode(value).decode('utf-8') for key, value in params_dict.items()} + response = {str(param.get('key')): str(param.get('value')) for param in params_dict.get("params", [])} + response |= {"params": json.dumps(params_dict.get("params", []))} # Upcoming for settings v1 + return response except Exception as e: cloudlog.exception("sunnylinkd.getParams.exception", e) @@ -252,14 +267,19 @@ def main(exit_event: threading.Event = None): handle_long_poll(ws, exit_event) except (KeyboardInterrupt, SystemExit): break - except (ConnectionError, TimeoutError, WebSocketException): + except Exception as e: conn_retries += 1 params.remove("LastSunnylinkPingTime") - except Exception: - cloudlog.exception("sunnylinkd.main.exception") - conn_retries += 1 - params.remove("LastSunnylinkPingTime") + if isinstance(e, (ConnectionError, TimeoutError, WebSocketException)): + cloudlog.warning(f"sunnylinkd.main.{type(e).__name__}") + elif isinstance(e, OSError): + name = errno.errorcode.get(e.errno or -1, "UNKNOWN") + msg = f"sunnylinkd.main.OSError.{name} ({e.errno})" + is_expected_error = e.errno in (errno.ENETDOWN, errno.ENETRESET, errno.ENETUNREACH) + cloudlog.warning(msg) if is_expected_error else cloudlog.exception(msg) + else: + cloudlog.exception("sunnylinkd.main.exception") time.sleep(backoff(conn_retries)) diff --git a/sunnypilot/sunnylink/backups/manager.py b/sunnypilot/sunnylink/backups/manager.py index f98088a1fb..315300c73c 100644 --- a/sunnypilot/sunnylink/backups/manager.py +++ b/sunnypilot/sunnylink/backups/manager.py @@ -20,6 +20,7 @@ from openpilot.system.version import get_version from cereal import messaging, custom from sunnypilot.sunnylink.api import SunnylinkApi from sunnypilot.sunnylink.backups.utils import decrypt_compressed_data, encrypt_compress_data, SnakeCaseEncoder +from sunnypilot.sunnylink.utils import get_param_as_byte class OperationType(Enum): @@ -74,7 +75,7 @@ class BackupManagerSP: config_data = {} params_to_backup = [k.decode('utf-8') for k in self.params.all_keys(ParamKeyFlag.BACKUP)] for param in params_to_backup: - value = str(self.params.get(param)).encode('utf-8') + value = get_param_as_byte(param) if value is not None: config_data[param] = base64.b64encode(value).decode('utf-8') return config_data @@ -113,6 +114,7 @@ class BackupManagerSP: payload = json.loads(json.dumps(backup_info.to_dict(), cls=SnakeCaseEncoder)) self._update_progress(75.0, OperationType.BACKUP) + cloudlog.debug(f"Uploading backup with payload: {json.dumps(payload)}") # Upload to sunnylink result = self.api.api_get( f"backup/{self.device_id}", @@ -124,9 +126,11 @@ class BackupManagerSP: if result: self.backup_status = custom.BackupManagerSP.Status.completed self._update_progress(100.0, OperationType.BACKUP) + cloudlog.info("Backup successfully created and uploaded") else: self.backup_status = custom.BackupManagerSP.Status.failed self.last_error = "Failed to upload backup" + cloudlog.error(result) self._report_status() return bool(self.backup_status == custom.BackupManagerSP.Status.completed) @@ -264,8 +268,8 @@ class BackupManagerSP: # Check for backup command if self.params.get_bool("BackupManager_CreateBackup"): try: - await self.create_backup() - reset_progress = True + if await self.create_backup(): + reset_progress = True finally: self.params.remove("BackupManager_CreateBackup") diff --git a/sunnypilot/sunnylink/utils.py b/sunnypilot/sunnylink/utils.py index 35714eafe3..569afd26b6 100644 --- a/sunnypilot/sunnylink/utils.py +++ b/sunnypilot/sunnylink/utils.py @@ -1,5 +1,6 @@ +import json from sunnypilot.sunnylink.api import SunnylinkApi, UNREGISTERED_SUNNYLINK_DONGLE_ID -from openpilot.common.params import Params +from openpilot.common.params import Params, ParamKeyType from openpilot.system.version import is_prebuilt @@ -55,3 +56,15 @@ def get_api_token(): sunnylink_api = SunnylinkApi(sunnylink_dongle_id) token = sunnylink_api.get_token() print(f"API Token: {token}") + + +def get_param_as_byte(param_name: str) -> bytes: + params = Params() + param = params.get(param_name) + param_type = params.get_type(param_name) + + if param_type == ParamKeyType.BYTES: + return bytes(param) + elif param_type == ParamKeyType.JSON: + return json.dumps(param).encode('utf-8') + return str(param).encode('utf-8') diff --git a/sunnypilot/system/hardware/c3/launch_env.sh b/sunnypilot/system/hardware/c3/launch_env.sh index 593aacc078..4c011c6ac0 100755 --- a/sunnypilot/system/hardware/c3/launch_env.sh +++ b/sunnypilot/system/hardware/c3/launch_env.sh @@ -9,3 +9,5 @@ export VECLIB_MAXIMUM_THREADS=1 if [ -z "$AGNOS_VERSION" ]; then export AGNOS_VERSION="12.8" fi + +export STAGING_ROOT="/data/safe_staging" diff --git a/system/hardware/hardwared.py b/system/hardware/hardwared.py index d702334fa8..df8ae23ab5 100755 --- a/system/hardware/hardwared.py +++ b/system/hardware/hardwared.py @@ -24,7 +24,7 @@ from openpilot.system.statsd import statlog from openpilot.common.swaglog import cloudlog from openpilot.system.hardware.power_monitoring import PowerMonitoring from openpilot.system.hardware.fan_controller import TiciFanController -from openpilot.system.version import terms_version, training_version +from openpilot.system.version import terms_version, training_version, get_build_metadata ThermalStatus = log.DeviceState.ThermalStatus NetworkType = log.DeviceState.NetworkType @@ -326,6 +326,16 @@ def hardware_thread(end_event, hw_queue) -> None: startup_conditions["not_always_offroad"] = not offroad_mode onroad_conditions["not_always_offroad"] = not offroad_mode + # if an unsupported device and branch is detected, going onroad is blocked + # only allow going onroad when: + # - TIZI, or + # - TICI and channel_type is "tici" + build_metadata = get_build_metadata() + is_unsupported_combo = TICI and HARDWARE.get_device_type() == "tici" and build_metadata.channel_type != "tici" + startup_conditions["not_tici"] = not is_unsupported_combo + onroad_conditions["not_tici"] = not is_unsupported_combo + set_offroad_alert("Offroad_TiciSupport", is_unsupported_combo, extra_text=build_metadata.channel) + # if the temperature enters the danger zone, go offroad to cool down onroad_conditions["device_temp_good"] = thermal_status < ThermalStatus.danger extra_text = f"{offroad_comp_temp:.1f}C" diff --git a/system/updated/updated.py b/system/updated/updated.py index 11928bc24c..7ab9e070dc 100755 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -7,7 +7,6 @@ import psutil import shutil import signal import fcntl -import time import threading from collections import defaultdict from pathlib import Path @@ -19,7 +18,7 @@ from openpilot.common.markdown import parse_markdown from openpilot.common.swaglog import cloudlog from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.system.hardware import AGNOS, HARDWARE -from openpilot.system.version import get_build_metadata +from openpilot.system.version import get_build_metadata, SP_BRANCH_MIGRATIONS LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock") STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging") @@ -190,15 +189,6 @@ def finalize_update() -> None: run(["git", "reset", "--hard"], FINALIZED) run(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"], FINALIZED) - cloudlog.info("Starting git cleanup in finalized update") - t = time.monotonic() - try: - run(["git", "gc"], FINALIZED) - run(["git", "lfs", "prune"], FINALIZED) - cloudlog.event("Done git cleanup", duration=time.monotonic() - t) - except subprocess.CalledProcessError: - cloudlog.exception(f"Failed git cleanup, took {time.monotonic() - t:.3f} s") - set_consistent_flag(True) cloudlog.info("done finalizing overlay") @@ -242,9 +232,7 @@ class Updater: b: str | None = self.params.get("UpdaterTargetBranch") if b is None: b = self.get_branch(BASEDIR) - b = { - ("tici", "release3"): "release-tici" - }.get((HARDWARE.get_device_type(), b), b) + b = SP_BRANCH_MIGRATIONS.get((HARDWARE.get_device_type(), b), b) return b @property diff --git a/system/version.py b/system/version.py index 5aa8d0115f..87044b84a8 100755 --- a/system/version.py +++ b/system/version.py @@ -16,6 +16,13 @@ MASTER_SP_BRANCHES = ['master'] RELEASE_BRANCHES = ['release3-staging', 'release3', 'release-tici', 'nightly'] + RELEASE_SP_BRANCHES TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging', 'nightly-dev'] + TESTED_SP_BRANCHES +SP_BRANCH_MIGRATIONS = { + ("tici", "staging-c3-new"): "staging-tici", + ("tici", "dev-c3-new"): "staging-tici", + ("tici", "master"): "master-tici", + ("tici", "master-dev-c3-new"): "master-tici", +} + BUILD_METADATA_FILENAME = "build.json" training_version: str = "0.2.0" @@ -85,7 +92,8 @@ class OpenpilotMetadata: @property def sunnypilot_remote(self) -> bool: - return self.git_normalized_origin == "github.com/sunnypilot/sunnypilot" + return self.git_normalized_origin in ("github.com/sunnypilot/sunnypilot", + "github.com/sunnypilot/openpilot") @property def git_normalized_origin(self) -> str: @@ -127,7 +135,9 @@ class BuildMetadata: @property def channel_type(self) -> str: - if self.development_channel: + if self.channel.endswith("-tici"): + return "tici" + elif self.development_channel: return "development" elif self.channel.startswith("staging-"): return "staging"