From 3832c73da93be7061eb5be776c853583c3bf96a0 Mon Sep 17 00:00:00 2001 From: James <91348155+FrogAi@users.noreply.github.com> Date: Fri, 2 Jan 2026 18:14:19 -0700 Subject: [PATCH] Automatic backups --- frogpilot/common/frogpilot_backups.py | 103 +++++++ frogpilot/common/frogpilot_functions.py | 5 +- frogpilot/frogpilot_process.py | 11 +- frogpilot/ui/qt/offroad/data_settings.cc | 370 +++++++++++++++++++++++ system/manager/manager.py | 12 +- system/updated/updated.py | 5 +- 6 files changed, 497 insertions(+), 9 deletions(-) create mode 100644 frogpilot/common/frogpilot_backups.py diff --git a/frogpilot/common/frogpilot_backups.py b/frogpilot/common/frogpilot_backups.py new file mode 100644 index 00000000..8bd82bc9 --- /dev/null +++ b/frogpilot/common/frogpilot_backups.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import datetime +import shutil +import tarfile +import zstandard as zstd + +from pathlib import Path + +from openpilot.common.basedir import BASEDIR +from openpilot.common.params import Params + +from openpilot.frogpilot.common.frogpilot_utilities import delete_file +from openpilot.frogpilot.common.frogpilot_variables import EXCLUDED_KEYS, FROGPILOT_BACKUPS, TOGGLE_BACKUPS + + +def backup_frogpilot(build_metadata, params): + maximum_backups = 3 + cleanup_backups(FROGPILOT_BACKUPS, maximum_backups) + + today = datetime.datetime.now().date() + for backup in FROGPILOT_BACKUPS.glob("*_auto.tar.zst"): + if backup.name.endswith(f"_{build_metadata.channel}_auto.tar.zst"): + if datetime.datetime.fromtimestamp(backup.stat().st_mtime).date() == today: + if not backup.name.startswith(f"{build_metadata.openpilot.git_commit[:6]}_"): + delete_file(backup, report=False) + + _, _, free = shutil.disk_usage(FROGPILOT_BACKUPS) + minimum_backup_size = params.get("MinimumBackupSize") + if free > minimum_backup_size * maximum_backups: + destination = FROGPILOT_BACKUPS / f"{build_metadata.openpilot.git_commit}_{build_metadata.channel}_auto" + create_backup(Path(BASEDIR), destination, "Successfully backed up FrogPilot!", "Failed to backup FrogPilot...", params, minimum_backup_size, compressed=True) + + +def backup_toggles(params, boot_run=False): + params_backup = Params("/dev/shm/params_backup", return_defaults=True) + + changes_found = False + for key in params.all_keys(): + current_value = params.get(key) + if current_value is None: + continue + + if boot_run: + params_backup.put(key, current_value) + changes_found = True + elif current_value != params_backup.get(key): + params_backup.put(key, current_value) + changes_found |= key.decode("utf-8") not in EXCLUDED_KEYS + + maximum_backups = 5 + cleanup_backups(TOGGLE_BACKUPS, maximum_backups) + + if not changes_found or boot_run: + print("Toggles are identical to the previous backup. Aborting...") + return + + destination = TOGGLE_BACKUPS / f"{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_auto" + create_backup(Path(params_backup.get_param_path()), destination, "Successfully backed up toggles!", "Failed to backup toggles...", params) + + +def cleanup_backups(directory, limit): + directory.mkdir(parents=True, exist_ok=True) + + for backup in directory.glob("*_in_progress*"): + delete_file(backup, report=False) + + backups = sorted(directory.glob("*_auto*"), key=lambda f: f.stat().st_mtime, reverse=True) + for oldest_backup in backups[limit:]: + delete_file(oldest_backup, report=False) + + +def create_backup(backup, destination, success_message, fail_message, params, minimum_backup_size=0, compressed=False): + final_destination = destination.parent / f"{destination.name}.tar.zst" if compressed else destination + + if final_destination.exists(): + print("Backup already exists. Aborting...") + return + + if compressed: + compressed_temp = destination.parent / f"{destination.name}_in_progress.tar.zst" + + with open(compressed_temp, "wb") as f_out: + cctx = zstd.ZstdCompressor() + with cctx.stream_writer(f_out) as compressor: + with tarfile.open(fileobj=compressor, mode="w") as tar: + try: + tar.add(backup, arcname=destination.name) + except OSError: + pass + + compressed_temp.rename(final_destination) + + compressed_backup_size = final_destination.stat().st_size + if minimum_backup_size == 0 or compressed_backup_size < minimum_backup_size: + params.put("MinimumBackupSize", int(compressed_backup_size)) + else: + in_progress_destination = destination.parent / f"{destination.name}_in_progress" + + shutil.copytree(backup, in_progress_destination, symlinks=True) + + in_progress_destination.rename(destination) + + print(success_message) diff --git a/frogpilot/common/frogpilot_functions.py b/frogpilot/common/frogpilot_functions.py index 0da69573..6076838e 100644 --- a/frogpilot/common/frogpilot_functions.py +++ b/frogpilot/common/frogpilot_functions.py @@ -11,13 +11,14 @@ from openpilot.common.params import Params from openpilot.common.time_helpers import system_time_valid from openpilot.system.hardware import HARDWARE +from openpilot.frogpilot.common.frogpilot_backups import backup_frogpilot from openpilot.frogpilot.common.frogpilot_utilities import run_cmd from openpilot.frogpilot.common.frogpilot_variables import ( FrogPilotVariables ) -def frogpilot_boot_functions(params): +def frogpilot_boot_functions(build_metadata, params): params_memory = Params(memory=True) FrogPilotVariables() @@ -27,6 +28,8 @@ def frogpilot_boot_functions(params): print("Waiting for system time to become valid...") time.sleep(1) + backup_frogpilot(build_metadata, params) + threading.Thread(target=boot_thread, daemon=True).start() diff --git a/frogpilot/frogpilot_process.py b/frogpilot/frogpilot_process.py index 79f7c16b..a96fda74 100644 --- a/frogpilot/frogpilot_process.py +++ b/frogpilot/frogpilot_process.py @@ -8,6 +8,7 @@ from openpilot.common.params import Params from openpilot.common.realtime import DT_MDL, Priority, Ratekeeper, config_realtime_process from openpilot.common.time_helpers import system_time_valid +from openpilot.frogpilot.common.frogpilot_backups import backup_toggles from openpilot.frogpilot.common.frogpilot_utilities import ThreadManager, is_url_pingable from openpilot.frogpilot.common.frogpilot_variables import FrogPilotVariables from openpilot.frogpilot.controls.frogpilot_planner import FrogPilotPlanner @@ -32,10 +33,13 @@ def update_checks(now, thread_manager, params, params_memory, frogpilot_toggles, time.sleep(1) -def update_toggles(frogpilot_variables, started): +def update_toggles(frogpilot_variables, started, thread_manager, time_validated, params): frogpilot_variables.update(started) frogpilot_toggles = frogpilot_variables.frogpilot_toggles + if time_validated: + thread_manager.run_with_lock(backup_toggles, (params)) + return frogpilot_toggles def frogpilot_thread(): @@ -70,7 +74,7 @@ def frogpilot_thread(): started = sm["deviceState"].started if not started and started_previously: - frogpilot_toggles = update_toggles(frogpilot_variables, started) + frogpilot_toggles = update_toggles(frogpilot_variables, started, thread_manager, time_validated, params) transition_offroad(frogpilot_planner, thread_manager, time_validated, sm, params, frogpilot_toggles) run_update_checks = True @@ -96,7 +100,7 @@ def frogpilot_thread(): check_assets(thread_manager, params_memory, frogpilot_toggles) if params_memory.get_bool("FrogPilotTogglesUpdated"): - frogpilot_toggles = update_toggles(frogpilot_variables, started) + frogpilot_toggles = update_toggles(frogpilot_variables, started, thread_manager, time_validated, params) run_update_checks |= now.second == 0 and (now.minute % 60 == 0) run_update_checks &= time_validated @@ -110,6 +114,7 @@ def frogpilot_thread(): if not time_validated: continue + thread_manager.run_with_lock(backup_toggles, (params, True)) thread_manager.run_with_lock(send_stats, (params, frogpilot_toggles)) thread_manager.run_with_lock(update_checks, (now, thread_manager, params, params_memory, frogpilot_toggles, True)) diff --git a/frogpilot/ui/qt/offroad/data_settings.cc b/frogpilot/ui/qt/offroad/data_settings.cc index 9ce06c73..d83320d0 100644 --- a/frogpilot/ui/qt/offroad/data_settings.cc +++ b/frogpilot/ui/qt/offroad/data_settings.cc @@ -14,6 +14,376 @@ FrogPilotDataPanel::FrogPilotDataPanel(FrogPilotSettingsWindow *parent, bool for ScrollView *statsLabelsPanel = new ScrollView(statsLabelsList, this); dataLayout->addWidget(statsLabelsPanel); + FrogPilotButtonsControl *frogpilotBackupButton = new FrogPilotButtonsControl(tr("FrogPilot Backups"), tr("Create, delete, or restore FrogPilot backups."), "", {tr("BACKUP"), tr("DELETE"), tr("DELETE ALL"), tr("RESTORE")}); + QObject::connect(frogpilotBackupButton, &FrogPilotButtonsControl::buttonClicked, [=](int id) { + QDir backupDir("/data/backups"); + + QFileInfoList backupList = backupDir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot); + std::sort(backupList.begin(), backupList.end(), [](const QFileInfo &a, const QFileInfo &b) { + return a.lastModified() > b.lastModified(); + }); + + QStringList friendlyNames; + QMap backupMap; + + for (const QFileInfo &fileInfo : backupList) { + QString fileName = fileInfo.fileName(); + + if (fileName.contains("in_progress")) { + continue; + } + + QString friendlyName = fileName; + + if (fileName.endsWith("_auto.tar.zst")) { + QStringList parts = QString(fileName).remove(".tar.zst").split("_"); + + if (parts.size() >= 3) { + QDate date = fileInfo.lastModified().date(); + + if (date.isValid()) { + int day = date.day(); + QString suffix = (day >= 11 && day <= 13) ? "th" : + (day % 10 == 1) ? "st" : + (day % 10 == 2) ? "nd" : + (day % 10 == 3) ? "rd" : "th"; + + friendlyName = QString("%1 %2%3, %4 (%5)") + .arg(date.toString("MMMM")) + .arg(day) + .arg(suffix) + .arg(date.year()) + .arg(parts[1]); + } + } + } + + if (friendlyName == fileName) { + friendlyName.remove(".tar.zst"); + friendlyName.replace("_", " "); + } + + friendlyNames.append(friendlyName); + backupMap[friendlyName] = fileName; + } + + if (id == 0) { + QString name = InputDialog::getText(tr("Name your backup"), this, tr("Backup Name")).trimmed().replace(" ", "_"); + if (!name.isEmpty()) { + QStringList distinctFileNames = backupDir.entryList(QDir::Files | QDir::NoDotAndDotDot); + if (distinctFileNames.contains(name + ".tar.zst")) { + ConfirmationDialog::alert(tr("Name already in use. Please choose a different name!"), this); + return; + } + + std::thread([=]() { + parent->keepScreenOn = true; + + frogpilotBackupButton->setEnabled(false); + frogpilotBackupButton->setValue(tr("Backing up...")); + + frogpilotBackupButton->setVisibleButton(1, false); + frogpilotBackupButton->setVisibleButton(2, false); + frogpilotBackupButton->setVisibleButton(3, false); + + std::system(QString("tar --use-compress-program=zstd -cf %1 %2").arg(backupDir.absoluteFilePath(name + ".tar.zst"), "/data/openpilot").toStdString().c_str()); + + frogpilotBackupButton->setValue(tr("Backup created!")); + + util::sleep_for(2500); + + frogpilotBackupButton->setEnabled(true); + frogpilotBackupButton->setValue(""); + + frogpilotBackupButton->setVisibleButton(1, true); + frogpilotBackupButton->setVisibleButton(2, true); + frogpilotBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + + } else if (id == 1) { + QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to delete"), friendlyNames, "", this); + if (!selection.isEmpty()) { + if (ConfirmationDialog::confirm(tr("Delete this backup?"), tr("Delete"), this)) { + std::thread([=]() { + parent->keepScreenOn = true; + + frogpilotBackupButton->setEnabled(false); + frogpilotBackupButton->setValue(tr("Deleting...")); + + frogpilotBackupButton->setVisibleButton(0, false); + frogpilotBackupButton->setVisibleButton(2, false); + frogpilotBackupButton->setVisibleButton(3, false); + + QFile::remove(backupDir.absoluteFilePath(backupMap[selection])); + + frogpilotBackupButton->setValue(tr("Deleted!")); + + util::sleep_for(2500); + + frogpilotBackupButton->setEnabled(true); + frogpilotBackupButton->setValue(""); + + frogpilotBackupButton->setVisibleButton(0, true); + frogpilotBackupButton->setVisibleButton(2, true); + frogpilotBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + } + + } else if (id == 2) { + if (ConfirmationDialog::confirm(tr("Delete all FrogPilot backups?"), tr("Delete All"), this)) { + std::thread([=]() mutable { + parent->keepScreenOn = true; + + frogpilotBackupButton->setEnabled(false); + frogpilotBackupButton->setValue(tr("Deleting...")); + + frogpilotBackupButton->setVisibleButton(0, false); + frogpilotBackupButton->setVisibleButton(1, false); + frogpilotBackupButton->setVisibleButton(3, false); + + backupDir.removeRecursively(); + backupDir.mkpath("."); + + frogpilotBackupButton->setValue(tr("Deleted!")); + + util::sleep_for(2500); + + frogpilotBackupButton->setEnabled(true); + frogpilotBackupButton->setValue(""); + + frogpilotBackupButton->setVisibleButton(0, true); + frogpilotBackupButton->setVisibleButton(1, true); + frogpilotBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + + } else if (id == 3) { + QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to restore"), friendlyNames, "", this); + if (!selection.isEmpty()) { + if (FrogPilotConfirmationDialog::yesorno(tr("Restore this backup? This will overwrite your current installation and reboot the device."), this)) { + std::thread([=]() { + parent->keepScreenOn = true; + + frogpilotBackupButton->setEnabled(false); + frogpilotBackupButton->setValue(tr("Restoring...")); + + frogpilotBackupButton->setVisibleButton(0, false); + frogpilotBackupButton->setVisibleButton(1, false); + frogpilotBackupButton->setVisibleButton(2, false); + + std::system(QString("rm -rf /data/openpilot/* && tar --use-compress-program=zstd -xf %1 -C /").arg(backupDir.absoluteFilePath(backupMap[selection])).toStdString().c_str()); + QFile("/cache/on_backup").open(QIODevice::WriteOnly); + + frogpilotBackupButton->setValue(tr("Restored!")); + + util::sleep_for(2500); + + frogpilotBackupButton->setValue(tr("Rebooting...")); + + util::sleep_for(2500); + + Hardware::reboot(); + }).detach(); + } + } + } + }); + if (forceOpenDescriptions) { + frogpilotBackupButton->showDescription(); + } + dataMainList->addItem(frogpilotBackupButton); + + FrogPilotButtonsControl *toggleBackupButton = new FrogPilotButtonsControl(tr("Toggle Backups"), tr("Create, delete, or restore toggle backups."), "", {tr("BACKUP"), tr("DELETE"), tr("DELETE ALL"), tr("RESTORE")}); + QObject::connect(toggleBackupButton, &FrogPilotButtonsControl::buttonClicked, [=](int id) { + QDir backupDir("/data/toggle_backups"); + + QStringList backupNames = backupDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot); + std::sort(backupNames.begin(), backupNames.end(), std::greater()); + + QMap backupMap; + for (const QString &dirName : backupNames) { + if (dirName.contains("in_progress")) { + continue; + } + + QString friendlyName = dirName; + + if (dirName.endsWith("_auto")) { + QStringList parts = QString(dirName).remove("_auto").split("_"); + + if (parts.size() >= 2) { + QDate date = QDate::fromString(parts[0], "yyyy-MM-dd"); + QTime time = QTime::fromString(parts[1], "HH-mm-ss"); + + if (date.isValid() && time.isValid()) { + int day = date.day(); + QString suffix = (day >= 11 && day <= 13) ? "th" : + (day % 10 == 1) ? "st" : + (day % 10 == 2) ? "nd" : + (day % 10 == 3) ? "rd" : "th"; + + friendlyName = QString("%1 %2%3, %4 (%5)") + .arg(date.toString("MMMM")) + .arg(day) + .arg(suffix) + .arg(date.year()) + .arg(time.toString("h:mm AP")); + } + } + } + + if (friendlyName == dirName) { + friendlyName.replace("_", " "); + } + + backupMap[friendlyName] = dirName; + } + + if (id == 0) { + QString name = InputDialog::getText(tr("Name your backup"), this, tr("Backup Name")).trimmed().replace(" ", "_"); + if (!name.isEmpty()) { + if (backupNames.contains(name)) { + ConfirmationDialog::alert(tr("Name already in use. Please choose a different name!"), this); + return; + } + + std::thread([=]() { + parent->keepScreenOn = true; + + toggleBackupButton->setEnabled(false); + toggleBackupButton->setValue(tr("Backing up...")); + + toggleBackupButton->setVisibleButton(1, false); + toggleBackupButton->setVisibleButton(2, false); + toggleBackupButton->setVisibleButton(3, false); + + std::system(QString("cp -r /data/params/d/ %1").arg(backupDir.absoluteFilePath(name)).toStdString().c_str()); + + toggleBackupButton->setValue(tr("Backup created!")); + + util::sleep_for(2500); + + toggleBackupButton->setEnabled(true); + toggleBackupButton->setValue(""); + + toggleBackupButton->setVisibleButton(1, true); + toggleBackupButton->setVisibleButton(2, true); + toggleBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + + } else if (id == 1) { + QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to delete"), backupMap.keys(), "", this); + if (!selection.isEmpty()) { + if (ConfirmationDialog::confirm(tr("Delete this backup?"), tr("Delete"), this)) { + std::thread([=]() { + parent->keepScreenOn = true; + + toggleBackupButton->setEnabled(false); + toggleBackupButton->setValue(tr("Deleting...")); + + toggleBackupButton->setVisibleButton(0, false); + toggleBackupButton->setVisibleButton(2, false); + toggleBackupButton->setVisibleButton(3, false); + + QDir(backupDir.absoluteFilePath(backupMap[selection])).removeRecursively(); + + toggleBackupButton->setValue(tr("Deleted!")); + + util::sleep_for(2500); + + toggleBackupButton->setEnabled(true); + toggleBackupButton->setValue(""); + + toggleBackupButton->setVisibleButton(0, true); + toggleBackupButton->setVisibleButton(2, true); + toggleBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + } + + } else if (id == 2) { + if (ConfirmationDialog::confirm(tr("Delete all toggle backups?"), tr("Delete All"), this)) { + std::thread([=]() mutable { + parent->keepScreenOn = true; + + toggleBackupButton->setEnabled(false); + toggleBackupButton->setValue(tr("Deleting...")); + + toggleBackupButton->setVisibleButton(0, false); + toggleBackupButton->setVisibleButton(1, false); + toggleBackupButton->setVisibleButton(3, false); + + backupDir.removeRecursively(); + backupDir.mkpath("."); + + toggleBackupButton->setValue(tr("Deleted!")); + + util::sleep_for(2500); + + toggleBackupButton->setEnabled(true); + toggleBackupButton->setValue(""); + + toggleBackupButton->setVisibleButton(0, true); + toggleBackupButton->setVisibleButton(1, true); + toggleBackupButton->setVisibleButton(3, true); + + parent->keepScreenOn = false; + }).detach(); + } + + } else if (id == 3) { + QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to restore"), backupMap.keys(), "", this); + if (!selection.isEmpty()) { + if (FrogPilotConfirmationDialog::yesorno(tr("Restore this backup? This will overwrite your current settings!"), this)) { + std::thread([=]() { + parent->keepScreenOn = true; + + toggleBackupButton->setEnabled(false); + toggleBackupButton->setValue(tr("Restoring...")); + + toggleBackupButton->setVisibleButton(0, false); + toggleBackupButton->setVisibleButton(1, false); + toggleBackupButton->setVisibleButton(2, false); + + std::system(QString("cp -r %1/* /data/params/d/").arg(backupDir.absoluteFilePath(backupMap[selection])).toStdString().c_str()); + + updateFrogPilotToggles(); + + toggleBackupButton->setValue(tr("Restored!")); + + util::sleep_for(2500); + + toggleBackupButton->setEnabled(true); + toggleBackupButton->setValue(""); + + toggleBackupButton->setVisibleButton(0, true); + toggleBackupButton->setVisibleButton(1, true); + toggleBackupButton->setVisibleButton(2, true); + + parent->keepScreenOn = false; + }).detach(); + } + } + } + }); + if (forceOpenDescriptions) { + toggleBackupButton->showDescription(); + } + dataMainList->addItem(toggleBackupButton); + FrogPilotButtonsControl *viewStatsButton = new FrogPilotButtonsControl(tr("FrogPilot Stats"), tr("View your collected FrogPilot stats."), "", {tr("RESET"), tr("VIEW")}); QObject::connect(viewStatsButton, &FrogPilotButtonsControl::buttonClicked, [dataLayout, statsLabelsPanel, this](int id) { if (id == 0) { diff --git a/system/manager/manager.py b/system/manager/manager.py index f5fbec1c..3a8583f3 100644 --- a/system/manager/manager.py +++ b/system/manager/manager.py @@ -46,9 +46,13 @@ def manager_init() -> None: # set unset params to their default value for k in params.all_keys(): - default_value = params.get_default_value(k) - if default_value is not None and params.get(k) is None: - params.put(k, default_value) + current_value = params.get(k) + if current_value is None: + cached_value = params_cache.get(k) + if cached_value is not None: + params.put(k, cached_value) + else: + params_cache.put(k, current_value) # Create folders needed for msgq try: @@ -101,7 +105,7 @@ def manager_init() -> None: # FrogPilot variables install_frogpilot(params) - frogpilot_boot_functions(params) + frogpilot_boot_functions(build_metadata, params) def manager_cleanup() -> None: diff --git a/system/updated/updated.py b/system/updated/updated.py index 08d08ccf..6704d0a2 100644 --- a/system/updated/updated.py +++ b/system/updated/updated.py @@ -22,7 +22,7 @@ from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert from openpilot.system.hardware import AGNOS, HARDWARE from openpilot.system.version import get_build_metadata -from openpilot.frogpilot.common.frogpilot_variables import get_frogpilot_toggles +from openpilot.frogpilot.common.frogpilot_variables import BACKUP_PATH, get_frogpilot_toggles LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock") STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging") @@ -414,6 +414,9 @@ class Updater: cloudlog.info("finalize success!") # FrogPilot variables + if os.path.isfile(BACKUP_PATH): + os.remove(BACKUP_PATH) + self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo("America/Phoenix")).strftime("%B %d, %Y - %I:%M%p")) def main() -> None: