Automatic backups

This commit is contained in:
James
2026-01-02 18:14:19 -07:00
parent e97b9aca53
commit 3832c73da9
6 changed files with 497 additions and 9 deletions
+103
View File
@@ -0,0 +1,103 @@
#!/usr/bin/env python3
import datetime
import shutil
import tarfile
import zstandard as zstd
from pathlib import Path
from openpilot.common.basedir import BASEDIR
from openpilot.common.params import Params
from openpilot.frogpilot.common.frogpilot_utilities import delete_file
from openpilot.frogpilot.common.frogpilot_variables import EXCLUDED_KEYS, FROGPILOT_BACKUPS, TOGGLE_BACKUPS
def backup_frogpilot(build_metadata, params):
maximum_backups = 3
cleanup_backups(FROGPILOT_BACKUPS, maximum_backups)
today = datetime.datetime.now().date()
for backup in FROGPILOT_BACKUPS.glob("*_auto.tar.zst"):
if backup.name.endswith(f"_{build_metadata.channel}_auto.tar.zst"):
if datetime.datetime.fromtimestamp(backup.stat().st_mtime).date() == today:
if not backup.name.startswith(f"{build_metadata.openpilot.git_commit[:6]}_"):
delete_file(backup, report=False)
_, _, free = shutil.disk_usage(FROGPILOT_BACKUPS)
minimum_backup_size = params.get("MinimumBackupSize")
if free > minimum_backup_size * maximum_backups:
destination = FROGPILOT_BACKUPS / f"{build_metadata.openpilot.git_commit}_{build_metadata.channel}_auto"
create_backup(Path(BASEDIR), destination, "Successfully backed up FrogPilot!", "Failed to backup FrogPilot...", params, minimum_backup_size, compressed=True)
def backup_toggles(params, boot_run=False):
params_backup = Params("/dev/shm/params_backup", return_defaults=True)
changes_found = False
for key in params.all_keys():
current_value = params.get(key)
if current_value is None:
continue
if boot_run:
params_backup.put(key, current_value)
changes_found = True
elif current_value != params_backup.get(key):
params_backup.put(key, current_value)
changes_found |= key.decode("utf-8") not in EXCLUDED_KEYS
maximum_backups = 5
cleanup_backups(TOGGLE_BACKUPS, maximum_backups)
if not changes_found or boot_run:
print("Toggles are identical to the previous backup. Aborting...")
return
destination = TOGGLE_BACKUPS / f"{datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")}_auto"
create_backup(Path(params_backup.get_param_path()), destination, "Successfully backed up toggles!", "Failed to backup toggles...", params)
def cleanup_backups(directory, limit):
directory.mkdir(parents=True, exist_ok=True)
for backup in directory.glob("*_in_progress*"):
delete_file(backup, report=False)
backups = sorted(directory.glob("*_auto*"), key=lambda f: f.stat().st_mtime, reverse=True)
for oldest_backup in backups[limit:]:
delete_file(oldest_backup, report=False)
def create_backup(backup, destination, success_message, fail_message, params, minimum_backup_size=0, compressed=False):
final_destination = destination.parent / f"{destination.name}.tar.zst" if compressed else destination
if final_destination.exists():
print("Backup already exists. Aborting...")
return
if compressed:
compressed_temp = destination.parent / f"{destination.name}_in_progress.tar.zst"
with open(compressed_temp, "wb") as f_out:
cctx = zstd.ZstdCompressor()
with cctx.stream_writer(f_out) as compressor:
with tarfile.open(fileobj=compressor, mode="w") as tar:
try:
tar.add(backup, arcname=destination.name)
except OSError:
pass
compressed_temp.rename(final_destination)
compressed_backup_size = final_destination.stat().st_size
if minimum_backup_size == 0 or compressed_backup_size < minimum_backup_size:
params.put("MinimumBackupSize", int(compressed_backup_size))
else:
in_progress_destination = destination.parent / f"{destination.name}_in_progress"
shutil.copytree(backup, in_progress_destination, symlinks=True)
in_progress_destination.rename(destination)
print(success_message)
+4 -1
View File
@@ -11,13 +11,14 @@ from openpilot.common.params import Params
from openpilot.common.time_helpers import system_time_valid
from openpilot.system.hardware import HARDWARE
from openpilot.frogpilot.common.frogpilot_backups import backup_frogpilot
from openpilot.frogpilot.common.frogpilot_utilities import run_cmd
from openpilot.frogpilot.common.frogpilot_variables import (
FrogPilotVariables
)
def frogpilot_boot_functions(params):
def frogpilot_boot_functions(build_metadata, params):
params_memory = Params(memory=True)
FrogPilotVariables()
@@ -27,6 +28,8 @@ def frogpilot_boot_functions(params):
print("Waiting for system time to become valid...")
time.sleep(1)
backup_frogpilot(build_metadata, params)
threading.Thread(target=boot_thread, daemon=True).start()
+8 -3
View File
@@ -8,6 +8,7 @@ from openpilot.common.params import Params
from openpilot.common.realtime import DT_MDL, Priority, Ratekeeper, config_realtime_process
from openpilot.common.time_helpers import system_time_valid
from openpilot.frogpilot.common.frogpilot_backups import backup_toggles
from openpilot.frogpilot.common.frogpilot_utilities import ThreadManager, is_url_pingable
from openpilot.frogpilot.common.frogpilot_variables import FrogPilotVariables
from openpilot.frogpilot.controls.frogpilot_planner import FrogPilotPlanner
@@ -32,10 +33,13 @@ def update_checks(now, thread_manager, params, params_memory, frogpilot_toggles,
time.sleep(1)
def update_toggles(frogpilot_variables, started):
def update_toggles(frogpilot_variables, started, thread_manager, time_validated, params):
frogpilot_variables.update(started)
frogpilot_toggles = frogpilot_variables.frogpilot_toggles
if time_validated:
thread_manager.run_with_lock(backup_toggles, (params))
return frogpilot_toggles
def frogpilot_thread():
@@ -70,7 +74,7 @@ def frogpilot_thread():
started = sm["deviceState"].started
if not started and started_previously:
frogpilot_toggles = update_toggles(frogpilot_variables, started)
frogpilot_toggles = update_toggles(frogpilot_variables, started, thread_manager, time_validated, params)
transition_offroad(frogpilot_planner, thread_manager, time_validated, sm, params, frogpilot_toggles)
run_update_checks = True
@@ -96,7 +100,7 @@ def frogpilot_thread():
check_assets(thread_manager, params_memory, frogpilot_toggles)
if params_memory.get_bool("FrogPilotTogglesUpdated"):
frogpilot_toggles = update_toggles(frogpilot_variables, started)
frogpilot_toggles = update_toggles(frogpilot_variables, started, thread_manager, time_validated, params)
run_update_checks |= now.second == 0 and (now.minute % 60 == 0)
run_update_checks &= time_validated
@@ -110,6 +114,7 @@ def frogpilot_thread():
if not time_validated:
continue
thread_manager.run_with_lock(backup_toggles, (params, True))
thread_manager.run_with_lock(send_stats, (params, frogpilot_toggles))
thread_manager.run_with_lock(update_checks, (now, thread_manager, params, params_memory, frogpilot_toggles, True))
+370
View File
@@ -14,6 +14,376 @@ FrogPilotDataPanel::FrogPilotDataPanel(FrogPilotSettingsWindow *parent, bool for
ScrollView *statsLabelsPanel = new ScrollView(statsLabelsList, this);
dataLayout->addWidget(statsLabelsPanel);
FrogPilotButtonsControl *frogpilotBackupButton = new FrogPilotButtonsControl(tr("FrogPilot Backups"), tr("<b>Create, delete, or restore FrogPilot backups.</b>"), "", {tr("BACKUP"), tr("DELETE"), tr("DELETE ALL"), tr("RESTORE")});
QObject::connect(frogpilotBackupButton, &FrogPilotButtonsControl::buttonClicked, [=](int id) {
QDir backupDir("/data/backups");
QFileInfoList backupList = backupDir.entryInfoList(QDir::Files | QDir::NoDotAndDotDot);
std::sort(backupList.begin(), backupList.end(), [](const QFileInfo &a, const QFileInfo &b) {
return a.lastModified() > b.lastModified();
});
QStringList friendlyNames;
QMap<QString, QString> backupMap;
for (const QFileInfo &fileInfo : backupList) {
QString fileName = fileInfo.fileName();
if (fileName.contains("in_progress")) {
continue;
}
QString friendlyName = fileName;
if (fileName.endsWith("_auto.tar.zst")) {
QStringList parts = QString(fileName).remove(".tar.zst").split("_");
if (parts.size() >= 3) {
QDate date = fileInfo.lastModified().date();
if (date.isValid()) {
int day = date.day();
QString suffix = (day >= 11 && day <= 13) ? "th" :
(day % 10 == 1) ? "st" :
(day % 10 == 2) ? "nd" :
(day % 10 == 3) ? "rd" : "th";
friendlyName = QString("%1 %2%3, %4 (%5)")
.arg(date.toString("MMMM"))
.arg(day)
.arg(suffix)
.arg(date.year())
.arg(parts[1]);
}
}
}
if (friendlyName == fileName) {
friendlyName.remove(".tar.zst");
friendlyName.replace("_", " ");
}
friendlyNames.append(friendlyName);
backupMap[friendlyName] = fileName;
}
if (id == 0) {
QString name = InputDialog::getText(tr("Name your backup"), this, tr("Backup Name")).trimmed().replace(" ", "_");
if (!name.isEmpty()) {
QStringList distinctFileNames = backupDir.entryList(QDir::Files | QDir::NoDotAndDotDot);
if (distinctFileNames.contains(name + ".tar.zst")) {
ConfirmationDialog::alert(tr("Name already in use. Please choose a different name!"), this);
return;
}
std::thread([=]() {
parent->keepScreenOn = true;
frogpilotBackupButton->setEnabled(false);
frogpilotBackupButton->setValue(tr("Backing up..."));
frogpilotBackupButton->setVisibleButton(1, false);
frogpilotBackupButton->setVisibleButton(2, false);
frogpilotBackupButton->setVisibleButton(3, false);
std::system(QString("tar --use-compress-program=zstd -cf %1 %2").arg(backupDir.absoluteFilePath(name + ".tar.zst"), "/data/openpilot").toStdString().c_str());
frogpilotBackupButton->setValue(tr("Backup created!"));
util::sleep_for(2500);
frogpilotBackupButton->setEnabled(true);
frogpilotBackupButton->setValue("");
frogpilotBackupButton->setVisibleButton(1, true);
frogpilotBackupButton->setVisibleButton(2, true);
frogpilotBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
} else if (id == 1) {
QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to delete"), friendlyNames, "", this);
if (!selection.isEmpty()) {
if (ConfirmationDialog::confirm(tr("Delete this backup?"), tr("Delete"), this)) {
std::thread([=]() {
parent->keepScreenOn = true;
frogpilotBackupButton->setEnabled(false);
frogpilotBackupButton->setValue(tr("Deleting..."));
frogpilotBackupButton->setVisibleButton(0, false);
frogpilotBackupButton->setVisibleButton(2, false);
frogpilotBackupButton->setVisibleButton(3, false);
QFile::remove(backupDir.absoluteFilePath(backupMap[selection]));
frogpilotBackupButton->setValue(tr("Deleted!"));
util::sleep_for(2500);
frogpilotBackupButton->setEnabled(true);
frogpilotBackupButton->setValue("");
frogpilotBackupButton->setVisibleButton(0, true);
frogpilotBackupButton->setVisibleButton(2, true);
frogpilotBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
}
} else if (id == 2) {
if (ConfirmationDialog::confirm(tr("Delete all FrogPilot backups?"), tr("Delete All"), this)) {
std::thread([=]() mutable {
parent->keepScreenOn = true;
frogpilotBackupButton->setEnabled(false);
frogpilotBackupButton->setValue(tr("Deleting..."));
frogpilotBackupButton->setVisibleButton(0, false);
frogpilotBackupButton->setVisibleButton(1, false);
frogpilotBackupButton->setVisibleButton(3, false);
backupDir.removeRecursively();
backupDir.mkpath(".");
frogpilotBackupButton->setValue(tr("Deleted!"));
util::sleep_for(2500);
frogpilotBackupButton->setEnabled(true);
frogpilotBackupButton->setValue("");
frogpilotBackupButton->setVisibleButton(0, true);
frogpilotBackupButton->setVisibleButton(1, true);
frogpilotBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
} else if (id == 3) {
QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to restore"), friendlyNames, "", this);
if (!selection.isEmpty()) {
if (FrogPilotConfirmationDialog::yesorno(tr("Restore this backup? This will overwrite your current installation and reboot the device."), this)) {
std::thread([=]() {
parent->keepScreenOn = true;
frogpilotBackupButton->setEnabled(false);
frogpilotBackupButton->setValue(tr("Restoring..."));
frogpilotBackupButton->setVisibleButton(0, false);
frogpilotBackupButton->setVisibleButton(1, false);
frogpilotBackupButton->setVisibleButton(2, false);
std::system(QString("rm -rf /data/openpilot/* && tar --use-compress-program=zstd -xf %1 -C /").arg(backupDir.absoluteFilePath(backupMap[selection])).toStdString().c_str());
QFile("/cache/on_backup").open(QIODevice::WriteOnly);
frogpilotBackupButton->setValue(tr("Restored!"));
util::sleep_for(2500);
frogpilotBackupButton->setValue(tr("Rebooting..."));
util::sleep_for(2500);
Hardware::reboot();
}).detach();
}
}
}
});
if (forceOpenDescriptions) {
frogpilotBackupButton->showDescription();
}
dataMainList->addItem(frogpilotBackupButton);
FrogPilotButtonsControl *toggleBackupButton = new FrogPilotButtonsControl(tr("Toggle Backups"), tr("<b>Create, delete, or restore toggle backups.</b>"), "", {tr("BACKUP"), tr("DELETE"), tr("DELETE ALL"), tr("RESTORE")});
QObject::connect(toggleBackupButton, &FrogPilotButtonsControl::buttonClicked, [=](int id) {
QDir backupDir("/data/toggle_backups");
QStringList backupNames = backupDir.entryList(QDir::Dirs | QDir::NoDotAndDotDot);
std::sort(backupNames.begin(), backupNames.end(), std::greater<QString>());
QMap<QString, QString> backupMap;
for (const QString &dirName : backupNames) {
if (dirName.contains("in_progress")) {
continue;
}
QString friendlyName = dirName;
if (dirName.endsWith("_auto")) {
QStringList parts = QString(dirName).remove("_auto").split("_");
if (parts.size() >= 2) {
QDate date = QDate::fromString(parts[0], "yyyy-MM-dd");
QTime time = QTime::fromString(parts[1], "HH-mm-ss");
if (date.isValid() && time.isValid()) {
int day = date.day();
QString suffix = (day >= 11 && day <= 13) ? "th" :
(day % 10 == 1) ? "st" :
(day % 10 == 2) ? "nd" :
(day % 10 == 3) ? "rd" : "th";
friendlyName = QString("%1 %2%3, %4 (%5)")
.arg(date.toString("MMMM"))
.arg(day)
.arg(suffix)
.arg(date.year())
.arg(time.toString("h:mm AP"));
}
}
}
if (friendlyName == dirName) {
friendlyName.replace("_", " ");
}
backupMap[friendlyName] = dirName;
}
if (id == 0) {
QString name = InputDialog::getText(tr("Name your backup"), this, tr("Backup Name")).trimmed().replace(" ", "_");
if (!name.isEmpty()) {
if (backupNames.contains(name)) {
ConfirmationDialog::alert(tr("Name already in use. Please choose a different name!"), this);
return;
}
std::thread([=]() {
parent->keepScreenOn = true;
toggleBackupButton->setEnabled(false);
toggleBackupButton->setValue(tr("Backing up..."));
toggleBackupButton->setVisibleButton(1, false);
toggleBackupButton->setVisibleButton(2, false);
toggleBackupButton->setVisibleButton(3, false);
std::system(QString("cp -r /data/params/d/ %1").arg(backupDir.absoluteFilePath(name)).toStdString().c_str());
toggleBackupButton->setValue(tr("Backup created!"));
util::sleep_for(2500);
toggleBackupButton->setEnabled(true);
toggleBackupButton->setValue("");
toggleBackupButton->setVisibleButton(1, true);
toggleBackupButton->setVisibleButton(2, true);
toggleBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
} else if (id == 1) {
QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to delete"), backupMap.keys(), "", this);
if (!selection.isEmpty()) {
if (ConfirmationDialog::confirm(tr("Delete this backup?"), tr("Delete"), this)) {
std::thread([=]() {
parent->keepScreenOn = true;
toggleBackupButton->setEnabled(false);
toggleBackupButton->setValue(tr("Deleting..."));
toggleBackupButton->setVisibleButton(0, false);
toggleBackupButton->setVisibleButton(2, false);
toggleBackupButton->setVisibleButton(3, false);
QDir(backupDir.absoluteFilePath(backupMap[selection])).removeRecursively();
toggleBackupButton->setValue(tr("Deleted!"));
util::sleep_for(2500);
toggleBackupButton->setEnabled(true);
toggleBackupButton->setValue("");
toggleBackupButton->setVisibleButton(0, true);
toggleBackupButton->setVisibleButton(2, true);
toggleBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
}
} else if (id == 2) {
if (ConfirmationDialog::confirm(tr("Delete all toggle backups?"), tr("Delete All"), this)) {
std::thread([=]() mutable {
parent->keepScreenOn = true;
toggleBackupButton->setEnabled(false);
toggleBackupButton->setValue(tr("Deleting..."));
toggleBackupButton->setVisibleButton(0, false);
toggleBackupButton->setVisibleButton(1, false);
toggleBackupButton->setVisibleButton(3, false);
backupDir.removeRecursively();
backupDir.mkpath(".");
toggleBackupButton->setValue(tr("Deleted!"));
util::sleep_for(2500);
toggleBackupButton->setEnabled(true);
toggleBackupButton->setValue("");
toggleBackupButton->setVisibleButton(0, true);
toggleBackupButton->setVisibleButton(1, true);
toggleBackupButton->setVisibleButton(3, true);
parent->keepScreenOn = false;
}).detach();
}
} else if (id == 3) {
QString selection = MultiOptionDialog::getSelection(tr("Choose a backup to restore"), backupMap.keys(), "", this);
if (!selection.isEmpty()) {
if (FrogPilotConfirmationDialog::yesorno(tr("Restore this backup? This will overwrite your current settings!"), this)) {
std::thread([=]() {
parent->keepScreenOn = true;
toggleBackupButton->setEnabled(false);
toggleBackupButton->setValue(tr("Restoring..."));
toggleBackupButton->setVisibleButton(0, false);
toggleBackupButton->setVisibleButton(1, false);
toggleBackupButton->setVisibleButton(2, false);
std::system(QString("cp -r %1/* /data/params/d/").arg(backupDir.absoluteFilePath(backupMap[selection])).toStdString().c_str());
updateFrogPilotToggles();
toggleBackupButton->setValue(tr("Restored!"));
util::sleep_for(2500);
toggleBackupButton->setEnabled(true);
toggleBackupButton->setValue("");
toggleBackupButton->setVisibleButton(0, true);
toggleBackupButton->setVisibleButton(1, true);
toggleBackupButton->setVisibleButton(2, true);
parent->keepScreenOn = false;
}).detach();
}
}
}
});
if (forceOpenDescriptions) {
toggleBackupButton->showDescription();
}
dataMainList->addItem(toggleBackupButton);
FrogPilotButtonsControl *viewStatsButton = new FrogPilotButtonsControl(tr("FrogPilot Stats"), tr("<b>View your collected FrogPilot stats.</b>"), "", {tr("RESET"), tr("VIEW")});
QObject::connect(viewStatsButton, &FrogPilotButtonsControl::buttonClicked, [dataLayout, statsLabelsPanel, this](int id) {
if (id == 0) {
+8 -4
View File
@@ -46,9 +46,13 @@ def manager_init() -> None:
# set unset params to their default value
for k in params.all_keys():
default_value = params.get_default_value(k)
if default_value is not None and params.get(k) is None:
params.put(k, default_value)
current_value = params.get(k)
if current_value is None:
cached_value = params_cache.get(k)
if cached_value is not None:
params.put(k, cached_value)
else:
params_cache.put(k, current_value)
# Create folders needed for msgq
try:
@@ -101,7 +105,7 @@ def manager_init() -> None:
# FrogPilot variables
install_frogpilot(params)
frogpilot_boot_functions(params)
frogpilot_boot_functions(build_metadata, params)
def manager_cleanup() -> None:
+4 -1
View File
@@ -22,7 +22,7 @@ from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
from openpilot.system.hardware import AGNOS, HARDWARE
from openpilot.system.version import get_build_metadata
from openpilot.frogpilot.common.frogpilot_variables import get_frogpilot_toggles
from openpilot.frogpilot.common.frogpilot_variables import BACKUP_PATH, get_frogpilot_toggles
LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
@@ -414,6 +414,9 @@ class Updater:
cloudlog.info("finalize success!")
# FrogPilot variables
if os.path.isfile(BACKUP_PATH):
os.remove(BACKUP_PATH)
self.params.put("Updated", datetime.datetime.now().astimezone(ZoneInfo("America/Phoenix")).strftime("%B %d, %Y - %I:%M%p"))
def main() -> None: