add hash check and validation for model download

This commit is contained in:
DevTekVE
2024-02-12 12:07:42 +00:00
parent d6abb30453
commit ecfb60feff
5 changed files with 235 additions and 56 deletions
@@ -0,0 +1,28 @@
import requests
import hashlib
import json
# Function to calculate SHA-256 hash of a file downloaded from a given URI
def calculate_hash(download_uri):
response = requests.get(download_uri, stream=True)
hash_sha256 = hashlib.sha256()
for chunk in response.iter_content(chunk_size=1024):
hash_sha256.update(chunk)
return hash_sha256.hexdigest()
# Download the models_v3.json file
models_v3_url = "https://docs.sunnypilot.ai/models_v3.json"
response = requests.get(models_v3_url)
models_data = response.json()
# Iterate through each model and calculate SHA-256 for its files
for model_key, model_info in models_data.items():
# Calculate and insert the SHA-256 hash for each download URI
model_info['download_uri'] = {'url': model_info['download_uri'], 'sha256': calculate_hash(model_info['download_uri'])}
model_info['download_uri_nav'] = {'url': model_info['download_uri_nav'], 'sha256': calculate_hash(model_info['download_uri_nav'])}
model_info['download_uri_metadata'] = {'url': model_info['download_uri_metadata'], 'sha256': calculate_hash(model_info['download_uri_metadata'])}
# Save the updated JSON with hashes
updated_json_file_path = 'updated_models_v3.json' # Path for the updated JSON file
with open(updated_json_file_path, 'w') as file:
json.dump(models_data, file, indent=4)
@@ -4,15 +4,51 @@ ModelsFetcher::ModelsFetcher(QObject* parent) : QObject(parent) {
manager = new QNetworkAccessManager(this);
}
void ModelsFetcher::download(const QUrl& url, const QString& filename, const QString& destinationPath) {
if (!QDir(destinationPath).exists() && !QDir().mkpath(destinationPath)) {
LOGE("Failed to create directory: [%s]", destinationPath.toStdString().c_str());
QByteArray ModelsFetcher::verifyFileHash(const QString& filePath, const QString& expectedHash, bool& hashMatches) {
hashMatches = false; // Default to false
QByteArray fileData;
if (expectedHash.isEmpty()) {
// If no hash is provided, assume verification isn't required but return the file data
hashMatches = true;
} else {
QFile file(filePath);
if (file.open(QIODevice::ReadOnly)) {
QCryptographicHash hash(QCryptographicHash::Sha256); // Or your chosen algorithm
fileData = file.readAll(); // Read the file data once
hash.addData(fileData);
file.close();
QString currentHash = QString(hash.result().toHex());
hashMatches = (currentHash == expectedHash);
}
}
const QNetworkRequest request(url);
// Return the file data if hash matches or no hash was provided; empty otherwise
return hashMatches ? fileData : QByteArray();
}
void ModelsFetcher::download(const DownloadInfo& downloadInfo, const QString& filename, const QString& destinationPath) {
QString fullPath = destinationPath + "/" + filename;
QFileInfo fileInfo(fullPath);
bool hashMatches = false;
QByteArray data = verifyFileHash(fullPath, downloadInfo.sha256, hashMatches);
if (fileInfo.exists() && hashMatches) {
// Hash matches or no hash provided, and we have the file data
LOGD("File already downloaded and verified: %s", filename.toStdString().c_str());
emit downloadProgress(100);
emit downloadComplete(data, true); // Use the data returned from verifyFileHash
return; // Exit early
}
// Proceed with download if file does not exist or hash verification failed
QNetworkRequest request(downloadInfo.url);
QNetworkReply* reply = manager->get(request);
connect(reply, &QNetworkReply::downloadProgress, this, &ModelsFetcher::onDownloadProgress);
connect(reply, &QNetworkReply::finished, this, [this, reply, destinationPath, filename]() {
onFinished(reply, destinationPath, filename);
connect(reply, &QNetworkReply::finished, this, [this, reply, destinationPath, filename, downloadInfo]() {
onFinished(reply, destinationPath, filename, downloadInfo.sha256);
});
}
@@ -31,10 +67,10 @@ QString extractFileName(const QString& contentDisposition) {
return filename;
}
void ModelsFetcher::onFinished(QNetworkReply* reply, const QString& destinationPath, const QString& filename) {
void ModelsFetcher::onFinished(QNetworkReply* reply, const QString& destinationPath, const QString& filename, const QString& expectedHash) {
// Handle download error
if (reply->error()) {
return;
return; // Possibly emit a signal or log an error as per your error handling policy
}
const QByteArray data = reply->readAll();
@@ -46,16 +82,30 @@ void ModelsFetcher::onFinished(QNetworkReply* reply, const QString& destinationP
QString finalPath = QDir(destinationPath).filePath(finalFilename);
// handle file open error
// Save the downloaded file
QFile file(finalPath);
if (!file.open(QIODevice::WriteOnly)) {
return;
return; // Consider emitting a signal or logging an error here as well
}
file.write(data);
file.close();
emit downloadComplete(data);
bool hashMatches = false;
verifyFileHash(finalPath, expectedHash, hashMatches);
// Verify the file hash if expectedHash is provided
if (!expectedHash.isEmpty() && !hashMatches) {
LOGE("The downloaded file didn't pass the hash validation!: %s", filename.toStdString().c_str());
// Hash verification failed, handle accordingly
// This could involve deleting the file, logging an error, or emitting a failure signal
QFile::remove(finalPath); // Example action: Remove the invalid file
emit downloadFailed(filename);
return; // Stop further processing
}
emit downloadComplete(data, false); // Emit your success signal
}
void ModelsFetcher::onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal) {
@@ -17,38 +17,71 @@
static const QString MODELS_PATH = Hardware::PC() ? QDir::homePath() + "/.comma/media/0/models/" : "/data/media/0/models/";
struct DownloadInfo {
QString url;
QString sha256;
};
// New class ModelsFetcher with a new function that handles web requests and JSON parsing for the new JSON structure
class Model {
public:
explicit Model(const QJsonObject&json) {
explicit Model(const QJsonObject &json) {
displayName = json["display_name"].toString();
fullName = json["full_name"].toString();
fileName = json["file_name"].toString();
downloadUri = json["download_uri"].toString();
// Parse downloadUri as an object
QJsonObject downloadUriObj = json["download_uri"].toObject();
downloadUri.url = downloadUriObj["url"].toString();
downloadUri.sha256 = downloadUriObj["sha256"].toString();
fullNameNav = json["full_name_nav"].toString();
fileNameNav = json["file_name_nav"].toString();
downloadUriNav = json["download_uri_nav"].toString();
// Parse downloadUriNav as an object
QJsonObject downloadUriNavObj = json["download_uri_nav"].toObject();
downloadUriNav.url = downloadUriNavObj["url"].toString();
downloadUriNav.sha256 = downloadUriNavObj["sha256"].toString();
fullNameMetadata = json["full_name_metadata"].toString();
fileNameMetadata = json["file_name_metadata"].toString();
downloadUriMetadata = json["download_uri_metadata"].toString();
// Parse downloadUriMetadata as an object
QJsonObject downloadUriMetadataObj = json["download_uri_metadata"].toObject();
downloadUriMetadata.url = downloadUriMetadataObj["url"].toString();
downloadUriMetadata.sha256 = downloadUriMetadataObj["sha256"].toString();
index = json["index"].toString();
environment = json["environment"].toString();
generation = json["generation"].toString();
}
// Method to convert model back to QJsonObject, if needed
QJsonObject toJson() const {
QJsonObject json;
json["display_name"] = displayName;
json["full_name"] = fullName;
json["file_name"] = fileName;
json["download_uri"] = downloadUri;
QJsonObject uriObj;
uriObj["url"] = downloadUri.url;
uriObj["sha256"] = downloadUri.sha256;
json["download_uri"] = uriObj;
QJsonObject uriNavObj;
uriNavObj["url"] = downloadUriNav.url;
uriNavObj["sha256"] = downloadUriNav.sha256;
json["download_uri_nav"] = uriNavObj;
QJsonObject uriMetadataObj;
uriMetadataObj["url"] = downloadUriMetadata.url;
uriMetadataObj["sha256"] = downloadUriMetadata.sha256;
json["download_uri_metadata"] = uriMetadataObj;
json["full_name_nav"] = fullNameNav;
json["file_name_nav"] = fileNameNav;
json["download_uri_nav"] = downloadUriNav;
json["full_name_metadata"] = fullNameMetadata;
json["file_name_metadata"] = fileNameMetadata;
json["download_uri_metadata"] = downloadUriMetadata;
json["index"] = index;
json["environment"] = environment;
json["generation"] = generation;
@@ -58,16 +91,17 @@ public:
QString displayName;
QString fullName;
QString fileName;
QString downloadUri;
DownloadInfo downloadUri;
DownloadInfo downloadUriNav;
DownloadInfo downloadUriMetadata;
QString fullNameNav;
QString fileNameNav;
QString downloadUriNav;
QString fullNameMetadata;
QString fileNameMetadata;
QString index;
QString environment;
QString generation;
QString fullNameMetadata;
QString fileNameMetadata;
QString downloadUriMetadata;
};
class ModelsFetcher : public QObject {
@@ -75,18 +109,21 @@ class ModelsFetcher : public QObject {
public:
explicit ModelsFetcher(QObject* parent = nullptr);
void download(const QUrl&url, const QString& filename = "", const QString&destinationPath = MODELS_PATH);
void download(const DownloadInfo&url, const QString& filename = "", const QString&destinationPath = MODELS_PATH);
static std::vector<Model> getModelsFromURL(const QUrl&url);
static std::vector<Model> getModelsFromURL(const QString&url);
static std::vector<Model> getModelsFromURL();
signals:
void downloadProgress(double percentage);
void downloadComplete(const QByteArray&data);
void downloadComplete(const QByteArray&data, bool fromCache = false);
void downloadFailed(const QString &filename);
private:
// static bool verifyFileHash(const QString& filePath, const QString& expectedHash);
static QByteArray verifyFileHash(const QString& filePath, const QString& expectedHash, bool& hashMatches);
void onDownloadProgress(qint64 bytesReceived, qint64 bytesTotal);
void onFinished(QNetworkReply* reply, const QString&destinationPath, const QString&filename);
void onFinished(QNetworkReply* reply, const QString&destinationPath, const QString&filename, const QString& expectedHash);
QNetworkAccessManager* manager;
};
@@ -1,6 +1,6 @@
#include "selfdrive/ui/qt/offroad/sunnypilot/software_settings_sp.h"
SoftwarePanelSP::SoftwarePanelSP(QWidget* parent) : SoftwarePanel(parent) {
SoftwarePanelSP::SoftwarePanelSP(QWidget *parent) : SoftwarePanel(parent) {
// Get current model name and create new ButtonControl
const auto current_model = GetModelName();
currentModelLblBtn = new ButtonControl(tr("Driving Model"), tr("SELECT"), current_model);
@@ -9,14 +9,34 @@ SoftwarePanelSP::SoftwarePanelSP(QWidget* parent) : SoftwarePanel(parent) {
connect(&models_fetcher, &ModelsFetcher::downloadProgress, this, [this](const double progress) {
handleDownloadProgress(progress, "driving");
});
connect(&nav_models_fetcher, &ModelsFetcher::downloadProgress, this, [this](const double progress) {
handleDownloadProgress(progress, "navigation");
handleDownloadProgress(progress, "navigation");
});
connect(&metadata_fetcher, &ModelsFetcher::downloadProgress, this, [this](const double progress) {
handleDownloadProgress(progress, "metadata");
});
connect(&models_fetcher, &ModelsFetcher::downloadComplete, this, [this](const QByteArray&data, bool fromCache = false) {
modelFromCache = fromCache;
updateLabels();
});
connect(&nav_models_fetcher, &ModelsFetcher::downloadComplete, this, [this](const QByteArray&data, bool fromCache = false) {
navModelFromCache = fromCache;
updateLabels();
});
connect(&metadata_fetcher, &ModelsFetcher::downloadComplete, this, [this](const QByteArray&data, bool fromCache = false) {
metadataFromCache = fromCache;
updateLabels();
});
connect(&models_fetcher, &ModelsFetcher::downloadFailed, this, &SoftwarePanelSP::handleDownloadFailed);
connect(&nav_models_fetcher, &ModelsFetcher::downloadFailed, this, &SoftwarePanelSP::handleDownloadFailed);
connect(&metadata_fetcher, &ModelsFetcher::downloadFailed, this, &SoftwarePanelSP::handleDownloadFailed);
// Connect click event from currentModelLblBtn to local slot
connect(currentModelLblBtn, &ButtonControl::clicked, this, &SoftwarePanelSP::handleCurrentModelLblBtnClicked);
@@ -24,6 +44,13 @@ SoftwarePanelSP::SoftwarePanelSP(QWidget* parent) : SoftwarePanel(parent) {
ReplaceOrAddWidget(currentModelLbl, currentModelLblBtn);
}
void SoftwarePanelSP::handleDownloadFailed(const QString &modelType) {
model_download_failed = true;
LOGE("Download failed reported for [%s]", modelType.toStdString().c_str());
failed_downloads_description += "["+ modelType +"]\n";
HandleModelDownloadProgressReport(); // Call to update UI based on the new status
}
void SoftwarePanelSP::handleDownloadProgress(const double progress, const QString &modelType) {
if (modelType == "driving") {
modelDownloadProgress = progress;
@@ -32,6 +59,7 @@ void SoftwarePanelSP::handleDownloadProgress(const double progress, const QStrin
} else if (modelType == "metadata") {
metadataDownloadProgress = progress;
}
HandleModelDownloadProgressReport();
}
@@ -51,7 +79,7 @@ QString SoftwarePanelSP::GetNavModelName() {
if (selectedNavModelToDownload.has_value()) {
return selectedNavModelToDownload->fullNameNav;
}
return QString::fromStdString(params.get("NavModelText"));
}
@@ -78,52 +106,71 @@ void SoftwarePanelSP::HandleModelDownloadProgressReport() {
// Driving model status
if (isDownloadingModel()) {
description += downloadingTemplate.arg("Driving", drivingModelName, QString::number(modelDownloadProgress, 'f', 2));
description += downloadingTemplate.arg("Driving", drivingModelName, QString::number(modelDownloadProgress.value_or(0.0), 'f', 2));
} else {
if (modelFromCache) drivingModelName += " (CACHED)";
description += downloadedTemplate.arg("Driving", drivingModelName);
}
// Navigation model status
if (isDownloadingNavModel()) {
if (!description.isEmpty()) description += "\n"; // Add newline if driving model status is already appended
description += downloadingTemplate.arg("Navigation", navModelName, QString::number(navModelDownloadProgress, 'f', 2));
description += downloadingTemplate.arg("Navigation", navModelName,
QString::number(navModelDownloadProgress.value_or(0.0), 'f', 2));
} else {
if (navModelFromCache) navModelName += " (CACHED)";
if (!description.isEmpty()) description += "\n"; // Ensure newline separation
description += downloadedTemplate.arg("Navigation", navModelName);
}
if (isDownloadingMetadata()) {
if (!description.isEmpty()) description += "\n";
description += downloadingTemplate.arg("Metadata", metadataName, QString::number(metadataDownloadProgress, 'f', 2));
description += downloadingTemplate.arg("Metadata", metadataName, QString::number(metadataDownloadProgress.value_or(0.0), 'f', 2));
} else {
if (metadataFromCache) metadataName += " (CACHED)";
if (!description.isEmpty()) description += "\n";
description += downloadedTemplate.arg("Metadata", metadataName);
}
if (model_download_failed) {
description = "Downloads have failed, please try swapping the model!\n"
"Failed:\n" + failed_downloads_description;
LOGE("MODEL DOWNLOADS FAILED!!!");
}
currentModelLblBtn->setDescription(description);
currentModelLblBtn->showDescription();
currentModelLblBtn->setEnabled(!(is_onroad || isDownloadingModel()));
currentModelLblBtn->setEnabled(
!(is_onroad || (isDownloadingModel() || isDownloadingMetadata() || isDownloadingNavModel())));
// If not downloading and there is a selected model, update parameters
if (!isDownloadingNavModel() && !isDownloadingModel() && selectedModelToDownload.has_value()) {
if (!isDownloadingModel() && modelDownloadProgress.has_value()) {
params.put("DrivingModelText", selectedModelToDownload->fullName.toStdString());
params.put("DrivingModelName", selectedModelToDownload->displayName.toStdString());
//params.put("DrivingModelUrl", selectedModelToDownload->downloadUri.toStdString()); // TODO: Placeholder for future implementation
LOGD("Resetting selectedModelToDownload");
selectedModelToDownload.reset();
params.putBool("CustomDrivingModel", true);
modelDownloadProgress.reset();
modelFromCache = false;
params.putBool("CustomDrivingModel", !model_download_failed);
}
// If not downloading and there is a selected model, update parameters
if (!isDownloadingNavModel() && selectedNavModelToDownload.has_value()) {
params.put("DrivingModelGeneration", selectedNavModelToDownload->generation.toStdString());
params.put("NavModelText", selectedNavModelToDownload->fullNameNav.toStdString());
selectedNavModelToDownload.reset();
if (!isDownloadingNavModel() && navModelDownloadProgress.has_value()) {
params.put("DrivingModelGeneration", selectedNavModelToDownload->generation.toStdString());
params.put("NavModelText", selectedNavModelToDownload->fullNameNav.toStdString());
LOGD("Resetting selectedNavModelToDownload");
selectedNavModelToDownload.reset();
navModelDownloadProgress.reset();
navModelFromCache = false;
}
if (!isDownloadingModel() && !isDownloadingNavModel() && !isDownloadingMetadata() && selectedMetadataToDownload.has_value()) {
if (!isDownloadingMetadata() && metadataDownloadProgress.has_value()) {
params.put("ModelMetadataText", selectedMetadataToDownload->fullNameMetadata.toStdString());
LOGD("Resetting selectedMetadataToDownload");
selectedMetadataToDownload.reset();
metadataDownloadProgress.reset();
metadataFromCache = false;
}
}
@@ -141,7 +188,7 @@ void SoftwarePanelSP::handleCurrentModelLblBtnClicked() {
QMap<QString, QString> index_to_model;
// Collecting indices with display names
for (const auto &model : models) {
for (const auto &model: models) {
if ((is_release_sp && model.environment == "release") || !is_release_sp) {
index_to_model.insert(model.index, model.displayName);
}
@@ -153,14 +200,15 @@ void SoftwarePanelSP::handleCurrentModelLblBtnClicked() {
return index1.toInt() > index2.toInt();
});
for (const QString &index : indices) {
for (const QString &index: indices) {
modelNames.push_back(index_to_model[index]);
}
currentModelLblBtn->setEnabled(!is_onroad);
currentModelLblBtn->setValue(GetModelName());
const QString selectedModelName = MultiOptionDialog::getSelection(tr("Select a Driving Model"), modelNames, currentModelName, this);
const QString selectedModelName = MultiOptionDialog::getSelection(tr("Select a Driving Model"), modelNames,
currentModelName, this);
// Bail if no selected model or the user doesn't want to continue while on metered
if (selectedModelName.isEmpty() || !canContinueOnMeteredDialog()) {
@@ -180,16 +228,19 @@ void SoftwarePanelSP::handleCurrentModelLblBtnClicked() {
// If decision is to download and there is a selected model, update UI and begin downloading
if (selectedModelToDownload.has_value()) {
model_download_failed = false;
currentModelLblBtn->setValue(selectedModelToDownload->displayName);
currentModelLblBtn->setDescription(selectedModelToDownload->displayName);
models_fetcher.download(selectedModelToDownload->downloadUri, selectedModelToDownload->fileName);
nav_models_fetcher.download(selectedNavModelToDownload->downloadUriNav, selectedNavModelToDownload->fileNameNav);
metadata_fetcher.download(selectedMetadataToDownload->downloadUriMetadata, selectedMetadataToDownload->fileNameMetadata);
metadata_fetcher.download(selectedMetadataToDownload->downloadUriMetadata,
selectedMetadataToDownload->fileNameMetadata);
// Disable select button until download completes
currentModelLblBtn->setEnabled(false);
showResetParamsDialog();
showResetParamsDialog();
}
updateLabels();
}
void SoftwarePanelSP::checkNetwork() {
@@ -204,14 +255,18 @@ void SoftwarePanelSP::updateLabels() {
if (!isVisible()) {
return;
}
if(!model_download_failed)
failed_downloads_description = "";
checkNetwork();
currentModelLblBtn->setEnabled(!is_onroad);
currentModelLblBtn->setEnabled(!(is_onroad || (isDownloadingModel() || isDownloadingMetadata() || isDownloadingNavModel())));
SoftwarePanel::updateLabels();
}
void SoftwarePanelSP::showResetParamsDialog() {
const auto confirmMsg = tr("Download has started in the background.\nWe STRONGLY suggest you to reset calibration, would you like to do that now?");
const auto confirmMsg = tr(
"Download has started in the background.\nWe STRONGLY suggest you to reset calibration, would you like to do that now?");
const auto button_text = tr("Reset Calibration");
// If user confirms, remove specified parameters
@@ -19,15 +19,18 @@ private:
void checkNetwork();
bool isDownloadingModel() const {
return selectedModelToDownload.has_value() && modelDownloadProgress > 0.0 && modelDownloadProgress < 100.0;
LOGD("isDownloadingModel: selectedModelToDownload.has_value() [%s] && modelDownloadProgress [%f]",selectedModelToDownload.has_value() ?"true": "false", modelDownloadProgress.value_or(0.0));
return selectedModelToDownload.has_value() && modelDownloadProgress.value_or(0.0) > 0.0 && modelDownloadProgress.value_or(0.0) < 100.0;
}
bool isDownloadingNavModel() const {
return selectedNavModelToDownload.has_value() && navModelDownloadProgress > 0.0 && navModelDownloadProgress < 100.0;
LOGD("isDownloadingNavModel: selectedNavModelToDownload.has_value() [%s] && navModelDownloadProgress [%f]",selectedNavModelToDownload.has_value() ?"true": "false", navModelDownloadProgress.value_or(0.0));
return selectedNavModelToDownload.has_value() && navModelDownloadProgress.value_or(0.0) > 0.0 && navModelDownloadProgress.value_or(0.0) < 100.0;
}
bool isDownloadingMetadata() const {
return selectedMetadataToDownload.has_value() && metadataDownloadProgress > 0.0 && metadataDownloadProgress < 100.0;
LOGD("isDownloadingMetadata: selectedMetadataToDownload.has_value() [%s] && metadataDownloadProgress [%f]",selectedMetadataToDownload.has_value() ?"true": "false", metadataDownloadProgress.value_or(0.0));
return selectedMetadataToDownload.has_value() && metadataDownloadProgress.value_or(0.0) > 0.0 && metadataDownloadProgress.value_or(0.0) < 100.0;
}
// UI update related methods
@@ -36,6 +39,7 @@ private:
void HandleModelDownloadProgressReport();
void handleDownloadProgress(double progress, const QString&modelType);
void HandleNavModelDownloadProgressReport();
void handleDownloadFailed(const QString &modelType);
void showResetParamsDialog();
bool canContinueOnMeteredDialog() {
if (!is_metered) return true;
@@ -56,9 +60,12 @@ private:
bool is_metered{};
bool is_wifi{};
double modelDownloadProgress = 0.0;
double navModelDownloadProgress = 0.0;
double metadataDownloadProgress = 0.0;
bool modelFromCache;
bool navModelFromCache;
bool metadataFromCache;
std::optional<double> modelDownloadProgress;
std::optional<double> navModelDownloadProgress;
std::optional<double> metadataDownloadProgress;
std::optional<Model> selectedModelToDownload;
std::optional<Model> selectedNavModelToDownload;
std::optional<Model> selectedMetadataToDownload;
@@ -66,4 +73,6 @@ private:
ModelsFetcher models_fetcher;
ModelsFetcher nav_models_fetcher;
ModelsFetcher metadata_fetcher;
bool model_download_failed;
QString failed_downloads_description = "";
};