StarPilot
This commit is contained in:
firestar5683
Executable
+121
@@ -0,0 +1,121 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
derive_feasible_params.py
|
||||
|
||||
Dynamically parses the OpenPilot/StarPilot codebase to cross-reference logically
|
||||
registered Param keys with UI string literals. This ensures that no hidden or
|
||||
dynamically-instantiated UI toggles are missed, outputting a highly accurate "Golden List"
|
||||
of parameters that can be safely modified by The Pond or other configuration interfaces.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
def get_repo_root() -> str:
|
||||
# Resolves to the root of the StarPilot repository based on this script's location
|
||||
return os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
|
||||
|
||||
# Constants
|
||||
REPO_ROOT = get_repo_root()
|
||||
PARAMS_CC_PATH = os.path.join(REPO_ROOT, 'common/params.cc')
|
||||
UI_DIRECTORIES = [
|
||||
os.path.join(REPO_ROOT, 'selfdrive/ui'),
|
||||
os.path.join(REPO_ROOT, 'frogpilot/ui')
|
||||
]
|
||||
|
||||
# A curated list of parameters that are known to be strictly readable state metadata
|
||||
# rather than user-toggled configurations.
|
||||
KNOWN_READ_ONLY = {
|
||||
"ApiCache_Device", "ApiCache_DriveStats", "ApiCache_NavDestinations",
|
||||
"CarMake", "CarModel", "CarModelName", "CarParamsPersistent", "CarVin",
|
||||
"ClusterOffset", "Compass", "DeveloperSidebarMetric1", "DeveloperSidebarMetric2",
|
||||
"DeveloperSidebarMetric3", "DeveloperSidebarMetric4", "DeveloperSidebarMetric5",
|
||||
"DeveloperSidebarMetric6", "DeveloperSidebarMetric7", "DongleId",
|
||||
"FrogPilotCarParamsPersistent", "FrogPilotDrives", "FrogPilotKilometers",
|
||||
"FrogPilotMinutes", "GitBranch", "GitCommit", "GitCommitDate", "GitDiff",
|
||||
"GitRemote", "GithubSshKeys", "GithubUsername", "HardwareSerial", "IMEI",
|
||||
"InstallDate", "IsRhdDetected", "KonikMinutes", "LastGPSPosition",
|
||||
"LastMapsUpdate", "LastUpdateTime", "ModelDrivesAndScores", "ModelReleasedDates",
|
||||
"ModelVersions", "PrimeType", "TermsVersion", "TrainingVersion", "Version",
|
||||
"openpilotMinutes", "CompletedTrainingVersion"
|
||||
}
|
||||
|
||||
def extract_registered_keys(params_path: str) -> set:
|
||||
"""Extracts all legally registered parameter keys from common/params.cc"""
|
||||
registered_keys = set()
|
||||
try:
|
||||
with open(params_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
# Isolate the keys `unordered_map` block
|
||||
keys_block_match = re.search(r'unordered_map<std::string, uint32_t> keys = \{(.*?)\};', content, re.DOTALL)
|
||||
if not keys_block_match:
|
||||
print("Error: Could not locate 'keys' map in params.cc")
|
||||
return registered_keys
|
||||
|
||||
# Extract {"KeyName", FLAG} entries
|
||||
for match in re.finditer(r'\{"([A-Za-z0-9_]+)",\s*([^}]+)\}', keys_block_match.group(1)):
|
||||
key, flag = match.group(1), match.group(2)
|
||||
# Remove keys that are strictly internal ephemeral states
|
||||
if 'CLEAR_ON_MANAGER_START' not in flag:
|
||||
registered_keys.add(key)
|
||||
|
||||
except FileNotFoundError:
|
||||
print(f"Error: Could not find params source file at {params_path}")
|
||||
|
||||
return registered_keys
|
||||
|
||||
def extract_ui_string_literals(ui_dirs: list) -> set:
|
||||
"""Recursively walks UI directories to extract every string literal."""
|
||||
ui_strings = set()
|
||||
valid_extensions = ('.cc', '.h', '.cpp', '.hpp', '.qml')
|
||||
|
||||
for directory in ui_dirs:
|
||||
if not os.path.exists(directory):
|
||||
print(f"Warning: UI directory not found {directory}")
|
||||
continue
|
||||
|
||||
for root, _, files in os.walk(directory):
|
||||
for file in files:
|
||||
if file.endswith(valid_extensions):
|
||||
filepath = os.path.join(root, file)
|
||||
with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
|
||||
# Extract all "StringLiterals" block
|
||||
matches = re.findall(r'\"([A-Za-z0-9_]+)\"', f.read())
|
||||
ui_strings.update(matches)
|
||||
|
||||
return ui_strings
|
||||
|
||||
def main():
|
||||
print(f"Starting parameter derivation inside {REPO_ROOT}...")
|
||||
|
||||
# 1. Fetch
|
||||
registered_keys = extract_registered_keys(PARAMS_CC_PATH)
|
||||
ui_strings = extract_ui_string_literals(UI_DIRECTORIES)
|
||||
|
||||
# 2. Intersect
|
||||
feasible_keys = registered_keys.intersection(ui_strings)
|
||||
|
||||
# 3. Filter Read-Only
|
||||
editable_keys = feasible_keys - KNOWN_READ_ONLY
|
||||
|
||||
# 4. Export
|
||||
output_path = os.path.join(os.path.dirname(__file__), 'feasibleparams.txt')
|
||||
try:
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
f.write("Dynamically Derived Feasible Param Candidates (The Golden List)\n")
|
||||
f.write("===============================================================\n\n")
|
||||
f.write(f"Total globally registered C++ keys: {len(registered_keys)}\n")
|
||||
f.write(f"Total explicit UI string references: {len(feasible_keys)}\n")
|
||||
f.write(f"Total Editable/Toggleable targets: {len(editable_keys)}\n\n")
|
||||
|
||||
for key in sorted(list(editable_keys)):
|
||||
f.write(f"{key}\n")
|
||||
|
||||
print(f"Successfully derived {len(editable_keys)} highly feasible parameter targets.")
|
||||
print(f"Report exported to: {output_path}")
|
||||
except Exception as e:
|
||||
print(f"Error writing to output file: {e}")
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,337 @@
|
||||
Dynamically Derived Feasible Param Candidates (The Golden List)
|
||||
===============================================================
|
||||
|
||||
Total globally registered C++ keys: 417
|
||||
Total explicit UI string references: 369
|
||||
Total Editable/Toggleable targets: 331
|
||||
|
||||
AMapKey1
|
||||
AMapKey2
|
||||
AccelerationPath
|
||||
AccelerationProfile
|
||||
AdjacentLeadsUI
|
||||
AdjacentPath
|
||||
AdjacentPathMetrics
|
||||
AdvancedCustomUI
|
||||
AdvancedLateralTune
|
||||
AdvancedLongitudinalTune
|
||||
AggressiveFollow
|
||||
AggressiveFollowHigh
|
||||
AggressiveJerkAcceleration
|
||||
AggressiveJerkDanger
|
||||
AggressiveJerkDeceleration
|
||||
AggressiveJerkSpeed
|
||||
AggressiveJerkSpeedDecrease
|
||||
AggressivePersonalityProfile
|
||||
AlertVolumeControl
|
||||
AlwaysOnLateral
|
||||
AlwaysOnLateralLKAS
|
||||
AlwaysOnLateralMain
|
||||
AutomaticUpdates
|
||||
AutomaticallyDownloadModels
|
||||
AvailableModelNames
|
||||
AvailableModelSeries
|
||||
AvailableModels
|
||||
BigMap
|
||||
BlacklistedModels
|
||||
BlindSpotMetrics
|
||||
BlindSpotPath
|
||||
BootLogo
|
||||
BorderMetrics
|
||||
CECurves
|
||||
CECurvesLead
|
||||
CELead
|
||||
CEModelStopTime
|
||||
CENavigation
|
||||
CENavigationIntersections
|
||||
CENavigationLead
|
||||
CENavigationTurns
|
||||
CESignalLaneDetection
|
||||
CESignalSpeed
|
||||
CESlowerLead
|
||||
CESpeed
|
||||
CESpeedLead
|
||||
CEStatus
|
||||
CEStoppedLead
|
||||
CalibratedLateralAcceleration
|
||||
CalibrationParams
|
||||
CalibrationProgress
|
||||
CameraView
|
||||
CommunityFavorites
|
||||
ConditionalExperimental
|
||||
CurvatureData
|
||||
CurveSpeedController
|
||||
CustomAlerts
|
||||
CustomColors
|
||||
CustomCruise
|
||||
CustomCruiseLong
|
||||
CustomDistanceIcons
|
||||
CustomIcons
|
||||
CustomPersonalities
|
||||
CustomSignals
|
||||
CustomSounds
|
||||
CustomUI
|
||||
DebugMode
|
||||
DecelerationProfile
|
||||
DeveloperMetrics
|
||||
DeveloperSidebar
|
||||
DeveloperUI
|
||||
DeveloperWidgets
|
||||
DeviceManagement
|
||||
DeviceShutdown
|
||||
DisableOnroadUploads
|
||||
DisableOpenpilotLongitudinal
|
||||
DiscordUsername
|
||||
DisengageOnAccelerator
|
||||
DisengageVolume
|
||||
DistanceButtonControl
|
||||
DoToggleReset
|
||||
DoToggleResetStock
|
||||
DownloadableBootLogos
|
||||
DownloadableColors
|
||||
DownloadableDistanceIcons
|
||||
DownloadableIcons
|
||||
DownloadableSignals
|
||||
DownloadableSounds
|
||||
DownloadableWheels
|
||||
DriverCamera
|
||||
DynamicPathWidth
|
||||
DynamicPedalsOnUI
|
||||
EVTuning
|
||||
EngageVolume
|
||||
ExperimentalLongitudinalEnabled
|
||||
ExperimentalMode
|
||||
ExperimentalModeConfirmed
|
||||
FPSCounter
|
||||
Fahrenheit
|
||||
FavoriteDestinations
|
||||
ForceAutoTune
|
||||
ForceAutoTuneOff
|
||||
ForceFingerprint
|
||||
ForceMPHDashboard
|
||||
ForceStops
|
||||
ForceTorqueController
|
||||
FrogPilotStats
|
||||
FrogsGoMoosTweak
|
||||
FullMap
|
||||
GMPedalLongitudinal
|
||||
GoatScream
|
||||
GreenLightAlert
|
||||
GsmApn
|
||||
GsmMetered
|
||||
GsmRoaming
|
||||
HasAcceptedTerms
|
||||
HideAlerts
|
||||
HideLeadMarker
|
||||
HideMap
|
||||
HideMapIcon
|
||||
HideMaxSpeed
|
||||
HideSpeed
|
||||
HideSpeedLimit
|
||||
HigherBitrate
|
||||
HolidayThemes
|
||||
HumanAcceleration
|
||||
HumanFollowing
|
||||
IncreaseThermalLimits
|
||||
IncreasedStoppedDistance
|
||||
IsDriverViewEnabled
|
||||
IsLdwEnabled
|
||||
IsMetric
|
||||
LKASButtonControl
|
||||
LaneChangeTime
|
||||
LaneChanges
|
||||
LaneDetectionWidth
|
||||
LaneLinesWidth
|
||||
LanguageSetting
|
||||
LateralTune
|
||||
LeadDepartingAlert
|
||||
LeadDetectionThreshold
|
||||
LeadInfo
|
||||
LiveDelay
|
||||
LiveParameters
|
||||
LiveTorqueParameters
|
||||
LockDoors
|
||||
LockDoorsTimer
|
||||
LongDistanceButtonControl
|
||||
LongPitch
|
||||
LongitudinalActuatorDelay
|
||||
LongitudinalActuatorDelayStock
|
||||
LongitudinalPersonality
|
||||
LongitudinalTune
|
||||
LoudBlindspotAlert
|
||||
LowVoltageShutdown
|
||||
MapAcceleration
|
||||
MapDeceleration
|
||||
MapGears
|
||||
MapStyle
|
||||
MapboxPublicKey
|
||||
MapboxSecretKey
|
||||
MapsSelected
|
||||
MaxDesiredAcceleration
|
||||
MinimumLaneChangeSpeed
|
||||
Model
|
||||
ModelRandomizer
|
||||
ModelUI
|
||||
ModelVersion
|
||||
NNFF
|
||||
NNFFLite
|
||||
NavPastDestinations
|
||||
NavSettingLeftSide
|
||||
NavSettingTime24h
|
||||
NavigationUI
|
||||
NewLongAPI
|
||||
NoLogging
|
||||
NoUploads
|
||||
NudgelessLaneChange
|
||||
NumericalTemp
|
||||
OSMDownloadLocations
|
||||
Offset1
|
||||
Offset2
|
||||
Offset3
|
||||
Offset4
|
||||
Offset5
|
||||
Offset6
|
||||
Offset7
|
||||
OneLaneChange
|
||||
OnroadDistanceButton
|
||||
OpenpilotEnabledToggle
|
||||
OverpassRequests
|
||||
PathEdgeWidth
|
||||
PathWidth
|
||||
PauseAOLOnBrake
|
||||
PauseLateralOnSignal
|
||||
PauseLateralSpeed
|
||||
PedalsOnUI
|
||||
PersonalizeOpenpilot
|
||||
PreferredSchedule
|
||||
PromptDistractedVolume
|
||||
PromptVolume
|
||||
QOLLateral
|
||||
QOLLongitudinal
|
||||
QOLVisuals
|
||||
RadarTracksUI
|
||||
RainbowPath
|
||||
RandomEvents
|
||||
RandomThemes
|
||||
RecordFront
|
||||
RecordFrontLock
|
||||
RecoveryPower
|
||||
RedPanda
|
||||
RefuseVolume
|
||||
RelaxedFollow
|
||||
RelaxedFollowHigh
|
||||
RelaxedJerkAcceleration
|
||||
RelaxedJerkDanger
|
||||
RelaxedJerkDeceleration
|
||||
RelaxedJerkSpeed
|
||||
RelaxedJerkSpeedDecrease
|
||||
RelaxedPersonalityProfile
|
||||
RemoteStartBootsComma
|
||||
ReverseCruise
|
||||
RoadEdgesWidth
|
||||
RoadNameUI
|
||||
RotatingWheel
|
||||
SLCConfirmation
|
||||
SLCConfirmationHigher
|
||||
SLCConfirmationLower
|
||||
SLCFallback
|
||||
SLCLookaheadHigher
|
||||
SLCLookaheadLower
|
||||
SLCMapboxFiller
|
||||
SLCOverride
|
||||
SNGHack
|
||||
ScreenBrightness
|
||||
ScreenBrightnessOnroad
|
||||
ScreenManagement
|
||||
ScreenRecorder
|
||||
ScreenTimeout
|
||||
ScreenTimeoutOnroad
|
||||
SearchInput
|
||||
SetSpeedLimit
|
||||
SetSpeedOffset
|
||||
ShowCEMStatus
|
||||
ShowCPU
|
||||
ShowCSCStatus
|
||||
ShowGPU
|
||||
ShowIP
|
||||
ShowMemoryUsage
|
||||
ShowSLCOffset
|
||||
ShowSpeedLimits
|
||||
ShowSteering
|
||||
ShowStoppingPoint
|
||||
ShowStoppingPointMetrics
|
||||
ShowStorageLeft
|
||||
ShowStorageUsed
|
||||
ShownToggleDescriptions
|
||||
Sidebar
|
||||
SignalMetrics
|
||||
SpeedLimitChangedAlert
|
||||
SpeedLimitController
|
||||
SpeedLimitFiller
|
||||
SpeedLimitSources
|
||||
SshEnabled
|
||||
StandardFollow
|
||||
StandardFollowHigh
|
||||
StandardJerkAcceleration
|
||||
StandardJerkDanger
|
||||
StandardJerkDeceleration
|
||||
StandardJerkSpeed
|
||||
StandardJerkSpeedDecrease
|
||||
StandardPersonalityProfile
|
||||
StandbyMode
|
||||
StartAccel
|
||||
StartAccelStock
|
||||
StartupMessageBottom
|
||||
StartupMessageTop
|
||||
StaticPedalsOnUI
|
||||
SteerDelay
|
||||
SteerDelayStock
|
||||
SteerFriction
|
||||
SteerFrictionStock
|
||||
SteerKP
|
||||
SteerKPStock
|
||||
SteerLatAccel
|
||||
SteerLatAccelStock
|
||||
SteerOffset
|
||||
SteerOffsetStock
|
||||
SteerRatio
|
||||
SteerRatioStock
|
||||
StopAccel
|
||||
StopAccelStock
|
||||
StopDistance
|
||||
StoppedTimer
|
||||
StoppingDecelRate
|
||||
StoppingDecelRateStock
|
||||
TacoTune
|
||||
TacoTuneHacks
|
||||
TetheringEnabled
|
||||
ToyotaDoors
|
||||
TrafficFollow
|
||||
TrafficJerkAcceleration
|
||||
TrafficJerkDanger
|
||||
TrafficJerkDeceleration
|
||||
TrafficJerkSpeed
|
||||
TrafficJerkSpeedDecrease
|
||||
TrafficPersonalityProfile
|
||||
TrailerLoad
|
||||
TruckTuning
|
||||
TuningLevel
|
||||
TuningLevelConfirmed
|
||||
TurnDesires
|
||||
UnlimitedLength
|
||||
UnlockDoors
|
||||
UpdaterAvailableBranches
|
||||
UseKonikServer
|
||||
UseSI
|
||||
UseVienna
|
||||
UserFavorites
|
||||
VEgoStarting
|
||||
VEgoStartingStock
|
||||
VEgoStopping
|
||||
VEgoStoppingStock
|
||||
VeryLongDistanceButtonControl
|
||||
VoltSNG
|
||||
WarningImmediateVolume
|
||||
WarningSoftVolume
|
||||
WheelIcon
|
||||
WheelSpeed
|
||||
Executable
+550
@@ -0,0 +1,550 @@
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import json
|
||||
import ast
|
||||
|
||||
REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
|
||||
|
||||
CATEGORIES = [
|
||||
{"file": "lateral_settings.cc", "name": "Lateral (Steering)", "icon": "bi-arrows-move"},
|
||||
{"file": "longitudinal_settings.cc", "name": "Longitudinal (Speed & Following)", "icon": "bi-speedometer2"},
|
||||
{"file": "visual_settings.cc", "name": "Visual (Display & UI)", "icon": "bi-eye"},
|
||||
{"file": "sounds_settings.cc", "name": "Sounds & Alerts", "icon": "bi-volume-up"},
|
||||
{"file": "vehicle_settings.cc", "name": "Vehicle", "icon": "bi-car-front"},
|
||||
{"file": "device_settings.cc", "name": "Device & Data", "icon": "bi-hdd"},
|
||||
{"file": "model_settings.cc", "name": "Model & Customization", "icon": "bi-cpu"},
|
||||
]
|
||||
|
||||
DROPDOWN_MAPPING = {
|
||||
"SelectModel": {
|
||||
"key": "DrivingModel",
|
||||
"options_endpoint": "/api/models/installed"
|
||||
}
|
||||
}
|
||||
|
||||
# Custom controls implemented outside the tuple vectors in Qt settings panels.
|
||||
# Inject these so regenerated pond layouts retain equivalent functionality.
|
||||
INJECTED_SECTION_PARAMS = {
|
||||
"Vehicle": [
|
||||
{
|
||||
"key": "CarMake",
|
||||
"label": "Car Make",
|
||||
"description": "Select your car make.",
|
||||
"data_type": "string",
|
||||
"ui_type": "dropdown",
|
||||
"options_endpoint": "/api/fingerprints/makes",
|
||||
},
|
||||
{
|
||||
"key": "CarModel",
|
||||
"label": "Car Model (Fingerprint)",
|
||||
"description": "Choose the fingerprint platform to use when automatic detection is disabled.",
|
||||
"data_type": "string",
|
||||
"ui_type": "dropdown",
|
||||
"options_endpoint": "/api/fingerprints/models?make={CarMake}",
|
||||
},
|
||||
{
|
||||
"key": "ForceFingerprint",
|
||||
"label": "Disable Automatic Fingerprint Detection",
|
||||
"description": "Force the selected fingerprint and prevent it from changing automatically.",
|
||||
"data_type": "bool",
|
||||
"ui_type": "toggle",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
# Keys explicitly hidden from The Pond's generic settings UI.
|
||||
HIDDEN_KEYS = {
|
||||
"FrogsGoMoosTweak",
|
||||
"LockDoorsTimer",
|
||||
"NewLongAPI",
|
||||
"ToyotaDoors",
|
||||
}
|
||||
|
||||
# Keys that are boolean toggles despite ambiguous defaults in frogpilot_variables.py.
|
||||
FORCE_BOOL_KEYS = {"EVTuning"}
|
||||
|
||||
DEVELOPER_SIDEBAR_METRIC_KEYS = {
|
||||
"DeveloperSidebarMetric1",
|
||||
"DeveloperSidebarMetric2",
|
||||
"DeveloperSidebarMetric3",
|
||||
"DeveloperSidebarMetric4",
|
||||
"DeveloperSidebarMetric5",
|
||||
"DeveloperSidebarMetric6",
|
||||
"DeveloperSidebarMetric7",
|
||||
}
|
||||
|
||||
DEVELOPER_SIDEBAR_METRIC_OPTIONS = [
|
||||
{"value": 0, "label": "None"},
|
||||
{"value": 1, "label": "Acceleration: Current"},
|
||||
{"value": 2, "label": "Acceleration: Max"},
|
||||
{"value": 3, "label": "Auto Tune: Actuator Delay"},
|
||||
{"value": 4, "label": "Auto Tune: Friction"},
|
||||
{"value": 5, "label": "Auto Tune: Lateral Acceleration"},
|
||||
{"value": 6, "label": "Auto Tune: Steer Ratio"},
|
||||
{"value": 7, "label": "Auto Tune: Stiffness Factor"},
|
||||
{"value": 8, "label": "Engagement %: Lateral"},
|
||||
{"value": 9, "label": "Engagement %: Longitudinal"},
|
||||
{"value": 10, "label": "Lateral Control: Steering Angle"},
|
||||
{"value": 11, "label": "Lateral Control: Torque % Used"},
|
||||
{"value": 12, "label": "Longitudinal Control: Actuator Acceleration Output"},
|
||||
{"value": 13, "label": "Longitudinal MPC Jerk: Acceleration"},
|
||||
{"value": 14, "label": "Longitudinal MPC Jerk: Danger Zone"},
|
||||
{"value": 15, "label": "Longitudinal MPC Jerk: Speed Control"},
|
||||
{"value": 16, "label": "Driving Model: Current"},
|
||||
]
|
||||
|
||||
PARENT_KEYS_MAPPING = {
|
||||
"device_settings.cc": {
|
||||
"deviceManagementKeys": "DeviceManagement",
|
||||
"screenKeys": "ScreenManagement"
|
||||
},
|
||||
"lateral_settings.cc": {
|
||||
"advancedLateralTuneKeys": "AdvancedLateralTune",
|
||||
"aolKeys": "AlwaysOnLateral",
|
||||
"laneChangeKeys": "LaneChanges",
|
||||
"lateralTuneKeys": "LateralTune",
|
||||
"qolKeys": "QOLLateral"
|
||||
},
|
||||
"longitudinal_settings.cc": {
|
||||
"advancedLongitudinalTuneKeys": "AdvancedLongitudinalTune",
|
||||
"aggressivePersonalityKeys": "AggressivePersonalityProfile",
|
||||
"conditionalExperimentalKeys": "ConditionalExperimental",
|
||||
"curveSpeedKeys": "CurveSpeedController",
|
||||
"customDrivingPersonalityKeys": "CustomPersonalities",
|
||||
"longitudinalTuneKeys": "LongitudinalTune",
|
||||
"qolKeys": "QOLLongitudinal",
|
||||
"relaxedPersonalityKeys": "RelaxedPersonalityProfile",
|
||||
"speedLimitControllerKeys": "SpeedLimitController",
|
||||
"speedLimitControllerOffsetsKeys": "SpeedLimitController",
|
||||
"speedLimitControllerQOLKeys": "SpeedLimitController",
|
||||
"speedLimitControllerVisualKeys": "SpeedLimitController",
|
||||
"standardPersonalityKeys": "StandardPersonalityProfile",
|
||||
"trafficPersonalityKeys": "TrafficPersonalityProfile"
|
||||
},
|
||||
"sounds_settings.cc": {
|
||||
"alertVolumeControlKeys": "AlertVolumeControl",
|
||||
"customAlertsKeys": "CustomAlerts"
|
||||
},
|
||||
"theme_settings.cc": {
|
||||
"customThemeKeys": "CustomTheme"
|
||||
},
|
||||
"visual_settings.cc": {
|
||||
"advancedCustomOnroadUIKeys": "AdvancedCustomUI",
|
||||
"customOnroadUIKeys": "CustomUI",
|
||||
"developerMetricKeys": "DeveloperMetrics",
|
||||
"developerSidebarKeys": "DeveloperSidebar",
|
||||
"developerUIKeys": "DeveloperUI",
|
||||
"developerWidgetKeys": "DeveloperWidgets",
|
||||
"modelUIKeys": "ModelUI",
|
||||
"navigationUIKeys": "NavigationUI",
|
||||
"qualityOfLifeKeys": "QOLVisuals"
|
||||
},
|
||||
"vehicle_settings.cc": {}
|
||||
}
|
||||
|
||||
ALL_PARENT_KEYS = set()
|
||||
for cmap in PARENT_KEYS_MAPPING.values():
|
||||
for parent in cmap.values():
|
||||
ALL_PARENT_KEYS.add(parent)
|
||||
|
||||
def get_variables_data():
|
||||
filepath = os.path.join(REPO_ROOT, "frogpilot/common/frogpilot_variables.py")
|
||||
excluded = set()
|
||||
defaults = {}
|
||||
if not os.path.exists(filepath):
|
||||
return excluded, defaults
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
tree = ast.parse(f.read())
|
||||
|
||||
def parse_params_list(value_node):
|
||||
try:
|
||||
if isinstance(value_node, ast.List):
|
||||
for elt in value_node.elts:
|
||||
if isinstance(elt, ast.Tuple) and len(elt.elts) >= 2:
|
||||
key_node = elt.elts[0]
|
||||
val_node = elt.elts[1]
|
||||
if isinstance(key_node, ast.Constant):
|
||||
key = key_node.value
|
||||
if isinstance(val_node, ast.Constant):
|
||||
val = val_node.value
|
||||
if isinstance(val, (str, bytes)):
|
||||
v = val.decode('utf-8') if isinstance(val, bytes) else str(val)
|
||||
if v in ("0", "1"):
|
||||
defaults[key] = "bool"
|
||||
elif "." in v and v.replace(".", "", 1).isdigit():
|
||||
defaults[key] = "float"
|
||||
elif v.isdigit():
|
||||
defaults[key] = "int"
|
||||
else:
|
||||
defaults[key] = "string"
|
||||
else:
|
||||
defaults[key] = "unknown"
|
||||
elif isinstance(val_node, ast.Call) and isinstance(val_node.func, ast.Name) and val_node.func.id == "str":
|
||||
# str(<numeric expression>) is used for several numeric defaults.
|
||||
defaults[key] = "float"
|
||||
else:
|
||||
defaults[key] = "unknown"
|
||||
except:
|
||||
pass
|
||||
|
||||
for node in tree.body:
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if getattr(target, 'id', '') == 'EXCLUDED_KEYS':
|
||||
try:
|
||||
excluded = ast.literal_eval(node.value)
|
||||
except:
|
||||
pass
|
||||
elif getattr(target, 'id', '') in ('frogpilot_default_params', 'misc_tuning_levels'):
|
||||
parse_params_list(node.value)
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
if getattr(node.target, 'id', '') in ('frogpilot_default_params', 'misc_tuning_levels'):
|
||||
parse_params_list(node.value)
|
||||
|
||||
return excluded, defaults
|
||||
|
||||
EXCLUDED_KEYS, DEFAULT_TYPES = get_variables_data()
|
||||
|
||||
def get_editable_keys():
|
||||
filepath = os.path.join(REPO_ROOT, "frogpilot/common/frogpilot_variables.py")
|
||||
editable = set()
|
||||
if not os.path.exists(filepath):
|
||||
return editable
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
tree = ast.parse(f.read())
|
||||
|
||||
for node in tree.body:
|
||||
value_node = None
|
||||
if isinstance(node, ast.Assign):
|
||||
for target in node.targets:
|
||||
if getattr(target, 'id', '') == 'frogpilot_default_params':
|
||||
value_node = node.value
|
||||
break
|
||||
elif isinstance(node, ast.AnnAssign):
|
||||
if getattr(node.target, 'id', '') == 'frogpilot_default_params':
|
||||
value_node = node.value
|
||||
|
||||
if isinstance(value_node, ast.List):
|
||||
for elt in value_node.elts:
|
||||
if isinstance(elt, ast.Tuple) and elt.elts and isinstance(elt.elts[0], ast.Constant):
|
||||
editable.add(elt.elts[0].value)
|
||||
|
||||
return editable
|
||||
|
||||
EDITABLE_KEYS = get_editable_keys()
|
||||
|
||||
|
||||
def parse_params_keys_h():
|
||||
filepath = os.path.join(REPO_ROOT, "common/params_keys.h")
|
||||
keys = set()
|
||||
types = {}
|
||||
if not os.path.exists(filepath):
|
||||
return keys, types
|
||||
|
||||
type_map = {
|
||||
"BOOL": "bool",
|
||||
"INT": "int",
|
||||
"FLOAT": "float",
|
||||
"STRING": "string",
|
||||
"JSON": "string",
|
||||
"BYTES": "string",
|
||||
}
|
||||
pattern = re.compile(r'\{"([A-Za-z0-9_]+)",\s*\{[^,]+,\s*([A-Z]+)')
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
match = pattern.search(line)
|
||||
if not match:
|
||||
continue
|
||||
key, ptype = match.groups()
|
||||
keys.add(key)
|
||||
types[key] = type_map.get(ptype, "unknown")
|
||||
|
||||
return keys, types
|
||||
|
||||
|
||||
if not DEFAULT_TYPES or not EDITABLE_KEYS:
|
||||
parsed_keys, parsed_types = parse_params_keys_h()
|
||||
if not EDITABLE_KEYS:
|
||||
EDITABLE_KEYS = parsed_keys
|
||||
for key, value in parsed_types.items():
|
||||
DEFAULT_TYPES.setdefault(key, value)
|
||||
|
||||
def get_param_type(key):
|
||||
return DEFAULT_TYPES.get(key, "unknown")
|
||||
|
||||
def extract_bracket_block(text, start_idx):
|
||||
if text[start_idx] != '{': return ""
|
||||
depth = 0
|
||||
in_str = False
|
||||
escape = False
|
||||
for i in range(start_idx, len(text)):
|
||||
char = text[i]
|
||||
if escape:
|
||||
escape = False
|
||||
continue
|
||||
if char == '\\':
|
||||
escape = True
|
||||
continue
|
||||
if char == '"':
|
||||
in_str = not in_str
|
||||
continue
|
||||
if not in_str:
|
||||
if char == '{': depth += 1
|
||||
elif char == '}':
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
return text[start_idx:i+1]
|
||||
return ""
|
||||
|
||||
def parse_cpp_file(filename):
|
||||
filepath = os.path.join(REPO_ROOT, "frogpilot/ui/qt/offroad", filename)
|
||||
if not os.path.exists(filepath): return []
|
||||
|
||||
with open(filepath, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
|
||||
vector_match = re.search(
|
||||
r'(?:const\s+)?std::vector<\s*std::tuple<QString,\s*QString,\s*QString,\s*QString>\s*>\s*\w+\s*\{',
|
||||
content,
|
||||
re.DOTALL,
|
||||
)
|
||||
if not vector_match: return []
|
||||
|
||||
start_idx = vector_match.end() - 1
|
||||
vector_content = extract_bracket_block(content, start_idx)
|
||||
|
||||
local_parent_map = PARENT_KEYS_MAPPING.get(filename, {})
|
||||
child_to_parent = {}
|
||||
child_to_qsets = {}
|
||||
|
||||
header_filename = filename.replace(".cc", ".h")
|
||||
header_filepath = os.path.join(REPO_ROOT, "frogpilot/ui/qt/offroad", header_filename)
|
||||
full_source = content
|
||||
if os.path.exists(header_filepath):
|
||||
with open(header_filepath, 'r', encoding='utf-8') as fh:
|
||||
full_source += "\n" + fh.read()
|
||||
|
||||
for qset_match in re.finditer(r'QSet<QString>\s+(\w+)\s*(?:=\s*)?\{([^}]+)\};', full_source):
|
||||
qset_name = qset_match.group(1)
|
||||
if qset_name in local_parent_map:
|
||||
parent_key = local_parent_map[qset_name]
|
||||
children_str = qset_match.group(2)
|
||||
children = [c.strip().strip('"') for c in children_str.split(',') if c.strip()]
|
||||
for child in children:
|
||||
child_to_parent[child] = parent_key
|
||||
child_to_qsets.setdefault(child, []).append(qset_name)
|
||||
|
||||
items = []
|
||||
|
||||
idx = 0
|
||||
while True:
|
||||
idx = vector_content.find('{"', idx)
|
||||
if idx == -1: break
|
||||
|
||||
block = extract_bracket_block(vector_content, idx)
|
||||
if not block:
|
||||
idx += 1
|
||||
continue
|
||||
|
||||
row_match = re.search(r'\{"([A-Za-z0-9_]+)"\s*,\s*(.*?)\s*\}$', block, re.DOTALL)
|
||||
if not row_match:
|
||||
idx += len(block)
|
||||
continue
|
||||
|
||||
key = row_match.group(1)
|
||||
rest = row_match.group(2)
|
||||
idx += len(block)
|
||||
|
||||
if key in HIDDEN_KEYS or key in EXCLUDED_KEYS or key.startswith("IgnoreMe"):
|
||||
continue
|
||||
|
||||
strings = re.findall(r'tr\("((?:[^"\\]|\\.)+)"\)|"((?:[^"\\]|\\.)+)"', rest)
|
||||
valid_strings = [s[0] or s[1] for s in strings if s[0] or s[1]]
|
||||
|
||||
if not valid_strings: continue
|
||||
|
||||
title = valid_strings[0]
|
||||
desc = valid_strings[1] if len(valid_strings) > 1 else ""
|
||||
options_endpoint = None
|
||||
dropdown_options = None
|
||||
|
||||
if key in DEVELOPER_SIDEBAR_METRIC_KEYS:
|
||||
if key not in EDITABLE_KEYS:
|
||||
continue
|
||||
widget_type = "dropdown"
|
||||
data_type = "int"
|
||||
dropdown_options = DEVELOPER_SIDEBAR_METRIC_OPTIONS
|
||||
elif key in DROPDOWN_MAPPING:
|
||||
m = DROPDOWN_MAPPING[key]
|
||||
key = m["key"]
|
||||
widget_type = "dropdown"
|
||||
options_endpoint = m["options_endpoint"]
|
||||
data_type = "string"
|
||||
else:
|
||||
if key not in EDITABLE_KEYS:
|
||||
continue
|
||||
data_type = get_param_type(key)
|
||||
if data_type == "unknown": continue
|
||||
widget_type = "toggle"
|
||||
min_val, max_val, step = None, None, None
|
||||
|
||||
for i in range(1, 10):
|
||||
placeholder = f"%{i}"
|
||||
if placeholder in desc and len(valid_strings) > i + 1:
|
||||
desc = desc.replace(placeholder, valid_strings[i + 1])
|
||||
|
||||
desc = re.sub(r'<br\s*/?>', '\n', desc, flags=re.IGNORECASE)
|
||||
desc = re.sub(r'<[^>]+>', '', desc)
|
||||
desc = desc.replace('\\"', '"').strip()
|
||||
title = re.sub(r'\s*\(\s*Default:\s*%\d\s*\)', '', title)
|
||||
title = re.sub(r'%\d', '', title).strip()
|
||||
desc = re.sub(r'\s*\(\s*Default:\s*%\d\s*\)', '', desc)
|
||||
desc = re.sub(r'%\d', '', desc).strip()
|
||||
|
||||
if widget_type == "toggle":
|
||||
snippet_match = None
|
||||
|
||||
# Let's match the original's regex for finding the Toggle = assignment line
|
||||
search_patterns = [r'param\s*==\s*"' + key + r'"']
|
||||
for qset_name in child_to_qsets.get(key, []):
|
||||
search_patterns.append(r'(?:' + qset_name + r'\.contains\(param\))')
|
||||
|
||||
for pattern in search_patterns:
|
||||
match = re.search(pattern + r'.*?[a-zA-Z]+Toggle\s*=\s*(.*?);', content, re.DOTALL)
|
||||
if match:
|
||||
snippet_match = match
|
||||
break
|
||||
|
||||
if snippet_match:
|
||||
assignment = snippet_match.group(1)
|
||||
if "FrogPilotParamValueControl" in assignment or "FrogPilotParamValueButtonControl" in assignment:
|
||||
widget_type = "numeric"
|
||||
if data_type in ("string", "bool", "unknown"):
|
||||
data_type = "float"
|
||||
|
||||
if "alertVolumeControlKeys" in child_to_qsets.get(key, []):
|
||||
if key in ["WarningImmediateVolume", "WarningSoftVolume"]:
|
||||
min_val, max_val, step = "25", "101", "1"
|
||||
else:
|
||||
min_val, max_val, step = "0", "101", "1"
|
||||
else:
|
||||
args_match = re.search(r'Control[^(]*\(([^;]+)\)', assignment)
|
||||
if args_match:
|
||||
args_str = args_match.group(1)
|
||||
num_match = re.search(r'icon\s*,\s*([-\d.]+)\s*,\s*([-\d.]+)\s*,(?:[^,]*,){2}\s*([-\d.]+)', args_str)
|
||||
if num_match:
|
||||
min_val, max_val, step = num_match.group(1), num_match.group(2), num_match.group(3)
|
||||
else:
|
||||
num_match = re.search(r'icon\s*,\s*([-\d.]+)\s*,\s*([-\d.]+)', args_str)
|
||||
if num_match:
|
||||
min_val, max_val = num_match.group(1), num_match.group(2)
|
||||
step_match = re.search(r'(?:std::map<float,\s*QString>\(\)|[a-zA-Z0-9_]+Labels)\s*,\s*([-\d.]+)', args_str)
|
||||
if step_match:
|
||||
step = step_match.group(1)
|
||||
|
||||
# CESpeed is rendered in Qt with a dual numeric control (CESpeed + CESpeedLead),
|
||||
# so the generic assignment matcher cannot infer it reliably.
|
||||
if key == "CESpeed":
|
||||
widget_type = "numeric"
|
||||
data_type = "int"
|
||||
min_val, max_val, step = "0", "99", "1"
|
||||
|
||||
if key in FORCE_BOOL_KEYS:
|
||||
data_type = "bool"
|
||||
|
||||
precision = None
|
||||
precision_match = re.search(r"QString::number\([^,]+,\s*'f'\s*,\s*(\d+)\)", rest)
|
||||
if precision_match:
|
||||
precision = int(precision_match.group(1))
|
||||
|
||||
if data_type == "float" and step and float(step).is_integer():
|
||||
data_type = "int"
|
||||
|
||||
# Generic pond UI can't faithfully represent non-boolean button/multi-option controls.
|
||||
if widget_type == "toggle" and data_type != "bool":
|
||||
continue
|
||||
|
||||
s = {
|
||||
"key": key,
|
||||
"label": title,
|
||||
"description": desc,
|
||||
"data_type": data_type,
|
||||
"ui_type": widget_type
|
||||
}
|
||||
if widget_type == "numeric":
|
||||
if min_val is not None: s["min"] = float(min_val)
|
||||
if max_val is not None: s["max"] = float(max_val)
|
||||
if step is not None: s["step"] = float(step)
|
||||
if precision is not None: s["precision"] = precision
|
||||
elif widget_type == "dropdown":
|
||||
if options_endpoint: s["options_endpoint"] = options_endpoint
|
||||
if dropdown_options: s["options"] = dropdown_options
|
||||
if key in child_to_parent: s["parent_key"] = child_to_parent[key]
|
||||
if key in ALL_PARENT_KEYS: s["is_parent_toggle"] = True
|
||||
|
||||
if key == "CELead":
|
||||
s["is_parent_toggle"] = True
|
||||
|
||||
items.append(s)
|
||||
|
||||
# Mirror CELead's split sub-toggles from FrogPilotButtonToggleControl.
|
||||
if key == "CELead":
|
||||
items.extend([
|
||||
{
|
||||
"key": "CESlowerLead",
|
||||
"label": "Slower Lead",
|
||||
"description": "Switch to \"Experimental Mode\" when a slower lead vehicle is detected ahead.",
|
||||
"data_type": "bool",
|
||||
"ui_type": "toggle",
|
||||
"parent_key": "CELead",
|
||||
},
|
||||
{
|
||||
"key": "CEStoppedLead",
|
||||
"label": "Stopped Lead",
|
||||
"description": "Switch to \"Experimental Mode\" when a stopped lead vehicle is detected ahead.",
|
||||
"data_type": "bool",
|
||||
"ui_type": "toggle",
|
||||
"parent_key": "CELead",
|
||||
},
|
||||
])
|
||||
|
||||
# Mirror CESpeed's dual slider (with-lead variant) from Qt.
|
||||
if key == "CESpeed":
|
||||
items.append({
|
||||
"key": "CESpeedLead",
|
||||
"label": "Below (With Lead)",
|
||||
"description": "Switch to \"Experimental Mode\" when driving below this speed with a lead.",
|
||||
"data_type": "int",
|
||||
"ui_type": "numeric",
|
||||
"min": 0.0,
|
||||
"max": 99.0,
|
||||
"step": 1.0,
|
||||
"parent_key": "ConditionalExperimental",
|
||||
})
|
||||
|
||||
return items
|
||||
|
||||
def main():
|
||||
layout = []
|
||||
for cat in CATEGORIES:
|
||||
items = parse_cpp_file(cat["file"])
|
||||
injected = INJECTED_SECTION_PARAMS.get(cat["name"], [])
|
||||
if injected:
|
||||
existing_keys = {item["key"] for item in items}
|
||||
items = [dict(item) for item in injected if item["key"] not in existing_keys] + items
|
||||
if items:
|
||||
layout.append({
|
||||
"name": cat["name"],
|
||||
"icon": cat["icon"],
|
||||
"params": items
|
||||
})
|
||||
output_path = os.path.join(REPO_ROOT, "frogpilot/system/the_pond/assets/components/tools/device_settings_layout.json")
|
||||
with open(output_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(layout, f, indent=2)
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
File diff suppressed because it is too large
Load Diff
@@ -15,6 +15,12 @@ if ! command -v "uv" > /dev/null 2>&1; then
|
||||
PATH="$UV_BIN:$PATH"
|
||||
fi
|
||||
|
||||
# Homebrew llvm can break extension builds on macOS by masking system SDK headers.
|
||||
if [[ "$(uname)" == 'Darwin' ]]; then
|
||||
export CC=/usr/bin/clang
|
||||
export CXX=/usr/bin/clang++
|
||||
fi
|
||||
|
||||
echo "updating uv..."
|
||||
# ok to fail, can also fail due to installing with brew
|
||||
uv self update || true
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
FROM ghcr.io/commaai/openpilot-base-aarch64:latest
|
||||
|
||||
ENV PYTHONUNBUFFERED=1
|
||||
ENV UV_LINK_MODE=copy
|
||||
WORKDIR /work
|
||||
|
||||
# Keep this image minimal; dependency resolution is done by uv inside the workspace.
|
||||
# rsync/ssh-client are needed for optional sysroot sync from a comma device.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
rsync \
|
||||
openssh-client \
|
||||
e2fsprogs \
|
||||
android-sdk-libsparse-utils \
|
||||
qemu-user-static \
|
||||
gcc-aarch64-linux-gnu \
|
||||
g++-aarch64-linux-gnu \
|
||||
qtbase5-private-dev \
|
||||
python3-pip \
|
||||
xz-utils \
|
||||
git \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN /usr/bin/python3 -m pip install --no-cache-dir --break-system-packages uv
|
||||
|
||||
# Fix for "dubious ownership" git errors when mounting folders on Linux hosts.
|
||||
RUN git config --global --add safe.directory /work
|
||||
@@ -0,0 +1,290 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import lzma
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
REQUIRED_DIRS = [
|
||||
("usr/local/lib", "/usr/local/lib"),
|
||||
("usr/local/include", "/usr/local/include"),
|
||||
("lib/aarch64-linux-gnu", "/lib/aarch64-linux-gnu"),
|
||||
("usr/lib/aarch64-linux-gnu", "/usr/lib/aarch64-linux-gnu"),
|
||||
("usr/include", "/usr/include"),
|
||||
]
|
||||
OPTIONAL_DIRS = [
|
||||
("usr/lib/qt5/bin", "/usr/lib/qt5/bin"),
|
||||
]
|
||||
VENDOR_CANDIDATES = ["/system/vendor/lib64", "/vendor/lib64"]
|
||||
OPTIONAL_INCLUDE_DIRS: list[str] = []
|
||||
DEBUGFS_NOT_FOUND_MARKERS = (
|
||||
"file not found by ext2_lookup",
|
||||
"while trying to resolve filename",
|
||||
"couldn't allocate",
|
||||
"no such file",
|
||||
"not found",
|
||||
)
|
||||
ANDROID_SPARSE_MAGIC = 0xED26FF3A
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Extract TICI sysroot dirs from AGNOS system image")
|
||||
p.add_argument("--manifest", default="system/hardware/tici/agnos.json", help="Path to AGNOS manifest JSON")
|
||||
p.add_argument("--output-dir", required=True, help="Destination sysroot directory")
|
||||
p.add_argument("--cache-dir", default=".cache/agnos", help="Cache directory for downloaded images")
|
||||
p.add_argument("--url", default=None, help="Override AGNOS system image URL")
|
||||
p.add_argument("--force-download", action="store_true", help="Redownload system image even if cached")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def get_system_url(manifest_path: Path) -> str:
|
||||
data = json.loads(manifest_path.read_text())
|
||||
for entry in data:
|
||||
if entry.get("name") != "system":
|
||||
continue
|
||||
if isinstance(entry.get("url"), str):
|
||||
return entry["url"]
|
||||
alt = entry.get("alt") if isinstance(entry, dict) else None
|
||||
if isinstance(alt, dict) and isinstance(alt.get("url"), str):
|
||||
return alt["url"]
|
||||
raise RuntimeError(f"No system entry in manifest: {manifest_path}")
|
||||
|
||||
|
||||
def download(url: str, dst: Path) -> None:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
tmp = dst.with_suffix(dst.suffix + ".part")
|
||||
print(f"Downloading {url} -> {dst}", flush=True)
|
||||
with urllib.request.urlopen(url) as src, open(tmp, "wb") as out:
|
||||
shutil.copyfileobj(src, out, length=1024 * 1024)
|
||||
tmp.replace(dst)
|
||||
|
||||
|
||||
def download_and_prepare_image(url: str, cache_dir: Path, force_download: bool) -> Path:
|
||||
cache_dir.mkdir(parents=True, exist_ok=True)
|
||||
compressed = cache_dir / "agnos_system.img.xz"
|
||||
raw_image = cache_dir / "agnos_system.img"
|
||||
ext4_image = cache_dir / "agnos_system.ext4.img"
|
||||
|
||||
if force_download:
|
||||
compressed.unlink(missing_ok=True)
|
||||
raw_image.unlink(missing_ok=True)
|
||||
ext4_image.unlink(missing_ok=True)
|
||||
|
||||
# Prefer an already-cached raw image to avoid unnecessary multi-GB downloads.
|
||||
if raw_image.exists():
|
||||
return raw_image
|
||||
|
||||
if url.endswith(".xz"):
|
||||
if not compressed.exists():
|
||||
download(url, compressed)
|
||||
if not raw_image.exists():
|
||||
print(f"Decompressing {compressed} -> {raw_image}", flush=True)
|
||||
with lzma.open(compressed, "rb") as f_in, open(raw_image, "wb") as f_out:
|
||||
shutil.copyfileobj(f_in, f_out, length=1024 * 1024)
|
||||
else:
|
||||
if not raw_image.exists():
|
||||
download(url, raw_image)
|
||||
|
||||
return raw_image
|
||||
|
||||
|
||||
def is_android_sparse(image_path: Path) -> bool:
|
||||
with open(image_path, "rb") as f:
|
||||
header = f.read(4)
|
||||
if len(header) != 4:
|
||||
return False
|
||||
magic = int.from_bytes(header, "little")
|
||||
return magic == ANDROID_SPARSE_MAGIC
|
||||
|
||||
|
||||
def ensure_debugfs_readable_image(image_path: Path, cache_dir: Path) -> Path:
|
||||
if not is_android_sparse(image_path):
|
||||
return image_path
|
||||
|
||||
if shutil.which("simg2img") is None:
|
||||
raise RuntimeError(
|
||||
f"{image_path} is an Android sparse image, but simg2img is not installed."
|
||||
)
|
||||
|
||||
converted = cache_dir / "agnos_system.ext4.img"
|
||||
needs_convert = not converted.exists()
|
||||
if converted.exists():
|
||||
needs_convert = os.path.getmtime(converted) < os.path.getmtime(image_path)
|
||||
|
||||
if needs_convert:
|
||||
print(f"Converting sparse image {image_path} -> {converted}", flush=True)
|
||||
proc = subprocess.run(["simg2img", str(image_path), str(converted)], capture_output=True, text=True)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"simg2img failed:\n{proc.stdout}\n{proc.stderr}")
|
||||
return converted
|
||||
|
||||
|
||||
def _has_content(path: Path) -> bool:
|
||||
if not path.exists():
|
||||
return False
|
||||
if path.is_file():
|
||||
return path.stat().st_size > 0
|
||||
return any(path.iterdir())
|
||||
|
||||
|
||||
def _pick_extracted_root(tmp_path: Path, src_path: str) -> Path | None:
|
||||
rel = Path(src_path.lstrip("/"))
|
||||
candidates = [
|
||||
tmp_path / rel,
|
||||
tmp_path / rel.name,
|
||||
tmp_path,
|
||||
]
|
||||
for c in candidates:
|
||||
if _has_content(c):
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def run_debugfs(image_path: Path, src_path: str, dst_path: Path) -> None:
|
||||
attempts: list[str] = []
|
||||
for candidate_src in (src_path, src_path.lstrip("/")):
|
||||
if not candidate_src:
|
||||
continue
|
||||
with tempfile.TemporaryDirectory(prefix="sysroot_extract_") as tmp:
|
||||
tmp_path = Path(tmp)
|
||||
cmd = ["debugfs", "-R", f"rdump {candidate_src} {tmp_path}", str(image_path)]
|
||||
proc = subprocess.run(cmd, capture_output=True, text=True)
|
||||
out = f"{proc.stdout}\n{proc.stderr}".lower()
|
||||
attempts.append(f"{' '.join(cmd)}\n{proc.stdout}\n{proc.stderr}")
|
||||
|
||||
if proc.returncode != 0:
|
||||
continue
|
||||
if any(marker in out for marker in DEBUGFS_NOT_FOUND_MARKERS):
|
||||
continue
|
||||
|
||||
source_to_copy = _pick_extracted_root(tmp_path, candidate_src)
|
||||
if source_to_copy is None:
|
||||
continue
|
||||
|
||||
if dst_path.exists():
|
||||
shutil.rmtree(dst_path)
|
||||
dst_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(source_to_copy, dst_path, symlinks=True, dirs_exist_ok=True)
|
||||
return
|
||||
|
||||
joined_attempts = "\n---\n".join(attempts)
|
||||
raise RuntimeError(f"debugfs could not extract '{src_path}'. Attempts:\n{joined_attempts}")
|
||||
|
||||
|
||||
def is_non_empty_dir(path: Path) -> bool:
|
||||
return path.is_dir() and any(path.iterdir())
|
||||
|
||||
|
||||
def populate_optional_host_includes(output_dir: Path) -> None:
|
||||
dst_root = output_dir / "usr/include"
|
||||
if not dst_root.exists():
|
||||
return
|
||||
|
||||
include_roots = [
|
||||
Path("/usr/include/aarch64-linux-gnu"),
|
||||
Path("/usr/include"),
|
||||
Path("/usr/include/x86_64-linux-gnu"),
|
||||
]
|
||||
|
||||
for include_dir in OPTIONAL_INCLUDE_DIRS:
|
||||
dst = dst_root / include_dir
|
||||
if dst.exists():
|
||||
continue
|
||||
for root in include_roots:
|
||||
src = root / include_dir
|
||||
if not src.is_dir():
|
||||
continue
|
||||
shutil.copytree(src, dst, symlinks=True)
|
||||
break
|
||||
|
||||
openssl_dst = dst_root / "openssl"
|
||||
for src_dir in (
|
||||
Path("/usr/include/aarch64-linux-gnu/openssl"),
|
||||
Path("/usr/include/x86_64-linux-gnu/openssl"),
|
||||
):
|
||||
if not src_dir.is_dir():
|
||||
continue
|
||||
openssl_dst.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copytree(src_dir, openssl_dst, symlinks=True, dirs_exist_ok=True)
|
||||
break
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if shutil.which("debugfs") is None:
|
||||
raise RuntimeError("debugfs is required (install e2fsprogs)")
|
||||
|
||||
manifest_path = Path(args.manifest).resolve()
|
||||
output_dir = Path(args.output_dir).resolve()
|
||||
cache_dir = Path(args.cache_dir).resolve()
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
url = args.url or get_system_url(manifest_path)
|
||||
image_path = download_and_prepare_image(url, cache_dir, args.force_download)
|
||||
image_path = ensure_debugfs_readable_image(image_path, cache_dir)
|
||||
|
||||
for rel_dst, src_path in REQUIRED_DIRS:
|
||||
dst = output_dir / rel_dst
|
||||
print(f"Extracting {src_path} -> {dst}", flush=True)
|
||||
run_debugfs(image_path, src_path, dst)
|
||||
|
||||
for rel_dst, src_path in OPTIONAL_DIRS:
|
||||
dst = output_dir / rel_dst
|
||||
try:
|
||||
print(f"Extracting {src_path} -> {dst}", flush=True)
|
||||
run_debugfs(image_path, src_path, dst)
|
||||
except RuntimeError:
|
||||
print(f"WARN: optional path not found in AGNOS image: {src_path}", flush=True)
|
||||
|
||||
vendor_ok = False
|
||||
vendor_dst = output_dir / "system/vendor/lib64"
|
||||
for src_path in VENDOR_CANDIDATES:
|
||||
try:
|
||||
print(f"Extracting {src_path} -> {vendor_dst}", flush=True)
|
||||
run_debugfs(image_path, src_path, vendor_dst)
|
||||
vendor_ok = True
|
||||
break
|
||||
except RuntimeError:
|
||||
continue
|
||||
|
||||
if not vendor_ok:
|
||||
print(
|
||||
"WARN: vendor libs not found in AGNOS image at /system/vendor/lib64 or /vendor/lib64; "
|
||||
"falling back to usr/lib/aarch64-linux-gnu",
|
||||
flush=True,
|
||||
)
|
||||
if vendor_dst.is_symlink() or vendor_dst.is_file():
|
||||
vendor_dst.unlink()
|
||||
elif vendor_dst.exists():
|
||||
shutil.rmtree(vendor_dst)
|
||||
vendor_dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
os.symlink("../../usr/lib/aarch64-linux-gnu", vendor_dst, target_is_directory=True)
|
||||
|
||||
populate_optional_host_includes(output_dir)
|
||||
|
||||
missing = []
|
||||
for rel in ("usr/local/lib", "usr/local/include", "lib/aarch64-linux-gnu", "usr/lib/aarch64-linux-gnu", "usr/include", "system/vendor/lib64"):
|
||||
if not is_non_empty_dir(output_dir / rel):
|
||||
missing.append(rel)
|
||||
if missing:
|
||||
raise RuntimeError(f"Extracted sysroot is incomplete, missing content in: {', '.join(missing)}")
|
||||
|
||||
print(f"Sysroot ready: {output_dir}", flush=True)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
raise SystemExit(main())
|
||||
except KeyboardInterrupt:
|
||||
raise
|
||||
except Exception as exc:
|
||||
print(f"ERROR: {exc}", file=sys.stderr)
|
||||
raise SystemExit(1)
|
||||
Reference in New Issue
Block a user