diff --git a/openpilot/starpilot/flm/offroad_daemon.py b/openpilot/starpilot/flm/offroad_daemon.py index a701d0e7a5..4c501326b6 100644 --- a/openpilot/starpilot/flm/offroad_daemon.py +++ b/openpilot/starpilot/flm/offroad_daemon.py @@ -21,14 +21,56 @@ class StatusReport: self.clock_ns = clock_ns self.started = self.clock_ns() self.current: dict[str, Any] = {'status': 'collecting', 'reason': 'waiting_for_completed_drive', 'vehicle': ''} + self.previous: dict[str, Any] = {} self.next_ns = 0 def set(self, value): changed = (self.current.get('status'), self.current.get('reason')) != (value.get('status'), value.get('reason')) self.current = dict(value) + if value.get('completed_route') and value.get('status') != 'fitting': + self.previous = dict(value) if changed: self.next_ns = 0 + def analyzing(self, value): + # Historical findings survive a new fit. Never carry a saved/current map, + # enabled flag or application evidence forward from a previous report. + fields = ('completed_route', 'vehicle', 'analysis_counts', 'recorded_tracking', 'source_sha256', 'assessment_sha256', + 'recommendations', 'suggestions', 'suggestion_count') + previous = {key: self.previous[key] for key in fields if key in self.previous} + if previous: + previous['retained_decision'] = self.previous.get('reason', 'waiting_for_completed_drive') + self.set({**(previous or value), 'status': 'fitting', 'reason': value['reason'], 'retained_analysis': bool(previous)}) + + def restore(self, cache): + """Recover historical display data, never a current tune or permission.""" + from types import SimpleNamespace + from openpilot.starpilot.flm.offroad_worker import _recorded_values, recorded_recommendations + from openpilot.starpilot.flm.offroad_surface import route_values + + try: + latest = cache.latest() + if latest is None or type(latest.get('result')) is not dict: + return + result = dict(latest['result']) + if result.get('completed_route') and 'recommendations' not in result: + saved = cache.get(result['assessment_sha256']) + assessment = saved['assessment'] + route = assessment['route'] + if (saved['kind'] != 'completed_assessment' or route['route_id'] != result['completed_route'] + or route['source_sha256'] != result['source_sha256']): + return + groups = assessment.get('groups', ()) + if not groups or any(g['original_metadata']['cp_sha256'] != route['cp_sha256'] for g in groups): + return + baselines = [route_values(_recorded_values(g['original_metadata'])) for g in groups] + if baselines and all(values == baselines[0] for values in baselines): + inputs = SimpleNamespace(groups=groups, diagnostic_groups=assessment.get('diagnostic_groups', ())) + result['recommendations'] = recorded_recommendations(inputs, baselines[0], assessment) + self.previous = result + except (OSError, KeyError, ValueError, TypeError, AttributeError): + pass # The normal worker reports unreadable inputs; historical display is optional. + def publish(self): now = self.clock_ns() if now < self.next_ns: @@ -51,6 +93,7 @@ class StatusReport: from openpilot.starpilot.flm.offroad_surface import CURVE_NAMES, route_values if 'completed_route' in value: + details['Earlier analysis shown while updating'] = value.get('retained_analysis') is True details['Completed drive analyzed'] = value['completed_route'] details['Recorded steering observations'] = count('recorded_tracking') details['Joined steering observations'] = count('joined_tracking') @@ -59,7 +102,7 @@ class StatusReport: details['Original controller observations'] = count('original_control_metadata_rows') details['Controller eligibility refusals'] = count('controller_fit_refusals') details['Automatic configuration changed'] = bool(value.get('configuration_changed', False)) - details['Analysis decision'] = value['reason'] + details['Analysis decision'] = value.get('retained_decision', value['reason']) details['Saved route adjustment count'] = value.get('saved_adjustments') details['Recorded steering response'] = value.get('recorded_tracking') checks = value.get('qualification_summary') @@ -70,6 +113,18 @@ class StatusReport: details['Recorded route suggestions'] = [[s['parameter'], s['delta'], s['whole_windows'], s['qualification']] for s in suggestions[:16]] details['Recorded suggestion count'] = value.get('suggestion_count', len(suggestions)) details['More recorded suggestions omitted'] = value.get('suggestion_count', len(suggestions)) > len(suggestions) + if 'recommendations' in value: + recommendations = value['recommendations'][:35] + details['Recorded coordinate recommendations'] = [ + [s['parameter'], s['baseline'], s['target'], s['whole_windows'], s['reason']] + for s in recommendations + ] + details['Recorded route suggestions'] = [ + [s['parameter'], s['target'] - s['baseline'], s['whole_windows'], 'controller_native_checks_not_passed'] + for s in recommendations if s['target'] is not None + ][:16] + details['Recorded suggestion count'] = len(recommendations) + details['More recorded suggestions omitted'] = False proposal = value.get('suggestion') if proposal is not None: details['Full surface coordinates'] = [[n, proposal['values'][n]] for n in NAMES] @@ -222,8 +277,8 @@ def main(): Paths.log_root(), cache, permitted=permitted, - on_analysis=lambda: report.set({'status': 'fitting', 'reason': 'analyzing_last_completed_drive'}), - on_assessment=lambda a: report.set( + on_analysis=lambda: report.analyzing({'status': 'fitting', 'reason': 'analyzing_last_completed_drive'}), + on_assessment=lambda a: report.analyzing( { 'status': 'fitting', 'reason': 'checking_recorded_steering_recommendations', @@ -233,7 +288,9 @@ def main(): 'recorded_tracking': a.recorded_tracking, } ), + on_recommendations=lambda value: report.set({**value, 'status': 'fitting', 'reason': 'checking_recorded_steering_recommendations'}), ) + report.restore(worker.cache) next_analysis = 0.0 while not stopped: if not params.get_bool('IsOffroad'): diff --git a/openpilot/starpilot/flm/offroad_worker.py b/openpilot/starpilot/flm/offroad_worker.py index 11fcabcd56..87317f5e99 100644 --- a/openpilot/starpilot/flm/offroad_worker.py +++ b/openpilot/starpilot/flm/offroad_worker.py @@ -10,6 +10,7 @@ import json import math import os from pathlib import Path +from typing import Any from openpilot.starpilot.flm.offroad_surface import ROUTE_NAMES as NAMES from openpilot.starpilot.flm.learned_lifecycle import decode @@ -18,7 +19,7 @@ from openpilot.starpilot.flm.offroad_checks import qualify from openpilot.starpilot.flm.offroad_processor import analyze_latest, CompletedAssessment from openpilot.starpilot.flm.offroad_route import latest_route from openpilot.starpilot.flm.offroad_runtime import configuration_scope -from openpilot.starpilot.flm.offroad_store import RouteStorage, KEY, MAX_BYTES, NAMESPACE +from openpilot.starpilot.flm.offroad_store import RouteStorage, KEY, MAX_BYTES, MAX_ROUTE_STEP, NAMESPACE from openpilot.starpilot.flm.offroad_surface import REVISION from openpilot.starpilot.flm.offroad_tracking import TrackingEstimate, compare_observed from openpilot.starpilot.saved_source import read_saved @@ -231,6 +232,57 @@ def assess_next_drive(trial, assessment, scope, *, trial_started=True): } +def recorded_recommendations(assessment, previous, report): + """Publish original candidates independently of automatic qualification. + + These are read-only starting points, never an approval, Params write or a + replacement profile. Disagreement retains a finding without inventing a + single suggested setting. All original fitting/held-out/native gates remain. + """ + from openpilot.starpilot.flm.offroad_surface import route_bounds + + found: dict[str, dict[str, Any]] = {} + + def add(proposal, windows, reason): + if type(proposal) is not dict or type(windows) is not int or windows <= 0: + return + values = proposal.get('values') + if type(values) is not dict: + name = proposal.get('parameter', '') + if type(name) is not str: + return + name = name.removeprefix('stable:') + delta = proposal.get('delta') + if name not in NAMES or type(delta) not in (int, float) or not math.isfinite(delta): + return + values = {name: previous[name] + delta} + for name in NAMES: + base, target = previous[name], values.get(name, previous[name]) + if type(target) not in (int, float) or not math.isfinite(target): + continue + low, high = route_bounds(name) + if not low <= target <= high or not 1e-12 < abs(target - base) <= MAX_ROUTE_STEP + 1e-12: + continue + if name in found: + row = found[name] + row['whole_windows'] = max(windows, row['whole_windows']) + if row['target'] is None or abs(row['target'] - target) > 1e-12: + row.update(target=None, reason='conflicting_recorded_candidates') + else: + found[name] = {'parameter': name, 'baseline': base, 'target': target, 'whole_windows': windows, 'reason': reason} + + for group in (*assessment.groups, *assessment.diagnostic_groups): + for cohort in group['cohorts']: + add(cohort.get('proposal'), cohort.get('whole_windows', 0), 'recorded_candidate_requires_automatic_checks') + for proof in (*report.get('stable_profile_reports', ()), *report.get('short_turn_profile_reports', ())): + roles = proof.get('roles', {}) + windows = sum(len(roles.get(role, {}).get('original_windows', ())) for role in ('training', 'heldout')) + add(proof.get('frozen_training_profile'), windows, proof.get('reason', 'recorded_candidate_requires_automatic_checks')) + for proposal in roles.get('training', {}).get('coordinate_candidates', ()): + add(proposal, windows, 'training_candidate_not_independently_confirmed') + return [found[name] for name in NAMES if name in found] + + def qualification_summary(checks): refusals = {} unperformed = checks.get('unperformed_windows', 0) @@ -283,6 +335,7 @@ class OffroadWorker: scope_for=_current_scope, on_analysis=None, on_assessment=None, + on_recommendations=None, prior_provider=None, stable_prior_provider=None, ): @@ -294,6 +347,7 @@ class OffroadWorker: self.pending_failure_data = None self.on_analysis = on_analysis self.on_assessment = on_assessment + self.on_recommendations = on_recommendations self.prior_provider = prior_provider self.stable_prior_provider = stable_prior_provider @@ -378,18 +432,24 @@ class OffroadWorker: raise ValueError('Source or settings changed after fitting') # A qualification failure must not erase completed, source-bound measurements. # Keep their original authority check; no suggestion, save or application is implied. - self.pending_failure_data = ( - authorized, - { - 'completed_route': assessment.receipt.route_id, - 'source_sha256': assessment.receipt.source_sha256, - 'vehicle': str(assessment.cp.carFingerprint), - 'analysis_counts': dict(assessment.counts), - 'recorded_tracking': assessment.recorded_tracking, - }, - ) + recorded: dict[str, Any] = { + 'completed_route': assessment.receipt.route_id, + 'source_sha256': assessment.receipt.source_sha256, + 'vehicle': str(assessment.cp.carFingerprint), + 'analysis_counts': dict(assessment.counts), + 'recorded_tracking': assessment.recorded_tracking, + } + self.pending_failure_data = (authorized, recorded) report = assessment.report() report.update(policy=POLICY, current_scope=scope.document(), road_improvement_confirmed=False) + current = dict(ZERO) if row is None else dict(row['values']) + + def recommendations_ready(): + recorded['recommendations'] = recorded_recommendations(assessment, current, report) + if self.on_recommendations is not None: + self.on_recommendations(dict(recorded)) + + recommendations_ready() reason = 'more_independent_eligible_tracking_required' status = 'collecting' saved = False @@ -539,6 +599,7 @@ class OffroadWorker: choices.append((group, {**cohort, 'stable_profile_proof': proof}, alternative)) # Freeze input-only ranking; training may select a candidate, validation may not. choices.sort(key=lambda v: (-input_coverage(v[1]), -v[1]['whole_windows'], v[1]['cohort'], v[0]['original_context_sha256'])) + recommendations_ready() training_seen = set() training_spans = set() for group, cohort, proposal in choices: @@ -708,6 +769,7 @@ class OffroadWorker: 'vehicle': str(assessment.cp.carFingerprint), 'recorded_tracking': report.get('recorded_tracking'), 'suggestion_count': len(diagnostics) + len(profile_suggestions), + 'recommendations': recorded_recommendations(assessment, current, report), 'suggestions': profile_suggestions + [ {'parameter': v['parameter'], 'delta': v['delta'], 'whole_windows': c['whole_windows'], 'qualification': 'controller_native_checks_not_passed'} @@ -727,6 +789,8 @@ class OffroadWorker: if configured is not None: result['configured_values'] = dict(configured['values']) result['configuration_state'] = configured['state'] + result['recommendations'] = [r for r in result['recommendations'] if r['target'] is None or r['target'] != configured['values'][r['parameter']]] + result['recommendation_count'] = len(result['recommendations']) # A new started/changed source cannot be marked processed under its new ID. if not authorized(): raise ValueError('Offroad source changed before checkpoint') diff --git a/openpilot/starpilot/flm/tests/test_offroad_coordinate_policy.py b/openpilot/starpilot/flm/tests/test_offroad_coordinate_policy.py index 67cefa7a33..10f187853e 100644 --- a/openpilot/starpilot/flm/tests/test_offroad_coordinate_policy.py +++ b/openpilot/starpilot/flm/tests/test_offroad_coordinate_policy.py @@ -578,3 +578,208 @@ class TestBoundedInconclusiveTrialLifecycle(unittest.TestCase): self.assertEqual(row['values'], original['values']) self.assertEqual(row['previous'], values()) self.assertEqual(cache.get(digest), original) + + +class TestRecordedAdvisoryVisibility(unittest.TestCase): + """The advisory survives a refused automatic save; no hardware acceptance fixture.""" + + @staticmethod + def worker(directory): + from types import SimpleNamespace + from openpilot.starpilot.flm.offroad_cache import AssessmentCache, directory_for_params + from openpilot.starpilot.flm.offroad_processor import CompletedAssessment + from openpilot.starpilot.flm.offroad_route import RouteReceipt + from openpilot.starpilot.flm.offroad_worker import OffroadWorker + + params = Params(directory) + cache = AssessmentCache(directory_for_params(params)) + _, observed = TestComparableNextDriveOutcomes.documents() + group = observed.stable_groups[0] + group['original_metadata']['route_values'] = list(values().items()) + group['cohorts'][0]['proposal'] = {'parameter': 'highway_center_taper_max', 'delta': 0.005, 'values': dict(values(), highway_center_taper_max=0.005)} + inventory = {'routeId': 'declared-advisory-route'} + receipt = RouteReceipt(inventory['routeId'], 'd' * 64, (), 'a' * 64, 1, 2, 100, ()) + assessment = CompletedAssessment(receipt, (group,), {}, SimpleNamespace(carFingerprint='declared-unit-car'), (), inventory) + + def assessed(params, root, prior, *, permitted): + prior(assessment.cp) + return assessment + + def refused(*args, **kwargs): + return {'qualified': False, 'windows': [], 'policy': 'declared-unit-refusal'} + + worker = OffroadWorker(params, directory, cache, permitted=lambda: True, assessor=assessed, + checker=refused, scope_for=lambda cp, params: scope()) + return worker, params, inventory + + def test_refused_automatic_change_retains_its_per_dial_recommendation(self): + from openpilot.starpilot.flm import offroad_worker as module + + with tempfile.TemporaryDirectory() as directory: + worker, params, inventory = self.worker(directory) + with patch.object(module, 'latest_route', return_value=inventory): + result = worker.run_once() + rows = result.get('recommendations', []) + self.assertEqual([r['parameter'] for r in rows], ['highway_center_taper_max']) + self.assertEqual(rows[0]['baseline'], 0) + self.assertEqual(rows[0]['target'], 0.005) + self.assertFalse(result['configuration_changed']) + self.assertFalse(result['actual_application_proven']) + self.assertIsNone(read_saved(params, KEY, MAX_BYTES)[0], 'An advisory must not save the refused tune') + + +class TestRecordedRecommendationMetadata(unittest.TestCase): + @staticmethod + def assessment(proposals=()): + from types import SimpleNamespace + + cohorts = [{'whole_windows': 6, 'proposal': proposal} for proposal in proposals] + return SimpleNamespace(groups=({'cohorts': cohorts},), diagnostic_groups=()) + + def test_captured_unconfirmed_profile_is_visible_without_promoting_it(self): + from openpilot.starpilot.flm.offroad_worker import recorded_recommendations + + # Exact four targets from the retained last-drive assessment; no original + # route/window/VIN identifiers or invented controller qualification. + targets = dict(values(), friction_threshold_low=0.002, friction_threshold_mid=0.004, + friction_threshold_fast=0.005, friction_threshold_highway=0.006) + proof = {'reason': 'heldout_profile_does_not_support_training_step', 'frozen_training_profile': {'values': targets}, + 'roles': {role: {'original_windows': list(range(8)), 'coordinate_candidates': []} for role in ('training', 'heldout')}} + result = recorded_recommendations(self.assessment(), values(), {'stable_profile_reports': [proof]}) + self.assertEqual({r['parameter']: r['target'] for r in result}, {n: v for n, v in targets.items() if v}) + self.assertEqual(len(result), 4) + self.assertTrue(all(r['whole_windows'] == 16 and r['reason'] == proof['reason'] for r in result)) + self.assertEqual(proof['frozen_training_profile']['values'], targets) + + def test_all_thirty_five_coordinates_survive_without_a_sixteen_row_limit(self): + from openpilot.starpilot.flm.offroad_surface import route_bounds + from openpilot.starpilot.flm.offroad_worker import recorded_recommendations + + targets = {name: 0.005 if route_bounds(name)[1] >= 0.005 else -0.005 for name in ROUTE_NAMES} + result = recorded_recommendations(self.assessment([{'values': targets}]), values(), {}) + self.assertEqual([r['parameter'] for r in result], list(ROUTE_NAMES)) + self.assertEqual(len(result), 35) + self.assertTrue(all(r['baseline'] == 0 for r in result)) + + def test_conflicting_candidates_do_not_invent_a_best_setting(self): + from openpilot.starpilot.flm.offroad_worker import recorded_recommendations + + proposals = [{'values': {'center_taper_max': v}} for v in (0.005, 0.006)] + result = recorded_recommendations(self.assessment(proposals), values(), {}) + self.assertEqual(len(result), 1) + self.assertIsNone(result[0]['target']) + self.assertEqual(result[0]['reason'], 'conflicting_recorded_candidates') + self.assertEqual(result[0]['whole_windows'], 6, 'Shared windows cannot be double-counted') + + def test_invalid_or_out_of_envelope_candidates_are_not_suggestions(self): + from openpilot.starpilot.flm.offroad_worker import recorded_recommendations + + proposals = [{'values': {'center_taper_max': v}} for v in (True, float('nan'), 0.25)] + self.assertEqual(recorded_recommendations(self.assessment(proposals), values(), {}), []) + + def test_training_recipe_without_a_repeated_full_map_is_reported(self): + from openpilot.starpilot.flm.offroad_worker import recorded_recommendations + + proof = {'roles': {'training': {'original_windows': ['a', 'b', 'c'], + 'coordinate_candidates': [{'parameter': 'stable:turn_in_boost_left', 'delta': 0.005}]}, + 'heldout': {'original_windows': ['d', 'e', 'f']}}} + result = recorded_recommendations(self.assessment(), values(), {'short_turn_profile_reports': [proof]}) + self.assertEqual(result[0]['parameter'], 'turn_in_boost_left') + self.assertEqual(result[0]['target'], 0.005) + self.assertEqual(result[0]['reason'], 'training_candidate_not_independently_confirmed') + + def test_failed_checker_keeps_already_published_candidates(self): + from openpilot.starpilot.flm import offroad_worker as module + + with tempfile.TemporaryDirectory() as directory: + worker, params, inventory = TestRecordedAdvisoryVisibility.worker(directory) + snapshots = [] + worker.on_recommendations = snapshots.append + + def failed(*args, **kwargs): + raise ValueError('declared controller-check failure') + + worker.checker = failed + with patch.object(module, 'latest_route', return_value=inventory): + with self.assertRaisesRegex(ValueError, 'declared controller-check failure') as error: + worker.run_once() + result = worker.record_failure(error.exception) + self.assertIsNotNone(result) + assert result is not None + self.assertEqual(result['recommendations'][0]['target'], 0.005) + self.assertEqual(snapshots[0]['recommendations'][0]['parameter'], 'highway_center_taper_max') + self.assertIsNone(read_saved(params, KEY, MAX_BYTES)[0]) + + +class TestRetainedDriveRecommendationReport(unittest.TestCase): + def test_another_analysis_keeps_historical_findings_but_never_current_permissions(self): + from openpilot.starpilot.flm.offroad_daemon import StatusReport + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'report.json' + status = StatusReport(path=path, clock_ns=lambda: 1_000_000_000) + status.set({'status': 'collecting', 'reason': 'previous', 'completed_route': 'declared-last-drive', 'vehicle': 'declared-car', + 'analysis_counts': {'recorded_tracking': 120}, 'configuration_state': 'ready_for_next_drive', 'configured_values': values(0.01), + 'enabled': True, 'actual_coordinates': list(values(0.01).items()), 'actual_control_ns': 1_000_000_000, + 'recommendations': [{'parameter': 'ff_gain_left', 'baseline': 0, 'target': 0.005, 'whole_windows': 6, 'reason': 'unconfirmed'}]}) + status.analyzing({'reason': 'analyzing_last_completed_drive'}) + status.publish() + report = json.loads(path.read_bytes()) + details = report['current']['details'] + self.assertTrue(details['Earlier analysis shown while updating']) + self.assertEqual(details['Completed drive analyzed'], 'declared-last-drive') + self.assertEqual(details['Analysis decision'], 'previous') + self.assertEqual(details['Recorded coordinate recommendations'][0][2], 0.005) + self.assertNotIn('Saved route configuration state', details) + self.assertNotIn('Actual full surface coordinates', details) + self.assertFalse(report['current']['application']['applied']) + self.assertIsNone(report['current']['enabled']) + + def test_full_recommendation_wire_is_bounded_and_backward_compatible(self): + from openpilot.starpilot.flm.offroad_daemon import StatusReport + from openpilot.starpilot.flm.training_report import READ_MAX_BYTES, read_report + + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'report.json' + status = StatusReport(path=path, clock_ns=lambda: 1_000_000_000) + status.set({'status': 'collecting', 'reason': 'declared', 'completed_route': 'declared-last-drive', + 'recommendations': [{'parameter': n, 'baseline': 0, 'target': 0.005, 'whole_windows': 6, 'reason': 'unconfirmed'} for n in ROUTE_NAMES]}) + status.publish() + report = read_report(path, now_ns=1_000_000_000) + self.assertIsNotNone(report['current']) + assert report['current'] is not None + details = report['current']['details'] + self.assertEqual(len(details['Recorded coordinate recommendations']), 35) + self.assertEqual(len(details['Recorded route suggestions']), 16) + self.assertEqual(details['Recorded suggestion count'], 35) + self.assertFalse(details['More recorded suggestions omitted']) + self.assertLess(path.stat().st_size, READ_MAX_BYTES) + self.assertFalse(report['current']['application']['applied']) + + def test_startup_recovers_existing_candidates_without_rewriting_the_cache(self): + from openpilot.starpilot.flm.offroad_cache import AssessmentCache + from openpilot.starpilot.flm.offroad_daemon import StatusReport + + with tempfile.TemporaryDirectory() as directory: + cache = AssessmentCache(Path(directory) / 'cache') + assessment = {'route': {'route_id': 'declared-old-drive', 'source_sha256': 'c' * 64, 'cp_sha256': 'a' * 64}, + 'groups': [{'original_metadata': {'cp_sha256': 'a' * 64, 'route_revision': REVISION, 'route_values': list(values().items())}, + 'cohorts': [{'whole_windows': 148, 'proposal': {'values': dict(values(), highway_center_taper_max=0.005)}}]}]} + digest = cache.put({'kind': 'completed_assessment', 'assessment': assessment}) + cache.checkpoint({'input_fingerprint': 'f' * 64, 'result': {'status': 'collecting', 'completed_route': 'declared-old-drive', + 'source_sha256': 'c' * 64, 'assessment_sha256': digest, 'reason': 'no_new_native_feasible_training_step'}}) + before = {p.name: p.read_bytes() for p in cache.directory.iterdir() if p.is_file()} + path = Path(directory) / 'report.json' + status = StatusReport(path=path, clock_ns=lambda: 1_000_000_000) + status.restore(cache) + status.analyzing({'reason': 'analyzing_last_completed_drive'}) + status.publish() + report = json.loads(path.read_bytes()) + self.assertEqual(report['current']['details']['Recorded coordinate recommendations'][0][2], 0.005) + self.assertFalse(report['current']['application']['applied']) + self.assertEqual({p.name: p.read_bytes() for p in cache.directory.iterdir() if p.is_file()}, before) + wrong = StatusReport(path=path, clock_ns=lambda: 1_000_000_000) + cache.checkpoint({'input_fingerprint': 'f' * 64, 'result': {'status': 'collecting', 'completed_route': 'wrong-route', + 'source_sha256': 'c' * 64, 'assessment_sha256': digest, 'reason': 'declared'}}) + wrong.restore(cache) + self.assertEqual(wrong.previous, {}, 'Historical candidate inputs must match their exact completed source') diff --git a/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs b/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs index 2c3910f73d..c847317da3 100644 --- a/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs +++ b/openpilot/starpilot/galaxy/tests/test_flm_tracking_results.mjs @@ -2,7 +2,7 @@ import assert from 'node:assert/strict' import { compile } from '../web/vendor/vue/vue.esm-browser.js' import { FlmPage } from '../web/js/flm.js' import { GalaxySettingRow } from '../web/js/galaxy-setting-row.js' -import { SURFACE_NAMES, PID_UNSUPPORTED, validTrainingResults, validTrackingResults, trackingView, surfaceRows, measurementText, validTrackingValues, trackingCoordinates, validTrackingPlot, trackingPlotView, sourceSeconds, tunerStatus } from '../web/js/flm-training-results.js' +import { SURFACE_NAMES, FRICTION_NAMES, offroadView, offroadCoordinateEvidence, recommendationText, PID_UNSUPPORTED, validTrainingResults, validTrackingResults, trackingView, surfaceRows, measurementText, validTrackingValues, trackingCoordinates, validTrackingPlot, trackingPlotView, sourceSeconds, tunerStatus } from '../web/js/flm-training-results.js' const coordinates = amount => SURFACE_NAMES.map((name, i) => [name, i === 0 ? amount : 0]) const tracking = () => ({ version: 1, status: 'trial_active', reason: 'scoring_future_tracking', group: 'feedforward', @@ -357,3 +357,70 @@ for (const reason of ['trial_started_waiting_for_comparable_drive', 'trial_measu assert.equal(ctx.currentApplied, false, 'A pending retry cannot fabricate applied or accepted steering') } console.log('Inconclusive next-drive trials remain visibly provisional without fabricated application') + +// Complete, read-only dial advice is independent of automatic-tune approval. +const advisoryNames = [...SURFACE_NAMES, ...FRICTION_NAMES] +const advisoryReport = { version: 1, vehicle: 'DECLARED_CAR', history: [], current: { fresh: true, ageSeconds: 0, + enabled: null, status: 'collecting', reason: 'no_new_native_feasible_training_step', progress: { samples: 100, blocks: 6, requiredBlocks: 6 }, + fit: null, application: { applied: false, reason: 'not_applied' }, details: { + 'Completed drive analyzed': 'declared-last-drive', 'Recorded steering observations': 100, + 'Analysis decision': 'no_new_native_feasible_training_step', 'Recorded route suggestions': [], + 'Recorded coordinate recommendations': advisoryNames.map(name => [name, 0, .005, 6, 'training_candidate_not_independently_confirmed']), + } } } +assert(validTrainingResults(advisoryReport)) +assert.equal(offroadView(advisoryReport).suggestions.length, 35, 'All thirty dials and five curve points survive the public wire') +assert.equal(surfaceRows(advisoryReport).filter(row => row.recommendations.length).length, 30) +assert(offroadCoordinateEvidence(advisoryReport, 'ff_gain_left', false, '%').includes('suggested +0.5% (from 0%; change +0.5%)')) +assert.equal(advisoryReport.current.application.applied, false) +manualContext.report = advisoryReport +manualContext.now = 0 +manualContext.receivedAt = 0 +manualContext.manualState = { status: 'ready', pending: null, error: '', data: { rows: [{ + coordinate: 'ff_gain_left', label: 'Steering gain', reason: manualHelp, available: true, action: true, + value: 0, defaultValue: 0, resetAvailable: true, minimum: -25, maximum: 25, step: .25, unit: '%', +}] } } +const advisoryTree = render.call(manualContext, manualContext, []) +assert(text(advisoryTree).includes('suggested +0.5%'), 'The suggested slider-scale value must appear beside the actual manual dial') +assert(text(advisoryTree).includes('not saved')) +const untouchedManual = vnodeRows(advisoryTree).find(node => node.props?.row?.coordinate === 'ff_gain_left') +assert.equal(untouchedManual.props.row.value, 0, 'Showing advice cannot move a manual slider or enable an override') +const singleAdvisory = structuredClone(advisoryReport) +singleAdvisory.current.details['Recorded coordinate recommendations'] = singleAdvisory.current.details['Recorded coordinate recommendations'].slice(0, 1) +assert(offroadCoordinateEvidence(singleAdvisory, 'unwind_taper_right').includes('No recommendation from this analyzed drive')) +const conflictingAdvisory = structuredClone(singleAdvisory) +conflictingAdvisory.current.details['Recorded coordinate recommendations'][0][2] = null +conflictingAdvisory.current.details['Recorded coordinate recommendations'][0][4] = 'conflicting_recorded_candidates' +assert(validTrainingResults(conflictingAdvisory)) +assert(recommendationText(offroadView(conflictingAdvisory).suggestions[0]).includes('no single suggested setting')) +for (const row of [ ['unknown', 0, .005, 6, 'unconfirmed'], ['ff_gain_left', 0, NaN, 6, 'unconfirmed'], + ['ff_gain_left', 0, .02, 6, 'unconfirmed'], ['ff_gain_left', true, .005, 6, 'unconfirmed'], + ['ff_gain_left', 0, .005, -1, 'unconfirmed'], ['ff_gain_left', 0, null, 6, 'unconfirmed'] ]) { + const invalid = structuredClone(singleAdvisory) + invalid.current.details['Recorded coordinate recommendations'] = [row] + assert(!validTrainingResults(invalid), 'Invalid advice cannot pass the public report boundary') + assert.equal(offroadView(invalid), null) +} +const duplicateAdvisory = structuredClone(singleAdvisory) +duplicateAdvisory.current.details['Recorded coordinate recommendations'].push(...duplicateAdvisory.current.details['Recorded coordinate recommendations']) +assert(!validTrainingResults(duplicateAdvisory)) +const arbitraryList = structuredClone(advisoryReport) +arbitraryList.current.details['Unknown detail'] = Array(17).fill(1) +assert(!validTrainingResults(arbitraryList), 'The dedicated recommendation wire must not loosen unrelated telemetry bounds') +const historicalAdvisory = structuredClone(singleAdvisory) +historicalAdvisory.current.status = 'fitting' +historicalAdvisory.current.reason = 'analyzing_last_completed_drive' +historicalAdvisory.current.details['Earlier analysis shown while updating'] = true +assert(tunerStatus(historicalAdvisory, false, false, '', true).description.includes('Earlier completed-drive findings')) +assert(offroadCoordinateEvidence(historicalAdvisory, 'ff_gain_left').startsWith('Earlier analyzed drive')) +historicalAdvisory.preference = { version: 1, valid: true, enabled: true, parked: true, vehicle: 'OTHER_CAR' } +assert.equal(offroadView(historicalAdvisory), null, 'Earlier advice must not be relabeled for another reported vehicle') +console.log('All thirty dial targets, five curve points, honest conflicts and retained findings render without automatic approval or manual writes') + +for (const name of SURFACE_NAMES) assert(recommendationText({ parameter: name, baseline: 0, target: .005, delta: .005 }).includes('+0.5%'), 'Advice must use the same percent scale as TorqueFeature.manual_rows for every dial') +const nowSavedAdvisory = structuredClone(advisoryReport) +nowSavedAdvisory.current.details['Saved route configuration state'] = 'ready_for_next_drive' +nowSavedAdvisory.current.details['Full surface coordinates'] = coordinates(.005) +assert.equal(offroadView(nowSavedAdvisory).suggestions.filter(row => row.parameter === 'ff_gain_left').length, 0) +assert(offroadCoordinateEvidence(nowSavedAdvisory, 'ff_gain_left', false, '%').includes('Auto Learn saved: +0.5% for the next drive')) +assert(!offroadCoordinateEvidence(nowSavedAdvisory, 'ff_gain_left', false, '%').includes('Previous-drive recommendation')) +assert.equal(nowSavedAdvisory.current.application.applied, false, 'A saved recommendation is still not current application') diff --git a/openpilot/starpilot/galaxy/web/js/flm-training-results.js b/openpilot/starpilot/galaxy/web/js/flm-training-results.js index f0cd33c95c..d3f4082bf3 100644 --- a/openpilot/starpilot/galaxy/web/js/flm-training-results.js +++ b/openpilot/starpilot/galaxy/web/js/flm-training-results.js @@ -53,7 +53,8 @@ export function validTrainingResults(value) { key === 'Applied correction coordinates' && Array.isArray(item) && item.length === 30 ? item.every(number => typeof number === 'number' && Number.isFinite(number) && Math.abs(number) <= .25) : key === 'GLOBAL source observation refusals' ? validSourceRefusals(item) : - key === 'Recorded steering response' ? item === null || recordedSteeringView(item) !== null : detailValue(item))) + key === 'Recorded steering response' ? item === null || recordedSteeringView(item) !== null : + key === 'Recorded coordinate recommendations' ? validCoordinateRecommendations(item) : detailValue(item))) } const labels = { collecting: "Gathering driving examples", unavailable: "Waiting for usable driving data", qualifiedFit: "Fit passed validation", qualified: "Fit passed validation", rejected: "Fit did not pass validation", @@ -149,16 +150,27 @@ const frictionCoordinates = rows => Array.isArray(rows) && rows.length === 5 && Array.isArray(row) && row.length === 2 && row[0] === FRICTION_NAMES[i] && typeof row[1] === 'number' && Number.isFinite(row[1]) && Math.abs(row[1]) <= .25) ? Object.fromEntries(rows) : null -export function offroadCoordinateEvidence(report, name, live = false) { +export function recommendationText(row, unit = null) { + if (row.target === null) return 'Conflicting recorded candidates; no single suggested setting' + if (row.target === undefined) return `change ${coordinateText(row.delta)}` + // The setting owner displays every canonical coordinate as percent, including window edges. + const suffix = unit ?? '%' + const display = value => `${coordinateText(suffix === '%' ? value * 100 : value)}${suffix === 'm/s' ? ' ' : ''}${suffix}` + return `suggested ${display(row.target)} (from ${display(row.baseline)}; change ${display(row.delta)})` +} + +export function offroadCoordinateEvidence(report, name, live = false, unit = null) { const route = offroadView(report), row = surfaceRows(report, live).find(row => row.name === name) const statements = [] + const display = value => unit === '%' ? `${coordinateText(value * 100)}%` : coordinateText(value) + (unit || '') if (row?.applied !== null && row?.applied !== undefined && row.applied !== 0) - statements.push(`Auto Learn applied now: ${coordinateText(row.applied)}.`) + statements.push(`Auto Learn applied now: ${display(row.applied)}.`) const saved = route?.savedValues?.[name] if (saved !== undefined && saved !== 0) - statements.push(`Auto Learn saved: ${coordinateText(saved)}${route.configurationState === 'ready_for_next_drive' ? ' for the next drive' : ''}.`) + statements.push(`Auto Learn saved: ${display(saved)}${route.configurationState === 'ready_for_next_drive' ? ' for the next drive' : ''}.`) for (const suggestion of route?.suggestions || []) if (suggestion.parameter === name) - statements.push(`Previous-drive recommendation: change ${coordinateText(suggestion.delta)}; not saved.`) + statements.push(`${route.retained ? 'Earlier analyzed drive' : 'Previous-drive recommendation'}: ${recommendationText(suggestion, unit)}; not saved.${suggestion.reason ? ` ${learningReason(suggestion.reason)}` : ''}`) + if (route && !statements.length) statements.push('No recommendation from this analyzed drive for this dial.') return statements.join(' ') } @@ -179,6 +191,11 @@ export const SURFACE_GROUPS = Object.freeze([ ]) const REASONS = { + heldout_profile_does_not_support_training_step: "Part of the drive suggested this setting; separate windows did not confirm it.", + independent_support_for_frozen_training_profile: "Separate recorded windows support this starting point. Automatic approval is separate.", + training_candidate_not_independently_confirmed: "Training windows suggested this setting; it is not independently confirmed.", + recorded_candidate_requires_automatic_checks: "A recorded-drive starting point, not approved for automatic saving.", + conflicting_recorded_candidates: "Different windows suggested different settings. No single setting is recommended.", trial_started_waiting_for_comparable_drive: "The trial ran, but this drive lacked matching comparison windows. The same bounded trial will be checked on another drive.", trial_measurement_inconclusive: "The comparison did not establish improvement or worsening. The same bounded trial will be checked on another drive.", trial_inconclusive_drive_limit: "Three trial drives did not establish an improvement. The prior tune was restored.", @@ -230,15 +247,29 @@ const HISTORY_LABELS = { collecting: 'Collecting samples', calibrating: 'Measuri trial_active: 'Evaluating a trial', confirmed: 'Correction confirmed', rollback: 'Returning to baseline', paused: 'Learning paused' } export const historyTitle = status => Object.hasOwn(HISTORY_LABELS, status) ? HISTORY_LABELS[status] : readable(status) +export function validCoordinateRecommendations(rows) { + const names = [...SURFACE_NAMES, ...FRICTION_NAMES] + return Array.isArray(rows) && rows.length <= names.length && new Set(rows.map(row => row?.[0])).size === rows.length && + rows.every(row => Array.isArray(row) && row.length === 5 && names.includes(row[0]) && + typeof row[1] === 'number' && Number.isFinite(row[1]) && Math.abs(row[1]) <= 1000 && + (row[2] === null && row[4] === 'conflicting_recorded_candidates' || + typeof row[2] === 'number' && Number.isFinite(row[2]) && Math.abs(row[2]) <= 1000 && + Math.abs(row[2] - row[1]) > 1e-12 && Math.abs(row[2] - row[1]) <= .01 + 1e-12) && + stamp(row[3]) && row[3] > 0 && text(row[4], 240)) +} + export function offroadView(report) { const details = report?.current?.details if (!record(details) || !text(details['Completed drive analyzed'], 180) || !details['Completed drive analyzed'] || !stamp(details['Recorded steering observations']) || !text(details['Analysis decision'])) return null + if (report.preference?.valid && report.vehicle && report.preference.vehicle !== report.vehicle) return null const rows = details['Recorded route suggestions'] ?? [] if (!Array.isArray(rows) || rows.length > 16 || rows.some(row => !Array.isArray(row) || row.length !== 4 || !text(row[0], 500) || typeof row[1] !== 'number' || !Number.isFinite(row[1]) || Math.abs(row[1]) > .25 || !stamp(row[2]) || row[3] !== 'controller_native_checks_not_passed')) return null const observed = recordedSteeringView(details['Recorded steering response']) + const recommendations = details['Recorded coordinate recommendations'] + if (recommendations !== undefined && !validCoordinateRecommendations(recommendations)) return null const savedCount = details['Saved route adjustment count'] const checked = details['Controller and encoder check windows'], refused = details['Automatic tune refusal reasons'] const checks = Array.isArray(checked) && checked.length === 2 && checked.every(stamp) && checked[0] <= checked[1] && checked[1] <= 36 && @@ -252,7 +283,11 @@ export function offroadView(report) { reason: details['Analysis decision'], saved: details['Automatic configuration changed'] === true, configurationState, savedValues: savedSurface ? { ...savedSurface, ...(savedFriction || {}) } : null, observed, checks, savedCount: stamp(savedCount) && savedCount <= 35 ? savedCount : null, - suggestions: rows.map(([parameter, delta, windows, qualification]) => ({ parameter, delta, windows, qualification })), + retained: details['Earlier analysis shown while updating'] === true, + suggestions: recommendations === undefined ? rows.map(([parameter, delta, windows, qualification]) => ({ parameter, delta, windows, qualification })) : + recommendations.map(([parameter, baseline, target, windows, reason]) => ({ parameter, baseline, target, + delta: target === null ? null : target - baseline, windows, reason, qualification: 'controller_native_checks_not_passed' })) + .filter(row => row.target === null || row.target !== { ...savedSurface, ...savedFriction }[row.parameter]), moreSuggestions: details['More recorded suggestions omitted'] === true } } @@ -280,7 +315,7 @@ export function tunerStatus(report, live = false, choiceLive = false, error = "" if (preference?.valid && preference.enabled === false || !preference && live && current?.enabled === false) return { key: "off", tone: "muted", title: "Auto Learn is off", description: "No driving samples are being collected by the learner. Turn it on in steering settings to begin." } if (reportFresh && current?.status === 'fitting' && ['analyzing_last_completed_drive', 'checking_recorded_steering_recommendations'].includes(current.reason)) - return { key: 'fitting', tone: 'active', title: 'Analyzing your completed drive', description: learningReason(current.reason) } + return { key: 'fitting', tone: 'active', title: 'Analyzing your completed drive', description: learningReason(current.reason) + (offroadView(report)?.retained ? ' Earlier completed-drive findings remain below while this analysis runs.' : '') } const route = reportFresh && (!preference || preference.parked) ? offroadView(report) : null if (route) { if (route.configurationState === 'ready_for_next_drive') diff --git a/openpilot/starpilot/galaxy/web/js/flm.js b/openpilot/starpilot/galaxy/web/js/flm.js index a8782c7a99..23fa6d2178 100644 --- a/openpilot/starpilot/galaxy/web/js/flm.js +++ b/openpilot/starpilot/galaxy/web/js/flm.js @@ -6,7 +6,7 @@ import { GalaxySettingRow, settingControl } from "./galaxy-setting-row.js" import { PollTimer, connectionError } from "./polling.js" import { requestJson } from "./startup.js" import { PlotGraph, PlotsPage, chartGeometry } from "./plots.js" -import { validTrainingResults, readable, detailText, surfaceRows, coordinateText, trackingView, measurementText, validTrackingValues, trackingPlotView, sourceSeconds, validLearningPreference, SURFACE_GROUPS, tunerStatus, learningReason, sourceCaptureCounts, historyTitle, offroadView, FRICTION_NAMES, offroadCoordinateEvidence } from "./flm-training-results.js" +import { validTrainingResults, readable, detailText, surfaceRows, coordinateText, trackingView, measurementText, validTrackingValues, trackingPlotView, sourceSeconds, validLearningPreference, SURFACE_GROUPS, tunerStatus, learningReason, sourceCaptureCounts, historyTitle, offroadView, FRICTION_NAMES, offroadCoordinateEvidence, recommendationText } from "./flm-training-results.js" const VIEWS = ["overview", "tune", "activity", "plots"] const PARAMETER_LABELS = { @@ -77,6 +77,7 @@ export const FlmPage = { trackingPlot() { return trackingPlotView(this.report, this.now - this.receivedAt) }, offroad() { return offroadView(this.report) }, savedAdjustments() { return Object.entries(this.offroad?.savedValues || {}).filter(([, value]) => value !== 0).map(([name, value]) => ({ name, value })) }, + frictionRecommendations() { return (this.offroad?.suggestions || []).filter(row => FRICTION_NAMES.includes(row.parameter)) }, savedFriction() { return this.savedAdjustments.filter(row => FRICTION_NAMES.includes(row.name)) }, recordedTrackingChart() { return chartGeometry(this.offroad?.observed?.points || [], ['desired', 'actual']) }, pairedTrackingChart() { return chartGeometry(this.trackingPlot?.points || [], ["desiredForceAccel", "measuredForceAccel"]) }, @@ -103,8 +104,8 @@ export const FlmPage = { }, }, methods: { - readable, detailText, coordinateText, measurementText, sourceSeconds, learningReason, historyTitle, - manualEvidence(name) { return offroadCoordinateEvidence(this.report, name, this.live) }, + readable, detailText, coordinateText, measurementText, sourceSeconds, learningReason, historyTitle, recommendationText, + manualEvidence(name, unit) { return offroadCoordinateEvidence(this.report, name, this.live, unit) }, detailLabel(key) { return ['Fit attempts', 'Fit results', 'Fit in progress', 'Submitted fit jobs (reporter lifetime)'].includes(key) ? 'Legacy fitter · ' + key : readable(key) }, amount(value) { return value === null || value === undefined ? "—" : value.toLocaleString() }, parameterLabel(name) { return PARAMETER_LABELS[name] || readable(name) }, @@ -202,7 +203,7 @@ export const FlmPage = {

{{ offroad ? 'RECORDED-DRIVE ANALYSIS' : live ? 'THE CURRENT EXPERIMENT' : 'LAST REPORTED EXPERIMENT' }}

{{ offroad ? 'Next-drive tuning' : tracking?.acknowledged && tracking.status === 'trial_active' ? 'Trial in progress' : 'Trial results' }}

{{ learningReason(tracking.reason) }}

{{ learningReason(offroad.reason) }}

Waiting for driving samples. Auto Learn compares steering error before and during a trial.

- +
{{ offroad ? 'Recorded baseline' : 'Before' }}{{ measurementText(offroad?.observed?.errorRms ?? tracking?.baselineErrorRms ?? null) }}
Trial{{ offroad ? offroad.configurationState === 'ready_for_next_drive' ? 'Awaiting next-drive measurement' : 'No pending trial reported' : measurementText(tracking?.futureErrorRms ?? null) }}

Tracking error RMS · {{ offroad?.observed?.units ?? 'm/s²' }} · lower is better

{{ Math.abs(improvement).toFixed(1) }}% {{ improvement >= 0 ? 'lower' : 'higher' }} observed error{{ state.key === 'confirmed' ? 'Comparison passed; controller acknowledged' : 'Observation so far, not a confirmed improvement' }}

Comparable trial windows{{ quota.completed }} / {{ quota.required }}
@@ -211,25 +212,25 @@ export const FlmPage = {

Your tune

Thirty steering parameters, grouped by behavior.

Manual + Auto Learn
-

Auto Learn friction adjustments

These five-point curve adjustments are separate from the thirty manual sliders.

+

Auto Learn friction adjustments

These five-point curve adjustments are separate from the thirty manual sliders.

{{ selectedGroup.label }}

{{ selectedGroup.description }}

Manual tune

Parked edits apply to the next drive. Auto Learn pauses while manual overrides are set. Zero removes an override.

{{ manualState.error }}

{{ manualState.status === 'loading' ? 'Loading saved manual tune…' : 'Manual tuning unavailable. Reconnecting…' }}

{{ manualState.data.rows[0]?.reason || 'Manual tuning is unavailable for this controller.' }}

- +

Auto Learn evidence

These reported results are separate from your manual settings.

-

{{ parameterLabel(row.name) }}

Torque only · PID unsupported
{{ offroad ? 'Saved' : trackingValues ? 'Accepted' : 'Proposal' }}{{ coordinateText(offroad ? row.saved : trackingValues ? row.accepted : row.learned) }}
{{ offroad ? 'Recommendation · not saved' : 'Trial' }}{{ coordinateText(row.pending) }}
Applied now{{ coordinateText(row.applied) }}
+

{{ parameterLabel(row.name) }}

Torque only · PID unsupported
{{ offroad ? 'Saved' : trackingValues ? 'Accepted' : 'Proposal' }}{{ coordinateText(offroad ? row.saved : trackingValues ? row.accepted : row.learned) }}
{{ offroad ? 'Recommendation · not saved' : 'Trial' }}{{ coordinateText(row.pending) }}
Applied now{{ coordinateText(row.applied) }}
How to read these values

Zero is the original baseline. These are correction coordinates, not raw degrees or torque. Speed-window coordinates scale the original speed range; deadband and angle assistance use the controller's physical normalizer. A trial is an offer, not proof of application. All 30 stages exist for torque; 26 exist for torque-actuated PID.

LEARNING ACTIVITY

What the tuner found

Reported observations
- +
  1. {{ historyTitle(row.status) }}

    {{ learningReason(row.reason) }}

    {{ row.time }}

No findings reported yet

This will show real collection, trial and rollback events as the learner reports them.

diff --git a/prebuilt.json b/prebuilt.json index 550f3634d4..071f1ed7db 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "c1637af30fb219debc4ae608f0789e09d5be5e0fea63055823ce49c9225eb6d9", "schema_version": 1} +{"protected_inventory_sha256": "f8e900a800d10c8abcfda066821d1ed3c59aa2a61839bd79e8753add4182c0cc", "schema_version": 1}