diff --git a/openpilot/starpilot/galaxy/drive_analysis.py b/openpilot/starpilot/galaxy/drive_analysis.py index cb9da5065c..18468bc484 100644 --- a/openpilot/starpilot/galaxy/drive_analysis.py +++ b/openpilot/starpilot/galaxy/drive_analysis.py @@ -25,7 +25,7 @@ MAX_ANALYSIS_SECONDS = 30.0 MAX_PAIR_SECONDS = 1.0 MAX_SPEED_MPS = 70.0 MAX_ROUTE_COMPRESSED_BYTES = 256 * 1024 * 1024 -ANALYSIS_VERSION = 4 +ANALYSIS_VERSION = 5 MAX_MODEL_METADATA_SECONDS = 4.0 DISTRACTED_EVENT = 'driverDistracted2' UNRESPONSIVE_EVENT = 'driverUnresponsive3' @@ -441,7 +441,7 @@ def analyze_route(root: Path, route: dict, *, permitted: Callable[[], bool], pre return result except (_Incomplete, LocalLogUnavailable, LogDecodeError, OSError, ValueError) as error: result['reason'] = str(error) or 'Route analysis unavailable' - if (not prefer_rlog and result['reason'] in ('Route samples are missing or gapped', 'Route has no observed duration') and + if (not prefer_rlog and result['reason'] in ('Route samples are missing or gapped', 'Route has no observed duration', 'End of route was not recorded') and isinstance(route, dict) and route.get('segments') and all(segment.get('files', {}).get('rlog') is True for segment in route['segments']) and permitted()): return analyze_route(root, route, permitted=allowed, prefer_rlog=True) diff --git a/openpilot/starpilot/galaxy/drive_stats.py b/openpilot/starpilot/galaxy/drive_stats.py index 3a24206c8d..b091514415 100644 --- a/openpilot/starpilot/galaxy/drive_stats.py +++ b/openpilot/starpilot/galaxy/drive_stats.py @@ -18,6 +18,9 @@ from openpilot.starpilot.galaxy.drive_history import DriveHistory, MAX_ROOT_ENTR from openpilot.starpilot.galaxy.drive_analysis import ANALYSIS_VERSION from openpilot.starpilot.storage import galaxy_storage_root +SOURCE_LIMITATIONS = frozenset({'Route has no observed duration', 'End of route was not recorded', + 'Route has missing segments or logs', 'Route samples are missing or gapped'}) + SCHEMA_VERSION = 1 MAX_CACHED_ROUTES = 2048 MAX_CACHE_BYTES = 4_000_000 @@ -306,7 +309,7 @@ class DriveStatsOwner: continue if failed and failed['signature'] == signature and failed.get('analysisVersion', 0) == ANALYSIS_VERSION: now = self.clock() - if (failed['attempts'] >= MAX_RETRY_ATTEMPTS or now - failed['firstAt'] >= MAX_RETRY_AGE_SECONDS or + if (failed['reason'] in SOURCE_LIMITATIONS or failed['attempts'] >= MAX_RETRY_ATTEMPTS or now - failed['firstAt'] >= MAX_RETRY_AGE_SECONDS or failed['firstAt'] <= now < failed['retryAt']): continue candidates.append((route, route_id, signature, manifest)) @@ -514,7 +517,7 @@ class DriveStatsOwner: pending = max(self._pending, int(not self._scanned)) state = ('analyzing' if running else 'waitingParked' if pending and not self._allowed() else 'queued' if pending else 'retrying' if any( - f['attempts'] < MAX_RETRY_ATTEMPTS and self.clock() - f['firstAt'] < MAX_RETRY_AGE_SECONDS + f['reason'] not in SOURCE_LIMITATIONS and f['attempts'] < MAX_RETRY_ATTEMPTS and self.clock() - f['firstAt'] < MAX_RETRY_AGE_SECONDS for f in self._failures.values()) else 'failed' if self._failures else 'ready') def public(record): return {key: value for key, value in record.items() if key not in ('sourceSignature', 'sourceManifest')} @@ -522,10 +525,10 @@ class DriveStatsOwner: return {'schemaVersion': SCHEMA_VERSION, 'isMetric': is_metric, 'analysis': {'running': running, 'pending': pending, 'state': state, 'failed': len(self._failures), - 'quarantined': sum(f['attempts'] >= MAX_RETRY_ATTEMPTS or + 'quarantined': sum(f['reason'] in SOURCE_LIMITATIONS or f['attempts'] >= MAX_RETRY_ATTEMPTS or self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS for f in self._failures.values()), 'failures': [{'routeId': key, 'reason': f['reason'], 'attempts': f['attempts'], - 'retryStopped': f['attempts'] >= MAX_RETRY_ATTEMPTS or + 'retryStopped': f['reason'] in SOURCE_LIMITATIONS or f['attempts'] >= MAX_RETRY_ATTEMPTS or self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS} for key, f in self._failures.items()], 'scanIncomplete': self._scan_incomplete}, diff --git a/openpilot/starpilot/galaxy/tests/test_drive_analysis.py b/openpilot/starpilot/galaxy/tests/test_drive_analysis.py index 8befc28cf9..b74bd61d18 100644 --- a/openpilot/starpilot/galaxy/tests/test_drive_analysis.py +++ b/openpilot/starpilot/galaxy/tests/test_drive_analysis.py @@ -81,6 +81,17 @@ class DriveAnalysisTest(unittest.TestCase): self.assertIsNone(result['aolSeconds']) self.assertIsNone(result['aolPercent']) + def test_missing_qlog_end_marker_recovers_from_complete_rlog(self): + samples = [*self.sample(BASE, 5, True), *self.sample(BASE + 500_000_000, 5, True)] + segment = write(self.root, 0, samples, kind='qlog') + full = [*samples, event('sentinel', BASE + 500_000_000, type='endOfRoute')] + (self.root / segment['segmentName'] / 'rlog.zst').write_bytes( + zstd.compress(b''.join(item.as_reader().as_builder().to_bytes() for item in full))) + segment['files']['rlog'] = True + result = analyze_route(self.root, selection(segment), permitted=lambda: True) + self.assertTrue(result['complete'], result) + self.assertEqual(result['distanceMeters'], 2.5) + def test_complete_contiguous_route_uses_observed_values_and_event_edges(self): first = [event('clocks', BASE, wallTimeNanos=1_790_334_400_000_000_000), *self.sample(BASE, 0, True), diff --git a/openpilot/starpilot/galaxy/tests/test_drive_stats.py b/openpilot/starpilot/galaxy/tests/test_drive_stats.py index a2c5ffb114..82a146e376 100644 --- a/openpilot/starpilot/galaxy/tests/test_drive_stats.py +++ b/openpilot/starpilot/galaxy/tests/test_drive_stats.py @@ -408,6 +408,34 @@ class DriveStatsTest(unittest.TestCase): owner.snapshot() self.assertEqual(self.completed(owner, scans=3)['totals']['drives'], 5) + def test_incomplete_recording_waits_for_changed_source_without_repeated_analysis(self): + for reason in drive_stats.SOURCE_LIMITATIONS: + with self.subTest(reason=reason): + analyzer = mock.Mock(side_effect=lambda *args, reason=reason, **kwargs: { + **analyzed(*args, **kwargs), 'complete': False, 'reason': reason}) + owner = self.owner(analyzer=analyzer) + owner._start() + owner.snapshot() + result = self.completed(owner) + self.assertTrue(result['analysis']['failures'][0]['retryStopped']) + self.assertEqual(result['totals']['drives'], 0) + self.now[0] += 3600 + owner._start() + owner.snapshot() + self.completed(owner, scans=self.history.scans) + self.assertEqual(analyzer.call_count, 1) + self.history.routes = [route('abc', number=self.history.scans)] + owner.analyzer = mock.Mock(wraps=analyzed) + self.now[0] += 61 + owner._start() + owner.snapshot() + recovered = self.completed(owner, scans=self.history.scans) + owner.analyzer.assert_called_once() + self.assertEqual(recovered['analysis']['failed'], 0) + self.assertEqual(recovered['totals']['drives'], 1) + owner.close() + self.store.unlink() + def test_incomplete_source_waits_for_retry_or_revision(self): calls = [] def incomplete(root, selected, permitted): diff --git a/openpilot/starpilot/galaxy/tests/test_home.mjs b/openpilot/starpilot/galaxy/tests/test_home.mjs index e0d8936bef..4c8c8ccd2c 100644 --- a/openpilot/starpilot/galaxy/tests/test_home.mjs +++ b/openpilot/starpilot/galaxy/tests/test_home.mjs @@ -76,16 +76,16 @@ assert.equal(driving.records[0].display, "12.0 km") const incomplete = { ...route, complete: false, reason: "End of route was not recorded" } const paused = driveSummary({ status: "ready", data: { ...drives, lastDrive: incomplete, recentDrives: [incomplete], analysis: { ...drives.analysis, failures: [{ routeId: route.routeId, reason: incomplete.reason, retryStopped: true }] } } }) -assert.equal(paused.last.analysisLabel, "Analysis paused") -assert.equal(paused.recent[0].analysisError, incomplete.reason) -assert.equal(paused.recent[0].analysisLabel, "Analysis paused") +assert.equal(paused.last.analysisLabel, "Incomplete recording") +assert.equal(paused.recent[0].analysisError, undefined) +assert.equal(paused.recent[0].analysisLabel, "Incomplete recording") const retrying = driveSummary({ status: "ready", data: { ...drives, recentDrives: [incomplete], analysis: { ...drives.analysis, failures: [{ routeId: route.routeId, reason: "TimeoutError", retryStopped: false }] } } }) -assert.equal(retrying.recent[0].analysisLabel, "Retry scheduled") -assert.equal(retrying.recent[0].analysisError, "TimeoutError") +assert.equal(retrying.recent[0].analysisLabel, "Incomplete recording") +assert.equal(retrying.recent[0].analysisError, undefined) const pendingAnalysis = driveSummary({ status: "ready", data: { ...drives, recentDrives: [incomplete] } }) -assert.equal(pendingAnalysis.recent[0].analysisLabel, "Analysis pending") -assert.equal(pendingAnalysis.recent[0].analysisError, incomplete.reason) +assert.equal(pendingAnalysis.recent[0].analysisLabel, "Incomplete recording") +assert.equal(pendingAnalysis.recent[0].analysisError, undefined) assert.doesNotMatch(Home.template, /'Analyzing'|still being analyzed|Waiting for analysis/) const imperial = driveSummary({ status: "ready", data: { ...drives, isMetric: false } }) assert.equal(imperial.last.distance, "7.5 mi") @@ -193,3 +193,5 @@ assert.match(icon, /fill="#fff"/) assert.match(icon, /fill="#6f42a6"/) assert.doesNotMatch(icon, / { - const failure = data.analysis.failures?.find((item) => item.routeId === drive.routeId) - return { analysisLabel: failure?.retryStopped ? "Analysis paused" : failure ? "Retry scheduled" : "Analysis pending", - analysisError: failure?.reason || drive.reason || "" } - } + const analysisStatus = () => ({ analysisLabel: "Incomplete recording" }) const maximum = Math.max(1, ...data.week.days.map((day) => day.distanceMeters ?? 0)) const models = data.models.slice(0, 3) const modelTotal = models.reduce((sum, model) => sum + (model.distanceMeters ?? 0), 0) @@ -189,7 +185,7 @@ export const Home = {

No saved drives yet.