Quiet incomplete drive history and recover full logs

openpilot/starpilot/galaxy/drive_analysis.py

@@ -25,7 +25,7 @@ MAX_ANALYSIS_SECONDS = 30.0
 MAX_PAIR_SECONDS = 1.0
 MAX_SPEED_MPS = 70.0
 MAX_ROUTE_COMPRESSED_BYTES = 256 * 1024 * 1024
-ANALYSIS_VERSION = 4
+ANALYSIS_VERSION = 5
 MAX_MODEL_METADATA_SECONDS = 4.0
 DISTRACTED_EVENT = 'driverDistracted2'
 UNRESPONSIVE_EVENT = 'driverUnresponsive3'
@@ -441,7 +441,7 @@ def analyze_route(root: Path, route: dict, *, permitted: Callable[[], bool], pre
     return result
   except (_Incomplete, LocalLogUnavailable, LogDecodeError, OSError, ValueError) as error:
     result['reason'] = str(error) or 'Route analysis unavailable'
-    if (not prefer_rlog and result['reason'] in ('Route samples are missing or gapped', 'Route has no observed duration') and
+    if (not prefer_rlog and result['reason'] in ('Route samples are missing or gapped', 'Route has no observed duration', 'End of route was not recorded') and
         isinstance(route, dict) and route.get('segments') and
         all(segment.get('files', {}).get('rlog') is True for segment in route['segments']) and permitted()):
       return analyze_route(root, route, permitted=allowed, prefer_rlog=True)

openpilot/starpilot/galaxy/drive_stats.py

@@ -18,6 +18,9 @@ from openpilot.starpilot.galaxy.drive_history import DriveHistory, MAX_ROOT_ENTR
 from openpilot.starpilot.galaxy.drive_analysis import ANALYSIS_VERSION
 from openpilot.starpilot.storage import galaxy_storage_root
 
+SOURCE_LIMITATIONS = frozenset({'Route has no observed duration', 'End of route was not recorded',
+                                'Route has missing segments or logs', 'Route samples are missing or gapped'})
+
 SCHEMA_VERSION = 1
 MAX_CACHED_ROUTES = 2048
 MAX_CACHE_BYTES = 4_000_000
@@ -306,7 +309,7 @@ class DriveStatsOwner:
             continue
           if failed and failed['signature'] == signature and failed.get('analysisVersion', 0) == ANALYSIS_VERSION:
             now = self.clock()
-            if (failed['attempts'] >= MAX_RETRY_ATTEMPTS or now - failed['firstAt'] >= MAX_RETRY_AGE_SECONDS or
+            if (failed['reason'] in SOURCE_LIMITATIONS or failed['attempts'] >= MAX_RETRY_ATTEMPTS or now - failed['firstAt'] >= MAX_RETRY_AGE_SECONDS or
                 failed['firstAt'] <= now < failed['retryAt']):
               continue
           candidates.append((route, route_id, signature, manifest))
@@ -514,7 +517,7 @@ class DriveStatsOwner:
     pending = max(self._pending, int(not self._scanned))
     state = ('analyzing' if running else 'waitingParked' if pending and not self._allowed() else
              'queued' if pending else 'retrying' if any(
-               f['attempts'] < MAX_RETRY_ATTEMPTS and self.clock() - f['firstAt'] < MAX_RETRY_AGE_SECONDS
+               f['reason'] not in SOURCE_LIMITATIONS and f['attempts'] < MAX_RETRY_ATTEMPTS and self.clock() - f['firstAt'] < MAX_RETRY_AGE_SECONDS
                for f in self._failures.values()) else 'failed' if self._failures else 'ready')
     def public(record):
       return {key: value for key, value in record.items() if key not in ('sourceSignature', 'sourceManifest')}
@@ -522,10 +525,10 @@ class DriveStatsOwner:
     return {'schemaVersion': SCHEMA_VERSION, 'isMetric': is_metric,
             'analysis': {'running': running, 'pending': pending, 'state': state,
                          'failed': len(self._failures),
-                         'quarantined': sum(f['attempts'] >= MAX_RETRY_ATTEMPTS or
+                         'quarantined': sum(f['reason'] in SOURCE_LIMITATIONS or f['attempts'] >= MAX_RETRY_ATTEMPTS or
                                             self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS for f in self._failures.values()),
                          'failures': [{'routeId': key, 'reason': f['reason'], 'attempts': f['attempts'],
-                                       'retryStopped': f['attempts'] >= MAX_RETRY_ATTEMPTS or
+                                       'retryStopped': f['reason'] in SOURCE_LIMITATIONS or f['attempts'] >= MAX_RETRY_ATTEMPTS or
                                                        self.clock() - f['firstAt'] >= MAX_RETRY_AGE_SECONDS}
                                       for key, f in self._failures.items()],
                          'scanIncomplete': self._scan_incomplete},

openpilot/starpilot/galaxy/tests/test_drive_analysis.py

@@ -81,6 +81,17 @@ class DriveAnalysisTest(unittest.TestCase):
     self.assertIsNone(result['aolSeconds'])
     self.assertIsNone(result['aolPercent'])
 
+  def test_missing_qlog_end_marker_recovers_from_complete_rlog(self):
+    samples = [*self.sample(BASE, 5, True), *self.sample(BASE + 500_000_000, 5, True)]
+    segment = write(self.root, 0, samples, kind='qlog')
+    full = [*samples, event('sentinel', BASE + 500_000_000, type='endOfRoute')]
+    (self.root / segment['segmentName'] / 'rlog.zst').write_bytes(
+      zstd.compress(b''.join(item.as_reader().as_builder().to_bytes() for item in full)))
+    segment['files']['rlog'] = True
+    result = analyze_route(self.root, selection(segment), permitted=lambda: True)
+    self.assertTrue(result['complete'], result)
+    self.assertEqual(result['distanceMeters'], 2.5)
+
   def test_complete_contiguous_route_uses_observed_values_and_event_edges(self):
     first = [event('clocks', BASE, wallTimeNanos=1_790_334_400_000_000_000),
              *self.sample(BASE, 0, True),

openpilot/starpilot/galaxy/tests/test_drive_stats.py

@@ -408,6 +408,34 @@ class DriveStatsTest(unittest.TestCase):
     owner.snapshot()
     self.assertEqual(self.completed(owner, scans=3)['totals']['drives'], 5)
 
+  def test_incomplete_recording_waits_for_changed_source_without_repeated_analysis(self):
+    for reason in drive_stats.SOURCE_LIMITATIONS:
+      with self.subTest(reason=reason):
+        analyzer = mock.Mock(side_effect=lambda *args, reason=reason, **kwargs: {
+          **analyzed(*args, **kwargs), 'complete': False, 'reason': reason})
+        owner = self.owner(analyzer=analyzer)
+        owner._start()
+        owner.snapshot()
+        result = self.completed(owner)
+        self.assertTrue(result['analysis']['failures'][0]['retryStopped'])
+        self.assertEqual(result['totals']['drives'], 0)
+        self.now[0] += 3600
+        owner._start()
+        owner.snapshot()
+        self.completed(owner, scans=self.history.scans)
+        self.assertEqual(analyzer.call_count, 1)
+        self.history.routes = [route('abc', number=self.history.scans)]
+        owner.analyzer = mock.Mock(wraps=analyzed)
+        self.now[0] += 61
+        owner._start()
+        owner.snapshot()
+        recovered = self.completed(owner, scans=self.history.scans)
+        owner.analyzer.assert_called_once()
+        self.assertEqual(recovered['analysis']['failed'], 0)
+        self.assertEqual(recovered['totals']['drives'], 1)
+        owner.close()
+        self.store.unlink()
+
   def test_incomplete_source_waits_for_retry_or_revision(self):
     calls = []
     def incomplete(root, selected, permitted):

openpilot/starpilot/galaxy/tests/test_home.mjs

@@ -76,16 +76,16 @@ assert.equal(driving.records[0].display, "12.0 km")
 const incomplete = { ...route, complete: false, reason: "End of route was not recorded" }
 const paused = driveSummary({ status: "ready", data: { ...drives, lastDrive: incomplete, recentDrives: [incomplete],
   analysis: { ...drives.analysis, failures: [{ routeId: route.routeId, reason: incomplete.reason, retryStopped: true }] } } })
-assert.equal(paused.last.analysisLabel, "Analysis paused")
-assert.equal(paused.recent[0].analysisError, incomplete.reason)
-assert.equal(paused.recent[0].analysisLabel, "Analysis paused")
+assert.equal(paused.last.analysisLabel, "Incomplete recording")
+assert.equal(paused.recent[0].analysisError, undefined)
+assert.equal(paused.recent[0].analysisLabel, "Incomplete recording")
 const retrying = driveSummary({ status: "ready", data: { ...drives, recentDrives: [incomplete],
   analysis: { ...drives.analysis, failures: [{ routeId: route.routeId, reason: "TimeoutError", retryStopped: false }] } } })
-assert.equal(retrying.recent[0].analysisLabel, "Retry scheduled")
-assert.equal(retrying.recent[0].analysisError, "TimeoutError")
+assert.equal(retrying.recent[0].analysisLabel, "Incomplete recording")
+assert.equal(retrying.recent[0].analysisError, undefined)
 const pendingAnalysis = driveSummary({ status: "ready", data: { ...drives, recentDrives: [incomplete] } })
-assert.equal(pendingAnalysis.recent[0].analysisLabel, "Analysis pending")
-assert.equal(pendingAnalysis.recent[0].analysisError, incomplete.reason)
+assert.equal(pendingAnalysis.recent[0].analysisLabel, "Incomplete recording")
+assert.equal(pendingAnalysis.recent[0].analysisError, undefined)
 assert.doesNotMatch(Home.template, /'Analyzing'|still being analyzed|Waiting for analysis/)
 const imperial = driveSummary({ status: "ready", data: { ...drives, isMetric: false } })
 assert.equal(imperial.last.distance, "7.5 mi")
@@ -193,3 +193,5 @@ assert.match(icon, /fill="#fff"/)
 assert.match(icon, /fill="#6f42a6"/)
 assert.doesNotMatch(icon, /<image|<script/)
 console.log("Galaxy Home source, lifecycle, and route checks passed")
+
+assert.doesNotMatch(Home.template, /drive analysis failure|failure.reason|analysisError/)

openpilot/starpilot/galaxy/web/js/home.js

@@ -51,11 +51,7 @@ export function driveSummary(feed) {
   if (!data) return null
   const isMetric = data.isMetric
   const last = data.lastDrive
-  const analysisStatus = (drive) => {
-    const failure = data.analysis.failures?.find((item) => item.routeId === drive.routeId)
-    return { analysisLabel: failure?.retryStopped ? "Analysis paused" : failure ? "Retry scheduled" : "Analysis pending",
-      analysisError: failure?.reason || drive.reason || "" }
-  }
+  const analysisStatus = () => ({ analysisLabel: "Incomplete recording" })
   const maximum = Math.max(1, ...data.week.days.map((day) => day.distanceMeters ?? 0))
   const models = data.models.slice(0, 3)
   const modelTotal = models.reduce((sum, model) => sum + (model.distanceMeters ?? 0), 0)
@@ -189,7 +185,7 @@ export const Home = {
           <p v-else-if="!driving.last">No saved drives yet.</p>
           <template v-else>
             <p class="gx-home__date">{{ driving.last.date }}</p>
-            <p v-if="!driving.last.complete" class="gx-home__subtle">{{ driving.last.analysisLabel }}{{ driving.last.analysisError ? ": " + driving.last.analysisError : "" }}</p>
+            <p v-if="!driving.last.complete" class="gx-home__subtle">{{ driving.last.analysisLabel }}</p>
             <div class="gx-home__four"><div><strong>{{ driving.last.distance }}</strong><span>distance</span></div>
               <div><strong>{{ driving.last.duration }}</strong><span>duration</span></div>
               <div><strong>{{ driving.last.engaged }}</strong><span>engaged</span></div>
@@ -224,15 +220,12 @@ export const Home = {
               <div v-for="record in driving.records" :key="record.id" class="gx-home__record"><span class="gx-home__record-art" aria-hidden="true"><svg viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.7" stroke-linecap="round" stroke-linejoin="round"><path v-for="(path, index) in recordArt(record.id)" :key="index" :d="path" /></svg></span><span class="gx-home__record-label">{{ record.label }}</span><strong>{{ record.display }}</strong></div>
             </section>
           </div>
-          <GxNotice tone="danger" v-if="driving.analysis.failed">{{ driving.analysis.failed }} drive analysis failure{{ driving.analysis.failed === 1 ? '' : 's' }}{{ driving.analysis.quarantined ? ' · automatic retries paused for ' + driving.analysis.quarantined : '' }}.
-            <div v-for="failure in driving.analysis.failures" :key="failure.routeId">{{ failure.routeId }}: {{ failure.reason }}</div>
-          </GxNotice>
           <section class="gx-card gx-home__recent"><div class="gx-section__header"><i class="bi bi-clock-history"></i><h2 class="gx-section__title">Recent Drives</h2></div>
             <p v-if="!driving.recent.length" class="gx-home__subtle">No saved drives yet.</p>
             <div v-for="drive in driving.recent" :key="drive.routeId" class="gx-home__drive" :class="{ 'is-ignored': drive.ignored }">
               <div><strong>{{ drive.date }}</strong><small>{{ drive.model }}</small></div>
               <div><strong>{{ drive.complete ? drive.distance : drive.analysisLabel }}</strong><small>{{ drive.duration }} · {{ drive.engaged }} engaged</small><small>{{ drive.aol }} AOL steering only</small></div>
-              <div><small>{{ drive.ignored ? 'Excluded from totals' : drive.complete ? 'Included in totals' : drive.analysisError || drive.analysisLabel }}</small>
+              <div><small>{{ drive.ignored ? 'Excluded from totals' : drive.complete ? 'Included in totals' : drive.analysisLabel }}</small>
                 <button type="button" class="gx-home__link" :inert="drives.busy" :aria-busy="drives.busy || undefined" @click="ignoreDrive(drive)">
                   {{ drive.ignored ? 'Include in stats' : 'Ignore drive stats' }}</button></div>
             </div>

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "b3bc73e2dce346a8b9563a9661cfe8fab670b7872893e4dc01615b6063040ae2", "schema_version": 1}
+{"protected_inventory_sha256": "c3e01d80eff2b69cad37bab0e35e0ee5cef040205d73aae08752c4838b04a9ae", "schema_version": 1}