Recover Galaxy updates after gateway disconnects

openpilot/starpilot/galaxy/tests/test_software_status.mjs

@@ -608,7 +608,7 @@ for (const [state, outcome] of [['failed', undefined], ['complete', 'up_to_date'
 // Gateway error envelopes are not software snapshots, even when the tunnel uses HTTP 200.
 const unreachableMessage = 'Your comma cannot be reached. Check your connection and try again'
 for (const action of ['update', 'fast', 'rollback', 'install', 'version']) {
-  for (const status of [200, 400, 500, 503]) {
+  for (const status of [200, 400, 500, 503]) for (const code of [undefined, 'connection_unavailable']) {
     for (const field of ['error', 'message']) {
       for (const duringPoll of [false, true]) {
         const restarts = []
@@ -623,7 +623,7 @@ for (const action of ['update', 'fast', 'rollback', 'install', 'version']) {
           await f.reply(1, snapshot({ operations: { ...operations, request: request(action, 'pending', 'Dom') } }))
           f.fire(1000)
         }
-        await f.reply(duringPoll ? 2 : 1, { [field]: unreachableMessage }, status)
+        await f.reply(duringPoll ? 2 : 1, { [field]: unreachableMessage, code }, status)
         assert.ok(restarts.length, `${action} ${status} ${field} ${duringPoll ? 'poll' : 'POST'} must reconnect`)
         assert.equal(restarts.at(-1).action, action)
         assert.equal(f.feed.installBaseline.action, action)
@@ -637,7 +637,7 @@ for (const action of ['update', 'fast', 'rollback', 'install', 'version']) {
 for (const status of [200, 400, 500]) {
   const f = fixture(true, () => assert.fail('ordinary status failure must not show update recovery'))
   f.feed.start()
-  await f.reply(0, { message: unreachableMessage }, status)
+  await f.reply(0, { message: unreachableMessage, code: 'connection_unavailable' }, status)
   assert.equal(f.states.at(-1).error, unreachableMessage)
   f.feed.stop()
 }

openpilot/starpilot/galaxy/tests/test_update_restart.mjs

@@ -1,6 +1,6 @@
 import assert from "node:assert/strict"
 import { readFileSync } from "node:fs"
-import { UpdateRestart, UPDATE_RESTART_KEY, UPDATE_RESTART_PREPARED_KEY } from "../web/js/update-restart.js"
+import { UpdateRestart, UPDATE_RESTART_KEY, UPDATE_RESTART_PREPARED_KEY, UPDATE_RESTART_TIMEOUT, UPDATE_RESTART_TIMEOUT_MESSAGE } from "../web/js/update-restart.js"
 import { SoftwareStatusFeed } from "../web/js/software-status.js"
 
 const baseline = { commit: 'a'.repeat(40), branch: 'Dom', target: 'Dom', action: 'fast' }
@@ -26,14 +26,16 @@ const flush = async () => { for (let i = 0; i < 12; i++) await Promise.resolve()
 function fixture(saved = null) {
   const values = new Map(saved ? [[UPDATE_RESTART_KEY, JSON.stringify(saved)]] : [])
   const storage = { getItem: key => values.get(key), setItem: (key, value) => values.set(key, value), removeItem: key => values.delete(key) }
-  const states = [], timers = new Map(), requests = []
+  const states = [], timers = new Map(), requests = [], errors = []
   let nextTimer = 0
   let reloads = 0
-  const recovery = new UpdateRestart({ storage, publish: value => states.push(value),
+  const recovery = new UpdateRestart({ storage, publish: (value, error) => { states.push(value); errors.push(error) },
     request: () => new Promise((resolve, reject) => requests.push({ resolve, reject })),
-    later: (fn, ms) => { const id = ++nextTimer; timers.set(id, fn); return id },
+    later: (fn, ms) => { const id = ++nextTimer; timers.set(id, { fn, ms }); return id },
     cancel: id => timers.delete(id), reload: () => reloads++ })
-  return { recovery, states, requests, timers, values, tick() { const [id, fn] = timers.entries().next().value; timers.delete(id); fn() }, get reloads() { return reloads } }
+  return { recovery, states, errors, requests, timers, values, tick(ms = 2000) {
+    const [id, timer] = [...timers.entries()].find(([, timer]) => timer.ms === ms); timers.delete(id); timer.fn()
+  }, get reloads() { return reloads } }
 }
 const normal = fixture()
 assert.deepEqual(normal.states, [false])
@@ -140,3 +142,39 @@ assert.equal(restarts.length, 0)
 feed.connectionLost = true
 feed.emit('ready')
 assert.deepEqual(restarts, [baseline])
+
+// Recovery is bounded even if a transport request never settles.
+const timedOut = fixture()
+timedOut.recovery.prepare(baseline)
+timedOut.recovery.start(baseline)
+timedOut.tick(UPDATE_RESTART_TIMEOUT)
+assert.equal(timedOut.states.at(-1), false)
+assert.equal(timedOut.errors.at(-1), UPDATE_RESTART_TIMEOUT_MESSAGE)
+assert.equal(timedOut.recovery.baseline, null)
+assert.equal(timedOut.values.has(UPDATE_RESTART_KEY), false)
+assert.equal(timedOut.values.has(UPDATE_RESTART_PREPARED_KEY), false)
+assert.equal(timedOut.reloads, 0, 'timeout is not proof of a completed update')
+timedOut.recovery.start(baseline)
+assert.equal(timedOut.states.at(-1), false, 'repeated feed errors cannot reopen expired recovery')
+timedOut.requests[0].resolve(status('b'.repeat(40)))
+await flush()
+assert.equal(timedOut.reloads, 0, 'a late response cannot undo the timeout')
+assert.equal(timedOut.timers.size, 0)
+timedOut.recovery.prepare(baseline)
+timedOut.recovery.start(baseline)
+assert.equal(timedOut.states.at(-1), true, 'an explicit new action can start recovery again')
+
+const expiredOnReload = fixture({ ...baseline, createdAt: Date.now() - UPDATE_RESTART_TIMEOUT - 1000 })
+assert.equal(expiredOnReload.states.at(-1), false)
+assert.equal(expiredOnReload.errors.at(-1), UPDATE_RESTART_TIMEOUT_MESSAGE)
+assert.equal(expiredOnReload.requests.length, 0)
+assert.equal(expiredOnReload.timers.size, 0)
+
+const repeats = fixture()
+repeats.recovery.start(baseline)
+const firstDeadline = repeats.recovery.baseline.createdAt
+repeats.recovery.now = () => Date.now() + 60000
+repeats.recovery.start({ ...baseline, createdAt: Date.now() + 60000 })
+assert.equal(repeats.recovery.baseline.createdAt, firstDeadline, 'repeated losses must not extend the deadline')
+assert.equal([...repeats.timers.values()].filter(timer => timer.ms === UPDATE_RESTART_TIMEOUT).length, 1)
+console.log('Update restart recovery regressions passed')

openpilot/starpilot/galaxy/web/js/app.js

@@ -72,9 +72,10 @@ const state = reactive({
   theme: stored(THEME_KEY) === "light" ? "light" : "dark",
   searchPage: "",
 })
-const updateRestart = new UpdateRestart({ publish: waiting => {
+const updateRestart = new UpdateRestart({ publish: (waiting, error) => {
   state.updateRestarting = waiting
   if (waiting) showRestartScreen()
+  else if (error) showBootScreen("failed", error, true)
   else if (bootScreenMode() === "restart" && !state.loading) dismissBootScreen()
 } })
 const deviceRestart = new DeviceRestartWatch({ paused: () => state.updateRestarting })

openpilot/starpilot/galaxy/web/js/software-status.js

@@ -198,7 +198,7 @@ export class SoftwareStatusFeed {
         return null
       }
       // A tunnel may return its own JSON error page with a successful HTTP status.
-      const deviceUnreachable = !payload?.code && [payload?.error, payload?.message].some(message =>
+      const deviceUnreachable = payload?.code === "connection_unavailable" || !payload?.code && [payload?.error, payload?.message].some(message =>
         typeof message === "string" && /^Your comma cannot be reached(?:\. Check your connection and try again\.?)?$/i.test(message.trim()))
       if (!response.ok || deviceUnreachable) {
         const error = new Error(payload?.error || payload?.message || "Software request failed. Refresh to try again.")

openpilot/starpilot/galaxy/web/js/update-restart.js

@@ -3,6 +3,8 @@ import { requestJson } from "./startup.js"
 export const UPDATE_RESTART_KEY = "galaxy-update-restart"
 export const UPDATE_RESTART_PREPARED_KEY = "galaxy-update-prepared"
 export const UPDATE_RESTART_MESSAGE = "StarPilot is restarting after an update. Galaxy is waiting for your device to come online and will reconnect automatically."
+export const UPDATE_RESTART_TIMEOUT = 5 * 60 * 1000
+export const UPDATE_RESTART_TIMEOUT_MESSAGE = "Galaxy could not verify the update after five minutes. Your device may still be updating. Check its connection and refresh to read its status before trying another update."
 
 export function savedUpdateRestart(storage = globalThis.sessionStorage) {
   try {
@@ -28,25 +30,30 @@ export class UpdateRestart {
     this.offline = !!this.baseline
     this.polling = false
     this.retry = null
+    this.deadline = null
+    this.expired = false
+    this.generation = 0
     this.publish(!!this.baseline)
-    if (this.baseline) this.poll()
+    if (this.baseline) { this.watchDeadline(); if (this.baseline) this.poll() }
   }
 
   start(baseline, { provisional = false } = {}) {
+    if (this.expired) return
     if (!provisional) this.cancelPreparation()
     if (this.baseline) {
-      this.baseline = { ...this.baseline, ...baseline, instance: this.baseline.instance ?? baseline.instance }
+      this.baseline = { ...this.baseline, ...baseline, instance: this.baseline.instance ?? baseline.instance, createdAt: this.baseline.createdAt }
       try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(this.baseline)) } catch {}
       return
     }
-    this.baseline = baseline
-    try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(baseline)) } catch {}
+    this.baseline = { ...baseline, createdAt: this.now() }
+    this.watchDeadline()
     this.publish(true)
     this.poll()
   }
 
   prepare(baseline) {
     if (this.baseline) return
+    this.expired = false
     this.prepared = { ...baseline, createdAt: this.now() }
     try { this.storage.setItem(UPDATE_RESTART_PREPARED_KEY, JSON.stringify(this.prepared)) } catch {}
   }
@@ -56,11 +63,35 @@ export class UpdateRestart {
     try { this.storage.removeItem(UPDATE_RESTART_PREPARED_KEY) } catch {}
   }
 
+  watchDeadline() {
+    const createdAt = this.baseline.createdAt
+    this.baseline.createdAt = Number.isFinite(createdAt) && createdAt <= this.now() ? createdAt : this.now()
+    try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(this.baseline)) } catch {}
+    const remaining = UPDATE_RESTART_TIMEOUT - (this.now() - this.baseline.createdAt)
+    if (remaining <= 0) { this.finish(true); return }
+    this.deadline = this.later(() => this.finish(true), remaining)
+  }
+
+  finish(expired = false) {
+    this.generation++
+    this.baseline = null
+    this.expired = expired
+    if (this.retry !== null) this.cancel(this.retry)
+    if (this.deadline !== null) this.cancel(this.deadline)
+    this.retry = this.deadline = null
+    try { this.storage.removeItem(UPDATE_RESTART_KEY) } catch {}
+    this.cancelPreparation()
+    this.publish(false, expired ? UPDATE_RESTART_TIMEOUT_MESSAGE : "")
+    if (!expired) this.reload()
+  }
+
   async poll() {
     if (this.polling || !this.baseline) return
     this.polling = true
+    const generation = this.generation
     try {
       const data = await this.request("./api/software/status?history=0")
+      if (!this.baseline || generation !== this.generation) return
       const installed = data?.installed, request = data?.operations?.request
       if (data?.schemaVersion !== 1 || !installed || !data.operations) throw new Error("Status unavailable")
       const changed = installed.commit && (installed.commit !== this.baseline.commit || installed.branch !== this.baseline.branch)
@@ -72,13 +103,7 @@ export class UpdateRestart {
       const notStarted = this.prepared && data.updater?.state === "idle" && !request &&
         !changed && !restarted && this.now() - this.prepared.createdAt > 30000
       if (finished || returned || notStarted) {
-        this.baseline = null
-        if (this.retry !== null) this.cancel(this.retry)
-        this.retry = null
-        try { this.storage.removeItem(UPDATE_RESTART_KEY) } catch {}
-        this.cancelPreparation()
-        this.publish(false)
-        this.reload() // Preserve the current hash and load the newly installed Galaxy assets.
+        this.finish()
       }
     } catch {
       this.offline = true

prebuilt.json

@@ -1 +1 @@
-{"protected_inventory_sha256": "df3ac40c36c1fa948ae90e0d5b8b23d4e0734825dbf9e5e196a3da730f68bf31", "schema_version": 1}
+{"protected_inventory_sha256": "bb641b5236deefc3d7706552949ef82fe0d8a32c7396f5f582fd849dd23a9ac5", "schema_version": 1}