diff --git a/openpilot/starpilot/galaxy/tests/test_software_status.mjs b/openpilot/starpilot/galaxy/tests/test_software_status.mjs index f8d43bc166..f9a73ee2a0 100644 --- a/openpilot/starpilot/galaxy/tests/test_software_status.mjs +++ b/openpilot/starpilot/galaxy/tests/test_software_status.mjs @@ -608,7 +608,7 @@ for (const [state, outcome] of [['failed', undefined], ['complete', 'up_to_date' // Gateway error envelopes are not software snapshots, even when the tunnel uses HTTP 200. const unreachableMessage = 'Your comma cannot be reached. Check your connection and try again' for (const action of ['update', 'fast', 'rollback', 'install', 'version']) { - for (const status of [200, 400, 500, 503]) { + for (const status of [200, 400, 500, 503]) for (const code of [undefined, 'connection_unavailable']) { for (const field of ['error', 'message']) { for (const duringPoll of [false, true]) { const restarts = [] @@ -623,7 +623,7 @@ for (const action of ['update', 'fast', 'rollback', 'install', 'version']) { await f.reply(1, snapshot({ operations: { ...operations, request: request(action, 'pending', 'Dom') } })) f.fire(1000) } - await f.reply(duringPoll ? 2 : 1, { [field]: unreachableMessage }, status) + await f.reply(duringPoll ? 2 : 1, { [field]: unreachableMessage, code }, status) assert.ok(restarts.length, `${action} ${status} ${field} ${duringPoll ? 'poll' : 'POST'} must reconnect`) assert.equal(restarts.at(-1).action, action) assert.equal(f.feed.installBaseline.action, action) @@ -637,7 +637,7 @@ for (const action of ['update', 'fast', 'rollback', 'install', 'version']) { for (const status of [200, 400, 500]) { const f = fixture(true, () => assert.fail('ordinary status failure must not show update recovery')) f.feed.start() - await f.reply(0, { message: unreachableMessage }, status) + await f.reply(0, { message: unreachableMessage, code: 'connection_unavailable' }, status) assert.equal(f.states.at(-1).error, unreachableMessage) f.feed.stop() } diff --git a/openpilot/starpilot/galaxy/tests/test_update_restart.mjs b/openpilot/starpilot/galaxy/tests/test_update_restart.mjs index 74cfa4443c..e4e0e88e6e 100644 --- a/openpilot/starpilot/galaxy/tests/test_update_restart.mjs +++ b/openpilot/starpilot/galaxy/tests/test_update_restart.mjs @@ -1,6 +1,6 @@ import assert from "node:assert/strict" import { readFileSync } from "node:fs" -import { UpdateRestart, UPDATE_RESTART_KEY, UPDATE_RESTART_PREPARED_KEY } from "../web/js/update-restart.js" +import { UpdateRestart, UPDATE_RESTART_KEY, UPDATE_RESTART_PREPARED_KEY, UPDATE_RESTART_TIMEOUT, UPDATE_RESTART_TIMEOUT_MESSAGE } from "../web/js/update-restart.js" import { SoftwareStatusFeed } from "../web/js/software-status.js" const baseline = { commit: 'a'.repeat(40), branch: 'Dom', target: 'Dom', action: 'fast' } @@ -26,14 +26,16 @@ const flush = async () => { for (let i = 0; i < 12; i++) await Promise.resolve() function fixture(saved = null) { const values = new Map(saved ? [[UPDATE_RESTART_KEY, JSON.stringify(saved)]] : []) const storage = { getItem: key => values.get(key), setItem: (key, value) => values.set(key, value), removeItem: key => values.delete(key) } - const states = [], timers = new Map(), requests = [] + const states = [], timers = new Map(), requests = [], errors = [] let nextTimer = 0 let reloads = 0 - const recovery = new UpdateRestart({ storage, publish: value => states.push(value), + const recovery = new UpdateRestart({ storage, publish: (value, error) => { states.push(value); errors.push(error) }, request: () => new Promise((resolve, reject) => requests.push({ resolve, reject })), - later: (fn, ms) => { const id = ++nextTimer; timers.set(id, fn); return id }, + later: (fn, ms) => { const id = ++nextTimer; timers.set(id, { fn, ms }); return id }, cancel: id => timers.delete(id), reload: () => reloads++ }) - return { recovery, states, requests, timers, values, tick() { const [id, fn] = timers.entries().next().value; timers.delete(id); fn() }, get reloads() { return reloads } } + return { recovery, states, errors, requests, timers, values, tick(ms = 2000) { + const [id, timer] = [...timers.entries()].find(([, timer]) => timer.ms === ms); timers.delete(id); timer.fn() + }, get reloads() { return reloads } } } const normal = fixture() assert.deepEqual(normal.states, [false]) @@ -140,3 +142,39 @@ assert.equal(restarts.length, 0) feed.connectionLost = true feed.emit('ready') assert.deepEqual(restarts, [baseline]) + +// Recovery is bounded even if a transport request never settles. +const timedOut = fixture() +timedOut.recovery.prepare(baseline) +timedOut.recovery.start(baseline) +timedOut.tick(UPDATE_RESTART_TIMEOUT) +assert.equal(timedOut.states.at(-1), false) +assert.equal(timedOut.errors.at(-1), UPDATE_RESTART_TIMEOUT_MESSAGE) +assert.equal(timedOut.recovery.baseline, null) +assert.equal(timedOut.values.has(UPDATE_RESTART_KEY), false) +assert.equal(timedOut.values.has(UPDATE_RESTART_PREPARED_KEY), false) +assert.equal(timedOut.reloads, 0, 'timeout is not proof of a completed update') +timedOut.recovery.start(baseline) +assert.equal(timedOut.states.at(-1), false, 'repeated feed errors cannot reopen expired recovery') +timedOut.requests[0].resolve(status('b'.repeat(40))) +await flush() +assert.equal(timedOut.reloads, 0, 'a late response cannot undo the timeout') +assert.equal(timedOut.timers.size, 0) +timedOut.recovery.prepare(baseline) +timedOut.recovery.start(baseline) +assert.equal(timedOut.states.at(-1), true, 'an explicit new action can start recovery again') + +const expiredOnReload = fixture({ ...baseline, createdAt: Date.now() - UPDATE_RESTART_TIMEOUT - 1000 }) +assert.equal(expiredOnReload.states.at(-1), false) +assert.equal(expiredOnReload.errors.at(-1), UPDATE_RESTART_TIMEOUT_MESSAGE) +assert.equal(expiredOnReload.requests.length, 0) +assert.equal(expiredOnReload.timers.size, 0) + +const repeats = fixture() +repeats.recovery.start(baseline) +const firstDeadline = repeats.recovery.baseline.createdAt +repeats.recovery.now = () => Date.now() + 60000 +repeats.recovery.start({ ...baseline, createdAt: Date.now() + 60000 }) +assert.equal(repeats.recovery.baseline.createdAt, firstDeadline, 'repeated losses must not extend the deadline') +assert.equal([...repeats.timers.values()].filter(timer => timer.ms === UPDATE_RESTART_TIMEOUT).length, 1) +console.log('Update restart recovery regressions passed') diff --git a/openpilot/starpilot/galaxy/web/js/app.js b/openpilot/starpilot/galaxy/web/js/app.js index d2b01b0e72..c012784c57 100644 --- a/openpilot/starpilot/galaxy/web/js/app.js +++ b/openpilot/starpilot/galaxy/web/js/app.js @@ -72,9 +72,10 @@ const state = reactive({ theme: stored(THEME_KEY) === "light" ? "light" : "dark", searchPage: "", }) -const updateRestart = new UpdateRestart({ publish: waiting => { +const updateRestart = new UpdateRestart({ publish: (waiting, error) => { state.updateRestarting = waiting if (waiting) showRestartScreen() + else if (error) showBootScreen("failed", error, true) else if (bootScreenMode() === "restart" && !state.loading) dismissBootScreen() } }) const deviceRestart = new DeviceRestartWatch({ paused: () => state.updateRestarting }) diff --git a/openpilot/starpilot/galaxy/web/js/software-status.js b/openpilot/starpilot/galaxy/web/js/software-status.js index dbd5fa5e26..894255871b 100644 --- a/openpilot/starpilot/galaxy/web/js/software-status.js +++ b/openpilot/starpilot/galaxy/web/js/software-status.js @@ -198,7 +198,7 @@ export class SoftwareStatusFeed { return null } // A tunnel may return its own JSON error page with a successful HTTP status. - const deviceUnreachable = !payload?.code && [payload?.error, payload?.message].some(message => + const deviceUnreachable = payload?.code === "connection_unavailable" || !payload?.code && [payload?.error, payload?.message].some(message => typeof message === "string" && /^Your comma cannot be reached(?:\. Check your connection and try again\.?)?$/i.test(message.trim())) if (!response.ok || deviceUnreachable) { const error = new Error(payload?.error || payload?.message || "Software request failed. Refresh to try again.") diff --git a/openpilot/starpilot/galaxy/web/js/update-restart.js b/openpilot/starpilot/galaxy/web/js/update-restart.js index 69ec5730b9..c184d5b1a7 100644 --- a/openpilot/starpilot/galaxy/web/js/update-restart.js +++ b/openpilot/starpilot/galaxy/web/js/update-restart.js @@ -3,6 +3,8 @@ import { requestJson } from "./startup.js" export const UPDATE_RESTART_KEY = "galaxy-update-restart" export const UPDATE_RESTART_PREPARED_KEY = "galaxy-update-prepared" export const UPDATE_RESTART_MESSAGE = "StarPilot is restarting after an update. Galaxy is waiting for your device to come online and will reconnect automatically." +export const UPDATE_RESTART_TIMEOUT = 5 * 60 * 1000 +export const UPDATE_RESTART_TIMEOUT_MESSAGE = "Galaxy could not verify the update after five minutes. Your device may still be updating. Check its connection and refresh to read its status before trying another update." export function savedUpdateRestart(storage = globalThis.sessionStorage) { try { @@ -28,25 +30,30 @@ export class UpdateRestart { this.offline = !!this.baseline this.polling = false this.retry = null + this.deadline = null + this.expired = false + this.generation = 0 this.publish(!!this.baseline) - if (this.baseline) this.poll() + if (this.baseline) { this.watchDeadline(); if (this.baseline) this.poll() } } start(baseline, { provisional = false } = {}) { + if (this.expired) return if (!provisional) this.cancelPreparation() if (this.baseline) { - this.baseline = { ...this.baseline, ...baseline, instance: this.baseline.instance ?? baseline.instance } + this.baseline = { ...this.baseline, ...baseline, instance: this.baseline.instance ?? baseline.instance, createdAt: this.baseline.createdAt } try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(this.baseline)) } catch {} return } - this.baseline = baseline - try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(baseline)) } catch {} + this.baseline = { ...baseline, createdAt: this.now() } + this.watchDeadline() this.publish(true) this.poll() } prepare(baseline) { if (this.baseline) return + this.expired = false this.prepared = { ...baseline, createdAt: this.now() } try { this.storage.setItem(UPDATE_RESTART_PREPARED_KEY, JSON.stringify(this.prepared)) } catch {} } @@ -56,11 +63,35 @@ export class UpdateRestart { try { this.storage.removeItem(UPDATE_RESTART_PREPARED_KEY) } catch {} } + watchDeadline() { + const createdAt = this.baseline.createdAt + this.baseline.createdAt = Number.isFinite(createdAt) && createdAt <= this.now() ? createdAt : this.now() + try { this.storage.setItem(UPDATE_RESTART_KEY, JSON.stringify(this.baseline)) } catch {} + const remaining = UPDATE_RESTART_TIMEOUT - (this.now() - this.baseline.createdAt) + if (remaining <= 0) { this.finish(true); return } + this.deadline = this.later(() => this.finish(true), remaining) + } + + finish(expired = false) { + this.generation++ + this.baseline = null + this.expired = expired + if (this.retry !== null) this.cancel(this.retry) + if (this.deadline !== null) this.cancel(this.deadline) + this.retry = this.deadline = null + try { this.storage.removeItem(UPDATE_RESTART_KEY) } catch {} + this.cancelPreparation() + this.publish(false, expired ? UPDATE_RESTART_TIMEOUT_MESSAGE : "") + if (!expired) this.reload() + } + async poll() { if (this.polling || !this.baseline) return this.polling = true + const generation = this.generation try { const data = await this.request("./api/software/status?history=0") + if (!this.baseline || generation !== this.generation) return const installed = data?.installed, request = data?.operations?.request if (data?.schemaVersion !== 1 || !installed || !data.operations) throw new Error("Status unavailable") const changed = installed.commit && (installed.commit !== this.baseline.commit || installed.branch !== this.baseline.branch) @@ -72,13 +103,7 @@ export class UpdateRestart { const notStarted = this.prepared && data.updater?.state === "idle" && !request && !changed && !restarted && this.now() - this.prepared.createdAt > 30000 if (finished || returned || notStarted) { - this.baseline = null - if (this.retry !== null) this.cancel(this.retry) - this.retry = null - try { this.storage.removeItem(UPDATE_RESTART_KEY) } catch {} - this.cancelPreparation() - this.publish(false) - this.reload() // Preserve the current hash and load the newly installed Galaxy assets. + this.finish() } } catch { this.offline = true diff --git a/prebuilt.json b/prebuilt.json index ef3a56496b..dae5194a27 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "df3ac40c36c1fa948ae90e0d5b8b23d4e0734825dbf9e5e196a3da730f68bf31", "schema_version": 1} +{"protected_inventory_sha256": "bb641b5236deefc3d7706552949ef82fe0d8a32c7396f5f582fd849dd23a9ac5", "schema_version": 1}