diff --git a/openpilot/starpilot/galaxy/README.md b/openpilot/starpilot/galaxy/README.md index 84adf0070f..e9f3971ac2 100644 --- a/openpilot/starpilot/galaxy/README.md +++ b/openpilot/starpilot/galaxy/README.md @@ -23,8 +23,9 @@ then follows edits under this package. An open page reloads itself after each edit; other changes here restart the server first. Refresh a page opened before the session once. Edits outside the package need `./dev sync` and a restart. The reload script exists only in the host cache, never in this directory. -Add `--no-autoreload` to keep following edits but refresh by hand. - +Unsaved drafts and pending saves defer automatic reload until they are resolved. +Add `--no-autoreload` to keep following edits but refresh by hand. Only one live +Galaxy session can own a shared host cache at a time, regardless of port. Run the package tests with `./dev pytest openpilot/starpilot/galaxy/tests`. Browser JavaScript tests use Node 24; set `STARPILOT_NODE` to an existing Node 24 executable when needed. A static preview can serve `web/` over loopback; diff --git a/openpilot/starpilot/galaxy/projection_layout.py b/openpilot/starpilot/galaxy/projection_layout.py index 5ec27b4ed9..d5fa1c1d69 100644 --- a/openpilot/starpilot/galaxy/projection_layout.py +++ b/openpilot/starpilot/galaxy/projection_layout.py @@ -122,8 +122,8 @@ def host_owner(params, parked, root=None, environ=os.environ): if environ.get('SP_HOST_RUNTIME') != '1': return None if root is None: - from openpilot.starpilot.storage import starpilot_storage_root - root = starpilot_storage_root() / 'android_auto' + from openpilot.starpilot.system.android_auto.identity import DATA_DIR + root = DATA_DIR root = Path(root) screen_path = root / 'screen.json' if not screen_path.exists(): diff --git a/openpilot/starpilot/galaxy/test_projection_layout.py b/openpilot/starpilot/galaxy/test_projection_layout.py index 289db7a267..2f5f891a53 100644 --- a/openpilot/starpilot/galaxy/test_projection_layout.py +++ b/openpilot/starpilot/galaxy/test_projection_layout.py @@ -1,7 +1,10 @@ """Actual file-owner tests, CPU-only, no IPC/Params runtime required.""" import copy import json +import os from pathlib import Path +import subprocess +import sys import tempfile import unittest @@ -197,6 +200,27 @@ class TestProjectionLayoutOwner(unittest.TestCase): self.assertEqual((root / 'screen.json').read_bytes(), screen) self.assertEqual(again.snapshot()['document'], payload['document']) + def test_host_editor_and_renderer_share_default_paths(self): + # Constants are initialized at import time, as in the actual launcher processes. + code = """ +import copy +import os +from pathlib import Path +from types import SimpleNamespace +from openpilot.starpilot.galaxy.projection_layout import host_owner +from openpilot.starpilot.system.android_auto.projection_layout_runtime import load_projection_layout +root = Path(os.environ['ANDROID_AUTO_DIR']) +params = SimpleNamespace(get_param_path=lambda key: str(root / 'params' / key), get_bool=lambda key: False) +owner = host_owner(params, lambda: True) +snapshot = owner.snapshot() +payload = {'revision': snapshot['revision'], 'document': copy.deepcopy(snapshot['document'])} +payload['document']['widgets']['current_speed']['x'] += 10 +owner.save(payload, session_valid=lambda: True) +assert load_projection_layout((1920, 1080)) == payload['document'] +""" + env = os.environ | {'SP_HOST_RUNTIME': '1', 'ANDROID_AUTO_DIR': str(self.root / 'host')} + subprocess.run([sys.executable, '-c', code], env=env, check=True, capture_output=True) + if __name__ == '__main__': unittest.main() diff --git a/openpilot/starpilot/galaxy/tests/test_controllers.mjs b/openpilot/starpilot/galaxy/tests/test_controllers.mjs index 9c23cb12bf..d03599192b 100644 --- a/openpilot/starpilot/galaxy/tests/test_controllers.mjs +++ b/openpilot/starpilot/galaxy/tests/test_controllers.mjs @@ -118,6 +118,8 @@ component.feed.stop() assert.equal(component.state.draft, null) const earlierDocument = globalThis.document +const earlierWindow = globalThis.window +globalThis.window = new EventTarget() const listeners = new Map(), lifecycle = [] globalThis.document = { hidden: false, addEventListener(name, fn) { listeners.set(name, fn) }, removeEventListener(name) { listeners.delete(name) } } @@ -133,4 +135,5 @@ ControllersPage.beforeUnmount.call(mounted) assert.deepEqual(lifecycle, ["start", "stop", "stop", "stop"]) assert.equal(listeners.size, 0) globalThis.document = earlierDocument +globalThis.window = earlierWindow console.log("Controller buttons: source status validation, draft preservation, polling, auth, parked denial and capture cleanup passed") diff --git a/openpilot/starpilot/galaxy/tests/test_unload_guard.mjs b/openpilot/starpilot/galaxy/tests/test_unload_guard.mjs new file mode 100644 index 0000000000..6faeb5a4af --- /dev/null +++ b/openpilot/starpilot/galaxy/tests/test_unload_guard.mjs @@ -0,0 +1,19 @@ +import assert from "node:assert/strict" +import { guardUnload } from "../web/js/unload-guard.js" + +globalThis.window = new EventTarget() +let dirty = true +const stop = guardUnload(() => dirty) +const attempt = () => { + const event = new Event("beforeunload", { cancelable: true }) + // Browsers expose a writable returnValue on BeforeUnloadEvent; Node's Event does not. + Object.defineProperty(event, "returnValue", { value: "", writable: true }) + return window.dispatchEvent(event) +} +assert.equal(attempt(), false, "Live reload must defer while a draft is dirty") +dirty = false +assert.equal(attempt(), true, "Saving or discarding allows the pending reload") +dirty = true +stop() +assert.equal(attempt(), true, "An unmounted page must release its guard") +console.log("Unsaved draft unload guard passed") diff --git a/openpilot/starpilot/galaxy/web/js/controllers.js b/openpilot/starpilot/galaxy/web/js/controllers.js index 4556160be7..c59533d3ba 100644 --- a/openpilot/starpilot/galaxy/web/js/controllers.js +++ b/openpilot/starpilot/galaxy/web/js/controllers.js @@ -1,3 +1,4 @@ +import { guardUnload } from "./unload-guard.js" import { GxNotice } from "./notice.js" import { reactive } from "../vendor/vue/vue.esm-browser.js" @@ -132,11 +133,12 @@ export const ControllersPage = { actions() { return this.state.status?.slots.slice(3) || [] }, }, mounted() { + this._stopUnloadGuard = guardUnload(() => this.dirty || this.state.busy) this.visibility = () => { if (this.mode === "local" && !document.hidden) this.feed.start(); else this.feed.stop() } document.addEventListener("visibilitychange", this.visibility) this.visibility() }, - beforeUnmount() { document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() }, + beforeUnmount() { this._stopUnloadGuard(); document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() }, watch: { mode() { this.visibility() } }, methods: { async reload() { diff --git a/openpilot/starpilot/galaxy/web/js/model-laboratory.js b/openpilot/starpilot/galaxy/web/js/model-laboratory.js index f2c2ad4cdf..1312b71ce8 100644 --- a/openpilot/starpilot/galaxy/web/js/model-laboratory.js +++ b/openpilot/starpilot/galaxy/web/js/model-laboratory.js @@ -1,3 +1,4 @@ +import { guardUnload } from "./unload-guard.js" import { GxSummary } from "./summary.js" import { GxNotice } from "./notice.js" import { GalaxySelect } from "./galaxy-select.js" @@ -122,11 +123,12 @@ export const LaboratoryPage = { } }) }, mounted() { + this._stopUnloadGuard = guardUnload(() => this.dirty || this.busy) this.visibility = () => { if (document.hidden) this.feed.stop(); else if (this.mode === "local") this.feed.start() } document.addEventListener("visibilitychange", this.visibility) if (this.mode === "local" && !document.hidden) this.feed.start() }, - beforeUnmount() { document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() }, + beforeUnmount() { this._stopUnloadGuard(); document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() }, methods: { statusLabel: laboratoryStatusLabel, can(action, model = null) { return !this.busy && laboratoryActionAllowed(this.status, action, model, this.configuration) }, diff --git a/openpilot/starpilot/galaxy/web/js/onroad-layout.js b/openpilot/starpilot/galaxy/web/js/onroad-layout.js index ab4b80da07..559b8666be 100644 --- a/openpilot/starpilot/galaxy/web/js/onroad-layout.js +++ b/openpilot/starpilot/galaxy/web/js/onroad-layout.js @@ -1,3 +1,4 @@ +import { guardUnload } from "./unload-guard.js" import { PollTimer, connectionError } from "./polling.js" import { reactive, watch } from "../vendor/vue/vue.esm-browser.js" import { LayoutPreviewFeed, PREVIEW_SCENES } from "./layout-preview.js" @@ -437,10 +438,9 @@ export const OnroadLayoutPage = { }, mounted() { if (this.mode === "local") this.feed.start() - this._beforeUnload = (event) => { if (this.dirty || this.state.status === "saving") { event.preventDefault(); event.returnValue = "" } } - window.addEventListener("beforeunload", this._beforeUnload) + this._stopUnloadGuard = guardUnload(() => this.dirty || this.state.status === "saving") }, - beforeUnmount() { this.cancelDrag(); this.feed.stop(); this.previewFeed.stop(); window.removeEventListener("beforeunload", this._beforeUnload) }, + beforeUnmount() { this.cancelDrag(); this.feed.stop(); this.previewFeed.stop(); this._stopUnloadGuard() }, methods: { finishColorEdit() { this.state.history.group = null }, recordChange(before, group = null) { diff --git a/openpilot/starpilot/galaxy/web/js/unload-guard.js b/openpilot/starpilot/galaxy/web/js/unload-guard.js new file mode 100644 index 0000000000..fd9d7dca3a --- /dev/null +++ b/openpilot/starpilot/galaxy/web/js/unload-guard.js @@ -0,0 +1,8 @@ +// Keep unsaved drafts and pending saves on the page, including during host live reload. +export function guardUnload(shouldBlock) { + const listener = event => { + if (shouldBlock()) { event.preventDefault(); event.returnValue = "" } + } + window.addEventListener("beforeunload", listener) + return () => window.removeEventListener("beforeunload", listener) +} diff --git a/tools/ci/tests/test_dev_runtime.py b/tools/ci/tests/test_dev_runtime.py index ce0b2664fe..c940b46827 100644 --- a/tools/ci/tests/test_dev_runtime.py +++ b/tools/ci/tests/test_dev_runtime.py @@ -1,15 +1,17 @@ import fcntl import hashlib import os +import py_compile from pathlib import Path import signal import socket import subprocess +import sys import tempfile import unittest -from unittest.mock import patch +from unittest.mock import Mock, patch -from tools.host_runtime import GALAXY, LIVE_RELOAD, LIVE_TAG, LIVE_VERSION, HostRuntime, main, parse +from tools.host_runtime import GALAXY, LIVE_RELOAD, LIVE_TAG, LIVE_VERSION, HostRuntime, file_lock, main, parse class TestDeveloperRuntime(unittest.TestCase): @@ -85,6 +87,7 @@ class TestDeveloperRuntime(unittest.TestCase): self.assertEqual(env['CC'], '/usr/bin/clang') self.assertTrue(Path(env['PARAMS_ROOT']).is_relative_to(self.runtime.cache)) self.assertNotEqual(env['OPENPILOT_PREFIX'], 'real') + self.assertEqual(Path(env['ANDROID_AUTO_DIR']), self.runtime.cache / 'android_auto') second = HostRuntime(self.root, 'shared', system='Darwin', machine='arm64') cabana = HostRuntime(self.root, 'cabana', system='Darwin', machine='arm64') self.assertEqual(second.prefix, self.runtime.prefix) @@ -166,7 +169,7 @@ class TestDeveloperRuntime(unittest.TestCase): real_popen = subprocess.Popen def spawn(argv, *args, **kwargs): - # Only the server is faked; git check-ignore must really run. + # Only the server is faked; source selection must really run. return real_popen(argv, *args, **kwargs) if argv[0] == 'git' else Child() def tick(_interval): @@ -235,9 +238,130 @@ class TestDeveloperRuntime(unittest.TestCase): self.assertEqual(page.read_text().count(LIVE_TAG), 1) self.assertNotIn(LIVE_TAG, (galaxy / 'web/index.html').read_text()) script = self.runtime.work / GALAXY / 'web' / LIVE_RELOAD - script.unlink() # An overlapping live session exited and cleaned up. + self.runtime.end_live_reload() self.runtime.live_reload() self.assertTrue(script.is_file()) + self.assertTrue((page.parent / LIVE_VERSION).is_file()) + + def test_live_mirror_removes_stale_bytecode_and_detects_preserved_timestamps(self): + self.galaxy_fixture() + module = GALAXY / 'sample.py' + source = self.root / module + source.write_text('value = 1\n') + self.runtime.mirror([module], []) + cached = self.runtime.work / module + py_compile.compile(str(cached), doraise=True) + timestamp = source.stat().st_mtime_ns + before = self.runtime.galaxy_sources() + source.write_text('value = 2\n') + os.utime(source, ns=(timestamp, timestamp)) + self.assertNotEqual(before[module], self.runtime.galaxy_sources()[module]) + self.runtime.mirror([module], []) + result = subprocess.check_output([sys.executable, '-B', '-c', 'import sample; print(sample.value)'], cwd=cached.parent, text=True) + self.assertEqual(result.strip(), '2') + self.assertFalse(list((cached.parent / '__pycache__').glob('sample.*.pyc'))) + + def test_mirror_replaces_files_without_truncating_existing_readers(self): + galaxy = self.galaxy_fixture() + module = GALAXY / 'web/app.js' + with (self.runtime.work / module).open() as reader: + (galaxy / 'web/app.js').write_text('new contents\n') + self.runtime.mirror([module], []) + self.assertEqual(reader.read(), 'app\n') + self.assertEqual((self.runtime.work / module).read_text(), 'new contents\n') + + def test_sync_removes_deleted_live_added_files(self): + galaxy = self.galaxy_fixture() + module = GALAXY / 'new.py' + (galaxy / 'new.py').write_text('new\n') + self.runtime.mirror([module], []) + (galaxy / 'new.py').unlink() + self.runtime.sync() + self.assertFalse((self.runtime.work / module).exists()) + + def test_live_sources_and_sync_agree_on_tracked_ignored_files_and_symlinks(self): + galaxy = self.galaxy_fixture() + (galaxy / '.gitignore').write_text('ignored.py\n') + (galaxy / 'ignored.py').write_text('tracked\n') + self.git('add', '-f', str(GALAXY / 'ignored.py')) + link = GALAXY / 'linked.py' + (self.root / link).symlink_to('ignored.py') + sources = self.runtime.galaxy_sources() + self.assertIn(GALAXY / 'ignored.py', sources) + self.assertIn(link, sources) + self.runtime.mirror([link, GALAXY / 'ignored.py'], []) + self.assertTrue((self.runtime.work / link).is_symlink()) + self.assertEqual((self.runtime.work / link).read_text(), 'tracked\n') + + def test_live_start_reconciles_edits_and_deletions_during_build(self): + galaxy = self.galaxy_fixture() + (galaxy / 'web/app.js').write_text('edited during build\n') + (galaxy / 'server.py').unlink() + + def check(): + self.assertEqual((self.runtime.work / GALAXY / 'web/app.js').read_text(), 'edited during build\n') + self.assertFalse((self.runtime.work / GALAXY / 'server.py').exists()) + + self.run_live([], check) + + def test_live_retries_failed_copy_without_another_edit(self): + galaxy = self.galaxy_fixture() + module = GALAXY / 'web/app.js' + original = self.runtime.copy_source + failures = [] + + def copy(name): + if name == module and (galaxy / 'web/app.js').read_text() == 'edited\n' and not failures: + failures.append(name) + raise FileNotFoundError('editor replaced the file') + original(name) + + with patch.object(self.runtime, 'copy_source', copy): + self.run_live([lambda: (galaxy / 'web/app.js').write_text('edited\n'), lambda: None], lambda: None) + self.assertEqual(failures, [module]) + self.assertEqual((self.runtime.work / module).read_text(), 'edited\n') + + def test_live_rejects_another_owner_of_the_same_cache(self): + self.galaxy_fixture() + with file_lock(self.runtime.cache / 'galaxy-live-lock'), patch('tools.host_runtime.subprocess.Popen') as spawn: + with self.assertRaisesRegex(RuntimeError, 'already owns'): + self.runtime.live_galaxy(['--port', '8099']) + spawn.assert_not_called() + + def test_live_child_exit_and_start_failure_clean_up(self): + self.galaxy_fixture() + web = self.runtime.work / GALAXY / 'web' + for code in (0, 2, -signal.SIGTERM): + child = Mock(returncode=code) + child.poll.return_value = code + with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \ + patch('tools.host_runtime.subprocess.Popen', return_value=child): + self.assertEqual(self.runtime.live_galaxy([]), code if code >= 0 else 128 - code) + self.assertFalse((web / LIVE_VERSION).exists()) + with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \ + patch('tools.host_runtime.subprocess.Popen', side_effect=OSError('spawn failed')): + with self.assertRaisesRegex(OSError, 'spawn failed'): + self.runtime.live_galaxy([]) + self.assertFalse((web / LIVE_VERSION).exists()) + self.assertNotIn(LIVE_TAG, (web / 'index.html').read_text()) + + def test_live_termination_stops_child_and_restores_handlers(self): + self.galaxy_fixture() + for signum in (signal.SIGTERM, signal.SIGHUP): + previous = signal.getsignal(signum) + child = Mock(returncode=None) + child.poll.return_value = None + + def tick(_interval, signum=signum): + signal.getsignal(signum)(signum, None) + + with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \ + patch('tools.host_runtime.subprocess.Popen', return_value=child), \ + patch('tools.host_runtime.time.sleep', tick), patch('tools.host_runtime.stop') as stop: + self.assertEqual(self.runtime.live_galaxy([]), 128 + signum) + stop.assert_called_once_with(child, interrupted=False) + self.assertEqual(signal.getsignal(signum), previous) + self.assertFalse((self.runtime.work / GALAXY / 'web' / LIVE_VERSION).exists()) def test_live_galaxy_refuses_a_taken_port_before_syncing(self): with socket.socket() as taken: diff --git a/tools/host_runtime.py b/tools/host_runtime.py index 18ebd140e2..9291caaeec 100755 --- a/tools/host_runtime.py +++ b/tools/host_runtime.py @@ -16,6 +16,7 @@ import signal import socket import subprocess import sys +import tempfile import time @@ -67,7 +68,11 @@ LIVE_SCRIPT = f"""// Added by ./dev galaxy --live to its host cache only: reload const response = await fetch('/{LIVE_VERSION}', {{ cache: 'no-store' }}) if (!response.ok) return const version = (await response.text()).trim() - if (seen !== null && version !== seen) location.reload() + if (seen !== null && version !== seen) {{ + const event = new Event('beforeunload', {{ cancelable: true }}) + if (!window.dispatchEvent(event)) return + location.reload() + }} seen = version }} catch {{}} }}, 500) @@ -121,6 +126,31 @@ def same_contents(source, destination): return True +@contextmanager +def file_lock(path, *, nonblocking=False): + with path.open('a+') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | (fcntl.LOCK_NB if nonblocking else 0)) + yield + + +@contextmanager +def atomic_destination(path): + """Publish a complete file or symlink without truncating readers of the old inode.""" + fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f'.{path.name}-') + os.close(fd) + temporary = Path(temporary) + try: + yield temporary + os.replace(temporary, path) + finally: + temporary.unlink(missing_ok=True) + + +def atomic_write(path, data): + with atomic_destination(path) as temporary: + temporary.write_bytes(data) + + class HostRuntime: def __init__(self, root, bucket, *, system=None, machine=None): self.root = Path(root).resolve() @@ -162,48 +192,75 @@ class HostRuntime: self.lock_fd = None fcntl.flock(lock, fcntl.LOCK_UN) + def source_paths(self, *pathspec): + # Include uncommitted development, exclude ignored native/device outputs. + listed = git(self.root, 'ls-files', '--cached', '--others', '--exclude-standard', '-z', '--', *pathspec).split('\0') + return {Path(name) for name in listed if name and not (set(Path(name).parts) & EXCLUDE_DIRS) + and not (set(Path(name).suffixes) & NATIVE_SUFFIXES) and (self.root / name).is_file()} + + def source_lock(self): + return file_lock(inside(self.cache / 'source-lock', self.cache)) + + def copy_source(self, name): + source = self.root / name + destination = inside(self.work / name, self.work) + destination.parent.mkdir(parents=True, exist_ok=True) + if destination.is_dir() and not destination.is_symlink(): + shutil.rmtree(destination) + if source.is_symlink(): + inside(source, self.root) + target = os.readlink(source) + if os.path.isabs(target): + raise RuntimeError(f'Absolute source symlink is not portable: {name}') + if destination.is_symlink() and os.readlink(destination) == target: + return + with atomic_destination(destination) as temporary: + temporary.unlink() + temporary.symlink_to(target) + elif destination.is_symlink() or not same_contents(source, destination): + old_mtime = destination.stat().st_mtime_ns if destination.exists() else None + with atomic_destination(destination) as temporary: + shutil.copy2(source, temporary) + if temporary.stat().st_mtime_ns == old_mtime: + os.utime(temporary, None) # SCons must notice edits even when an editor keeps mtime. + elif source.stat().st_mode != destination.stat().st_mode: + destination.chmod(source.stat().st_mode) + if name.suffix == '.py': + # Timestamp-based bytecode can accept same-size edits within the same second. + for bytecode in (destination.parent / '__pycache__').glob(f'{destination.stem}.*.pyc'): + inside(bytecode, self.work).unlink(missing_ok=True) + + def mirror(self, changed, removed): + """Serialize cache mutations and retain ownership of live-added files for later syncs.""" + with self.source_lock(): + return self._mirror(changed, removed) + + def _mirror(self, changed, removed): + manifest = inside(self.cache / 'source-files.json', self.cache) + paths = set(json.loads(manifest.read_text())) if manifest.exists() else set() + copied = [] + for name in removed: + path = inside(self.work / name, self.work) + path.unlink(missing_ok=True) + paths.discard(str(name)) + for name in changed: + try: + self.copy_source(name) + except FileNotFoundError: + continue # Do not acknowledge this edit; the watcher will retry it. + copied.append(name) + paths.add(str(name)) + atomic_write(manifest, (json.dumps(sorted(paths)) + '\n').encode()) + return copied + def sync(self): inside(self.work, self.cache).mkdir(parents=True, exist_ok=True) - # Include uncommitted development, exclude ignored native/device outputs. - listed = git(self.root, 'ls-files', '--cached', '--others', '--exclude-standard', '-z').split('\0') - paths = {name for name in listed if name and not (set(Path(name).parts) & EXCLUDE_DIRS) - and not (set(Path(name).suffixes) & NATIVE_SUFFIXES) - and (self.root / name).is_file()} - manifest = inside(self.cache / 'source-files.json', self.cache) - previous = set(json.loads(manifest.read_text())) if manifest.exists() else set() - for name in sorted(previous - paths): - path = inside(self.work / name, self.work) - if path.is_file() or path.is_symlink(): - path.unlink() - for name in sorted(paths): - source = self.root / name - destination = inside(self.work / name, self.work) - destination.parent.mkdir(parents=True, exist_ok=True) - if source.is_symlink(): - # Keep repository-relative links; never point the host cache at device builds. - inside(source, self.root) - target = os.readlink(source) - if os.path.isabs(target): - raise RuntimeError(f'Absolute source symlink is not portable: {name}') - if destination.is_symlink() and os.readlink(destination) == target: - continue - if destination.exists() or destination.is_symlink(): - destination.unlink() - destination.symlink_to(target) - elif destination.is_symlink() or not same_contents(source, destination): - old_mtime = destination.stat().st_mtime_ns if destination.exists() else None - if destination.is_symlink(): - destination.unlink() - elif destination.is_dir(): - shutil.rmtree(destination) - shutil.copy2(source, destination) - # SCons' MD5-timestamp decider must notice edits even if an editor kept mtime. - if destination.stat().st_mtime_ns == old_mtime: - os.utime(destination, None) - elif source.stat().st_mode != destination.stat().st_mode: - destination.chmod(source.stat().st_mode) - self.sync_git() - manifest.write_text(json.dumps(sorted(paths)) + '\n') + paths = self.source_paths() + with self.source_lock(): + manifest = inside(self.cache / 'source-files.json', self.cache) + previous = set(json.loads(manifest.read_text())) if manifest.exists() else set() + self._mirror(sorted(paths), [Path(name) for name in sorted(previous - set(map(str, paths)))]) + self.sync_git() def sync_git(self): # Same read-only object sharing as `git clone --shared`, independent index/refs. @@ -238,6 +295,7 @@ class HostRuntime: BASEDIR=str(self.work), PWD=str(self.work), PARAMS_ROOT=str(inside(self.cache / 'params', self.cache)), OPENPILOT_PREFIX=self.prefix, SP_HOST_PREFIX=self.prefix, SP_HOST_PARAMS_ROOT=str(self.cache / 'params'), SP_HOST_RUNTIME='1', + ANDROID_AUTO_DIR=str(inside(self.cache / 'android_auto', self.cache)), SP_SCONS_CACHE_DIR=str(self.cache / 'scons-cache'), SCONS_CACHE=str(self.cache / 'scons-cache'), NOBOARD='1', SIMULATION='1', SKIP_FW_QUERY='1', STARPILOT_UI_DEV='1', @@ -322,58 +380,31 @@ class HostRuntime: signal.signal(sig, handler) def galaxy_sources(self): - """Source Galaxy files with their (mtime, size), under the same rules as sync().""" + """Content identities using the same source selection as sync(), including relative symlinks.""" found = {} - for directory, names, files in os.walk(self.root / GALAXY): - names[:] = [name for name in names if name not in EXCLUDE_DIRS] - for name in files: - path = Path(directory, name) - if path.is_symlink() or set(path.suffixes) & NATIVE_SUFFIXES: - continue - try: - stat = path.stat() - except FileNotFoundError: - continue - found[path.relative_to(self.root)] = (stat.st_mtime_ns, stat.st_size) - return found - - def ignored(self, names): - """Paths sync() would skip because Git ignores them, such as editor swap files.""" - if not names: - return set() - env = {key: value for key, value in os.environ.items() if key not in REMOVE_ENV} - result = subprocess.run(['git', 'check-ignore', '-z', '--stdin'], cwd=self.root, env=env, capture_output=True, - input='\0'.join(map(str, names)), text=True) - if result.returncode not in (0, 1): # 1 means nothing was ignored. - raise RuntimeError(f'git check-ignore failed: {result.stderr.strip()}') - return {Path(name) for name in result.stdout.split('\0') if name} - - def mirror(self, changed, removed): - # Unlocked on purpose: a concurrent sync copies these same files from the same checkout. - for name in removed: - path = inside(self.work / name, self.work) - if path.is_file() or path.is_symlink(): - path.unlink() - for name in changed: - destination = inside(self.work / name, self.work) - destination.parent.mkdir(parents=True, exist_ok=True) + for name in self.source_paths(str(GALAXY)): + path = self.root / name try: - shutil.copy2(self.root / name, destination) + identity = os.readlink(path).encode() if path.is_symlink() else path.read_bytes() + found[name] = (path.is_symlink(), path.stat().st_mode, hashlib.sha256(identity).digest()) except FileNotFoundError: - pass # Editors may replace files mid-save; the next poll copies the result. + continue + return found def live_reload(self, version=None): """Keep the cached page loading the reload script; publish a new edit version if given.""" + with self.source_lock(): + self._live_reload(version) + + def _live_reload(self, version=None): web = self.work / GALAXY / 'web' + if version is None and not (web / LIVE_VERSION).is_file(): + version = time.time_ns() if version is not None: - # Replace atomically so the page never reads a half-written version and reloads early. - temporary = inside(web / f'.{LIVE_VERSION}.tmp', self.work) - temporary.write_text(f'{version}\n') - os.replace(temporary, inside(web / LIVE_VERSION, self.work)) - # Checked every poll, like the page tag below: an overlapping session's exit removes it. + atomic_write(inside(web / LIVE_VERSION, self.work), f'{version}\n'.encode()) script = inside(web / LIVE_RELOAD, self.work) if not script.is_file() or script.read_text() != LIVE_SCRIPT: - script.write_text(LIVE_SCRIPT) + atomic_write(script, LIVE_SCRIPT.encode()) page = inside(web / 'index.html', self.work) try: html = page.read_text() @@ -381,19 +412,26 @@ class HostRuntime: return # Another command's sync restores the checkout's page. if LIVE_TAG not in html and '' in html: - page.write_text(html.replace('', f' {LIVE_TAG}\n', 1)) + atomic_write(page, html.replace('', f' {LIVE_TAG}\n', 1).encode()) def end_live_reload(self): - web = self.work / GALAXY / 'web' - for name in (LIVE_RELOAD, LIVE_VERSION): - inside(web / name, self.work).unlink(missing_ok=True) - source = self.root / GALAXY / 'web/index.html' - if source.is_file(): - shutil.copy2(source, inside(web / 'index.html', self.work)) + with self.source_lock(): + web = self.work / GALAXY / 'web' + for name in (LIVE_RELOAD, LIVE_VERSION): + inside(web / name, self.work).unlink(missing_ok=True) + if (self.root / GALAXY / 'web/index.html').is_file(): + self.copy_source(GALAXY / 'web/index.html') def live_galaxy(self, arguments, *, autoreload=True, interval=0.5): - """Serve Galaxy without the bucket lock, following package edits until interrupted.""" - argv = [str(self.venv / 'bin/python'), '-m', 'openpilot.starpilot.galaxy.server', *arguments] + """One live owner per cache; desktop UI and replay may still run alongside.""" + try: + with file_lock(inside(self.cache / 'galaxy-live-lock', self.cache), nonblocking=True): + return self._live_galaxy(arguments, autoreload=autoreload, interval=interval) + except BlockingIOError: + raise RuntimeError('A Galaxy live session already owns this host cache; stop it before starting another.') from None + + def _live_galaxy(self, arguments, *, autoreload, interval): + argv = [str(self.venv / 'bin/python'), '-B', '-m', 'openpilot.starpilot.galaxy.server', *arguments] env = self.environment() | {'SP_HOST_PARKED': '1'} self.private_ipc() # Without autoreload the cached page is left exactly as synced. @@ -405,23 +443,42 @@ class HostRuntime: print(f'Galaxy live: refresh to see edits under {GALAXY}; Python edits restart the server.', flush=True) print('Changes outside Galaxy need ./dev sync and a restart. Stop with Ctrl+C.', flush=True) seen = self.galaxy_sources() - publish(time.time_ns()) - child = subprocess.Popen(argv, cwd=self.work, env=env) + # Reconcile additions, edits and deletions made during preparation/building. + with self.source_lock(): + manifest = self.cache / 'source-files.json' + previous_sources = set(map(Path, json.loads(manifest.read_text()))) + removed = [name for name in previous_sources - seen.keys() if name.is_relative_to(GALAXY)] + copied = self._mirror(list(seen), removed) + seen = {name: seen[name] for name in copied} + child = None interrupted = False + termination = None + + def terminate(signum, _frame): + nonlocal termination + termination = signum + + previous = {sig: signal.signal(sig, terminate) for sig in (signal.SIGTERM, signal.SIGHUP)} try: + publish(time.time_ns()) + child = subprocess.Popen(argv, cwd=self.work, env=env) while True: + if termination is not None: + return 128 + termination + if child.poll() is not None: + result = child.returncode + return result if result >= 0 else 128 - result time.sleep(interval) publish() current = self.galaxy_sources() changed = [name for name, identity in current.items() if seen.get(name) != identity] removed = [name for name in seen if name not in current] - seen = current - skipped = self.ignored(changed + removed) - changed = [name for name in changed if name not in skipped] - removed = [name for name in removed if name not in skipped] if not changed and not removed: continue - self.mirror(changed, removed) + changed = self.mirror(changed, removed) + for name in removed: + seen.pop(name, None) + seen.update({name: current[name] for name in changed}) backend = [name for name in changed + removed if not name.is_relative_to(GALAXY / 'web') and not name.is_relative_to(GALAXY / 'tests') @@ -438,9 +495,14 @@ class HostRuntime: interrupted = True return 130 finally: - stop(child, interrupted=interrupted) - if autoreload: - self.end_live_reload() + try: + if child is not None: + stop(child, interrupted=interrupted) + if autoreload: + self.end_live_reload() + finally: + for sig, handler in previous.items(): + signal.signal(sig, handler) def galaxy_port_free(arguments):