mirror of
https://github.com/firestar5683/StarPilot.git
synced 2026-11-08 13:33:42 +08:00
Unload guard for unsaved drafts and enhance live reload handling
This commit is contained in:
@@ -23,8 +23,9 @@ then follows edits under this package. An open page reloads itself after each
|
||||
edit; other changes here restart the server first. Refresh a page opened before
|
||||
the session once. Edits outside the package need `./dev sync` and a restart.
|
||||
The reload script exists only in the host cache, never in this directory.
|
||||
Add `--no-autoreload` to keep following edits but refresh by hand.
|
||||
|
||||
Unsaved drafts and pending saves defer automatic reload until they are resolved.
|
||||
Add `--no-autoreload` to keep following edits but refresh by hand. Only one live
|
||||
Galaxy session can own a shared host cache at a time, regardless of port.
|
||||
Run the package tests with `./dev pytest openpilot/starpilot/galaxy/tests`.
|
||||
Browser JavaScript tests use Node 24; set `STARPILOT_NODE` to an existing Node
|
||||
24 executable when needed. A static preview can serve `web/` over loopback;
|
||||
|
||||
@@ -122,8 +122,8 @@ def host_owner(params, parked, root=None, environ=os.environ):
|
||||
if environ.get('SP_HOST_RUNTIME') != '1':
|
||||
return None
|
||||
if root is None:
|
||||
from openpilot.starpilot.storage import starpilot_storage_root
|
||||
root = starpilot_storage_root() / 'android_auto'
|
||||
from openpilot.starpilot.system.android_auto.identity import DATA_DIR
|
||||
root = DATA_DIR
|
||||
root = Path(root)
|
||||
screen_path = root / 'screen.json'
|
||||
if not screen_path.exists():
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
"""Actual file-owner tests, CPU-only, no IPC/Params runtime required."""
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
@@ -197,6 +200,27 @@ class TestProjectionLayoutOwner(unittest.TestCase):
|
||||
self.assertEqual((root / 'screen.json').read_bytes(), screen)
|
||||
self.assertEqual(again.snapshot()['document'], payload['document'])
|
||||
|
||||
def test_host_editor_and_renderer_share_default_paths(self):
|
||||
# Constants are initialized at import time, as in the actual launcher processes.
|
||||
code = """
|
||||
import copy
|
||||
import os
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from openpilot.starpilot.galaxy.projection_layout import host_owner
|
||||
from openpilot.starpilot.system.android_auto.projection_layout_runtime import load_projection_layout
|
||||
root = Path(os.environ['ANDROID_AUTO_DIR'])
|
||||
params = SimpleNamespace(get_param_path=lambda key: str(root / 'params' / key), get_bool=lambda key: False)
|
||||
owner = host_owner(params, lambda: True)
|
||||
snapshot = owner.snapshot()
|
||||
payload = {'revision': snapshot['revision'], 'document': copy.deepcopy(snapshot['document'])}
|
||||
payload['document']['widgets']['current_speed']['x'] += 10
|
||||
owner.save(payload, session_valid=lambda: True)
|
||||
assert load_projection_layout((1920, 1080)) == payload['document']
|
||||
"""
|
||||
env = os.environ | {'SP_HOST_RUNTIME': '1', 'ANDROID_AUTO_DIR': str(self.root / 'host')}
|
||||
subprocess.run([sys.executable, '-c', code], env=env, check=True, capture_output=True)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
@@ -118,6 +118,8 @@ component.feed.stop()
|
||||
assert.equal(component.state.draft, null)
|
||||
|
||||
const earlierDocument = globalThis.document
|
||||
const earlierWindow = globalThis.window
|
||||
globalThis.window = new EventTarget()
|
||||
const listeners = new Map(), lifecycle = []
|
||||
globalThis.document = { hidden: false, addEventListener(name, fn) { listeners.set(name, fn) },
|
||||
removeEventListener(name) { listeners.delete(name) } }
|
||||
@@ -133,4 +135,5 @@ ControllersPage.beforeUnmount.call(mounted)
|
||||
assert.deepEqual(lifecycle, ["start", "stop", "stop", "stop"])
|
||||
assert.equal(listeners.size, 0)
|
||||
globalThis.document = earlierDocument
|
||||
globalThis.window = earlierWindow
|
||||
console.log("Controller buttons: source status validation, draft preservation, polling, auth, parked denial and capture cleanup passed")
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
import assert from "node:assert/strict"
|
||||
import { guardUnload } from "../web/js/unload-guard.js"
|
||||
|
||||
globalThis.window = new EventTarget()
|
||||
let dirty = true
|
||||
const stop = guardUnload(() => dirty)
|
||||
const attempt = () => {
|
||||
const event = new Event("beforeunload", { cancelable: true })
|
||||
// Browsers expose a writable returnValue on BeforeUnloadEvent; Node's Event does not.
|
||||
Object.defineProperty(event, "returnValue", { value: "", writable: true })
|
||||
return window.dispatchEvent(event)
|
||||
}
|
||||
assert.equal(attempt(), false, "Live reload must defer while a draft is dirty")
|
||||
dirty = false
|
||||
assert.equal(attempt(), true, "Saving or discarding allows the pending reload")
|
||||
dirty = true
|
||||
stop()
|
||||
assert.equal(attempt(), true, "An unmounted page must release its guard")
|
||||
console.log("Unsaved draft unload guard passed")
|
||||
@@ -1,3 +1,4 @@
|
||||
import { guardUnload } from "./unload-guard.js"
|
||||
import { GxNotice } from "./notice.js"
|
||||
import { reactive } from "../vendor/vue/vue.esm-browser.js"
|
||||
|
||||
@@ -132,11 +133,12 @@ export const ControllersPage = {
|
||||
actions() { return this.state.status?.slots.slice(3) || [] },
|
||||
},
|
||||
mounted() {
|
||||
this._stopUnloadGuard = guardUnload(() => this.dirty || this.state.busy)
|
||||
this.visibility = () => { if (this.mode === "local" && !document.hidden) this.feed.start(); else this.feed.stop() }
|
||||
document.addEventListener("visibilitychange", this.visibility)
|
||||
this.visibility()
|
||||
},
|
||||
beforeUnmount() { document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() },
|
||||
beforeUnmount() { this._stopUnloadGuard(); document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() },
|
||||
watch: { mode() { this.visibility() } },
|
||||
methods: {
|
||||
async reload() {
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { guardUnload } from "./unload-guard.js"
|
||||
import { GxSummary } from "./summary.js"
|
||||
import { GxNotice } from "./notice.js"
|
||||
import { GalaxySelect } from "./galaxy-select.js"
|
||||
@@ -122,11 +123,12 @@ export const LaboratoryPage = {
|
||||
} })
|
||||
},
|
||||
mounted() {
|
||||
this._stopUnloadGuard = guardUnload(() => this.dirty || this.busy)
|
||||
this.visibility = () => { if (document.hidden) this.feed.stop(); else if (this.mode === "local") this.feed.start() }
|
||||
document.addEventListener("visibilitychange", this.visibility)
|
||||
if (this.mode === "local" && !document.hidden) this.feed.start()
|
||||
},
|
||||
beforeUnmount() { document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() },
|
||||
beforeUnmount() { this._stopUnloadGuard(); document.removeEventListener("visibilitychange", this.visibility); this.feed.stop() },
|
||||
methods: {
|
||||
statusLabel: laboratoryStatusLabel,
|
||||
can(action, model = null) { return !this.busy && laboratoryActionAllowed(this.status, action, model, this.configuration) },
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { guardUnload } from "./unload-guard.js"
|
||||
import { PollTimer, connectionError } from "./polling.js"
|
||||
import { reactive, watch } from "../vendor/vue/vue.esm-browser.js"
|
||||
import { LayoutPreviewFeed, PREVIEW_SCENES } from "./layout-preview.js"
|
||||
@@ -437,10 +438,9 @@ export const OnroadLayoutPage = {
|
||||
},
|
||||
mounted() {
|
||||
if (this.mode === "local") this.feed.start()
|
||||
this._beforeUnload = (event) => { if (this.dirty || this.state.status === "saving") { event.preventDefault(); event.returnValue = "" } }
|
||||
window.addEventListener("beforeunload", this._beforeUnload)
|
||||
this._stopUnloadGuard = guardUnload(() => this.dirty || this.state.status === "saving")
|
||||
},
|
||||
beforeUnmount() { this.cancelDrag(); this.feed.stop(); this.previewFeed.stop(); window.removeEventListener("beforeunload", this._beforeUnload) },
|
||||
beforeUnmount() { this.cancelDrag(); this.feed.stop(); this.previewFeed.stop(); this._stopUnloadGuard() },
|
||||
methods: {
|
||||
finishColorEdit() { this.state.history.group = null },
|
||||
recordChange(before, group = null) {
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
// Keep unsaved drafts and pending saves on the page, including during host live reload.
|
||||
export function guardUnload(shouldBlock) {
|
||||
const listener = event => {
|
||||
if (shouldBlock()) { event.preventDefault(); event.returnValue = "" }
|
||||
}
|
||||
window.addEventListener("beforeunload", listener)
|
||||
return () => window.removeEventListener("beforeunload", listener)
|
||||
}
|
||||
@@ -1,15 +1,17 @@
|
||||
import fcntl
|
||||
import hashlib
|
||||
import os
|
||||
import py_compile
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from tools.host_runtime import GALAXY, LIVE_RELOAD, LIVE_TAG, LIVE_VERSION, HostRuntime, main, parse
|
||||
from tools.host_runtime import GALAXY, LIVE_RELOAD, LIVE_TAG, LIVE_VERSION, HostRuntime, file_lock, main, parse
|
||||
|
||||
|
||||
class TestDeveloperRuntime(unittest.TestCase):
|
||||
@@ -85,6 +87,7 @@ class TestDeveloperRuntime(unittest.TestCase):
|
||||
self.assertEqual(env['CC'], '/usr/bin/clang')
|
||||
self.assertTrue(Path(env['PARAMS_ROOT']).is_relative_to(self.runtime.cache))
|
||||
self.assertNotEqual(env['OPENPILOT_PREFIX'], 'real')
|
||||
self.assertEqual(Path(env['ANDROID_AUTO_DIR']), self.runtime.cache / 'android_auto')
|
||||
second = HostRuntime(self.root, 'shared', system='Darwin', machine='arm64')
|
||||
cabana = HostRuntime(self.root, 'cabana', system='Darwin', machine='arm64')
|
||||
self.assertEqual(second.prefix, self.runtime.prefix)
|
||||
@@ -166,7 +169,7 @@ class TestDeveloperRuntime(unittest.TestCase):
|
||||
real_popen = subprocess.Popen
|
||||
|
||||
def spawn(argv, *args, **kwargs):
|
||||
# Only the server is faked; git check-ignore must really run.
|
||||
# Only the server is faked; source selection must really run.
|
||||
return real_popen(argv, *args, **kwargs) if argv[0] == 'git' else Child()
|
||||
|
||||
def tick(_interval):
|
||||
@@ -235,9 +238,130 @@ class TestDeveloperRuntime(unittest.TestCase):
|
||||
self.assertEqual(page.read_text().count(LIVE_TAG), 1)
|
||||
self.assertNotIn(LIVE_TAG, (galaxy / 'web/index.html').read_text())
|
||||
script = self.runtime.work / GALAXY / 'web' / LIVE_RELOAD
|
||||
script.unlink() # An overlapping live session exited and cleaned up.
|
||||
self.runtime.end_live_reload()
|
||||
self.runtime.live_reload()
|
||||
self.assertTrue(script.is_file())
|
||||
self.assertTrue((page.parent / LIVE_VERSION).is_file())
|
||||
|
||||
def test_live_mirror_removes_stale_bytecode_and_detects_preserved_timestamps(self):
|
||||
self.galaxy_fixture()
|
||||
module = GALAXY / 'sample.py'
|
||||
source = self.root / module
|
||||
source.write_text('value = 1\n')
|
||||
self.runtime.mirror([module], [])
|
||||
cached = self.runtime.work / module
|
||||
py_compile.compile(str(cached), doraise=True)
|
||||
timestamp = source.stat().st_mtime_ns
|
||||
before = self.runtime.galaxy_sources()
|
||||
source.write_text('value = 2\n')
|
||||
os.utime(source, ns=(timestamp, timestamp))
|
||||
self.assertNotEqual(before[module], self.runtime.galaxy_sources()[module])
|
||||
self.runtime.mirror([module], [])
|
||||
result = subprocess.check_output([sys.executable, '-B', '-c', 'import sample; print(sample.value)'], cwd=cached.parent, text=True)
|
||||
self.assertEqual(result.strip(), '2')
|
||||
self.assertFalse(list((cached.parent / '__pycache__').glob('sample.*.pyc')))
|
||||
|
||||
def test_mirror_replaces_files_without_truncating_existing_readers(self):
|
||||
galaxy = self.galaxy_fixture()
|
||||
module = GALAXY / 'web/app.js'
|
||||
with (self.runtime.work / module).open() as reader:
|
||||
(galaxy / 'web/app.js').write_text('new contents\n')
|
||||
self.runtime.mirror([module], [])
|
||||
self.assertEqual(reader.read(), 'app\n')
|
||||
self.assertEqual((self.runtime.work / module).read_text(), 'new contents\n')
|
||||
|
||||
def test_sync_removes_deleted_live_added_files(self):
|
||||
galaxy = self.galaxy_fixture()
|
||||
module = GALAXY / 'new.py'
|
||||
(galaxy / 'new.py').write_text('new\n')
|
||||
self.runtime.mirror([module], [])
|
||||
(galaxy / 'new.py').unlink()
|
||||
self.runtime.sync()
|
||||
self.assertFalse((self.runtime.work / module).exists())
|
||||
|
||||
def test_live_sources_and_sync_agree_on_tracked_ignored_files_and_symlinks(self):
|
||||
galaxy = self.galaxy_fixture()
|
||||
(galaxy / '.gitignore').write_text('ignored.py\n')
|
||||
(galaxy / 'ignored.py').write_text('tracked\n')
|
||||
self.git('add', '-f', str(GALAXY / 'ignored.py'))
|
||||
link = GALAXY / 'linked.py'
|
||||
(self.root / link).symlink_to('ignored.py')
|
||||
sources = self.runtime.galaxy_sources()
|
||||
self.assertIn(GALAXY / 'ignored.py', sources)
|
||||
self.assertIn(link, sources)
|
||||
self.runtime.mirror([link, GALAXY / 'ignored.py'], [])
|
||||
self.assertTrue((self.runtime.work / link).is_symlink())
|
||||
self.assertEqual((self.runtime.work / link).read_text(), 'tracked\n')
|
||||
|
||||
def test_live_start_reconciles_edits_and_deletions_during_build(self):
|
||||
galaxy = self.galaxy_fixture()
|
||||
(galaxy / 'web/app.js').write_text('edited during build\n')
|
||||
(galaxy / 'server.py').unlink()
|
||||
|
||||
def check():
|
||||
self.assertEqual((self.runtime.work / GALAXY / 'web/app.js').read_text(), 'edited during build\n')
|
||||
self.assertFalse((self.runtime.work / GALAXY / 'server.py').exists())
|
||||
|
||||
self.run_live([], check)
|
||||
|
||||
def test_live_retries_failed_copy_without_another_edit(self):
|
||||
galaxy = self.galaxy_fixture()
|
||||
module = GALAXY / 'web/app.js'
|
||||
original = self.runtime.copy_source
|
||||
failures = []
|
||||
|
||||
def copy(name):
|
||||
if name == module and (galaxy / 'web/app.js').read_text() == 'edited\n' and not failures:
|
||||
failures.append(name)
|
||||
raise FileNotFoundError('editor replaced the file')
|
||||
original(name)
|
||||
|
||||
with patch.object(self.runtime, 'copy_source', copy):
|
||||
self.run_live([lambda: (galaxy / 'web/app.js').write_text('edited\n'), lambda: None], lambda: None)
|
||||
self.assertEqual(failures, [module])
|
||||
self.assertEqual((self.runtime.work / module).read_text(), 'edited\n')
|
||||
|
||||
def test_live_rejects_another_owner_of_the_same_cache(self):
|
||||
self.galaxy_fixture()
|
||||
with file_lock(self.runtime.cache / 'galaxy-live-lock'), patch('tools.host_runtime.subprocess.Popen') as spawn:
|
||||
with self.assertRaisesRegex(RuntimeError, 'already owns'):
|
||||
self.runtime.live_galaxy(['--port', '8099'])
|
||||
spawn.assert_not_called()
|
||||
|
||||
def test_live_child_exit_and_start_failure_clean_up(self):
|
||||
self.galaxy_fixture()
|
||||
web = self.runtime.work / GALAXY / 'web'
|
||||
for code in (0, 2, -signal.SIGTERM):
|
||||
child = Mock(returncode=code)
|
||||
child.poll.return_value = code
|
||||
with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \
|
||||
patch('tools.host_runtime.subprocess.Popen', return_value=child):
|
||||
self.assertEqual(self.runtime.live_galaxy([]), code if code >= 0 else 128 - code)
|
||||
self.assertFalse((web / LIVE_VERSION).exists())
|
||||
with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \
|
||||
patch('tools.host_runtime.subprocess.Popen', side_effect=OSError('spawn failed')):
|
||||
with self.assertRaisesRegex(OSError, 'spawn failed'):
|
||||
self.runtime.live_galaxy([])
|
||||
self.assertFalse((web / LIVE_VERSION).exists())
|
||||
self.assertNotIn(LIVE_TAG, (web / 'index.html').read_text())
|
||||
|
||||
def test_live_termination_stops_child_and_restores_handlers(self):
|
||||
self.galaxy_fixture()
|
||||
for signum in (signal.SIGTERM, signal.SIGHUP):
|
||||
previous = signal.getsignal(signum)
|
||||
child = Mock(returncode=None)
|
||||
child.poll.return_value = None
|
||||
|
||||
def tick(_interval, signum=signum):
|
||||
signal.getsignal(signum)(signum, None)
|
||||
|
||||
with patch.object(self.runtime, 'galaxy_sources', return_value=self.runtime.galaxy_sources()), \
|
||||
patch('tools.host_runtime.subprocess.Popen', return_value=child), \
|
||||
patch('tools.host_runtime.time.sleep', tick), patch('tools.host_runtime.stop') as stop:
|
||||
self.assertEqual(self.runtime.live_galaxy([]), 128 + signum)
|
||||
stop.assert_called_once_with(child, interrupted=False)
|
||||
self.assertEqual(signal.getsignal(signum), previous)
|
||||
self.assertFalse((self.runtime.work / GALAXY / 'web' / LIVE_VERSION).exists())
|
||||
|
||||
def test_live_galaxy_refuses_a_taken_port_before_syncing(self):
|
||||
with socket.socket() as taken:
|
||||
|
||||
+164
-102
@@ -16,6 +16,7 @@ import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import time
|
||||
|
||||
|
||||
@@ -67,7 +68,11 @@ LIVE_SCRIPT = f"""// Added by ./dev galaxy --live to its host cache only: reload
|
||||
const response = await fetch('/{LIVE_VERSION}', {{ cache: 'no-store' }})
|
||||
if (!response.ok) return
|
||||
const version = (await response.text()).trim()
|
||||
if (seen !== null && version !== seen) location.reload()
|
||||
if (seen !== null && version !== seen) {{
|
||||
const event = new Event('beforeunload', {{ cancelable: true }})
|
||||
if (!window.dispatchEvent(event)) return
|
||||
location.reload()
|
||||
}}
|
||||
seen = version
|
||||
}} catch {{}}
|
||||
}}, 500)
|
||||
@@ -121,6 +126,31 @@ def same_contents(source, destination):
|
||||
return True
|
||||
|
||||
|
||||
@contextmanager
|
||||
def file_lock(path, *, nonblocking=False):
|
||||
with path.open('a+') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | (fcntl.LOCK_NB if nonblocking else 0))
|
||||
yield
|
||||
|
||||
|
||||
@contextmanager
|
||||
def atomic_destination(path):
|
||||
"""Publish a complete file or symlink without truncating readers of the old inode."""
|
||||
fd, temporary = tempfile.mkstemp(dir=path.parent, prefix=f'.{path.name}-')
|
||||
os.close(fd)
|
||||
temporary = Path(temporary)
|
||||
try:
|
||||
yield temporary
|
||||
os.replace(temporary, path)
|
||||
finally:
|
||||
temporary.unlink(missing_ok=True)
|
||||
|
||||
|
||||
def atomic_write(path, data):
|
||||
with atomic_destination(path) as temporary:
|
||||
temporary.write_bytes(data)
|
||||
|
||||
|
||||
class HostRuntime:
|
||||
def __init__(self, root, bucket, *, system=None, machine=None):
|
||||
self.root = Path(root).resolve()
|
||||
@@ -162,48 +192,75 @@ class HostRuntime:
|
||||
self.lock_fd = None
|
||||
fcntl.flock(lock, fcntl.LOCK_UN)
|
||||
|
||||
def source_paths(self, *pathspec):
|
||||
# Include uncommitted development, exclude ignored native/device outputs.
|
||||
listed = git(self.root, 'ls-files', '--cached', '--others', '--exclude-standard', '-z', '--', *pathspec).split('\0')
|
||||
return {Path(name) for name in listed if name and not (set(Path(name).parts) & EXCLUDE_DIRS)
|
||||
and not (set(Path(name).suffixes) & NATIVE_SUFFIXES) and (self.root / name).is_file()}
|
||||
|
||||
def source_lock(self):
|
||||
return file_lock(inside(self.cache / 'source-lock', self.cache))
|
||||
|
||||
def copy_source(self, name):
|
||||
source = self.root / name
|
||||
destination = inside(self.work / name, self.work)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if destination.is_dir() and not destination.is_symlink():
|
||||
shutil.rmtree(destination)
|
||||
if source.is_symlink():
|
||||
inside(source, self.root)
|
||||
target = os.readlink(source)
|
||||
if os.path.isabs(target):
|
||||
raise RuntimeError(f'Absolute source symlink is not portable: {name}')
|
||||
if destination.is_symlink() and os.readlink(destination) == target:
|
||||
return
|
||||
with atomic_destination(destination) as temporary:
|
||||
temporary.unlink()
|
||||
temporary.symlink_to(target)
|
||||
elif destination.is_symlink() or not same_contents(source, destination):
|
||||
old_mtime = destination.stat().st_mtime_ns if destination.exists() else None
|
||||
with atomic_destination(destination) as temporary:
|
||||
shutil.copy2(source, temporary)
|
||||
if temporary.stat().st_mtime_ns == old_mtime:
|
||||
os.utime(temporary, None) # SCons must notice edits even when an editor keeps mtime.
|
||||
elif source.stat().st_mode != destination.stat().st_mode:
|
||||
destination.chmod(source.stat().st_mode)
|
||||
if name.suffix == '.py':
|
||||
# Timestamp-based bytecode can accept same-size edits within the same second.
|
||||
for bytecode in (destination.parent / '__pycache__').glob(f'{destination.stem}.*.pyc'):
|
||||
inside(bytecode, self.work).unlink(missing_ok=True)
|
||||
|
||||
def mirror(self, changed, removed):
|
||||
"""Serialize cache mutations and retain ownership of live-added files for later syncs."""
|
||||
with self.source_lock():
|
||||
return self._mirror(changed, removed)
|
||||
|
||||
def _mirror(self, changed, removed):
|
||||
manifest = inside(self.cache / 'source-files.json', self.cache)
|
||||
paths = set(json.loads(manifest.read_text())) if manifest.exists() else set()
|
||||
copied = []
|
||||
for name in removed:
|
||||
path = inside(self.work / name, self.work)
|
||||
path.unlink(missing_ok=True)
|
||||
paths.discard(str(name))
|
||||
for name in changed:
|
||||
try:
|
||||
self.copy_source(name)
|
||||
except FileNotFoundError:
|
||||
continue # Do not acknowledge this edit; the watcher will retry it.
|
||||
copied.append(name)
|
||||
paths.add(str(name))
|
||||
atomic_write(manifest, (json.dumps(sorted(paths)) + '\n').encode())
|
||||
return copied
|
||||
|
||||
def sync(self):
|
||||
inside(self.work, self.cache).mkdir(parents=True, exist_ok=True)
|
||||
# Include uncommitted development, exclude ignored native/device outputs.
|
||||
listed = git(self.root, 'ls-files', '--cached', '--others', '--exclude-standard', '-z').split('\0')
|
||||
paths = {name for name in listed if name and not (set(Path(name).parts) & EXCLUDE_DIRS)
|
||||
and not (set(Path(name).suffixes) & NATIVE_SUFFIXES)
|
||||
and (self.root / name).is_file()}
|
||||
manifest = inside(self.cache / 'source-files.json', self.cache)
|
||||
previous = set(json.loads(manifest.read_text())) if manifest.exists() else set()
|
||||
for name in sorted(previous - paths):
|
||||
path = inside(self.work / name, self.work)
|
||||
if path.is_file() or path.is_symlink():
|
||||
path.unlink()
|
||||
for name in sorted(paths):
|
||||
source = self.root / name
|
||||
destination = inside(self.work / name, self.work)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
if source.is_symlink():
|
||||
# Keep repository-relative links; never point the host cache at device builds.
|
||||
inside(source, self.root)
|
||||
target = os.readlink(source)
|
||||
if os.path.isabs(target):
|
||||
raise RuntimeError(f'Absolute source symlink is not portable: {name}')
|
||||
if destination.is_symlink() and os.readlink(destination) == target:
|
||||
continue
|
||||
if destination.exists() or destination.is_symlink():
|
||||
destination.unlink()
|
||||
destination.symlink_to(target)
|
||||
elif destination.is_symlink() or not same_contents(source, destination):
|
||||
old_mtime = destination.stat().st_mtime_ns if destination.exists() else None
|
||||
if destination.is_symlink():
|
||||
destination.unlink()
|
||||
elif destination.is_dir():
|
||||
shutil.rmtree(destination)
|
||||
shutil.copy2(source, destination)
|
||||
# SCons' MD5-timestamp decider must notice edits even if an editor kept mtime.
|
||||
if destination.stat().st_mtime_ns == old_mtime:
|
||||
os.utime(destination, None)
|
||||
elif source.stat().st_mode != destination.stat().st_mode:
|
||||
destination.chmod(source.stat().st_mode)
|
||||
self.sync_git()
|
||||
manifest.write_text(json.dumps(sorted(paths)) + '\n')
|
||||
paths = self.source_paths()
|
||||
with self.source_lock():
|
||||
manifest = inside(self.cache / 'source-files.json', self.cache)
|
||||
previous = set(json.loads(manifest.read_text())) if manifest.exists() else set()
|
||||
self._mirror(sorted(paths), [Path(name) for name in sorted(previous - set(map(str, paths)))])
|
||||
self.sync_git()
|
||||
|
||||
def sync_git(self):
|
||||
# Same read-only object sharing as `git clone --shared`, independent index/refs.
|
||||
@@ -238,6 +295,7 @@ class HostRuntime:
|
||||
BASEDIR=str(self.work), PWD=str(self.work), PARAMS_ROOT=str(inside(self.cache / 'params', self.cache)),
|
||||
OPENPILOT_PREFIX=self.prefix, SP_HOST_PREFIX=self.prefix,
|
||||
SP_HOST_PARAMS_ROOT=str(self.cache / 'params'), SP_HOST_RUNTIME='1',
|
||||
ANDROID_AUTO_DIR=str(inside(self.cache / 'android_auto', self.cache)),
|
||||
SP_SCONS_CACHE_DIR=str(self.cache / 'scons-cache'),
|
||||
SCONS_CACHE=str(self.cache / 'scons-cache'),
|
||||
NOBOARD='1', SIMULATION='1', SKIP_FW_QUERY='1', STARPILOT_UI_DEV='1',
|
||||
@@ -322,58 +380,31 @@ class HostRuntime:
|
||||
signal.signal(sig, handler)
|
||||
|
||||
def galaxy_sources(self):
|
||||
"""Source Galaxy files with their (mtime, size), under the same rules as sync()."""
|
||||
"""Content identities using the same source selection as sync(), including relative symlinks."""
|
||||
found = {}
|
||||
for directory, names, files in os.walk(self.root / GALAXY):
|
||||
names[:] = [name for name in names if name not in EXCLUDE_DIRS]
|
||||
for name in files:
|
||||
path = Path(directory, name)
|
||||
if path.is_symlink() or set(path.suffixes) & NATIVE_SUFFIXES:
|
||||
continue
|
||||
try:
|
||||
stat = path.stat()
|
||||
except FileNotFoundError:
|
||||
continue
|
||||
found[path.relative_to(self.root)] = (stat.st_mtime_ns, stat.st_size)
|
||||
return found
|
||||
|
||||
def ignored(self, names):
|
||||
"""Paths sync() would skip because Git ignores them, such as editor swap files."""
|
||||
if not names:
|
||||
return set()
|
||||
env = {key: value for key, value in os.environ.items() if key not in REMOVE_ENV}
|
||||
result = subprocess.run(['git', 'check-ignore', '-z', '--stdin'], cwd=self.root, env=env, capture_output=True,
|
||||
input='\0'.join(map(str, names)), text=True)
|
||||
if result.returncode not in (0, 1): # 1 means nothing was ignored.
|
||||
raise RuntimeError(f'git check-ignore failed: {result.stderr.strip()}')
|
||||
return {Path(name) for name in result.stdout.split('\0') if name}
|
||||
|
||||
def mirror(self, changed, removed):
|
||||
# Unlocked on purpose: a concurrent sync copies these same files from the same checkout.
|
||||
for name in removed:
|
||||
path = inside(self.work / name, self.work)
|
||||
if path.is_file() or path.is_symlink():
|
||||
path.unlink()
|
||||
for name in changed:
|
||||
destination = inside(self.work / name, self.work)
|
||||
destination.parent.mkdir(parents=True, exist_ok=True)
|
||||
for name in self.source_paths(str(GALAXY)):
|
||||
path = self.root / name
|
||||
try:
|
||||
shutil.copy2(self.root / name, destination)
|
||||
identity = os.readlink(path).encode() if path.is_symlink() else path.read_bytes()
|
||||
found[name] = (path.is_symlink(), path.stat().st_mode, hashlib.sha256(identity).digest())
|
||||
except FileNotFoundError:
|
||||
pass # Editors may replace files mid-save; the next poll copies the result.
|
||||
continue
|
||||
return found
|
||||
|
||||
def live_reload(self, version=None):
|
||||
"""Keep the cached page loading the reload script; publish a new edit version if given."""
|
||||
with self.source_lock():
|
||||
self._live_reload(version)
|
||||
|
||||
def _live_reload(self, version=None):
|
||||
web = self.work / GALAXY / 'web'
|
||||
if version is None and not (web / LIVE_VERSION).is_file():
|
||||
version = time.time_ns()
|
||||
if version is not None:
|
||||
# Replace atomically so the page never reads a half-written version and reloads early.
|
||||
temporary = inside(web / f'.{LIVE_VERSION}.tmp', self.work)
|
||||
temporary.write_text(f'{version}\n')
|
||||
os.replace(temporary, inside(web / LIVE_VERSION, self.work))
|
||||
# Checked every poll, like the page tag below: an overlapping session's exit removes it.
|
||||
atomic_write(inside(web / LIVE_VERSION, self.work), f'{version}\n'.encode())
|
||||
script = inside(web / LIVE_RELOAD, self.work)
|
||||
if not script.is_file() or script.read_text() != LIVE_SCRIPT:
|
||||
script.write_text(LIVE_SCRIPT)
|
||||
atomic_write(script, LIVE_SCRIPT.encode())
|
||||
page = inside(web / 'index.html', self.work)
|
||||
try:
|
||||
html = page.read_text()
|
||||
@@ -381,19 +412,26 @@ class HostRuntime:
|
||||
return
|
||||
# Another command's sync restores the checkout's page.
|
||||
if LIVE_TAG not in html and '</body>' in html:
|
||||
page.write_text(html.replace('</body>', f' {LIVE_TAG}\n</body>', 1))
|
||||
atomic_write(page, html.replace('</body>', f' {LIVE_TAG}\n</body>', 1).encode())
|
||||
|
||||
def end_live_reload(self):
|
||||
web = self.work / GALAXY / 'web'
|
||||
for name in (LIVE_RELOAD, LIVE_VERSION):
|
||||
inside(web / name, self.work).unlink(missing_ok=True)
|
||||
source = self.root / GALAXY / 'web/index.html'
|
||||
if source.is_file():
|
||||
shutil.copy2(source, inside(web / 'index.html', self.work))
|
||||
with self.source_lock():
|
||||
web = self.work / GALAXY / 'web'
|
||||
for name in (LIVE_RELOAD, LIVE_VERSION):
|
||||
inside(web / name, self.work).unlink(missing_ok=True)
|
||||
if (self.root / GALAXY / 'web/index.html').is_file():
|
||||
self.copy_source(GALAXY / 'web/index.html')
|
||||
|
||||
def live_galaxy(self, arguments, *, autoreload=True, interval=0.5):
|
||||
"""Serve Galaxy without the bucket lock, following package edits until interrupted."""
|
||||
argv = [str(self.venv / 'bin/python'), '-m', 'openpilot.starpilot.galaxy.server', *arguments]
|
||||
"""One live owner per cache; desktop UI and replay may still run alongside."""
|
||||
try:
|
||||
with file_lock(inside(self.cache / 'galaxy-live-lock', self.cache), nonblocking=True):
|
||||
return self._live_galaxy(arguments, autoreload=autoreload, interval=interval)
|
||||
except BlockingIOError:
|
||||
raise RuntimeError('A Galaxy live session already owns this host cache; stop it before starting another.') from None
|
||||
|
||||
def _live_galaxy(self, arguments, *, autoreload, interval):
|
||||
argv = [str(self.venv / 'bin/python'), '-B', '-m', 'openpilot.starpilot.galaxy.server', *arguments]
|
||||
env = self.environment() | {'SP_HOST_PARKED': '1'}
|
||||
self.private_ipc()
|
||||
# Without autoreload the cached page is left exactly as synced.
|
||||
@@ -405,23 +443,42 @@ class HostRuntime:
|
||||
print(f'Galaxy live: refresh to see edits under {GALAXY}; Python edits restart the server.', flush=True)
|
||||
print('Changes outside Galaxy need ./dev sync and a restart. Stop with Ctrl+C.', flush=True)
|
||||
seen = self.galaxy_sources()
|
||||
publish(time.time_ns())
|
||||
child = subprocess.Popen(argv, cwd=self.work, env=env)
|
||||
# Reconcile additions, edits and deletions made during preparation/building.
|
||||
with self.source_lock():
|
||||
manifest = self.cache / 'source-files.json'
|
||||
previous_sources = set(map(Path, json.loads(manifest.read_text())))
|
||||
removed = [name for name in previous_sources - seen.keys() if name.is_relative_to(GALAXY)]
|
||||
copied = self._mirror(list(seen), removed)
|
||||
seen = {name: seen[name] for name in copied}
|
||||
child = None
|
||||
interrupted = False
|
||||
termination = None
|
||||
|
||||
def terminate(signum, _frame):
|
||||
nonlocal termination
|
||||
termination = signum
|
||||
|
||||
previous = {sig: signal.signal(sig, terminate) for sig in (signal.SIGTERM, signal.SIGHUP)}
|
||||
try:
|
||||
publish(time.time_ns())
|
||||
child = subprocess.Popen(argv, cwd=self.work, env=env)
|
||||
while True:
|
||||
if termination is not None:
|
||||
return 128 + termination
|
||||
if child.poll() is not None:
|
||||
result = child.returncode
|
||||
return result if result >= 0 else 128 - result
|
||||
time.sleep(interval)
|
||||
publish()
|
||||
current = self.galaxy_sources()
|
||||
changed = [name for name, identity in current.items() if seen.get(name) != identity]
|
||||
removed = [name for name in seen if name not in current]
|
||||
seen = current
|
||||
skipped = self.ignored(changed + removed)
|
||||
changed = [name for name in changed if name not in skipped]
|
||||
removed = [name for name in removed if name not in skipped]
|
||||
if not changed and not removed:
|
||||
continue
|
||||
self.mirror(changed, removed)
|
||||
changed = self.mirror(changed, removed)
|
||||
for name in removed:
|
||||
seen.pop(name, None)
|
||||
seen.update({name: current[name] for name in changed})
|
||||
backend = [name for name in changed + removed
|
||||
if not name.is_relative_to(GALAXY / 'web')
|
||||
and not name.is_relative_to(GALAXY / 'tests')
|
||||
@@ -438,9 +495,14 @@ class HostRuntime:
|
||||
interrupted = True
|
||||
return 130
|
||||
finally:
|
||||
stop(child, interrupted=interrupted)
|
||||
if autoreload:
|
||||
self.end_live_reload()
|
||||
try:
|
||||
if child is not None:
|
||||
stop(child, interrupted=interrupted)
|
||||
if autoreload:
|
||||
self.end_live_reload()
|
||||
finally:
|
||||
for sig, handler in previous.items():
|
||||
signal.signal(sig, handler)
|
||||
|
||||
|
||||
def galaxy_port_free(arguments):
|
||||
|
||||
Reference in New Issue
Block a user