openpilot v0.11.1 release
date: 2026-06-04T09:49:56 master commit: c0ab3550eca2e9daf197c46b7e4b24aa9637cf2e
This commit is contained in:
Vehicle Researcher
Executable
+246
@@ -0,0 +1,246 @@
|
||||
#!/usr/bin/env python3
|
||||
import io
|
||||
import lzma
|
||||
import os
|
||||
import pathlib
|
||||
import struct
|
||||
import sys
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import defaultdict, namedtuple
|
||||
from collections.abc import Callable
|
||||
from typing import IO
|
||||
|
||||
import requests
|
||||
from Crypto.Hash import SHA512
|
||||
from openpilot.system.updated.casync import tar
|
||||
from openpilot.system.updated.casync.common import create_casync_tar_package
|
||||
|
||||
CA_FORMAT_INDEX = 0x96824d9c7b129ff9
|
||||
CA_FORMAT_TABLE = 0xe75b9e112f17417d
|
||||
CA_FORMAT_TABLE_TAIL_MARKER = 0xe75b9e112f17417
|
||||
FLAGS = 0xb000000000000000
|
||||
|
||||
CA_HEADER_LEN = 48
|
||||
CA_TABLE_HEADER_LEN = 16
|
||||
CA_TABLE_ENTRY_LEN = 40
|
||||
CA_TABLE_MIN_LEN = CA_TABLE_HEADER_LEN + CA_TABLE_ENTRY_LEN
|
||||
|
||||
CHUNK_DOWNLOAD_TIMEOUT = 60
|
||||
CHUNK_DOWNLOAD_RETRIES = 3
|
||||
|
||||
CAIBX_DOWNLOAD_TIMEOUT = 120
|
||||
|
||||
Chunk = namedtuple('Chunk', ['sha', 'offset', 'length'])
|
||||
ChunkDict = dict[bytes, Chunk]
|
||||
|
||||
|
||||
class ChunkReader(ABC):
|
||||
@abstractmethod
|
||||
def read(self, chunk: Chunk) -> bytes:
|
||||
...
|
||||
|
||||
|
||||
class BinaryChunkReader(ChunkReader):
|
||||
"""Reads chunks from a local file"""
|
||||
def __init__(self, file_like: IO[bytes]) -> None:
|
||||
super().__init__()
|
||||
self.f = file_like
|
||||
|
||||
def read(self, chunk: Chunk) -> bytes:
|
||||
self.f.seek(chunk.offset)
|
||||
return self.f.read(chunk.length)
|
||||
|
||||
|
||||
class FileChunkReader(BinaryChunkReader):
|
||||
def __init__(self, path: str) -> None:
|
||||
super().__init__(open(path, 'rb'))
|
||||
|
||||
def __del__(self):
|
||||
self.f.close()
|
||||
|
||||
|
||||
class RemoteChunkReader(ChunkReader):
|
||||
"""Reads lzma compressed chunks from a remote store"""
|
||||
|
||||
def __init__(self, url: str) -> None:
|
||||
super().__init__()
|
||||
self.url = url
|
||||
self.session = requests.Session()
|
||||
|
||||
def read(self, chunk: Chunk) -> bytes:
|
||||
sha_hex = chunk.sha.hex()
|
||||
url = os.path.join(self.url, sha_hex[:4], sha_hex + ".cacnk")
|
||||
|
||||
if os.path.isfile(url):
|
||||
with open(url, 'rb') as f:
|
||||
contents = f.read()
|
||||
else:
|
||||
for i in range(CHUNK_DOWNLOAD_RETRIES):
|
||||
try:
|
||||
resp = self.session.get(url, timeout=CHUNK_DOWNLOAD_TIMEOUT)
|
||||
break
|
||||
except Exception:
|
||||
if i == CHUNK_DOWNLOAD_RETRIES - 1:
|
||||
raise
|
||||
time.sleep(CHUNK_DOWNLOAD_TIMEOUT)
|
||||
|
||||
resp.raise_for_status()
|
||||
contents = resp.content
|
||||
|
||||
decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO)
|
||||
return decompressor.decompress(contents)
|
||||
|
||||
|
||||
class DirectoryTarChunkReader(BinaryChunkReader):
|
||||
"""creates a tar archive of a directory and reads chunks from it"""
|
||||
|
||||
def __init__(self, path: str, cache_file: str) -> None:
|
||||
create_casync_tar_package(pathlib.Path(path), pathlib.Path(cache_file))
|
||||
|
||||
self.f = open(cache_file, "rb")
|
||||
super().__init__(self.f)
|
||||
|
||||
def __del__(self):
|
||||
self.f.close()
|
||||
os.unlink(self.f.name)
|
||||
|
||||
|
||||
def parse_caibx(caibx_path: str) -> list[Chunk]:
|
||||
"""Parses the chunks from a caibx file. Can handle both local and remote files.
|
||||
Returns a list of chunks with hash, offset and length"""
|
||||
caibx: io.BufferedIOBase
|
||||
if os.path.isfile(caibx_path):
|
||||
caibx = open(caibx_path, 'rb')
|
||||
else:
|
||||
resp = requests.get(caibx_path, timeout=CAIBX_DOWNLOAD_TIMEOUT)
|
||||
resp.raise_for_status()
|
||||
caibx = io.BytesIO(resp.content)
|
||||
|
||||
caibx.seek(0, os.SEEK_END)
|
||||
caibx_len = caibx.tell()
|
||||
caibx.seek(0, os.SEEK_SET)
|
||||
|
||||
# Parse header
|
||||
length, magic, flags, min_size, _, max_size = struct.unpack("<QQQQQQ", caibx.read(CA_HEADER_LEN))
|
||||
assert flags == flags
|
||||
assert length == CA_HEADER_LEN
|
||||
assert magic == CA_FORMAT_INDEX
|
||||
|
||||
# Parse table header
|
||||
length, magic = struct.unpack("<QQ", caibx.read(CA_TABLE_HEADER_LEN))
|
||||
assert magic == CA_FORMAT_TABLE
|
||||
|
||||
# Parse chunks
|
||||
num_chunks = (caibx_len - CA_HEADER_LEN - CA_TABLE_MIN_LEN) // CA_TABLE_ENTRY_LEN
|
||||
chunks = []
|
||||
|
||||
offset = 0
|
||||
for i in range(num_chunks):
|
||||
new_offset = struct.unpack("<Q", caibx.read(8))[0]
|
||||
|
||||
sha = caibx.read(32)
|
||||
length = new_offset - offset
|
||||
|
||||
assert length <= max_size
|
||||
|
||||
# Last chunk can be smaller
|
||||
if i < num_chunks - 1:
|
||||
assert length >= min_size
|
||||
|
||||
chunks.append(Chunk(sha, offset, length))
|
||||
offset = new_offset
|
||||
|
||||
caibx.close()
|
||||
return chunks
|
||||
|
||||
|
||||
def build_chunk_dict(chunks: list[Chunk]) -> ChunkDict:
|
||||
"""Turn a list of chunks into a dict for faster lookups based on hash.
|
||||
Keep first chunk since it's more likely to be already downloaded."""
|
||||
r = {}
|
||||
for c in chunks:
|
||||
if c.sha not in r:
|
||||
r[c.sha] = c
|
||||
return r
|
||||
|
||||
|
||||
def extract(target: list[Chunk],
|
||||
sources: list[tuple[str, ChunkReader, ChunkDict]],
|
||||
out_path: str,
|
||||
progress: Callable[[int], None] | None = None):
|
||||
stats: dict[str, int] = defaultdict(int)
|
||||
|
||||
mode = 'rb+' if os.path.exists(out_path) else 'wb'
|
||||
with open(out_path, mode) as out:
|
||||
for cur_chunk in target:
|
||||
|
||||
# Find source for desired chunk
|
||||
for name, chunk_reader, store_chunks in sources:
|
||||
if cur_chunk.sha in store_chunks:
|
||||
bts = chunk_reader.read(store_chunks[cur_chunk.sha])
|
||||
|
||||
# Check length
|
||||
if len(bts) != cur_chunk.length:
|
||||
continue
|
||||
|
||||
# Check hash
|
||||
if SHA512.new(bts, truncate="256").digest() != cur_chunk.sha:
|
||||
continue
|
||||
|
||||
# Write to output
|
||||
out.seek(cur_chunk.offset)
|
||||
out.write(bts)
|
||||
|
||||
stats[name] += cur_chunk.length
|
||||
|
||||
if progress is not None:
|
||||
progress(sum(stats.values()))
|
||||
|
||||
break
|
||||
else:
|
||||
raise RuntimeError("Desired chunk not found in provided stores")
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def extract_directory(target: list[Chunk],
|
||||
sources: list[tuple[str, ChunkReader, ChunkDict]],
|
||||
out_path: str,
|
||||
tmp_file: str,
|
||||
progress: Callable[[int], None] | None = None):
|
||||
"""extract a directory stored as a casync tar archive"""
|
||||
|
||||
stats = extract(target, sources, tmp_file, progress)
|
||||
|
||||
with open(tmp_file, "rb") as f:
|
||||
tar.extract_tar_archive(f, pathlib.Path(out_path))
|
||||
|
||||
return stats
|
||||
|
||||
|
||||
def print_stats(stats: dict[str, int]):
|
||||
total_bytes = sum(stats.values())
|
||||
print(f"Total size: {total_bytes / 1024 / 1024:.2f} MB")
|
||||
for name, total in stats.items():
|
||||
print(f" {name}: {total / 1024 / 1024:.2f} MB ({total / total_bytes * 100:.1f}%)")
|
||||
|
||||
|
||||
def extract_simple(caibx_path, out_path, store_path):
|
||||
# (name, callback, chunks)
|
||||
target = parse_caibx(caibx_path)
|
||||
sources = [
|
||||
# (store_path, RemoteChunkReader(store_path), build_chunk_dict(target)),
|
||||
(store_path, FileChunkReader(store_path), build_chunk_dict(target)),
|
||||
]
|
||||
|
||||
return extract(target, sources, out_path)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
caibx = sys.argv[1]
|
||||
out = sys.argv[2]
|
||||
store = sys.argv[3]
|
||||
|
||||
stats = extract_simple(caibx, out, store)
|
||||
print_stats(stats)
|
||||
@@ -0,0 +1,61 @@
|
||||
import dataclasses
|
||||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
|
||||
from openpilot.system.version import BUILD_METADATA_FILENAME, BuildMetadata
|
||||
from openpilot.system.updated.casync import tar
|
||||
|
||||
|
||||
CASYNC_ARGS = ["--with=symlinks", "--with=permissions", "--compression=xz", "--chunk-size=16M"]
|
||||
CASYNC_FILES = [BUILD_METADATA_FILENAME]
|
||||
|
||||
|
||||
def run(cmd):
|
||||
return subprocess.check_output(cmd)
|
||||
|
||||
|
||||
def get_exclude_set(path) -> set[str]:
|
||||
exclude_set = set(CASYNC_FILES)
|
||||
|
||||
for file in path.rglob("*"):
|
||||
if file.is_file() or file.is_symlink():
|
||||
|
||||
while file.resolve() != path.resolve():
|
||||
exclude_set.add(str(file.relative_to(path)))
|
||||
|
||||
file = file.parent
|
||||
|
||||
return exclude_set
|
||||
|
||||
|
||||
def create_build_metadata_file(path: pathlib.Path, build_metadata: BuildMetadata):
|
||||
with open(path / BUILD_METADATA_FILENAME, "w") as f:
|
||||
build_metadata_dict = dataclasses.asdict(build_metadata)
|
||||
build_metadata_dict["openpilot"].pop("is_dirty") # this is determined at runtime
|
||||
build_metadata_dict.pop("channel") # channel is unrelated to the build itself
|
||||
f.write(json.dumps(build_metadata_dict))
|
||||
|
||||
|
||||
def is_not_git(path: pathlib.Path) -> bool:
|
||||
return ".git" not in path.parts
|
||||
|
||||
|
||||
def create_casync_tar_package(target_dir: pathlib.Path, output_path: pathlib.Path):
|
||||
tar.create_tar_archive(output_path, target_dir, is_not_git)
|
||||
|
||||
|
||||
def create_casync_from_file(file: pathlib.Path, output_dir: pathlib.Path, caibx_name: str):
|
||||
caibx_file = output_dir / f"{caibx_name}.caibx"
|
||||
run(["casync", "make", *CASYNC_ARGS, caibx_file, str(file)])
|
||||
|
||||
return caibx_file
|
||||
|
||||
|
||||
def create_casync_release(target_dir: pathlib.Path, output_dir: pathlib.Path, caibx_name: str):
|
||||
tar_file = output_dir / f"{caibx_name}.tar"
|
||||
create_casync_tar_package(target_dir, tar_file)
|
||||
caibx_file = create_casync_from_file(tar_file, output_dir, caibx_name)
|
||||
tar_file.unlink()
|
||||
digest = run(["casync", "digest", *CASYNC_ARGS, target_dir]).decode("utf-8").strip()
|
||||
return digest, caibx_file
|
||||
@@ -0,0 +1,39 @@
|
||||
import pathlib
|
||||
import tarfile
|
||||
from typing import IO
|
||||
from collections.abc import Callable
|
||||
|
||||
|
||||
def include_default(_) -> bool:
|
||||
return True
|
||||
|
||||
|
||||
def create_tar_archive(filename: pathlib.Path, directory: pathlib.Path, include: Callable[[pathlib.Path], bool] = include_default):
|
||||
"""Creates a tar archive of a directory"""
|
||||
|
||||
with tarfile.open(filename, 'w') as tar:
|
||||
for file in sorted(directory.rglob("*"), key=lambda f: f.stat().st_size if f.is_file() else 0, reverse=True):
|
||||
if not include(file):
|
||||
continue
|
||||
relative_path = str(file.relative_to(directory))
|
||||
if file.is_symlink():
|
||||
info = tarfile.TarInfo(relative_path)
|
||||
info.type = tarfile.SYMTYPE
|
||||
info.linkpath = str(file.readlink())
|
||||
tar.addfile(info)
|
||||
|
||||
elif file.is_file():
|
||||
info = tarfile.TarInfo(relative_path)
|
||||
info.size = file.stat().st_size
|
||||
info.type = tarfile.REGTYPE
|
||||
info.mode = file.stat().st_mode
|
||||
with file.open('rb') as f:
|
||||
tar.addfile(info, f)
|
||||
|
||||
|
||||
def extract_tar_archive(fh: IO[bytes], directory: pathlib.Path):
|
||||
"""Extracts a tar archive to a directory"""
|
||||
|
||||
tar = tarfile.open(fileobj=fh, mode='r')
|
||||
tar.extractall(str(directory), filter=lambda info, path: info)
|
||||
tar.close()
|
||||
@@ -0,0 +1,264 @@
|
||||
import pytest
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import subprocess
|
||||
|
||||
from openpilot.system.updated.casync import casync
|
||||
from openpilot.system.updated.casync import tar
|
||||
|
||||
# dd if=/dev/zero of=/tmp/img.raw bs=1M count=2
|
||||
# sudo losetup -f /tmp/img.raw
|
||||
# losetup -a | grep img.raw
|
||||
LOOPBACK = os.environ.get('LOOPBACK', None)
|
||||
|
||||
|
||||
@pytest.mark.skip("not used yet")
|
||||
class TestCasync:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.tmpdir = tempfile.TemporaryDirectory()
|
||||
|
||||
# Build example contents
|
||||
chunk_a = [i % 256 for i in range(1024)] * 512
|
||||
chunk_b = [(256 - i) % 256 for i in range(1024)] * 512
|
||||
zeroes = [0] * (1024 * 128)
|
||||
contents = chunk_a + chunk_b + zeroes + chunk_a
|
||||
|
||||
cls.contents = bytes(contents)
|
||||
|
||||
# Write to file
|
||||
cls.orig_fn = os.path.join(cls.tmpdir.name, 'orig.bin')
|
||||
with open(cls.orig_fn, 'wb') as f:
|
||||
f.write(cls.contents)
|
||||
|
||||
# Create casync files
|
||||
cls.manifest_fn = os.path.join(cls.tmpdir.name, 'orig.caibx')
|
||||
cls.store_fn = os.path.join(cls.tmpdir.name, 'store')
|
||||
subprocess.check_output(["casync", "make", "--compression=xz", "--store", cls.store_fn, cls.manifest_fn, cls.orig_fn])
|
||||
|
||||
target = casync.parse_caibx(cls.manifest_fn)
|
||||
hashes = [c.sha.hex() for c in target]
|
||||
|
||||
# Ensure we have chunk reuse
|
||||
assert len(hashes) > len(set(hashes))
|
||||
|
||||
def setup_method(self):
|
||||
# Clear target_lo
|
||||
if LOOPBACK is not None:
|
||||
self.target_lo = LOOPBACK
|
||||
with open(self.target_lo, 'wb') as f:
|
||||
f.write(b"0" * len(self.contents))
|
||||
|
||||
self.target_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names()))
|
||||
self.seed_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names()))
|
||||
|
||||
def teardown_method(self):
|
||||
for fn in [self.target_fn, self.seed_fn]:
|
||||
try:
|
||||
os.unlink(fn)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
def test_simple_extract(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
sources = [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as target_f:
|
||||
assert target_f.read() == self.contents
|
||||
|
||||
assert stats['remote'] == len(self.contents)
|
||||
|
||||
def test_seed(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
# Populate seed with half of the target contents
|
||||
with open(self.seed_fn, 'wb') as seed_f:
|
||||
seed_f.write(self.contents[:len(self.contents) // 2])
|
||||
|
||||
sources = [('seed', casync.FileChunkReader(self.seed_fn), casync.build_chunk_dict(target))]
|
||||
sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as target_f:
|
||||
assert target_f.read() == self.contents
|
||||
|
||||
assert stats['seed'] > 0
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
def test_already_done(self):
|
||||
"""Test that an already flashed target doesn't download any chunks"""
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
with open(self.target_fn, 'wb') as f:
|
||||
f.write(self.contents)
|
||||
|
||||
sources = [('target', casync.FileChunkReader(self.target_fn), casync.build_chunk_dict(target))]
|
||||
sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as f:
|
||||
assert f.read() == self.contents
|
||||
|
||||
assert stats['target'] == len(self.contents)
|
||||
|
||||
def test_chunk_reuse(self):
|
||||
"""Test that chunks that are reused are only downloaded once"""
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
# Ensure target exists
|
||||
with open(self.target_fn, 'wb'):
|
||||
pass
|
||||
|
||||
sources = [('target', casync.FileChunkReader(self.target_fn), casync.build_chunk_dict(target))]
|
||||
sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
|
||||
stats = casync.extract(target, sources, self.target_fn)
|
||||
|
||||
with open(self.target_fn, 'rb') as f:
|
||||
assert f.read() == self.contents
|
||||
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
@pytest.mark.skipif(not LOOPBACK, reason="requires loopback device")
|
||||
def test_lo_simple_extract(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
sources = [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
|
||||
stats = casync.extract(target, sources, self.target_lo)
|
||||
|
||||
with open(self.target_lo, 'rb') as target_f:
|
||||
assert target_f.read(len(self.contents)) == self.contents
|
||||
|
||||
assert stats['remote'] == len(self.contents)
|
||||
|
||||
@pytest.mark.skipif(not LOOPBACK, reason="requires loopback device")
|
||||
def test_lo_chunk_reuse(self):
|
||||
"""Test that chunks that are reused are only downloaded once"""
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
sources = [('target', casync.FileChunkReader(self.target_lo), casync.build_chunk_dict(target))]
|
||||
sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
|
||||
stats = casync.extract(target, sources, self.target_lo)
|
||||
|
||||
with open(self.target_lo, 'rb') as f:
|
||||
assert f.read(len(self.contents)) == self.contents
|
||||
|
||||
assert stats['remote'] < len(self.contents)
|
||||
|
||||
|
||||
@pytest.mark.skip("not used yet")
|
||||
class TestCasyncDirectory:
|
||||
"""Tests extracting a directory stored as a casync tar archive"""
|
||||
|
||||
NUM_FILES = 16
|
||||
|
||||
@classmethod
|
||||
def setup_cache(cls, directory, files=None):
|
||||
if files is None:
|
||||
files = range(cls.NUM_FILES)
|
||||
|
||||
chunk_a = [i % 256 for i in range(1024)] * 512
|
||||
chunk_b = [(256 - i) % 256 for i in range(1024)] * 512
|
||||
zeroes = [0] * (1024 * 128)
|
||||
cls.contents = chunk_a + chunk_b + zeroes + chunk_a
|
||||
cls.contents = bytes(cls.contents)
|
||||
|
||||
for i in files:
|
||||
with open(os.path.join(directory, f"file_{i}.txt"), "wb") as f:
|
||||
f.write(cls.contents)
|
||||
|
||||
os.symlink(f"file_{i}.txt", os.path.join(directory, f"link_{i}.txt"))
|
||||
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
cls.tmpdir = tempfile.TemporaryDirectory()
|
||||
|
||||
# Create casync files
|
||||
cls.manifest_fn = os.path.join(cls.tmpdir.name, 'orig.caibx')
|
||||
cls.store_fn = os.path.join(cls.tmpdir.name, 'store')
|
||||
|
||||
cls.directory_to_extract = tempfile.TemporaryDirectory()
|
||||
cls.setup_cache(cls.directory_to_extract.name)
|
||||
|
||||
cls.orig_fn = os.path.join(cls.tmpdir.name, 'orig.tar')
|
||||
tar.create_tar_archive(cls.orig_fn, pathlib.Path(cls.directory_to_extract.name))
|
||||
|
||||
subprocess.check_output(["casync", "make", "--compression=xz", "--store", cls.store_fn, cls.manifest_fn, cls.orig_fn])
|
||||
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
cls.tmpdir.cleanup()
|
||||
cls.directory_to_extract.cleanup()
|
||||
|
||||
def setup_method(self):
|
||||
self.cache_dir = tempfile.TemporaryDirectory()
|
||||
self.working_dir = tempfile.TemporaryDirectory()
|
||||
self.out_dir = tempfile.TemporaryDirectory()
|
||||
|
||||
def teardown_method(self):
|
||||
self.cache_dir.cleanup()
|
||||
self.working_dir.cleanup()
|
||||
self.out_dir.cleanup()
|
||||
|
||||
def run_test(self):
|
||||
target = casync.parse_caibx(self.manifest_fn)
|
||||
|
||||
cache_filename = os.path.join(self.working_dir.name, "cache.tar")
|
||||
tmp_filename = os.path.join(self.working_dir.name, "tmp.tar")
|
||||
|
||||
sources = [('cache', casync.DirectoryTarChunkReader(self.cache_dir.name, cache_filename), casync.build_chunk_dict(target))]
|
||||
sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))]
|
||||
|
||||
stats = casync.extract_directory(target, sources, pathlib.Path(self.out_dir.name), tmp_filename)
|
||||
|
||||
with open(os.path.join(self.out_dir.name, "file_0.txt"), "rb") as f:
|
||||
assert f.read() == self.contents
|
||||
|
||||
with open(os.path.join(self.out_dir.name, "link_0.txt"), "rb") as f:
|
||||
assert f.read() == self.contents
|
||||
assert os.readlink(os.path.join(self.out_dir.name, "link_0.txt")) == "file_0.txt"
|
||||
|
||||
return stats
|
||||
|
||||
def test_no_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, [])
|
||||
stats = self.run_test()
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] == 0
|
||||
|
||||
def test_full_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
stats = self.run_test()
|
||||
assert stats['remote'] == 0
|
||||
assert stats['cache'] > 0
|
||||
|
||||
def test_one_file_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(1))
|
||||
stats = self.run_test()
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] < stats['remote']
|
||||
|
||||
def test_one_file_incorrect_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
with open(os.path.join(self.cache_dir.name, "file_0.txt"), "wb") as f:
|
||||
f.write(b"1234")
|
||||
|
||||
stats = self.run_test()
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] > stats['remote']
|
||||
|
||||
def test_one_file_missing_cache(self):
|
||||
self.setup_cache(self.cache_dir.name, range(self.NUM_FILES))
|
||||
os.unlink(os.path.join(self.cache_dir.name, "file_12.txt"))
|
||||
|
||||
stats = self.run_test()
|
||||
assert stats['remote'] > 0
|
||||
assert stats['cache'] > 0
|
||||
assert stats['cache'] > stats['remote']
|
||||
@@ -0,0 +1,16 @@
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
|
||||
def get_consistent_flag(path: str) -> bool:
|
||||
consistent_file = pathlib.Path(os.path.join(path, ".overlay_consistent"))
|
||||
return consistent_file.is_file()
|
||||
|
||||
def set_consistent_flag(path: str, consistent: bool) -> None:
|
||||
os.sync()
|
||||
consistent_file = pathlib.Path(os.path.join(path, ".overlay_consistent"))
|
||||
if consistent:
|
||||
consistent_file.touch()
|
||||
elif not consistent:
|
||||
consistent_file.unlink(missing_ok=True)
|
||||
os.sync()
|
||||
@@ -0,0 +1,259 @@
|
||||
import os
|
||||
import pathlib
|
||||
import shutil
|
||||
import signal
|
||||
import stat
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
import pytest
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.manager.process import ManagerProcess
|
||||
from openpilot.selfdrive.test.helpers import processes_context
|
||||
|
||||
|
||||
def get_consistent_flag(path: str) -> bool:
|
||||
consistent_file = pathlib.Path(os.path.join(path, ".overlay_consistent"))
|
||||
return consistent_file.is_file()
|
||||
|
||||
|
||||
def run(args, **kwargs):
|
||||
return subprocess.check_output(args, **kwargs)
|
||||
|
||||
|
||||
def update_release(directory, name, version, agnos_version, release_notes):
|
||||
with open(directory / "RELEASES.md", "w") as f:
|
||||
f.write(release_notes)
|
||||
|
||||
(directory / "common").mkdir(exist_ok=True)
|
||||
|
||||
with open(directory / "common" / "version.h", "w") as f:
|
||||
f.write(f'#define COMMA_VERSION "{version}"')
|
||||
|
||||
launch_env = directory / "launch_env.sh"
|
||||
with open(launch_env, "w") as f:
|
||||
f.write(f'export AGNOS_VERSION="{agnos_version}"')
|
||||
|
||||
st = os.stat(launch_env)
|
||||
os.chmod(launch_env, st.st_mode | stat.S_IEXEC)
|
||||
|
||||
test_symlink = directory / "test_symlink"
|
||||
if not os.path.exists(str(test_symlink)):
|
||||
os.symlink("common/version.h", test_symlink)
|
||||
|
||||
|
||||
def get_version(path: str) -> str:
|
||||
with open(os.path.join(path, "common", "version.h")) as f:
|
||||
return f.read().split('"')[1]
|
||||
|
||||
|
||||
@pytest.mark.slow # TODO: can we test overlayfs in GHA?
|
||||
class TestBaseUpdate:
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
if "Base" in cls.__name__:
|
||||
pytest.skip()
|
||||
|
||||
def setup_method(self):
|
||||
self.tmpdir = tempfile.mkdtemp()
|
||||
|
||||
run(["sudo", "mount", "-t", "tmpfs", "tmpfs", self.tmpdir]) # overlayfs doesn't work inside of docker unless this is a tmpfs
|
||||
|
||||
self.mock_update_path = pathlib.Path(self.tmpdir)
|
||||
|
||||
self.params = Params()
|
||||
|
||||
self.basedir = self.mock_update_path / "openpilot"
|
||||
self.basedir.mkdir()
|
||||
|
||||
self.staging_root = self.mock_update_path / "safe_staging"
|
||||
self.staging_root.mkdir()
|
||||
|
||||
self.remote_dir = self.mock_update_path / "remote"
|
||||
self.remote_dir.mkdir()
|
||||
|
||||
os.environ["UPDATER_STAGING_ROOT"] = str(self.staging_root)
|
||||
os.environ["UPDATER_LOCK_FILE"] = str(self.mock_update_path / "safe_staging_overlay.lock")
|
||||
|
||||
self.MOCK_RELEASES = {
|
||||
"release3": ("0.1.2", "1.2", "0.1.2 release notes"),
|
||||
"master": ("0.1.3", "1.2", "0.1.3 release notes"),
|
||||
}
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_basedir(self, mocker):
|
||||
mocker.patch("openpilot.common.basedir.BASEDIR", self.basedir)
|
||||
|
||||
def set_target_branch(self, branch):
|
||||
self.params.put("UpdaterTargetBranch", branch, block=True)
|
||||
|
||||
def setup_basedir_release(self, release):
|
||||
self.params = Params()
|
||||
self.set_target_branch(release)
|
||||
|
||||
def update_remote_release(self, release):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def setup_remote_release(self, release):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def additional_context(self):
|
||||
raise NotImplementedError("")
|
||||
|
||||
def teardown_method(self):
|
||||
try:
|
||||
run(["sudo", "umount", "-l", str(self.staging_root / "merged")])
|
||||
run(["sudo", "umount", "-l", self.tmpdir])
|
||||
shutil.rmtree(self.tmpdir)
|
||||
except Exception:
|
||||
print("cleanup failed...")
|
||||
|
||||
def wait_for_condition(self, condition, timeout=12):
|
||||
start = time.monotonic()
|
||||
while True:
|
||||
waited = time.monotonic() - start
|
||||
if condition():
|
||||
print(f"waited {waited}s for condition ")
|
||||
return waited
|
||||
|
||||
if waited > timeout:
|
||||
raise TimeoutError("timed out waiting for condition")
|
||||
|
||||
time.sleep(1)
|
||||
|
||||
def _test_finalized_update(self, branch, version, agnos_version, release_notes):
|
||||
assert get_version(str(self.staging_root / "finalized")) == version
|
||||
assert get_consistent_flag(str(self.staging_root / "finalized"))
|
||||
assert os.access(str(self.staging_root / "finalized" / "launch_env.sh"), os.X_OK)
|
||||
|
||||
with open(self.staging_root / "finalized" / "test_symlink") as f:
|
||||
assert version in f.read()
|
||||
|
||||
class ParamsBaseUpdateTest(TestBaseUpdate):
|
||||
def _test_finalized_update(self, branch, version, agnos_version, release_notes):
|
||||
assert self.params.get("UpdaterNewDescription").startswith(f"{version} / {branch}")
|
||||
assert self.params.get("UpdaterNewReleaseNotes") == f"{release_notes}\n".encode()
|
||||
super()._test_finalized_update(branch, version, agnos_version, release_notes)
|
||||
|
||||
def send_check_for_updates_signal(self, updated: ManagerProcess):
|
||||
updated.signal(signal.SIGUSR1.value)
|
||||
|
||||
def send_download_signal(self, updated: ManagerProcess):
|
||||
updated.signal(signal.SIGHUP.value)
|
||||
|
||||
def _test_params(self, branch, fetch_available, update_available):
|
||||
assert self.params.get("UpdaterTargetBranch") == branch
|
||||
assert self.params.get_bool("UpdaterFetchAvailable") == fetch_available
|
||||
assert self.params.get_bool("UpdateAvailable") == update_available
|
||||
|
||||
def wait_for_idle(self):
|
||||
self.wait_for_condition(lambda: self.params.get("UpdaterState") == "idle")
|
||||
|
||||
def wait_for_failed(self):
|
||||
self.wait_for_condition(lambda: self.params.get("UpdateFailedCount") is not None and \
|
||||
self.params.get("UpdateFailedCount") > 0)
|
||||
|
||||
def wait_for_fetch_available(self):
|
||||
self.wait_for_condition(lambda: self.params.get_bool("UpdaterFetchAvailable"))
|
||||
|
||||
def wait_for_update_available(self):
|
||||
self.wait_for_condition(lambda: self.params.get_bool("UpdateAvailable"))
|
||||
|
||||
def test_no_update(self):
|
||||
# Start on release3, ensure we don't fetch any updates
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_idle()
|
||||
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
def test_new_release(self):
|
||||
# Start on release3, simulate a release3 commit, ensure we fetch that update properly
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.MOCK_RELEASES["release3"] = ("0.1.3", "1.2", "0.1.3 release notes")
|
||||
self.update_remote_release("release3")
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("release3", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("release3", False, True)
|
||||
self._test_finalized_update("release3", *self.MOCK_RELEASES["release3"])
|
||||
|
||||
def test_switch_branches(self):
|
||||
# Start on release3, request to switch to master manually, ensure we switched
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_remote_release("master")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.set_target_branch("master")
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("master", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("master", False, True)
|
||||
self._test_finalized_update("master", *self.MOCK_RELEASES["master"])
|
||||
|
||||
def test_agnos_update(self, mocker):
|
||||
# Start on release3, push an update with an agnos change
|
||||
self.setup_remote_release("release3")
|
||||
self.setup_basedir_release("release3")
|
||||
|
||||
with self.additional_context(), processes_context(["updated"]) as [updated]:
|
||||
mocker.patch("openpilot.system.hardware.AGNOS", "True")
|
||||
mocker.patch("openpilot.system.hardware.tici.hardware.Tici.get_os_version", "1.2")
|
||||
mocker.patch("openpilot.system.hardware.tici.agnos.get_target_slot_number")
|
||||
mocker.patch("openpilot.system.hardware.tici.agnos.flash_agnos_update")
|
||||
|
||||
self._test_params("release3", False, False)
|
||||
self.wait_for_idle()
|
||||
self._test_params("release3", False, False)
|
||||
|
||||
self.MOCK_RELEASES["release3"] = ("0.1.3", "1.3", "0.1.3 release notes")
|
||||
self.update_remote_release("release3")
|
||||
|
||||
self.send_check_for_updates_signal(updated)
|
||||
|
||||
self.wait_for_fetch_available()
|
||||
|
||||
self._test_params("release3", True, False)
|
||||
|
||||
self.send_download_signal(updated)
|
||||
|
||||
self.wait_for_update_available()
|
||||
|
||||
self._test_params("release3", False, True)
|
||||
self._test_finalized_update("release3", *self.MOCK_RELEASES["release3"])
|
||||
@@ -0,0 +1,22 @@
|
||||
import contextlib
|
||||
from openpilot.system.updated.tests.test_base import ParamsBaseUpdateTest, run, update_release
|
||||
|
||||
|
||||
class TestUpdateDGitStrategy(ParamsBaseUpdateTest):
|
||||
def update_remote_release(self, release):
|
||||
update_release(self.remote_dir, release, *self.MOCK_RELEASES[release])
|
||||
run(["git", "add", "."], cwd=self.remote_dir)
|
||||
run(["git", "commit", "-m", f"openpilot release {release}"], cwd=self.remote_dir)
|
||||
|
||||
def setup_remote_release(self, release):
|
||||
run(["git", "init"], cwd=self.remote_dir)
|
||||
run(["git", "checkout", "-b", release], cwd=self.remote_dir)
|
||||
self.update_remote_release(release)
|
||||
|
||||
def setup_basedir_release(self, release):
|
||||
super().setup_basedir_release(release)
|
||||
run(["git", "clone", "-b", release, self.remote_dir, self.basedir])
|
||||
|
||||
@contextlib.contextmanager
|
||||
def additional_context(self):
|
||||
yield
|
||||
@@ -0,0 +1,38 @@
|
||||
import pytest
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.system.updated.updated import Updater
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("device_type", "branch", "expected"), [
|
||||
("tizi", "release3", "release-tizi"),
|
||||
("tizi", "release3-staging", "release-tizi-staging"),
|
||||
("mici", "release3", "release-mici"),
|
||||
("mici", "release3-staging", "release-mici-staging"),
|
||||
])
|
||||
def test_target_branch_migration_from_current_branch(mocker, device_type, branch, expected):
|
||||
params = Params()
|
||||
params.remove("UpdaterTargetBranch")
|
||||
|
||||
mocker.patch("openpilot.system.updated.updated.HARDWARE.get_device_type", return_value=device_type)
|
||||
mocker.patch.object(Updater, "get_branch", return_value=branch)
|
||||
|
||||
assert Updater().target_branch == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("device_type", "branch", "expected"), [
|
||||
("tizi", "release3", "release-tizi"),
|
||||
("tizi", "release3-staging", "release-tizi-staging"),
|
||||
("mici", "release3", "release-mici"),
|
||||
("mici", "release3-staging", "release-mici-staging"),
|
||||
])
|
||||
def test_target_branch_migration_from_param(mocker, device_type, branch, expected):
|
||||
params = Params()
|
||||
params.put("UpdaterTargetBranch", branch, block=True)
|
||||
|
||||
mocker.patch("openpilot.system.updated.updated.HARDWARE.get_device_type", return_value=device_type)
|
||||
|
||||
try:
|
||||
assert Updater().target_branch == expected
|
||||
finally:
|
||||
params.remove("UpdaterTargetBranch")
|
||||
Executable
+517
@@ -0,0 +1,517 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import re
|
||||
import datetime
|
||||
import subprocess
|
||||
import psutil
|
||||
import shutil
|
||||
import signal
|
||||
import fcntl
|
||||
import time
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
from openpilot.common.basedir import BASEDIR
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.time_helpers import system_time_valid
|
||||
from openpilot.common.markdown import parse_markdown
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.selfdrive.selfdrived.alertmanager import set_offroad_alert
|
||||
from openpilot.system.hardware import AGNOS, HARDWARE
|
||||
from openpilot.system.version import get_build_metadata
|
||||
|
||||
LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
|
||||
STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")
|
||||
|
||||
OVERLAY_UPPER = os.path.join(STAGING_ROOT, "upper")
|
||||
OVERLAY_METADATA = os.path.join(STAGING_ROOT, "metadata")
|
||||
OVERLAY_MERGED = os.path.join(STAGING_ROOT, "merged")
|
||||
FINALIZED = os.path.join(STAGING_ROOT, "finalized")
|
||||
|
||||
OVERLAY_INIT = Path(os.path.join(BASEDIR, ".overlay_init"))
|
||||
|
||||
# do not allow to engage after this many hours onroad and this many routes
|
||||
HOURS_NO_CONNECTIVITY_MAX = 27
|
||||
ROUTES_NO_CONNECTIVITY_MAX = 84
|
||||
# send an offroad prompt after this many hours onroad and this many routes
|
||||
HOURS_NO_CONNECTIVITY_PROMPT = 23
|
||||
ROUTES_NO_CONNECTIVITY_PROMPT = 80
|
||||
|
||||
|
||||
class UserRequest:
|
||||
NONE = 0
|
||||
CHECK = 1
|
||||
FETCH = 2
|
||||
|
||||
class WaitTimeHelper:
|
||||
def __init__(self):
|
||||
self.ready_event = threading.Event()
|
||||
self.user_request = UserRequest.NONE
|
||||
signal.signal(signal.SIGHUP, self.update_now)
|
||||
signal.signal(signal.SIGUSR1, self.check_now)
|
||||
|
||||
def update_now(self, signum: int, frame) -> None:
|
||||
cloudlog.info("caught SIGHUP, attempting to downloading update")
|
||||
self.user_request = UserRequest.FETCH
|
||||
self.ready_event.set()
|
||||
|
||||
def check_now(self, signum: int, frame) -> None:
|
||||
cloudlog.info("caught SIGUSR1, checking for updates")
|
||||
self.user_request = UserRequest.CHECK
|
||||
self.ready_event.set()
|
||||
|
||||
def sleep(self, t: float) -> None:
|
||||
self.ready_event.wait(timeout=t)
|
||||
|
||||
def write_time_to_param(params, param) -> None:
|
||||
t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
params.put(param, t, block=True)
|
||||
|
||||
def run(cmd: list[str], cwd: str | None = None) -> str:
|
||||
return subprocess.check_output(cmd, cwd=cwd, stderr=subprocess.STDOUT, encoding='utf8')
|
||||
|
||||
|
||||
def set_consistent_flag(consistent: bool) -> None:
|
||||
os.sync()
|
||||
consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
|
||||
if consistent:
|
||||
consistent_file.touch()
|
||||
elif not consistent:
|
||||
consistent_file.unlink(missing_ok=True)
|
||||
os.sync()
|
||||
|
||||
def parse_release_notes(basedir: str) -> bytes:
|
||||
try:
|
||||
with open(os.path.join(basedir, "RELEASES.md"), "rb") as f:
|
||||
r = f.read().split(b'\n\n', 1)[0] # Slice latest release notes
|
||||
try:
|
||||
return bytes(parse_markdown(r.decode("utf-8")), encoding="utf-8")
|
||||
except Exception:
|
||||
return r + b"\n"
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
except Exception:
|
||||
cloudlog.exception("failed to parse release notes")
|
||||
return b""
|
||||
|
||||
def setup_git_options(cwd: str) -> None:
|
||||
# We sync FS object atimes (which NEOS doesn't use) and mtimes, but ctimes
|
||||
# are outside user control. Make sure Git is set up to ignore system ctimes,
|
||||
# because they change when we make hard links during finalize. Otherwise,
|
||||
# there is a lot of unnecessary churn. This appears to be a common need on
|
||||
# OSX as well: https://www.git-tower.com/blog/make-git-rebase-safe-on-osx/
|
||||
|
||||
# We are using copytree to copy the directory, which also changes
|
||||
# inode numbers. Ignore those changes too.
|
||||
|
||||
# Set protocol to the new version (default after git 2.26) to reduce data
|
||||
# usage on git fetch --dry-run from about 400KB to 18KB.
|
||||
git_cfg = [
|
||||
("core.trustctime", "false"),
|
||||
("core.checkStat", "minimal"),
|
||||
("protocol.version", "2"),
|
||||
("gc.auto", "0"),
|
||||
("gc.autoDetach", "false"),
|
||||
]
|
||||
for option, value in git_cfg:
|
||||
run(["git", "config", option, value], cwd)
|
||||
|
||||
|
||||
def dismount_overlay() -> None:
|
||||
if os.path.ismount(OVERLAY_MERGED):
|
||||
cloudlog.info("unmounting existing overlay")
|
||||
run(["sudo", "umount", "-l", OVERLAY_MERGED])
|
||||
|
||||
|
||||
def init_overlay() -> None:
|
||||
|
||||
# Re-create the overlay if BASEDIR/.git has changed since we created the overlay
|
||||
if OVERLAY_INIT.is_file() and os.path.ismount(OVERLAY_MERGED):
|
||||
git_dir_path = os.path.join(BASEDIR, ".git")
|
||||
new_files = run(["find", git_dir_path, "-newer", str(OVERLAY_INIT)])
|
||||
if not len(new_files.splitlines()):
|
||||
# A valid overlay already exists
|
||||
return
|
||||
else:
|
||||
cloudlog.info(".git directory changed, recreating overlay")
|
||||
|
||||
cloudlog.info("preparing new safe staging area")
|
||||
|
||||
params = Params()
|
||||
params.put_bool("UpdateAvailable", False, block=True)
|
||||
set_consistent_flag(False)
|
||||
dismount_overlay()
|
||||
run(["sudo", "rm", "-rf", STAGING_ROOT])
|
||||
if os.path.isdir(STAGING_ROOT):
|
||||
shutil.rmtree(STAGING_ROOT)
|
||||
|
||||
for dirname in [STAGING_ROOT, OVERLAY_UPPER, OVERLAY_METADATA, OVERLAY_MERGED]:
|
||||
os.mkdir(dirname, 0o755)
|
||||
|
||||
if os.lstat(BASEDIR).st_dev != os.lstat(OVERLAY_MERGED).st_dev:
|
||||
raise RuntimeError("base and overlay merge directories are on different filesystems; not valid for overlay FS!")
|
||||
|
||||
# Leave a timestamped canary in BASEDIR to check at startup. The device clock
|
||||
# should be correct by the time we get here. If the init file disappears, or
|
||||
# critical mtimes in BASEDIR are newer than .overlay_init, continue.sh can
|
||||
# assume that BASEDIR has used for local development or otherwise modified,
|
||||
# and skips the update activation attempt.
|
||||
consistent_file = Path(os.path.join(BASEDIR, ".overlay_consistent"))
|
||||
if consistent_file.is_file():
|
||||
consistent_file.unlink()
|
||||
OVERLAY_INIT.touch()
|
||||
|
||||
os.sync()
|
||||
overlay_opts = f"lowerdir={BASEDIR},upperdir={OVERLAY_UPPER},workdir={OVERLAY_METADATA}"
|
||||
|
||||
mount_cmd = ["mount", "-t", "overlay", "-o", overlay_opts, "none", OVERLAY_MERGED]
|
||||
run(["sudo"] + mount_cmd)
|
||||
run(["sudo", "chmod", "755", os.path.join(OVERLAY_METADATA, "work")])
|
||||
|
||||
git_diff = run(["git", "diff", "--submodule=diff"], OVERLAY_MERGED)
|
||||
params.put("GitDiff", git_diff, block=True)
|
||||
cloudlog.info(f"git diff output:\n{git_diff}")
|
||||
|
||||
|
||||
def finalize_update() -> None:
|
||||
"""Take the current OverlayFS merged view and finalize a copy outside of
|
||||
OverlayFS, ready to be swapped-in at BASEDIR. Copy using shutil.copytree"""
|
||||
|
||||
# Remove the update ready flag and any old updates
|
||||
cloudlog.info("creating finalized version of the overlay")
|
||||
set_consistent_flag(False)
|
||||
|
||||
# Copy the merged overlay view and set the update ready flag
|
||||
if os.path.exists(FINALIZED):
|
||||
shutil.rmtree(FINALIZED)
|
||||
shutil.copytree(OVERLAY_MERGED, FINALIZED, symlinks=True)
|
||||
|
||||
run(["git", "reset", "--hard"], FINALIZED)
|
||||
run(["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"], FINALIZED)
|
||||
|
||||
cloudlog.info("Starting git cleanup in finalized update")
|
||||
t = time.monotonic()
|
||||
try:
|
||||
run(["git", "gc"], FINALIZED)
|
||||
run(["git", "lfs", "prune"], FINALIZED)
|
||||
cloudlog.event("Done git cleanup", duration=time.monotonic() - t)
|
||||
except subprocess.CalledProcessError:
|
||||
cloudlog.exception(f"Failed git cleanup, took {time.monotonic() - t:.3f} s")
|
||||
|
||||
set_consistent_flag(True)
|
||||
cloudlog.info("done finalizing overlay")
|
||||
|
||||
|
||||
def handle_agnos_update() -> None:
|
||||
from openpilot.system.hardware.tici.agnos import flash_agnos_update, get_target_slot_number
|
||||
|
||||
cur_version = HARDWARE.get_os_version()
|
||||
updated_version = run(["bash", "-c", r"unset AGNOS_VERSION && source launch_env.sh && \
|
||||
echo -n $AGNOS_VERSION"], OVERLAY_MERGED).strip()
|
||||
|
||||
cloudlog.info(f"AGNOS version check: {cur_version} vs {updated_version}")
|
||||
if cur_version == updated_version:
|
||||
return
|
||||
|
||||
# prevent an openpilot getting swapped in with a mismatched or partially downloaded agnos
|
||||
set_consistent_flag(False)
|
||||
|
||||
cloudlog.info(f"Beginning background installation for AGNOS {updated_version}")
|
||||
set_offroad_alert("Offroad_NeosUpdate", True)
|
||||
|
||||
manifest_path = os.path.join(OVERLAY_MERGED, "system/hardware/tici/agnos.json")
|
||||
target_slot_number = get_target_slot_number()
|
||||
flash_agnos_update(manifest_path, target_slot_number, cloudlog)
|
||||
set_offroad_alert("Offroad_NeosUpdate", False)
|
||||
|
||||
|
||||
|
||||
class Updater:
|
||||
def __init__(self):
|
||||
self.params = Params()
|
||||
self.branches = defaultdict(str)
|
||||
self._has_internet: bool = False
|
||||
|
||||
@property
|
||||
def has_internet(self) -> bool:
|
||||
return self._has_internet
|
||||
|
||||
@property
|
||||
def target_branch(self) -> str:
|
||||
b: str | None = self.params.get("UpdaterTargetBranch")
|
||||
if b is None:
|
||||
b = self.get_branch(BASEDIR)
|
||||
b = {
|
||||
("tizi", "release3"): "release-tizi",
|
||||
("tizi", "release3-staging"): "release-tizi-staging",
|
||||
("mici", "release3"): "release-mici",
|
||||
("mici", "release3-staging"): "release-mici-staging",
|
||||
}.get((HARDWARE.get_device_type(), b), b)
|
||||
return b
|
||||
|
||||
@property
|
||||
def update_ready(self) -> bool:
|
||||
consistent_file = Path(os.path.join(FINALIZED, ".overlay_consistent"))
|
||||
if consistent_file.is_file():
|
||||
hash_mismatch = self.get_commit_hash(BASEDIR) != self.branches[self.target_branch]
|
||||
branch_mismatch = self.get_branch(BASEDIR) != self.target_branch
|
||||
on_target_branch = self.get_branch(FINALIZED) == self.target_branch
|
||||
return ((hash_mismatch or branch_mismatch) and on_target_branch)
|
||||
return False
|
||||
|
||||
@property
|
||||
def update_available(self) -> bool:
|
||||
if os.path.isdir(OVERLAY_MERGED) and len(self.branches) > 0:
|
||||
hash_mismatch = self.get_commit_hash(OVERLAY_MERGED) != self.branches[self.target_branch]
|
||||
branch_mismatch = self.get_branch(OVERLAY_MERGED) != self.target_branch
|
||||
return hash_mismatch or branch_mismatch
|
||||
return False
|
||||
|
||||
def get_branch(self, path: str) -> str:
|
||||
return run(["git", "rev-parse", "--abbrev-ref", "HEAD"], path).rstrip()
|
||||
|
||||
def get_commit_hash(self, path: str = OVERLAY_MERGED) -> str:
|
||||
return run(["git", "rev-parse", "HEAD"], path).rstrip()
|
||||
|
||||
def set_params(self, update_success: bool, failed_count: int, exception: str | None) -> None:
|
||||
self.params.put("UpdateFailedCount", failed_count, block=True)
|
||||
self.params.put("UpdaterTargetBranch", self.target_branch, block=True)
|
||||
|
||||
self.params.put_bool("UpdaterFetchAvailable", self.update_available, block=True)
|
||||
if len(self.branches):
|
||||
self.params.put("UpdaterAvailableBranches", ','.join(self.branches.keys()), block=True)
|
||||
|
||||
last_uptime_onroad = self.params.get("UptimeOnroad", return_default=True)
|
||||
last_route_count = self.params.get("RouteCount", return_default=True)
|
||||
if update_success:
|
||||
self.params.put("LastUpdateTime", datetime.datetime.now(datetime.UTC).replace(tzinfo=None), block=True)
|
||||
self.params.put("LastUpdateUptimeOnroad", last_uptime_onroad, block=True)
|
||||
self.params.put("LastUpdateRouteCount", last_route_count, block=True)
|
||||
else:
|
||||
last_uptime_onroad = self.params.get("LastUpdateUptimeOnroad", return_default=True)
|
||||
last_route_count = self.params.get("LastUpdateRouteCount", return_default=True)
|
||||
|
||||
if exception is None:
|
||||
self.params.remove("LastUpdateException")
|
||||
else:
|
||||
self.params.put("LastUpdateException", exception, block=True)
|
||||
|
||||
# Write out current and new version info
|
||||
def get_description(basedir: str) -> str:
|
||||
if not os.path.exists(basedir):
|
||||
return ""
|
||||
|
||||
version = ""
|
||||
branch = ""
|
||||
commit = ""
|
||||
commit_date = ""
|
||||
try:
|
||||
branch = self.get_branch(basedir)
|
||||
commit = self.get_commit_hash(basedir)[:7]
|
||||
with open(os.path.join(basedir, "common", "version.h")) as f:
|
||||
version = f.read().split('"')[1]
|
||||
|
||||
commit_unix_ts = run(["git", "show", "-s", "--format=%ct", "HEAD"], basedir).rstrip()
|
||||
dt = datetime.datetime.fromtimestamp(int(commit_unix_ts))
|
||||
commit_date = dt.strftime("%b %d")
|
||||
except Exception:
|
||||
cloudlog.exception("updater.get_description")
|
||||
return f"{version} / {branch} / {commit} / {commit_date}"
|
||||
self.params.put("UpdaterCurrentDescription", get_description(BASEDIR), block=True)
|
||||
self.params.put("UpdaterCurrentReleaseNotes", parse_release_notes(BASEDIR), block=True)
|
||||
self.params.put("UpdaterNewDescription", get_description(FINALIZED), block=True)
|
||||
self.params.put("UpdaterNewReleaseNotes", parse_release_notes(FINALIZED), block=True)
|
||||
self.params.put_bool("UpdateAvailable", self.update_ready, block=True)
|
||||
|
||||
# Handle user prompt
|
||||
for alert in ("Offroad_UpdateFailed", "Offroad_ConnectivityNeeded", "Offroad_ConnectivityNeededPrompt"):
|
||||
set_offroad_alert(alert, False)
|
||||
|
||||
dt_uptime_onroad = (self.params.get("UptimeOnroad", return_default=True) - last_uptime_onroad) / (60*60)
|
||||
dt_route_count = self.params.get("RouteCount", return_default=True) - last_route_count
|
||||
build_metadata = get_build_metadata()
|
||||
if failed_count > 15 and exception is not None and self.has_internet:
|
||||
if build_metadata.tested_channel:
|
||||
extra_text = "Ensure the software is correctly installed. Uninstall and re-install if this error persists."
|
||||
else:
|
||||
extra_text = exception
|
||||
set_offroad_alert("Offroad_UpdateFailed", True, extra_text=extra_text)
|
||||
elif failed_count > 0:
|
||||
if dt_uptime_onroad > HOURS_NO_CONNECTIVITY_MAX and dt_route_count > ROUTES_NO_CONNECTIVITY_MAX:
|
||||
set_offroad_alert("Offroad_ConnectivityNeeded", True)
|
||||
elif dt_uptime_onroad > HOURS_NO_CONNECTIVITY_PROMPT and dt_route_count > ROUTES_NO_CONNECTIVITY_PROMPT:
|
||||
remaining = max(HOURS_NO_CONNECTIVITY_MAX - dt_uptime_onroad, 1)
|
||||
set_offroad_alert("Offroad_ConnectivityNeededPrompt", True, extra_text=f"{remaining} hour{'' if remaining == 1 else 's'}.")
|
||||
|
||||
def check_for_update(self) -> None:
|
||||
cloudlog.info("checking for updates")
|
||||
|
||||
excluded_branches = ('release2', 'release2-staging')
|
||||
|
||||
try:
|
||||
run(["git", "ls-remote", "origin", "HEAD"], OVERLAY_MERGED)
|
||||
self._has_internet = True
|
||||
except subprocess.CalledProcessError:
|
||||
self._has_internet = False
|
||||
|
||||
setup_git_options(OVERLAY_MERGED)
|
||||
output = run(["git", "ls-remote", "--heads"], OVERLAY_MERGED)
|
||||
|
||||
self.branches = defaultdict(lambda: None)
|
||||
for line in output.split('\n'):
|
||||
ls_remotes_re = r'(?P<commit_sha>\b[0-9a-f]{5,40}\b)(\s+)(refs\/heads\/)(?P<branch_name>.*$)'
|
||||
x = re.fullmatch(ls_remotes_re, line.strip())
|
||||
if x is not None and x.group('branch_name') not in excluded_branches:
|
||||
self.branches[x.group('branch_name')] = x.group('commit_sha')
|
||||
|
||||
cur_branch = self.get_branch(OVERLAY_MERGED)
|
||||
cur_commit = self.get_commit_hash(OVERLAY_MERGED)
|
||||
new_branch = self.target_branch
|
||||
new_commit = self.branches[new_branch]
|
||||
if (cur_branch, cur_commit) != (new_branch, new_commit):
|
||||
cloudlog.info(f"update available, {cur_branch} ({str(cur_commit)[:7]}) -> {new_branch} ({str(new_commit)[:7]})")
|
||||
else:
|
||||
cloudlog.info(f"up to date on {cur_branch} ({str(cur_commit)[:7]})")
|
||||
|
||||
def fetch_update(self) -> None:
|
||||
cloudlog.info("attempting git fetch inside staging overlay")
|
||||
|
||||
self.params.put("UpdaterState", "downloading...", block=True)
|
||||
|
||||
# TODO: cleanly interrupt this and invalidate old update
|
||||
set_consistent_flag(False)
|
||||
self.params.put_bool("UpdateAvailable", False, block=True)
|
||||
|
||||
setup_git_options(OVERLAY_MERGED)
|
||||
|
||||
run(["git", "config", "--replace-all", "remote.origin.fetch", "+refs/heads/*:refs/remotes/origin/*"], OVERLAY_MERGED)
|
||||
|
||||
branch = self.target_branch
|
||||
git_fetch_output = run(["git", "fetch", "origin", branch], OVERLAY_MERGED)
|
||||
cloudlog.info("git fetch success: %s", git_fetch_output)
|
||||
|
||||
cloudlog.info("git reset in progress")
|
||||
cmds = [
|
||||
["git", "checkout", "--force", "--no-recurse-submodules", "-B", branch, "FETCH_HEAD"],
|
||||
["git", "branch", "--set-upstream-to", f"origin/{branch}"],
|
||||
["git", "reset", "--hard"],
|
||||
["git", "clean", "-xdff"],
|
||||
["git", "submodule", "sync"],
|
||||
["git", "submodule", "update", "--init", "--recursive"],
|
||||
["git", "submodule", "foreach", "--recursive", "git", "reset", "--hard"],
|
||||
]
|
||||
r = [run(cmd, OVERLAY_MERGED) for cmd in cmds]
|
||||
cloudlog.info("git reset success: %s", '\n'.join(r))
|
||||
|
||||
# TODO: show agnos download progress
|
||||
if AGNOS:
|
||||
handle_agnos_update()
|
||||
|
||||
# Create the finalized, ready-to-swap update
|
||||
self.params.put("UpdaterState", "finalizing update...", block=True)
|
||||
finalize_update()
|
||||
cloudlog.info("finalize success!")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
params = Params()
|
||||
|
||||
if params.get_bool("DisableUpdates"):
|
||||
cloudlog.warning("updates are disabled by the DisableUpdates param")
|
||||
exit(0)
|
||||
|
||||
with open(LOCK_FILE, 'w') as ov_lock_fd:
|
||||
try:
|
||||
fcntl.flock(ov_lock_fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
except OSError as e:
|
||||
raise RuntimeError("couldn't get overlay lock; is another instance running?") from e
|
||||
|
||||
# Set low io priority
|
||||
proc = psutil.Process()
|
||||
if psutil.LINUX:
|
||||
proc.ionice(psutil.IOPRIO_CLASS_BE, value=7)
|
||||
|
||||
# Check if we just performed an update
|
||||
if Path(os.path.join(STAGING_ROOT, "old_openpilot")).is_dir():
|
||||
cloudlog.event("update installed")
|
||||
|
||||
if not params.get("InstallDate"):
|
||||
t = datetime.datetime.now(datetime.UTC).replace(tzinfo=None)
|
||||
params.put("InstallDate", t, block=True)
|
||||
|
||||
updater = Updater()
|
||||
update_failed_count = 0 # TODO: Load from param?
|
||||
wait_helper = WaitTimeHelper()
|
||||
|
||||
# invalidate old finalized update
|
||||
set_consistent_flag(False)
|
||||
|
||||
# set initial state
|
||||
params.put("UpdaterState", "idle", block=True)
|
||||
|
||||
# Run the update loop
|
||||
first_run = True
|
||||
while True:
|
||||
wait_helper.ready_event.clear()
|
||||
|
||||
# Attempt an update
|
||||
exception = None
|
||||
try:
|
||||
# TODO: reuse overlay from previous updated instance if it looks clean
|
||||
init_overlay()
|
||||
|
||||
# ensure we have some params written soon after startup
|
||||
updater.set_params(False, update_failed_count, exception)
|
||||
|
||||
if not system_time_valid() or first_run:
|
||||
first_run = False
|
||||
wait_helper.sleep(60)
|
||||
continue
|
||||
|
||||
update_failed_count += 1
|
||||
|
||||
# check for update
|
||||
params.put("UpdaterState", "checking...", block=True)
|
||||
updater.check_for_update()
|
||||
|
||||
# download update
|
||||
last_fetch = params.get("UpdaterLastFetchTime")
|
||||
timed_out = last_fetch is None or (datetime.datetime.now(datetime.UTC).replace(tzinfo=None) - last_fetch > datetime.timedelta(days=3))
|
||||
user_requested_fetch = wait_helper.user_request == UserRequest.FETCH
|
||||
if params.get_bool("NetworkMetered") and not timed_out and not user_requested_fetch:
|
||||
cloudlog.info("skipping fetch, connection metered")
|
||||
elif wait_helper.user_request == UserRequest.CHECK:
|
||||
cloudlog.info("skipping fetch, only checking")
|
||||
else:
|
||||
updater.fetch_update()
|
||||
write_time_to_param(params, "UpdaterLastFetchTime")
|
||||
update_failed_count = 0
|
||||
except subprocess.CalledProcessError as e:
|
||||
cloudlog.event(
|
||||
"update process failed",
|
||||
cmd=e.cmd,
|
||||
output=e.output,
|
||||
returncode=e.returncode
|
||||
)
|
||||
exception = f"command failed: {e.cmd}\n{e.output}"
|
||||
OVERLAY_INIT.unlink(missing_ok=True)
|
||||
except Exception as e:
|
||||
cloudlog.exception("uncaught updated exception, shouldn't happen")
|
||||
exception = str(e)
|
||||
OVERLAY_INIT.unlink(missing_ok=True)
|
||||
|
||||
try:
|
||||
params.put("UpdaterState", "idle", block=True)
|
||||
update_successful = (update_failed_count == 0)
|
||||
updater.set_params(update_successful, update_failed_count, exception)
|
||||
except Exception:
|
||||
cloudlog.exception("uncaught updated exception while setting params, shouldn't happen")
|
||||
|
||||
# infrequent attempts if we successfully updated recently
|
||||
wait_helper.user_request = UserRequest.NONE
|
||||
wait_helper.sleep(5*60 if update_failed_count > 0 else 1.5*60*60)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user