From f48e99b33537c823e97a2da588ec441c2d5f3a78 Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 20 Jun 2026 14:19:25 -0700 Subject: [PATCH] updated: remove (never used) casync support (#38198) --- pyproject.toml | 2 +- system/hardware/tici/agnos.py | 52 +--- .../tici/tests/compare_casync_manifest.py | 73 ----- system/updated/casync/casync.py | 246 ---------------- system/updated/casync/common.py | 61 ---- system/updated/casync/tar.py | 39 --- system/updated/casync/tests/test_casync.py | 264 ------------------ 7 files changed, 2 insertions(+), 735 deletions(-) delete mode 100755 system/hardware/tici/tests/compare_casync_manifest.py delete mode 100755 system/updated/casync/casync.py delete mode 100644 system/updated/casync/common.py delete mode 100644 system/updated/casync/tar.py delete mode 100644 system/updated/casync/tests/test_casync.py diff --git a/pyproject.toml b/pyproject.toml index 2120e6a430..51844ef93c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -68,7 +68,7 @@ dependencies = [ # these should be removed "psutil", - "pycryptodome", # used in updated/casync, panda, body, and a test + "pycryptodome", # used in panda, body, and a test "setproctitle", # logreader diff --git a/system/hardware/tici/agnos.py b/system/hardware/tici/agnos.py index f5261953d5..c5ca7efb4a 100755 --- a/system/hardware/tici/agnos.py +++ b/system/hardware/tici/agnos.py @@ -10,10 +10,7 @@ from collections.abc import Generator import requests -import openpilot.system.updated.casync.casync as casync - SPARSE_CHUNK_FMT = struct.Struct('H2xI4x') -CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/" AGNOS_MANIFEST_FILE = "system/hardware/tici/agnos.json" @@ -187,50 +184,6 @@ def extract_compressed_image(target_slot_number: int, partition: dict, cloudlog) os.sync() -def extract_casync_image(target_slot_number: int, partition: dict, cloudlog): - path = get_partition_path(target_slot_number, partition) - seed_path = path[:-1] + ('b' if path[-1] == 'a' else 'a') - - target = casync.parse_caibx(partition['casync_caibx']) - - sources: list[tuple[str, casync.ChunkReader, casync.ChunkDict]] = [] - - # First source is the current partition. - try: - raw_hash = get_raw_hash(seed_path, partition['size']) - caibx_url = f"{CAIBX_URL}{partition['name']}-{raw_hash}.caibx" - - try: - cloudlog.info(f"casync fetching {caibx_url}") - sources += [('seed', casync.FileChunkReader(seed_path), casync.build_chunk_dict(casync.parse_caibx(caibx_url)))] - except requests.RequestException: - cloudlog.error(f"casync failed to load {caibx_url}") - except Exception: - cloudlog.exception("casync failed to hash seed partition") - - # Second source is the target partition, this allows for resuming - sources += [('target', casync.FileChunkReader(path), casync.build_chunk_dict(target))] - - # Finally we add the remote source to download any missing chunks - sources += [('remote', casync.RemoteChunkReader(partition['casync_store']), casync.build_chunk_dict(target))] - - last_p = 0 - - def progress(cur): - nonlocal last_p - p = int(cur / partition['size'] * 100) - if p != last_p: - last_p = p - print(f"Installing {partition['name']}: {p}", flush=True) - - stats = casync.extract(target, sources, path, progress) - cloudlog.error(f'casync done {json.dumps(stats)}') - - os.sync() - if not verify_partition(target_slot_number, partition, force_full_check=True): - raise Exception(f"Raw hash mismatch '{partition['hash_raw'].lower()}'") - - def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalone=False): cloudlog.info(f"Downloading and writing {partition['name']}") @@ -245,10 +198,7 @@ def flash_partition(target_slot_number: int, partition: dict, cloudlog, standalo path = get_partition_path(target_slot_number, partition) - if ('casync_caibx' in partition) and not standalone: - extract_casync_image(target_slot_number, partition, cloudlog) - else: - extract_compressed_image(target_slot_number, partition, cloudlog) + extract_compressed_image(target_slot_number, partition, cloudlog) # Write hash after successful flash if not full_check: diff --git a/system/hardware/tici/tests/compare_casync_manifest.py b/system/hardware/tici/tests/compare_casync_manifest.py deleted file mode 100755 index 7de66d91d0..0000000000 --- a/system/hardware/tici/tests/compare_casync_manifest.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python3 -import argparse -import collections -import multiprocessing -import os - -import requests -from tqdm import tqdm - -import openpilot.system.hardware.tici.casync as casync - - -def get_chunk_download_size(chunk): - sha = chunk.sha.hex() - path = os.path.join(remote_url, sha[:4], sha + ".cacnk") - if os.path.isfile(path): - return os.path.getsize(path) - else: - r = requests.head(path, timeout=10) - r.raise_for_status() - return int(r.headers['content-length']) - - -if __name__ == "__main__": - - parser = argparse.ArgumentParser(description='Compute overlap between two casync manifests') - parser.add_argument('frm') - parser.add_argument('to') - args = parser.parse_args() - - frm = casync.parse_caibx(args.frm) - to = casync.parse_caibx(args.to) - remote_url = args.to.replace('.caibx', '') - - most_common = collections.Counter(t.sha for t in to).most_common(1)[0][0] - - frm_dict = casync.build_chunk_dict(frm) - - # Get content-length for each chunk - with multiprocessing.Pool() as pool: - szs = list(tqdm(pool.imap(get_chunk_download_size, to), total=len(to))) - chunk_sizes = {t.sha: sz for (t, sz) in zip(to, szs, strict=True)} - - sources: dict[str, list[int]] = { - 'seed': [], - 'remote_uncompressed': [], - 'remote_compressed': [], - } - - for chunk in to: - # Assume most common chunk is the zero chunk - if chunk.sha == most_common: - continue - - if chunk.sha in frm_dict: - sources['seed'].append(chunk.length) - else: - sources['remote_uncompressed'].append(chunk.length) - sources['remote_compressed'].append(chunk_sizes[chunk.sha]) - - print() - print("Update statistics (excluding zeros)") - print() - print("Download only with no seed:") - print(f" Remote (uncompressed)\t\t{sum(sources['seed'] + sources['remote_uncompressed']) / 1000 / 1000:.2f} MB\tn = {len(to)}") - print(f" Remote (compressed download)\t{sum(chunk_sizes.values()) / 1000 / 1000:.2f} MB\tn = {len(to)}") - print() - print("Upgrade with seed partition:") - print(f" Seed (uncompressed)\t\t{sum(sources['seed']) / 1000 / 1000:.2f} MB\t\t\t\tn = {len(sources['seed'])}") - sz, n = sum(sources['remote_uncompressed']), len(sources['remote_uncompressed']) - print(f" Remote (uncompressed)\t\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}") - sz, n = sum(sources['remote_compressed']), len(sources['remote_compressed']) - print(f" Remote (compressed download)\t{sz / 1000 / 1000:.2f} MB\t(avg {sz / 1000 / 1000 / n:4f} MB)\tn = {n}") diff --git a/system/updated/casync/casync.py b/system/updated/casync/casync.py deleted file mode 100755 index 2bd46a1ffb..0000000000 --- a/system/updated/casync/casync.py +++ /dev/null @@ -1,246 +0,0 @@ -#!/usr/bin/env python3 -import io -import lzma -import os -import pathlib -import struct -import sys -import time -from abc import ABC, abstractmethod -from collections import defaultdict, namedtuple -from collections.abc import Callable -from typing import IO - -import requests -from Crypto.Hash import SHA512 -from openpilot.system.updated.casync import tar -from openpilot.system.updated.casync.common import create_casync_tar_package - -CA_FORMAT_INDEX = 0x96824d9c7b129ff9 -CA_FORMAT_TABLE = 0xe75b9e112f17417d -CA_FORMAT_TABLE_TAIL_MARKER = 0xe75b9e112f17417 -FLAGS = 0xb000000000000000 - -CA_HEADER_LEN = 48 -CA_TABLE_HEADER_LEN = 16 -CA_TABLE_ENTRY_LEN = 40 -CA_TABLE_MIN_LEN = CA_TABLE_HEADER_LEN + CA_TABLE_ENTRY_LEN - -CHUNK_DOWNLOAD_TIMEOUT = 60 -CHUNK_DOWNLOAD_RETRIES = 3 - -CAIBX_DOWNLOAD_TIMEOUT = 120 - -Chunk = namedtuple('Chunk', ['sha', 'offset', 'length']) -ChunkDict = dict[bytes, Chunk] - - -class ChunkReader(ABC): - @abstractmethod - def read(self, chunk: Chunk) -> bytes: - ... - - -class BinaryChunkReader(ChunkReader): - """Reads chunks from a local file""" - def __init__(self, file_like: IO[bytes]) -> None: - super().__init__() - self.f = file_like - - def read(self, chunk: Chunk) -> bytes: - self.f.seek(chunk.offset) - return self.f.read(chunk.length) - - -class FileChunkReader(BinaryChunkReader): - def __init__(self, path: str) -> None: - super().__init__(open(path, 'rb')) - - def __del__(self): - self.f.close() - - -class RemoteChunkReader(ChunkReader): - """Reads lzma compressed chunks from a remote store""" - - def __init__(self, url: str) -> None: - super().__init__() - self.url = url - self.session = requests.Session() - - def read(self, chunk: Chunk) -> bytes: - sha_hex = chunk.sha.hex() - url = os.path.join(self.url, sha_hex[:4], sha_hex + ".cacnk") - - if os.path.isfile(url): - with open(url, 'rb') as f: - contents = f.read() - else: - for i in range(CHUNK_DOWNLOAD_RETRIES): - try: - resp = self.session.get(url, timeout=CHUNK_DOWNLOAD_TIMEOUT) - break - except Exception: - if i == CHUNK_DOWNLOAD_RETRIES - 1: - raise - time.sleep(CHUNK_DOWNLOAD_TIMEOUT) - - resp.raise_for_status() - contents = resp.content - - decompressor = lzma.LZMADecompressor(format=lzma.FORMAT_AUTO) - return decompressor.decompress(contents) - - -class DirectoryTarChunkReader(BinaryChunkReader): - """creates a tar archive of a directory and reads chunks from it""" - - def __init__(self, path: str, cache_file: str) -> None: - create_casync_tar_package(pathlib.Path(path), pathlib.Path(cache_file)) - - self.f = open(cache_file, "rb") - super().__init__(self.f) - - def __del__(self): - self.f.close() - os.unlink(self.f.name) - - -def parse_caibx(caibx_path: str) -> list[Chunk]: - """Parses the chunks from a caibx file. Can handle both local and remote files. - Returns a list of chunks with hash, offset and length""" - caibx: io.BufferedIOBase - if os.path.isfile(caibx_path): - caibx = open(caibx_path, 'rb') - else: - resp = requests.get(caibx_path, timeout=CAIBX_DOWNLOAD_TIMEOUT) - resp.raise_for_status() - caibx = io.BytesIO(resp.content) - - caibx.seek(0, os.SEEK_END) - caibx_len = caibx.tell() - caibx.seek(0, os.SEEK_SET) - - # Parse header - length, magic, flags, min_size, _, max_size = struct.unpack("= min_size - - chunks.append(Chunk(sha, offset, length)) - offset = new_offset - - caibx.close() - return chunks - - -def build_chunk_dict(chunks: list[Chunk]) -> ChunkDict: - """Turn a list of chunks into a dict for faster lookups based on hash. - Keep first chunk since it's more likely to be already downloaded.""" - r = {} - for c in chunks: - if c.sha not in r: - r[c.sha] = c - return r - - -def extract(target: list[Chunk], - sources: list[tuple[str, ChunkReader, ChunkDict]], - out_path: str, - progress: Callable[[int], None] | None = None): - stats: dict[str, int] = defaultdict(int) - - mode = 'rb+' if os.path.exists(out_path) else 'wb' - with open(out_path, mode) as out: - for cur_chunk in target: - - # Find source for desired chunk - for name, chunk_reader, store_chunks in sources: - if cur_chunk.sha in store_chunks: - bts = chunk_reader.read(store_chunks[cur_chunk.sha]) - - # Check length - if len(bts) != cur_chunk.length: - continue - - # Check hash - if SHA512.new(bts, truncate="256").digest() != cur_chunk.sha: - continue - - # Write to output - out.seek(cur_chunk.offset) - out.write(bts) - - stats[name] += cur_chunk.length - - if progress is not None: - progress(sum(stats.values())) - - break - else: - raise RuntimeError("Desired chunk not found in provided stores") - - return stats - - -def extract_directory(target: list[Chunk], - sources: list[tuple[str, ChunkReader, ChunkDict]], - out_path: str, - tmp_file: str, - progress: Callable[[int], None] | None = None): - """extract a directory stored as a casync tar archive""" - - stats = extract(target, sources, tmp_file, progress) - - with open(tmp_file, "rb") as f: - tar.extract_tar_archive(f, pathlib.Path(out_path)) - - return stats - - -def print_stats(stats: dict[str, int]): - total_bytes = sum(stats.values()) - print(f"Total size: {total_bytes / 1024 / 1024:.2f} MB") - for name, total in stats.items(): - print(f" {name}: {total / 1024 / 1024:.2f} MB ({total / total_bytes * 100:.1f}%)") - - -def extract_simple(caibx_path, out_path, store_path): - # (name, callback, chunks) - target = parse_caibx(caibx_path) - sources = [ - # (store_path, RemoteChunkReader(store_path), build_chunk_dict(target)), - (store_path, FileChunkReader(store_path), build_chunk_dict(target)), - ] - - return extract(target, sources, out_path) - - -if __name__ == "__main__": - caibx = sys.argv[1] - out = sys.argv[2] - store = sys.argv[3] - - stats = extract_simple(caibx, out, store) - print_stats(stats) diff --git a/system/updated/casync/common.py b/system/updated/casync/common.py deleted file mode 100644 index 6979f5cb06..0000000000 --- a/system/updated/casync/common.py +++ /dev/null @@ -1,61 +0,0 @@ -import dataclasses -import json -import pathlib -import subprocess - -from openpilot.system.version import BUILD_METADATA_FILENAME, BuildMetadata -from openpilot.system.updated.casync import tar - - -CASYNC_ARGS = ["--with=symlinks", "--with=permissions", "--compression=xz", "--chunk-size=16M"] -CASYNC_FILES = [BUILD_METADATA_FILENAME] - - -def run(cmd): - return subprocess.check_output(cmd) - - -def get_exclude_set(path) -> set[str]: - exclude_set = set(CASYNC_FILES) - - for file in path.rglob("*"): - if file.is_file() or file.is_symlink(): - - while file.resolve() != path.resolve(): - exclude_set.add(str(file.relative_to(path))) - - file = file.parent - - return exclude_set - - -def create_build_metadata_file(path: pathlib.Path, build_metadata: BuildMetadata): - with open(path / BUILD_METADATA_FILENAME, "w") as f: - build_metadata_dict = dataclasses.asdict(build_metadata) - build_metadata_dict["openpilot"].pop("is_dirty") # this is determined at runtime - build_metadata_dict.pop("channel") # channel is unrelated to the build itself - f.write(json.dumps(build_metadata_dict)) - - -def is_not_git(path: pathlib.Path) -> bool: - return ".git" not in path.parts - - -def create_casync_tar_package(target_dir: pathlib.Path, output_path: pathlib.Path): - tar.create_tar_archive(output_path, target_dir, is_not_git) - - -def create_casync_from_file(file: pathlib.Path, output_dir: pathlib.Path, caibx_name: str): - caibx_file = output_dir / f"{caibx_name}.caibx" - run(["casync", "make", *CASYNC_ARGS, caibx_file, str(file)]) - - return caibx_file - - -def create_casync_release(target_dir: pathlib.Path, output_dir: pathlib.Path, caibx_name: str): - tar_file = output_dir / f"{caibx_name}.tar" - create_casync_tar_package(target_dir, tar_file) - caibx_file = create_casync_from_file(tar_file, output_dir, caibx_name) - tar_file.unlink() - digest = run(["casync", "digest", *CASYNC_ARGS, target_dir]).decode("utf-8").strip() - return digest, caibx_file diff --git a/system/updated/casync/tar.py b/system/updated/casync/tar.py deleted file mode 100644 index a5a8238bba..0000000000 --- a/system/updated/casync/tar.py +++ /dev/null @@ -1,39 +0,0 @@ -import pathlib -import tarfile -from typing import IO -from collections.abc import Callable - - -def include_default(_) -> bool: - return True - - -def create_tar_archive(filename: pathlib.Path, directory: pathlib.Path, include: Callable[[pathlib.Path], bool] = include_default): - """Creates a tar archive of a directory""" - - with tarfile.open(filename, 'w') as tar: - for file in sorted(directory.rglob("*"), key=lambda f: f.stat().st_size if f.is_file() else 0, reverse=True): - if not include(file): - continue - relative_path = str(file.relative_to(directory)) - if file.is_symlink(): - info = tarfile.TarInfo(relative_path) - info.type = tarfile.SYMTYPE - info.linkpath = str(file.readlink()) - tar.addfile(info) - - elif file.is_file(): - info = tarfile.TarInfo(relative_path) - info.size = file.stat().st_size - info.type = tarfile.REGTYPE - info.mode = file.stat().st_mode - with file.open('rb') as f: - tar.addfile(info, f) - - -def extract_tar_archive(fh: IO[bytes], directory: pathlib.Path): - """Extracts a tar archive to a directory""" - - tar = tarfile.open(fileobj=fh, mode='r') - tar.extractall(str(directory), filter=lambda info, path: info) - tar.close() diff --git a/system/updated/casync/tests/test_casync.py b/system/updated/casync/tests/test_casync.py deleted file mode 100644 index bc171e7432..0000000000 --- a/system/updated/casync/tests/test_casync.py +++ /dev/null @@ -1,264 +0,0 @@ -import pytest -import os -import pathlib -import tempfile -import subprocess - -from openpilot.system.updated.casync import casync -from openpilot.system.updated.casync import tar - -# dd if=/dev/zero of=/tmp/img.raw bs=1M count=2 -# sudo losetup -f /tmp/img.raw -# losetup -a | grep img.raw -LOOPBACK = os.environ.get('LOOPBACK', None) - - -@pytest.mark.skip("not used yet") -class TestCasync: - @classmethod - def setup_class(cls): - cls.tmpdir = tempfile.TemporaryDirectory() - - # Build example contents - chunk_a = [i % 256 for i in range(1024)] * 512 - chunk_b = [(256 - i) % 256 for i in range(1024)] * 512 - zeroes = [0] * (1024 * 128) - contents = chunk_a + chunk_b + zeroes + chunk_a - - cls.contents = bytes(contents) - - # Write to file - cls.orig_fn = os.path.join(cls.tmpdir.name, 'orig.bin') - with open(cls.orig_fn, 'wb') as f: - f.write(cls.contents) - - # Create casync files - cls.manifest_fn = os.path.join(cls.tmpdir.name, 'orig.caibx') - cls.store_fn = os.path.join(cls.tmpdir.name, 'store') - subprocess.check_output(["casync", "make", "--compression=xz", "--store", cls.store_fn, cls.manifest_fn, cls.orig_fn]) - - target = casync.parse_caibx(cls.manifest_fn) - hashes = [c.sha.hex() for c in target] - - # Ensure we have chunk reuse - assert len(hashes) > len(set(hashes)) - - def setup_method(self): - # Clear target_lo - if LOOPBACK is not None: - self.target_lo = LOOPBACK - with open(self.target_lo, 'wb') as f: - f.write(b"0" * len(self.contents)) - - self.target_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names())) - self.seed_fn = os.path.join(self.tmpdir.name, next(tempfile._get_candidate_names())) - - def teardown_method(self): - for fn in [self.target_fn, self.seed_fn]: - try: - os.unlink(fn) - except FileNotFoundError: - pass - - def test_simple_extract(self): - target = casync.parse_caibx(self.manifest_fn) - - sources = [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - stats = casync.extract(target, sources, self.target_fn) - - with open(self.target_fn, 'rb') as target_f: - assert target_f.read() == self.contents - - assert stats['remote'] == len(self.contents) - - def test_seed(self): - target = casync.parse_caibx(self.manifest_fn) - - # Populate seed with half of the target contents - with open(self.seed_fn, 'wb') as seed_f: - seed_f.write(self.contents[:len(self.contents) // 2]) - - sources = [('seed', casync.FileChunkReader(self.seed_fn), casync.build_chunk_dict(target))] - sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - stats = casync.extract(target, sources, self.target_fn) - - with open(self.target_fn, 'rb') as target_f: - assert target_f.read() == self.contents - - assert stats['seed'] > 0 - assert stats['remote'] < len(self.contents) - - def test_already_done(self): - """Test that an already flashed target doesn't download any chunks""" - target = casync.parse_caibx(self.manifest_fn) - - with open(self.target_fn, 'wb') as f: - f.write(self.contents) - - sources = [('target', casync.FileChunkReader(self.target_fn), casync.build_chunk_dict(target))] - sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - - stats = casync.extract(target, sources, self.target_fn) - - with open(self.target_fn, 'rb') as f: - assert f.read() == self.contents - - assert stats['target'] == len(self.contents) - - def test_chunk_reuse(self): - """Test that chunks that are reused are only downloaded once""" - target = casync.parse_caibx(self.manifest_fn) - - # Ensure target exists - with open(self.target_fn, 'wb'): - pass - - sources = [('target', casync.FileChunkReader(self.target_fn), casync.build_chunk_dict(target))] - sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - - stats = casync.extract(target, sources, self.target_fn) - - with open(self.target_fn, 'rb') as f: - assert f.read() == self.contents - - assert stats['remote'] < len(self.contents) - - @pytest.mark.skipif(not LOOPBACK, reason="requires loopback device") - def test_lo_simple_extract(self): - target = casync.parse_caibx(self.manifest_fn) - sources = [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - - stats = casync.extract(target, sources, self.target_lo) - - with open(self.target_lo, 'rb') as target_f: - assert target_f.read(len(self.contents)) == self.contents - - assert stats['remote'] == len(self.contents) - - @pytest.mark.skipif(not LOOPBACK, reason="requires loopback device") - def test_lo_chunk_reuse(self): - """Test that chunks that are reused are only downloaded once""" - target = casync.parse_caibx(self.manifest_fn) - - sources = [('target', casync.FileChunkReader(self.target_lo), casync.build_chunk_dict(target))] - sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - - stats = casync.extract(target, sources, self.target_lo) - - with open(self.target_lo, 'rb') as f: - assert f.read(len(self.contents)) == self.contents - - assert stats['remote'] < len(self.contents) - - -@pytest.mark.skip("not used yet") -class TestCasyncDirectory: - """Tests extracting a directory stored as a casync tar archive""" - - NUM_FILES = 16 - - @classmethod - def setup_cache(cls, directory, files=None): - if files is None: - files = range(cls.NUM_FILES) - - chunk_a = [i % 256 for i in range(1024)] * 512 - chunk_b = [(256 - i) % 256 for i in range(1024)] * 512 - zeroes = [0] * (1024 * 128) - cls.contents = chunk_a + chunk_b + zeroes + chunk_a - cls.contents = bytes(cls.contents) - - for i in files: - with open(os.path.join(directory, f"file_{i}.txt"), "wb") as f: - f.write(cls.contents) - - os.symlink(f"file_{i}.txt", os.path.join(directory, f"link_{i}.txt")) - - @classmethod - def setup_class(cls): - cls.tmpdir = tempfile.TemporaryDirectory() - - # Create casync files - cls.manifest_fn = os.path.join(cls.tmpdir.name, 'orig.caibx') - cls.store_fn = os.path.join(cls.tmpdir.name, 'store') - - cls.directory_to_extract = tempfile.TemporaryDirectory() - cls.setup_cache(cls.directory_to_extract.name) - - cls.orig_fn = os.path.join(cls.tmpdir.name, 'orig.tar') - tar.create_tar_archive(cls.orig_fn, pathlib.Path(cls.directory_to_extract.name)) - - subprocess.check_output(["casync", "make", "--compression=xz", "--store", cls.store_fn, cls.manifest_fn, cls.orig_fn]) - - @classmethod - def teardown_class(cls): - cls.tmpdir.cleanup() - cls.directory_to_extract.cleanup() - - def setup_method(self): - self.cache_dir = tempfile.TemporaryDirectory() - self.working_dir = tempfile.TemporaryDirectory() - self.out_dir = tempfile.TemporaryDirectory() - - def teardown_method(self): - self.cache_dir.cleanup() - self.working_dir.cleanup() - self.out_dir.cleanup() - - def run_test(self): - target = casync.parse_caibx(self.manifest_fn) - - cache_filename = os.path.join(self.working_dir.name, "cache.tar") - tmp_filename = os.path.join(self.working_dir.name, "tmp.tar") - - sources = [('cache', casync.DirectoryTarChunkReader(self.cache_dir.name, cache_filename), casync.build_chunk_dict(target))] - sources += [('remote', casync.RemoteChunkReader(self.store_fn), casync.build_chunk_dict(target))] - - stats = casync.extract_directory(target, sources, pathlib.Path(self.out_dir.name), tmp_filename) - - with open(os.path.join(self.out_dir.name, "file_0.txt"), "rb") as f: - assert f.read() == self.contents - - with open(os.path.join(self.out_dir.name, "link_0.txt"), "rb") as f: - assert f.read() == self.contents - assert os.readlink(os.path.join(self.out_dir.name, "link_0.txt")) == "file_0.txt" - - return stats - - def test_no_cache(self): - self.setup_cache(self.cache_dir.name, []) - stats = self.run_test() - assert stats['remote'] > 0 - assert stats['cache'] == 0 - - def test_full_cache(self): - self.setup_cache(self.cache_dir.name, range(self.NUM_FILES)) - stats = self.run_test() - assert stats['remote'] == 0 - assert stats['cache'] > 0 - - def test_one_file_cache(self): - self.setup_cache(self.cache_dir.name, range(1)) - stats = self.run_test() - assert stats['remote'] > 0 - assert stats['cache'] > 0 - assert stats['cache'] < stats['remote'] - - def test_one_file_incorrect_cache(self): - self.setup_cache(self.cache_dir.name, range(self.NUM_FILES)) - with open(os.path.join(self.cache_dir.name, "file_0.txt"), "wb") as f: - f.write(b"1234") - - stats = self.run_test() - assert stats['remote'] > 0 - assert stats['cache'] > 0 - assert stats['cache'] > stats['remote'] - - def test_one_file_missing_cache(self): - self.setup_cache(self.cache_dir.name, range(self.NUM_FILES)) - os.unlink(os.path.join(self.cache_dir.name, "file_12.txt")) - - stats = self.run_test() - assert stats['remote'] > 0 - assert stats['cache'] > 0 - assert stats['cache'] > stats['remote']