diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index a1bad1eef3..3ad7c4a26b 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -39,7 +39,7 @@ jobs: - name: Test dependency and packaging tools run: | python3 -m unittest discover -s tools/vendor/tests -v - python3 -m unittest discover -s tools/release/tests -v + python3 -m pytest -q tools/release/tests python3 -m unittest discover -s tools/ci/tests -v python3 -m unittest discover -s openpilot/system/updated/tests -v - name: Check setup and release shell syntax @@ -54,6 +54,7 @@ jobs: env: STRIPPED_DIR: /tmp/releasepilot PYTHONPATH: /tmp/releasepilot + STARPILOT_CI_BUILD_MANIFEST: ${{ runner.temp }}/starpilot-release-build-${{ github.run_id }}-${{ github.run_attempt }}.json steps: - uses: actions/checkout@v7 - uses: actions/setup-go@v7 diff --git a/SConstruct b/SConstruct index edde875d22..dc64170010 100644 --- a/SConstruct +++ b/SConstruct @@ -374,3 +374,28 @@ def progress_function(node): Progress(progress_function, interval=progress_interval) AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir) + +# Release CI rebuilds tracked device outputs for the host architecture. +if manifest_path := os.environ.get("STARPILOT_CI_BUILD_MANIFEST"): + from tools.release.build_manifest import capture + manifest_nodes = env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry) + manifest_seen = set() + manifest_outputs = set() + while manifest_nodes: + manifest_node = manifest_nodes.pop().disambiguate() + if manifest_node in manifest_seen: + continue + manifest_seen.add(manifest_node) + manifest_executor = manifest_node.get_executor() + if manifest_executor is not None: + manifest_nodes += manifest_executor.get_all_prerequisites() + manifest_executor.get_all_children() + for target in manifest_executor.get_all_targets(): + if target.is_derived() and hasattr(target, 'abspath'): + relative = os.path.relpath(target.abspath, Dir('#').abspath) + if not relative.startswith('../'): + manifest_outputs.add(relative) + capture(Dir('#').abspath, manifest_path, manifest_outputs, + {'arch': arch, 'targets': list(BUILD_TARGETS), 'extras': GetOption('extras'), + 'ccflags': GetOption('ccflags'), 'release': release, 'arguments': dict(ARGUMENTS), + 'toolchain': {key: str(env.get(key, '')) for key in + ('CC', 'CXX', 'CCFLAGS', 'CFLAGS', 'CXXFLAGS', 'LINKFLAGS')}}) diff --git a/panda/SConscript b/panda/SConscript index 8be82ef1f4..1eb3c3b671 100644 --- a/panda/SConscript +++ b/panda/SConscript @@ -158,20 +158,22 @@ base_project_h7 = { } # Common autogenerated includes -with open("board/obj/gitversion.h", "w") as f: - version = get_version(BUILDER, BUILD_TYPE) - f.write("#pragma once\n\n") - f.write(f'extern const uint8_t gitversion[{len(version)+1}];\n') - f.write(f'const uint8_t gitversion[{len(version)+1}] = "{version}";\n') - -with open("board/obj/version", "w") as f: - f.write(f'{get_version(BUILDER, BUILD_TYPE)}') - +version = get_version(BUILDER, BUILD_TYPE) +gitversion_header = ("#pragma once\n\n" + + f'extern const uint8_t gitversion[{len(version)+1}];\n' + + f'const uint8_t gitversion[{len(version)+1}] = "{version}";\n') certs = [get_key_header(n) for n in ["debug", "release"]] -with open("board/obj/cert.h", "w") as f: - f.write("#pragma once\n\n") - for cert in certs: - f.write("\n".join(cert) + "\n") +cert_header = "#pragma once\n\n" + "".join("\n".join(cert) + "\n" for cert in certs) + + +def write_generated(target, source, env): + Path(target[0].abspath).write_text(str(source[0])) + return 0 + + +generation_env = Environment(tools=[]) +for name, payload in (("gitversion.h", gitversion_header), ("version", version), ("cert.h", cert_header)): + generation_env.Command(File(f"./board/obj/{name}"), Value(payload), write_generated) # Packet version defines: SHA hash of the struct header files def version_hash(path): diff --git a/tools/ci/run_predeploy.py b/tools/ci/run_predeploy.py index 3d687ea735..d8a5db0e31 100755 --- a/tools/ci/run_predeploy.py +++ b/tools/ci/run_predeploy.py @@ -22,7 +22,8 @@ def commands(output, cache, download=False): ('schemas', [python, 'tools/ci/schema_policy.py'])] for name, path in (('vendor', 'tools/vendor/tests'), ('packaging', 'tools/release/tests'), ('ci-tools', 'tools/ci/tests'), ('updater', 'openpilot/system/updated/tests')): - source.append((name, [python, '-m', 'unittest', 'discover', '-s', path, '-v'])) + command = [python, '-m', 'pytest', '-q', path] if name == 'packaging' else [python, '-m', 'unittest', 'discover', '-s', path, '-v'] + source.append((name, command)) host = [('unittest', [python, 'tools/ci/run_host_tests.py', '--json-output', str(output / 'host.json')]), ('pytest', [python, 'tools/ci/run_host_tests.py', '--pytest'])] vehicles = [(suite, [python, 'tools/ci/run_vehicle_tests.py', '--suite', suite, '--output', str(output / suite)]) diff --git a/tools/release/build_manifest.py b/tools/release/build_manifest.py new file mode 100644 index 0000000000..e8d5d0fde4 --- /dev/null +++ b/tools/release/build_manifest.py @@ -0,0 +1,95 @@ +import argparse +import hashlib +import json +import os +from pathlib import Path +import stat +import subprocess + + +def git(root, *args): + return subprocess.check_output(['git', '-C', str(root), *args]) + + +def tracked(root): + return [os.fsdecode(path) for path in git(root, 'ls-files', '-z').split(b'\0') if path] + + +def fingerprint(path): + mode = path.lstat().st_mode + if stat.S_ISLNK(mode): + return {'kind': 'symlink', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(os.fsencode(os.readlink(path))).hexdigest()} + if not stat.S_ISREG(mode): + raise ValueError(f'Not a regular file: {path}') + return {'kind': 'file', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} + + +def read_manifest(path): + path = Path(path) + metadata = path.lstat() + if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600: + raise ValueError('Unsafe build manifest') + if metadata.st_size > 16 * 1024 * 1024: + raise ValueError('Oversized build manifest') + value = json.loads(path.read_text()) + if value.get('format') != 'starpilot-ci-build' or value.get('version') != 1: + raise ValueError('Unsupported build manifest') + return value + + +def capture(root, destination, outputs, config): + root = Path(root).resolve() + destination = Path(destination).absolute() + if destination.resolve().is_relative_to(root): + raise ValueError('Build manifest must be outside checkout') + revision = git(root, 'rev-parse', 'HEAD').decode().strip() + if destination.exists(): + existing = read_manifest(destination) + if existing['root'] != str(root) or existing['revision'] != revision or existing['config'] != config: + raise ValueError('Stale build manifest') + return + if git(root, 'status', '--porcelain', '--untracked-files=all'): + raise ValueError('Checkout must be clean before build') + files = {name: fingerprint(root / name) for name in tracked(root)} + selected = sorted(set(outputs) & files.keys()) + if any(files[name]['kind'] != 'file' for name in selected): + raise ValueError('Derived output is not a regular file') + value = {'format': 'starpilot-ci-build', 'version': 1, 'root': str(root), 'revision': revision, + 'config': config, 'files': files, 'outputs': selected} + with open(destination, 'x', opener=lambda path, flags: os.open(path, flags, 0o600)) as stream: + json.dump(value, stream, sort_keys=True) + stream.flush() + os.fsync(stream.fileno()) + + +def verify(root, manifest): + root = Path(root).resolve() + value = read_manifest(manifest) + if value['root'] != str(root) or value['revision'] != git(root, 'rev-parse', 'HEAD').decode().strip(): + raise ValueError('Build manifest belongs to a different checkout or revision') + files = value['files'] + outputs = set(value['outputs']) + if not outputs <= files.keys() or set(tracked(root)) != files.keys(): + raise ValueError('Tracked inventory changed') + if git(root, 'diff', '--cached', '--name-only'): + raise ValueError('Index changed during build') + if git(root, 'ls-files', '--others', '--exclude-standard', '-z'): + raise ValueError('Untracked files created during build') + for name, before in files.items(): + if Path(name).is_absolute() or '..' in Path(name).parts: + raise ValueError('Unsafe manifest path') + after = fingerprint(root / name) + if after['kind'] != before['kind'] or after['mode'] != before['mode']: + raise ValueError(f'File type or mode changed: {name}') + if name in outputs: + if after['kind'] != 'file' or (root / name).stat().st_size == 0: + raise ValueError(f'Missing or empty derived output: {name}') + elif after != before: + raise ValueError(f'Source or undeclared output changed: {name}') + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--manifest', required=True) + args = parser.parse_args() + verify(git(Path.cwd(), 'rev-parse', '--show-toplevel').decode().strip(), args.manifest) diff --git a/tools/release/check-dirty.sh b/tools/release/check-dirty.sh index 73d393f3c2..aca469f7b9 100755 --- a/tools/release/check-dirty.sh +++ b/tools/release/check-dirty.sh @@ -4,6 +4,11 @@ set -e DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)" cd "$DIR" +if [ -n "${STARPILOT_CI_BUILD_MANIFEST:-}" ]; then + python3 "$DIR/build_manifest.py" --manifest "$STARPILOT_CI_BUILD_MANIFEST" + exit 0 +fi + if [ ! -z "$(git status --porcelain)" ]; then echo "Dirty working tree after build:" git status --porcelain diff --git a/tools/release/tests/test_build_manifest.py b/tools/release/tests/test_build_manifest.py new file mode 100644 index 0000000000..caf0a7bf6b --- /dev/null +++ b/tools/release/tests/test_build_manifest.py @@ -0,0 +1,91 @@ +import subprocess + +import pytest + +from tools.release.build_manifest import capture, verify + + +@pytest.fixture +def checkout(tmp_path): + root = tmp_path / 'repo' + root.mkdir() + subprocess.run(['git', 'init', '-q', str(root)], check=True) + for name in ('source.py', 'output.bin', 'undeclared.bin'): + (root / name).write_text('original\n') + subprocess.run(['git', '-C', str(root), 'add', '.'], check=True) + subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid', + 'commit', '-qm', 'fixture'], check=True) + return root, tmp_path / 'manifest.json' + + +def test_declared_output_rebuilt(checkout): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + (root / 'output.bin').write_bytes(b'host output') + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + verify(root, manifest) + + +@pytest.mark.parametrize('change', ['source', 'undeclared', 'missing', 'empty', 'mode', 'symlink', 'untracked', 'index']) +def test_build_mutations_rejected(checkout, change): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'test'}) + target = root / 'output.bin' + if change in ('source', 'undeclared'): + (root / ('source.py' if change == 'source' else 'undeclared.bin')).write_text('changed') + elif change == 'missing': + target.unlink() + elif change == 'empty': + target.write_bytes(b'') + elif change == 'mode': + target.chmod(0o755) + elif change == 'symlink': + target.unlink() + target.symlink_to('source.py') + elif change == 'untracked': + (root / 'unexpected.py').write_text('created') + else: + target.write_text('changed') + subprocess.run(['git', '-C', str(root), 'add', 'output.bin'], check=True) + with pytest.raises((ValueError, FileNotFoundError)): + verify(root, manifest) + + +def test_dirty_capture_rejected(checkout): + root, manifest = checkout + (root / 'source.py').write_text('changed') + with pytest.raises(ValueError, match='clean'): + capture(root, manifest, ['output.bin'], {}) + + +def test_stale_config_and_root_rejected(checkout, tmp_path): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {'arch': 'one'}) + with pytest.raises(ValueError, match='Stale'): + capture(root, manifest, ['output.bin'], {'arch': 'two'}) + other = tmp_path / 'other' + subprocess.run(['git', 'clone', '-q', str(root), str(other)], check=True) + with pytest.raises(ValueError, match='different checkout'): + verify(other, manifest) + + +def test_unsafe_manifest_rejected(checkout): + root, manifest = checkout + capture(root, manifest, ['output.bin'], {}) + manifest.chmod(0o644) + with pytest.raises(ValueError, match='Unsafe'): + verify(root, manifest) + + +def test_manifest_inside_checkout_rejected(checkout): + root, _ = checkout + with pytest.raises(ValueError, match='outside'): + capture(root, root / 'manifest.json', ['output.bin'], {}) + + +def test_manifest_parent_symlink_inside_checkout_rejected(checkout, tmp_path): + root, _ = checkout + alias = tmp_path / 'alias' + alias.symlink_to(root, target_is_directory=True) + with pytest.raises(ValueError, match='outside'): + capture(root, alias / 'manifest.json', ['output.bin'], {})