Release Build Checks

.github/workflows/tests.yaml

@@ -39,7 +39,7 @@ jobs:
     - name: Test dependency and packaging tools
       run: |
         python3 -m unittest discover -s tools/vendor/tests -v
-        python3 -m unittest discover -s tools/release/tests -v
+        python3 -m pytest -q tools/release/tests
         python3 -m unittest discover -s tools/ci/tests -v
         python3 -m unittest discover -s openpilot/system/updated/tests -v
     - name: Check setup and release shell syntax
@@ -54,6 +54,7 @@ jobs:
     env:
       STRIPPED_DIR: /tmp/releasepilot
       PYTHONPATH: /tmp/releasepilot
+      STARPILOT_CI_BUILD_MANIFEST: ${{ runner.temp }}/starpilot-release-build-${{ github.run_id }}-${{ github.run_attempt }}.json
     steps:
     - uses: actions/checkout@v7
     - uses: actions/setup-go@v7

SConstruct

@@ -374,3 +374,28 @@ def progress_function(node):
 
 Progress(progress_function, interval=progress_interval)
 AddPostAction(BUILD_TARGETS or [Dir('.')], prune_cache_dir)
+
+# Release CI rebuilds tracked device outputs for the host architecture.
+if manifest_path := os.environ.get("STARPILOT_CI_BUILD_MANIFEST"):
+  from tools.release.build_manifest import capture
+  manifest_nodes = env.arg2nodes(BUILD_TARGETS or [Dir('.')], env.fs.Entry)
+  manifest_seen = set()
+  manifest_outputs = set()
+  while manifest_nodes:
+    manifest_node = manifest_nodes.pop().disambiguate()
+    if manifest_node in manifest_seen:
+      continue
+    manifest_seen.add(manifest_node)
+    manifest_executor = manifest_node.get_executor()
+    if manifest_executor is not None:
+      manifest_nodes += manifest_executor.get_all_prerequisites() + manifest_executor.get_all_children()
+      for target in manifest_executor.get_all_targets():
+        if target.is_derived() and hasattr(target, 'abspath'):
+          relative = os.path.relpath(target.abspath, Dir('#').abspath)
+          if not relative.startswith('../'):
+            manifest_outputs.add(relative)
+  capture(Dir('#').abspath, manifest_path, manifest_outputs,
+          {'arch': arch, 'targets': list(BUILD_TARGETS), 'extras': GetOption('extras'),
+           'ccflags': GetOption('ccflags'), 'release': release, 'arguments': dict(ARGUMENTS),
+           'toolchain': {key: str(env.get(key, '')) for key in
+                         ('CC', 'CXX', 'CCFLAGS', 'CFLAGS', 'CXXFLAGS', 'LINKFLAGS')}})

panda/SConscript

@@ -158,20 +158,22 @@ base_project_h7 = {
 }
 
 # Common autogenerated includes
-with open("board/obj/gitversion.h", "w") as f:
-  version = get_version(BUILDER, BUILD_TYPE)
-  f.write("#pragma once\n\n")
-  f.write(f'extern const uint8_t gitversion[{len(version)+1}];\n')
-  f.write(f'const uint8_t gitversion[{len(version)+1}] = "{version}";\n')
-
-with open("board/obj/version", "w") as f:
-  f.write(f'{get_version(BUILDER, BUILD_TYPE)}')
-
+version = get_version(BUILDER, BUILD_TYPE)
+gitversion_header = ("#pragma once\n\n" +
+                     f'extern const uint8_t gitversion[{len(version)+1}];\n' +
+                     f'const uint8_t gitversion[{len(version)+1}] = "{version}";\n')
 certs = [get_key_header(n) for n in ["debug", "release"]]
-with open("board/obj/cert.h", "w") as f:
-  f.write("#pragma once\n\n")
-  for cert in certs:
-    f.write("\n".join(cert) + "\n")
+cert_header = "#pragma once\n\n" + "".join("\n".join(cert) + "\n" for cert in certs)
+
+
+def write_generated(target, source, env):
+  Path(target[0].abspath).write_text(str(source[0]))
+  return 0
+
+
+generation_env = Environment(tools=[])
+for name, payload in (("gitversion.h", gitversion_header), ("version", version), ("cert.h", cert_header)):
+  generation_env.Command(File(f"./board/obj/{name}"), Value(payload), write_generated)
 
 # Packet version defines: SHA hash of the struct header files
 def version_hash(path):

tools/ci/run_predeploy.py

@@ -22,7 +22,8 @@ def commands(output, cache, download=False):
             ('schemas', [python, 'tools/ci/schema_policy.py'])]
   for name, path in (('vendor', 'tools/vendor/tests'), ('packaging', 'tools/release/tests'),
                      ('ci-tools', 'tools/ci/tests'), ('updater', 'openpilot/system/updated/tests')):
-    source.append((name, [python, '-m', 'unittest', 'discover', '-s', path, '-v']))
+    command = [python, '-m', 'pytest', '-q', path] if name == 'packaging' else [python, '-m', 'unittest', 'discover', '-s', path, '-v']
+    source.append((name, command))
   host = [('unittest', [python, 'tools/ci/run_host_tests.py', '--json-output', str(output / 'host.json')]),
           ('pytest', [python, 'tools/ci/run_host_tests.py', '--pytest'])]
   vehicles = [(suite, [python, 'tools/ci/run_vehicle_tests.py', '--suite', suite, '--output', str(output / suite)])

tools/release/build_manifest.py

@@ -0,0 +1,95 @@
+import argparse
+import hashlib
+import json
+import os
+from pathlib import Path
+import stat
+import subprocess
+
+
+def git(root, *args):
+  return subprocess.check_output(['git', '-C', str(root), *args])
+
+
+def tracked(root):
+  return [os.fsdecode(path) for path in git(root, 'ls-files', '-z').split(b'\0') if path]
+
+
+def fingerprint(path):
+  mode = path.lstat().st_mode
+  if stat.S_ISLNK(mode):
+    return {'kind': 'symlink', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(os.fsencode(os.readlink(path))).hexdigest()}
+  if not stat.S_ISREG(mode):
+    raise ValueError(f'Not a regular file: {path}')
+  return {'kind': 'file', 'mode': stat.S_IMODE(mode), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+
+
+def read_manifest(path):
+  path = Path(path)
+  metadata = path.lstat()
+  if not stat.S_ISREG(metadata.st_mode) or metadata.st_uid != os.getuid() or stat.S_IMODE(metadata.st_mode) != 0o600:
+    raise ValueError('Unsafe build manifest')
+  if metadata.st_size > 16 * 1024 * 1024:
+    raise ValueError('Oversized build manifest')
+  value = json.loads(path.read_text())
+  if value.get('format') != 'starpilot-ci-build' or value.get('version') != 1:
+    raise ValueError('Unsupported build manifest')
+  return value
+
+
+def capture(root, destination, outputs, config):
+  root = Path(root).resolve()
+  destination = Path(destination).absolute()
+  if destination.resolve().is_relative_to(root):
+    raise ValueError('Build manifest must be outside checkout')
+  revision = git(root, 'rev-parse', 'HEAD').decode().strip()
+  if destination.exists():
+    existing = read_manifest(destination)
+    if existing['root'] != str(root) or existing['revision'] != revision or existing['config'] != config:
+      raise ValueError('Stale build manifest')
+    return
+  if git(root, 'status', '--porcelain', '--untracked-files=all'):
+    raise ValueError('Checkout must be clean before build')
+  files = {name: fingerprint(root / name) for name in tracked(root)}
+  selected = sorted(set(outputs) & files.keys())
+  if any(files[name]['kind'] != 'file' for name in selected):
+    raise ValueError('Derived output is not a regular file')
+  value = {'format': 'starpilot-ci-build', 'version': 1, 'root': str(root), 'revision': revision,
+           'config': config, 'files': files, 'outputs': selected}
+  with open(destination, 'x', opener=lambda path, flags: os.open(path, flags, 0o600)) as stream:
+    json.dump(value, stream, sort_keys=True)
+    stream.flush()
+    os.fsync(stream.fileno())
+
+
+def verify(root, manifest):
+  root = Path(root).resolve()
+  value = read_manifest(manifest)
+  if value['root'] != str(root) or value['revision'] != git(root, 'rev-parse', 'HEAD').decode().strip():
+    raise ValueError('Build manifest belongs to a different checkout or revision')
+  files = value['files']
+  outputs = set(value['outputs'])
+  if not outputs <= files.keys() or set(tracked(root)) != files.keys():
+    raise ValueError('Tracked inventory changed')
+  if git(root, 'diff', '--cached', '--name-only'):
+    raise ValueError('Index changed during build')
+  if git(root, 'ls-files', '--others', '--exclude-standard', '-z'):
+    raise ValueError('Untracked files created during build')
+  for name, before in files.items():
+    if Path(name).is_absolute() or '..' in Path(name).parts:
+      raise ValueError('Unsafe manifest path')
+    after = fingerprint(root / name)
+    if after['kind'] != before['kind'] or after['mode'] != before['mode']:
+      raise ValueError(f'File type or mode changed: {name}')
+    if name in outputs:
+      if after['kind'] != 'file' or (root / name).stat().st_size == 0:
+        raise ValueError(f'Missing or empty derived output: {name}')
+    elif after != before:
+      raise ValueError(f'Source or undeclared output changed: {name}')
+
+
+if __name__ == '__main__':
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--manifest', required=True)
+  args = parser.parse_args()
+  verify(git(Path.cwd(), 'rev-parse', '--show-toplevel').decode().strip(), args.manifest)

tools/release/check-dirty.sh

@@ -4,6 +4,11 @@ set -e
 DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null && pwd)"
 cd "$DIR"
 
+if [ -n "${STARPILOT_CI_BUILD_MANIFEST:-}" ]; then
+  python3 "$DIR/build_manifest.py" --manifest "$STARPILOT_CI_BUILD_MANIFEST"
+  exit 0
+fi
+
 if [ ! -z "$(git status --porcelain)" ]; then
   echo "Dirty working tree after build:"
   git status --porcelain

tools/release/tests/test_build_manifest.py

@@ -0,0 +1,91 @@
+import subprocess
+
+import pytest
+
+from tools.release.build_manifest import capture, verify
+
+
+@pytest.fixture
+def checkout(tmp_path):
+  root = tmp_path / 'repo'
+  root.mkdir()
+  subprocess.run(['git', 'init', '-q', str(root)], check=True)
+  for name in ('source.py', 'output.bin', 'undeclared.bin'):
+    (root / name).write_text('original\n')
+  subprocess.run(['git', '-C', str(root), 'add', '.'], check=True)
+  subprocess.run(['git', '-C', str(root), '-c', 'user.name=Test', '-c', 'user.email=test@example.invalid',
+                  'commit', '-qm', 'fixture'], check=True)
+  return root, tmp_path / 'manifest.json'
+
+
+def test_declared_output_rebuilt(checkout):
+  root, manifest = checkout
+  capture(root, manifest, ['output.bin'], {'arch': 'test'})
+  (root / 'output.bin').write_bytes(b'host output')
+  capture(root, manifest, ['output.bin'], {'arch': 'test'})
+  verify(root, manifest)
+
+
+@pytest.mark.parametrize('change', ['source', 'undeclared', 'missing', 'empty', 'mode', 'symlink', 'untracked', 'index'])
+def test_build_mutations_rejected(checkout, change):
+  root, manifest = checkout
+  capture(root, manifest, ['output.bin'], {'arch': 'test'})
+  target = root / 'output.bin'
+  if change in ('source', 'undeclared'):
+    (root / ('source.py' if change == 'source' else 'undeclared.bin')).write_text('changed')
+  elif change == 'missing':
+    target.unlink()
+  elif change == 'empty':
+    target.write_bytes(b'')
+  elif change == 'mode':
+    target.chmod(0o755)
+  elif change == 'symlink':
+    target.unlink()
+    target.symlink_to('source.py')
+  elif change == 'untracked':
+    (root / 'unexpected.py').write_text('created')
+  else:
+    target.write_text('changed')
+    subprocess.run(['git', '-C', str(root), 'add', 'output.bin'], check=True)
+  with pytest.raises((ValueError, FileNotFoundError)):
+    verify(root, manifest)
+
+
+def test_dirty_capture_rejected(checkout):
+  root, manifest = checkout
+  (root / 'source.py').write_text('changed')
+  with pytest.raises(ValueError, match='clean'):
+    capture(root, manifest, ['output.bin'], {})
+
+
+def test_stale_config_and_root_rejected(checkout, tmp_path):
+  root, manifest = checkout
+  capture(root, manifest, ['output.bin'], {'arch': 'one'})
+  with pytest.raises(ValueError, match='Stale'):
+    capture(root, manifest, ['output.bin'], {'arch': 'two'})
+  other = tmp_path / 'other'
+  subprocess.run(['git', 'clone', '-q', str(root), str(other)], check=True)
+  with pytest.raises(ValueError, match='different checkout'):
+    verify(other, manifest)
+
+
+def test_unsafe_manifest_rejected(checkout):
+  root, manifest = checkout
+  capture(root, manifest, ['output.bin'], {})
+  manifest.chmod(0o644)
+  with pytest.raises(ValueError, match='Unsafe'):
+    verify(root, manifest)
+
+
+def test_manifest_inside_checkout_rejected(checkout):
+  root, _ = checkout
+  with pytest.raises(ValueError, match='outside'):
+    capture(root, root / 'manifest.json', ['output.bin'], {})
+
+
+def test_manifest_parent_symlink_inside_checkout_rejected(checkout, tmp_path):
+  root, _ = checkout
+  alias = tmp_path / 'alias'
+  alias.symlink_to(root, target_is_directory=True)
+  with pytest.raises(ValueError, match='outside'):
+    capture(root, alias / 'manifest.json', ['output.bin'], {})