diff --git a/prebuilt.json b/prebuilt.json index cf813cab75..61011256d4 100644 --- a/prebuilt.json +++ b/prebuilt.json @@ -1 +1 @@ -{"protected_inventory_sha256": "7108bf481055a45efb64e51483cd7069b51fdc7ff87a1cab991138088903e7e0", "schema_version": 1} +{"protected_inventory_sha256": "4e1b9257513a6301cec583c3e21e83ebbdd48f727008bcf0294fc8efdeab75a3", "schema_version": 1} diff --git a/tools/ci/lifecycle_build_outputs.json b/tools/ci/lifecycle_build_outputs.json new file mode 100644 index 0000000000..4136ba8b5d --- /dev/null +++ b/tools/ci/lifecycle_build_outputs.json @@ -0,0 +1,334 @@ +{ + "msgq_repo/libmsgq.a": { + "recipe": "msgq_repo/SConscript", + "required_tracked": true + }, + "msgq_repo/libvisionipc.a": { + "recipe": "msgq_repo/SConscript", + "required_tracked": true + }, + "msgq_repo/msgq/ipc_pyx.so": { + "recipe": "msgq_repo/SConscript", + "required_tracked": true + }, + "msgq_repo/msgq/visionipc/visionipc_pyx.so": { + "recipe": "msgq_repo/SConscript", + "required_tracked": true + }, + "opendbc_repo/opendbc/can/_raw_value.so": { + "recipe": "opendbc_repo/opendbc/can/SConscript", + "required_tracked": true + }, + "openpilot/cereal/libcereal.a": { + "recipe": "openpilot/cereal/SConscript", + "required_tracked": true + }, + "openpilot/cereal/libsocketmaster.a": { + "recipe": "openpilot/cereal/SConscript", + "required_tracked": true + }, + "openpilot/cereal/messaging/bridge": { + "recipe": "openpilot/cereal/SConscript", + "required_tracked": true + }, + "openpilot/common/libcommon.a": { + "recipe": "openpilot/common/SConscript", + "required_tracked": true + }, + "openpilot/common/libparams_c.so": { + "recipe": "openpilot/common/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/acados_ocp_long.json": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/Makefile": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.c": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/acados_ocp_solver_pyx.so": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libacados.so": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libacados_ocp_solver_long.so": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libblasfeo.so": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libhpipm.so": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/c_generated_code/libqpOASES_e.so.3.1": { + "recipe": "openpilot/selfdrive/controls/lib/longitudinal_mpc_lib/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/car.cpp": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/car.h": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/libcar.so": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/libpose.so": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/pose.cpp": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/locationd/models/generated/pose.h": { + "recipe": "openpilot/selfdrive/locationd/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/modeld/models/dm_warp_1344x760_tinygrad.pkl": { + "recipe": "openpilot/selfdrive/modeld/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/modeld/models/dm_warp_1928x1208_tinygrad.pkl": { + "recipe": "openpilot/selfdrive/modeld/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/modeld/models/driving_warp_1344x760_tinygrad.pkl": { + "recipe": "openpilot/selfdrive/modeld/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/modeld/models/driving_warp_1928x1208_tinygrad.pkl": { + "recipe": "openpilot/selfdrive/modeld/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/pandad/libpanda.a": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/pandad/pandad": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": true + }, + "openpilot/selfdrive/pandad/tests/test_panda_owner": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": false + }, + "openpilot/selfdrive/pandad/tests/test_peripheral_state": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": false + }, + "openpilot/selfdrive/pandad/tests/test_preap_pedal_startup": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": false + }, + "openpilot/selfdrive/pandad/tests/test_producer_sink": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": false + }, + "openpilot/selfdrive/pandad/tests/test_producer_timing": { + "recipe": "openpilot/selfdrive/pandad/SConscript", + "required_tracked": false + }, + "openpilot/starpilot/flm/_primitive_walk.binding.json": { + "recipe": "openpilot/starpilot/flm/SConscript", + "required_tracked": true + }, + "openpilot/starpilot/flm/_primitive_walk.so": { + "recipe": "openpilot/starpilot/flm/SConscript", + "required_tracked": true + }, + "openpilot/system/loggerd/bootlog": { + "recipe": "openpilot/system/loggerd/SConscript", + "required_tracked": true + }, + "openpilot/system/loggerd/encoderd": { + "recipe": "openpilot/system/loggerd/SConscript", + "required_tracked": true + }, + "openpilot/system/loggerd/liblogger.a": { + "recipe": "openpilot/system/loggerd/SConscript", + "required_tracked": true + }, + "openpilot/system/loggerd/loggerd": { + "recipe": "openpilot/system/loggerd/SConscript", + "required_tracked": true + }, + "openpilot/tools/replay/libmedia_mux.so": { + "recipe": "openpilot/tools/replay/SConscript", + "required_tracked": true + }, + "openpilot/tools/replay/replay": { + "recipe": "openpilot/tools/replay/SConscript", + "required_tracked": true + }, + "panda/board/obj/body_h7.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.body_h7.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_hkg_remote.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_hkg_remote_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_remote.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_remote_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_tesla_wake.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_h7_tesla_wake_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_hkg_remote.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_hkg_remote_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_remote.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_remote_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_tesla_wake.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/bootstub.panda_tesla_wake_can_ignition_only.bin": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/gitversion.h": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_hkg_remote.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_hkg_remote_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_remote.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_remote_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_tesla_wake.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_h7_tesla_wake_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_hkg_remote.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_hkg_remote_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_remote.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_remote_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_tesla_wake.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/panda_tesla_wake_can_ignition_only.bin.signed": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/board/obj/version": { + "recipe": "panda/SConscript", + "required_tracked": true + }, + "panda/tests/libpanda/libpanda.so": { + "recipe": "panda/tests/libpanda/SConscript", + "required_tracked": true + }, + "rednose_repo/rednose/helpers/ekf_sym_pyx.so": { + "recipe": "rednose_repo/rednose/SConscript", + "required_tracked": true + }, + "rednose_repo/rednose/helpers/libekf_sym.a": { + "recipe": "rednose_repo/rednose/SConscript", + "required_tracked": true + } +} diff --git a/tools/ci/lifecycle_shards.py b/tools/ci/lifecycle_shards.py index 692c1164b0..97d6eff0f0 100644 --- a/tools/ci/lifecycle_shards.py +++ b/tools/ci/lifecycle_shards.py @@ -3,9 +3,77 @@ import argparse import hashlib from collections import Counter import json +import os +import subprocess from pathlib import Path +def digest(data): + return hashlib.sha256(data).hexdigest() + + +def file_record(path): + if path.is_symlink(): + return {'sha256': digest(os.fsencode(os.readlink(path))), 'mode': 0o120000} + if not path.is_file(): + return None + return {'sha256': digest(path.read_bytes()), 'mode': path.stat().st_mode & 0o777} + + +def build_identity(root): + """Pin the complete committed tree plus actual changes; separate only named build outputs.""" + root = Path(root) + def git(*args): + return subprocess.check_output(['git', '--no-optional-locks', '-c', 'gc.auto=0', '-c', 'maintenance.auto=false', *args], cwd=root) + inventory_path = root / 'tools/ci/lifecycle_build_outputs.json' + inventory_bytes = inventory_path.read_bytes() + inventory = json.loads(inventory_bytes) + outputs = {} + for name, declaration in inventory.items(): + path = Path(name) + if path.is_absolute() or '..' in path.parts or set(declaration) != {'recipe', 'required_tracked'}: + raise ValueError('Invalid declared build output') + outputs[name] = file_record(root / name) + if declaration['required_tracked'] and outputs[name] is None: + raise ValueError(f'Missing declared tracked artifact: {name}') + if not (root / declaration['recipe']).is_file(): + raise ValueError(f'Missing build recipe for {name}') + changed = git('diff', '--name-only', '--no-renames', '-z', 'HEAD', '--').decode().split('\0') + changes = {name: file_record(root / name) for name in changed if name and name not in inventory} + untracked = git('ls-files', '--others', '--exclude-standard', '-z', '--', 'openpilot', 'opendbc_repo', 'panda', + 'msgq_repo', 'rednose_repo', 'tinygrad_repo', 'tinygrad_model_runtime', 'tools', 'scripts').decode().split('\0') + additions = {name: file_record(root / name) for name in untracked if name and name not in inventory} + recipes = {name: file_record(root / name) for name in sorted({'SConstruct', *[row['recipe'] for row in inventory.values()]})} + tracked = git('ls-files', '--cached', '-z').decode().split('\0') + source_files = {name: file_record(root / name) for name in tracked if name and name not in inventory} + if any(metadata is None for metadata in source_files.values()): + raise ValueError('Missing source input from the tracked build closure') + inputs = {'revision': git('rev-parse', 'HEAD').decode().strip(), 'tree': git('rev-parse', 'HEAD^{tree}').decode().strip(), + 'tracked_inputs': source_files, 'changes': changes, 'untracked': additions, 'recipes': recipes, + 'output_inventory_sha256': digest(inventory_bytes)} + return {'inputs': inputs, 'declared_outputs': outputs} + + +def same_build_inputs(first, second): + inventory = json.loads((Path(__file__).with_name('lifecycle_build_outputs.json')).read_text()) + expected = digest(Path(__file__).with_name('lifecycle_build_outputs.json').read_bytes()) + for record in (first, second): + if set(record) != {'inputs', 'declared_outputs'} or set(record['declared_outputs']) != set(inventory): + raise ValueError('Missing or forged build provenance') + if set(record['inputs']) != {'revision', 'tree', 'tracked_inputs', 'changes', 'untracked', 'recipes', 'output_inventory_sha256'}: + raise ValueError('Missing source input closure') + if not record['inputs']['tracked_inputs'] or not record['inputs']['recipes']: + raise ValueError('Empty source input closure') + if record['inputs']['output_inventory_sha256'] != expected: + raise ValueError('Build output inventory changed') + for name, metadata in record['declared_outputs'].items(): + if metadata is None and inventory[name]['required_tracked']: + raise ValueError('Missing native artifact provenance') + if metadata is not None and (set(metadata) != {'sha256', 'mode'} or len(metadata['sha256']) != 64): + raise ValueError('Malformed native artifact provenance') + return first['inputs'] == second['inputs'] + + def partition(targets, identities, count): if count < 1 or len(set(targets)) != len(targets) or len(set(identities)) != len(identities): raise ValueError('Invalid lifecycle catalog') @@ -47,17 +115,42 @@ def aggregate(directory, count): raise ValueError('Lifecycle shard errors') if doc['full_catalog_test_ids'] != full or doc['full_catalog_targets'] != targets or doc['targets'] != expected[index]: raise ValueError('Lifecycle catalog or partition changed') - for key in ('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256', 'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'): + for key in ('revision', 'manifest_sha256', 'dependencies', 'runner_sha256', 'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'): if doc['source'][key] != first['source'][key]: raise ValueError(f'Lifecycle source disagreement: {key}') + if not same_build_inputs(doc['source']['build_identity'], first['source']['build_identity']): + raise ValueError('Lifecycle source or native build inputs differ across shards') + if doc['source']['build_identity'] != doc['source_after']['build_identity']: + raise ValueError('Lifecycle source or declared native outputs changed during execution') if doc['source']['inputs'] != doc['source_after']['inputs']: raise ValueError('Lifecycle source changed during execution') + can_paths = {'opendbc_repo/opendbc/can/packer.py', 'opendbc_repo/opendbc/can/parser.py'} + can_sources = doc['native'].get('can_sources', {}) + if set(can_sources) != can_paths or can_sources != first['native'].get('can_sources'): + raise ValueError('Tracked CAN import origins changed') + for name, checksum in can_sources.items(): + if doc['source']['build_identity']['inputs']['tracked_inputs'][name]['sha256'] != checksum: + raise ValueError('Imported CAN implementation differs from source inputs') + capnp = doc['native'].get('pycapnp', {}) + if capnp.get('sha256') != first['native'].get('pycapnp', {}).get('sha256') or len(capnp.get('sha256', '')) != 64: + raise ValueError('Native pycapnp import provenance changed') for key in ('safety', 'aol_protocol'): if not doc['native'][key]['sha256']: raise ValueError('Missing genuine native build evidence') ids = doc['selected_test_ids'] coverage_path = next(path for path in Path(directory).rglob('coverage.json') if json.loads(path.read_text())['shard']['index'] == index) + build_input_digest = digest(json.dumps(doc['source']['build_identity']['inputs'], sort_keys=True).encode()) + for kind in ('safety', 'aol_protocol'): + native = doc['native'][kind] + if native['build_inputs_sha256'] != build_input_digest: + raise ValueError('Native build input source provenance changed') + artifact = coverage_path.with_name(Path(native['path']).name) + if not artifact.is_file() or digest(artifact.read_bytes()) != native['sha256']: + raise ValueError('Missing or modified genuine native artifact') + command = native.get('commands') if kind == 'safety' else native.get('command') + if not command: + raise ValueError('Missing actual native builder command') result_path = coverage_path.with_name('results.json') if doc['result_sha256'] != hashlib.sha256(result_path.read_bytes()).hexdigest(): raise ValueError('Lifecycle outcome artifact changed') diff --git a/tools/ci/run_vehicle_tests.py b/tools/ci/run_vehicle_tests.py index 3c2a8f0166..b35a65d894 100644 --- a/tools/ci/run_vehicle_tests.py +++ b/tools/ci/run_vehicle_tests.py @@ -442,7 +442,7 @@ SOURCE_INPUT_PATHS = ( def source_input_snapshot(): def git(*args): - return subprocess.check_output(["git", *args], cwd=ROOT) + return subprocess.check_output(["git", "--no-optional-locks", "-c", "gc.auto=0", "-c", "maintenance.auto=false", *args], cwd=ROOT) untracked = {} for raw_path in git("ls-files", "--others", "--exclude-standard", "-z", "--", *SOURCE_INPUT_PATHS).split(b"\0"): @@ -451,7 +451,7 @@ def source_input_snapshot(): path = ROOT / name data = os.fsencode(os.readlink(path)) if path.is_symlink() else path.read_bytes() untracked[name] = hashlib.sha256(data).hexdigest() - patch_bytes = git("diff", "--no-ext-diff", "--no-textconv", "--binary", "HEAD", "--", *SOURCE_INPUT_PATHS) + patch_bytes = git("diff", "--no-renames", "--no-ext-diff", "--no-textconv", "--binary", "HEAD", "--", *SOURCE_INPUT_PATHS) return {"revision": git("rev-parse", "HEAD").decode().strip(), "paths": list(SOURCE_INPUT_PATHS), "tracked_patch_sha256": hashlib.sha256(patch_bytes).hexdigest(), "untracked_sha256": untracked} @@ -557,10 +557,10 @@ def vehicle_inventory(): def source_provenance(): def git(*args): - return subprocess.check_output(["git", *args], cwd=ROOT).decode().strip() + return subprocess.check_output(["git", "--no-optional-locks", "-c", "gc.auto=0", "-c", "maintenance.auto=false", *args], cwd=ROOT).decode().strip() manifest = ROOT / "upstream-sync.json" return {"revision": git("rev-parse", "HEAD"), "status": git("status", "--porcelain"), - "inputs": source_input_snapshot(), + "inputs": source_input_snapshot(), "build_identity": lifecycle_shards.build_identity(ROOT), "manifest_sha256": sha256(manifest), "dependencies": json.loads(manifest.read_text())["dependencies"], "runner_sha256": sha256(ROOT / "tools/test_runner.py"), "suite_runner_sha256": sha256(__file__), "pytest_runner_sha256": sha256(ROOT / "tools/ci/vehicle_pytest.py"), @@ -654,6 +654,10 @@ def run(suite, output, collect_only=False, shard_index=None, shard_count=4): plan["native"]["safety"] = build_safety_library(suite == "safety-release", output) if suite == "aol-lifecycle" and not collect_only: plan["native"]["aol_protocol"] = build_aol_protocol(output) + build_input_digest = hashlib.sha256(json.dumps(plan["source"]["build_identity"]["inputs"], sort_keys=True).encode()).hexdigest() + for name in ('safety', 'aol_protocol'): + if name in plan["native"]: + plan["native"][name]["build_inputs_sha256"] = build_input_digest phase = "collection" unittest_targets, pytest_targets = vehicle_pytest.split_targets(ROOT, plan["targets"]) plan["runners"] = {"unittest": unittest_targets, "pytest": pytest_targets} diff --git a/tools/ci/tests/test_lifecycle_build_identity.py b/tools/ci/tests/test_lifecycle_build_identity.py new file mode 100644 index 0000000000..bd5406c07e --- /dev/null +++ b/tools/ci/tests/test_lifecycle_build_identity.py @@ -0,0 +1,92 @@ +import copy +import importlib.util +import json +from pathlib import Path +import subprocess +import tempfile +import unittest + +GIT = ['git', '--no-optional-locks', '-c', 'gc.auto=0', '-c', 'maintenance.auto=false'] + +spec = importlib.util.spec_from_file_location('private_build_identity', Path(__file__).parents[1] / 'lifecycle_shards.py') +assert spec is not None and spec.loader is not None +helper = importlib.util.module_from_spec(spec) +spec.loader.exec_module(helper) + + +class BuildIdentityTests(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.root = Path(self.directory.name) + self.inventory_bytes = Path(__file__).parents[1].joinpath('lifecycle_build_outputs.json').read_bytes() + self.inventory = json.loads(self.inventory_bytes) + inventory_path = self.root / 'tools/ci/lifecycle_build_outputs.json' + inventory_path.parent.mkdir(parents=True) + inventory_path.write_bytes(self.inventory_bytes) + for name in self.inventory: + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b'build output') + for name in {'SConstruct', *[row['recipe'] for row in self.inventory.values()], 'opendbc_repo/native_input.h'}: + path = self.root / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text('source input\n') + subprocess.run([*GIT, 'init', '-q'], cwd=self.root, check=True) + subprocess.run([*GIT, 'add', '.'], cwd=self.root, check=True) + subprocess.run([*GIT, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'Fixture'], cwd=self.root, check=True) + + def test_only_named_outputs_can_differ_with_identical_inputs(self): + first = helper.build_identity(self.root) + (self.root / 'msgq_repo/msgq/ipc_pyx.so').write_bytes(b'independent genuine build') + second = helper.build_identity(self.root) + self.assertTrue(helper.same_build_inputs(first, second)) + self.assertNotEqual(first['declared_outputs'], second['declared_outputs']) + self.assertNotEqual(first, second) # The per-job before/after guard still rejects an in-test mutation. + + def test_source_native_input_and_recipe_changes_are_denied(self): + first = helper.build_identity(self.root) + for name in ('opendbc_repo/native_input.h', 'SConstruct'): + path = self.root / name + original = path.read_bytes() + path.write_bytes(b'changed source') + self.assertFalse(helper.same_build_inputs(first, helper.build_identity(self.root))) + path.write_bytes(original) + + def test_undeclared_input_path_is_pinned(self): + first = helper.build_identity(self.root) + path = self.root / 'opendbc_repo/undeclared.so' + path.write_bytes(b'unregistered artifact') + self.assertFalse(helper.same_build_inputs(first, helper.build_identity(self.root))) + + def test_missing_declared_artifact_is_denied(self): + (self.root / 'msgq_repo/msgq/ipc_pyx.so').unlink() + with self.assertRaises(ValueError): + helper.build_identity(self.root) + + def test_nonoutput_symlink_target_is_pinned(self): + path = self.root / 'opendbc_repo/input-link.h' + path.symlink_to('native_input.h') + subprocess.run([*GIT, 'add', '.'], cwd=self.root, check=True) + subprocess.run([*GIT, '-c', 'user.name=Test', '-c', 'user.email=test@example.com', 'commit', '-qm', 'Symlink'], + cwd=self.root, check=True) + first = helper.build_identity(self.root) + path.unlink() + path.symlink_to('other_source.h') + self.assertFalse(helper.same_build_inputs(first, helper.build_identity(self.root))) + + def test_forged_or_stripped_provenance_is_denied(self): + first = helper.build_identity(self.root) + for field in ('inputs', 'declared_outputs'): + forged = copy.deepcopy(first) + del forged[field] + with self.assertRaises(ValueError): + helper.same_build_inputs(first, forged) + forged = copy.deepcopy(first) + forged['inputs']['output_inventory_sha256'] = 'f' * 64 + with self.assertRaises(ValueError): + helper.same_build_inputs(first, forged) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/ci/tests/test_lifecycle_shards.py b/tools/ci/tests/test_lifecycle_shards.py index 91a509f9b8..20a3da4e85 100644 --- a/tools/ci/tests/test_lifecycle_shards.py +++ b/tools/ci/tests/test_lifecycle_shards.py @@ -36,33 +36,67 @@ class LifecycleShardTests(unittest.TestCase): def documents(self): targets, full = self.catalog() partition = helper.partition(targets, full, 4) - source = dict.fromkeys(('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256', + source: dict = dict.fromkeys(('revision', 'inputs', 'manifest_sha256', 'dependencies', 'runner_sha256', 'suite_runner_sha256', 'pytest_runner_sha256', 'shard_runner_sha256'), 'same') + inventory_bytes = Path(__file__).parents[1].joinpath('lifecycle_build_outputs.json').read_bytes() + inventory = json.loads(inventory_bytes) + source['build_identity'] = {'inputs': {'revision': 'same', 'tree': 'same', 'tracked_inputs': {'source.c': {'sha256': 'c' * 64, 'mode': 420}}, + 'changes': {}, 'untracked': {}, 'recipes': {'SConstruct': {'sha256': 'd' * 64, 'mode': 420}}, + 'output_inventory_sha256': hashlib.sha256(inventory_bytes).hexdigest()}, + 'declared_outputs': {name: {'sha256': 'a' * 64, 'mode': 420} for name in inventory}} + can_sources = dict.fromkeys(('opendbc_repo/opendbc/can/packer.py', 'opendbc_repo/opendbc/can/parser.py'), 'e' * 64) + source['build_identity']['inputs']['tracked_inputs'].update({name: {'sha256': checksum, 'mode': 420} + for name, checksum in can_sources.items()}) docs = [] for index, assigned in enumerate(partition): ids = [identity for identity in full if identity.startswith(assigned[0].removesuffix('.py') + '.')] doc = {'suite': 'aol-lifecycle', 'status': 'completed', 'exit_code': 0, 'shard': {'index': index, 'count': 4}, 'targets': assigned, 'selected_test_ids': ids, 'full_catalog_targets': targets, 'full_catalog_test_ids': full, 'collection_errors': [], 'gate_errors': [], 'source': source, 'source_after': source, - 'native': {'safety': {'sha256': '1' * 64}, 'aol_protocol': {'sha256': '2' * 64}}} + 'native': {'can_sources': can_sources, 'pycapnp': {'sha256': 'b' * 64}, 'safety': {'sha256': '1' * 64}, 'aol_protocol': {'sha256': '2' * 64}}} result = {'selected_test_ids': ids, 'exit_code': 0, 'collection_errors': [], 'gate_errors': [], 'results': [{'id': identity, 'status': 'passed'} for identity in ids]} docs.append((doc, result)) return docs - def check(self, docs): + def check(self, docs, artifact_mutation=None): with tempfile.TemporaryDirectory() as directory: for index, (doc, result) in enumerate(docs): path = Path(directory) / str(index) path.mkdir() + digest = hashlib.sha256(json.dumps(doc['source']['build_identity']['inputs'], sort_keys=True).encode()).hexdigest() + for kind in ('safety', 'aol_protocol'): + native = doc['native'][kind] + name = 'libsafety-debug.so' if kind == 'safety' else 'test_aol_protocol' + artifact = path / name + artifact.write_bytes(kind.encode()) + native.update(path=str(artifact), sha256=hashlib.sha256(artifact.read_bytes()).hexdigest(), + build_inputs_sha256=digest, commands=[['cc', 'actual.c']] if kind == 'safety' else None, + command=['c++', 'actual.cc']) (path / 'results.json').write_text(json.dumps(result)) doc['result_sha256'] = hashlib.sha256((path / 'results.json').read_bytes()).hexdigest() (path / 'coverage.json').write_text(json.dumps(doc)) + if artifact_mutation is not None: + artifact_mutation(Path(directory)) return helper.aggregate(directory, 4) def test_complete_catalog_passes(self): self.assertEqual(self.check(self.documents())['cases_passed'], 12) + def test_missing_modified_or_forged_native_artifacts_are_denied(self): + def remove(root): + (root / '0/libsafety-debug.so').unlink() + def modify(root): + (root / '0/test_aol_protocol').write_bytes(b'foreign binary') + def forge(root): + path = root / '0/coverage.json' + data = json.loads(path.read_text()) + data['native']['safety']['build_inputs_sha256'] = 'f' * 64 + path.write_text(json.dumps(data)) + for mutate in (remove, modify, forge): + with self.subTest(mutation=mutate.__name__), self.assertRaises(ValueError): + self.check(self.documents(), mutate) + def test_bad_outcomes_and_provenance_are_denied(self): mutations = [lambda d: d.pop(), lambda d: d[0][0]['shard'].update(index=1), @@ -72,7 +106,9 @@ class LifecycleShardTests(unittest.TestCase): lambda d: d[0][0].update(targets=['foreign.py']), lambda d: d[0][0]['source'].update(revision='foreign'), lambda d: d[0][0]['source_after'].update(inputs='changed'), - lambda d: d[0][0]['native']['safety'].update(sha256=''), + lambda d: d[0][0]['source']['build_identity']['declared_outputs'].clear(), + lambda d: d[0][0]['native']['can_sources'].clear(), + lambda d: d[0][0]['native']['pycapnp'].update(sha256='f' * 64), lambda d: d[0][1]['results'].pop(), lambda d: d[0][1]['results'].append(d[0][1]['results'][0]), lambda d: d[0][1]['results'][0].update(status='skipped'),