StarPilot/tools/ci/tests/test_qcom_artifact_contract.py

import ast
import copy
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch

from tools.laptop_device_build import qcom_artifact_contract as contract


ROOT = Path(__file__).resolve().parents[3]


class TestQcomArtifactContract(unittest.TestCase):
  def setUp(self):
    temporary = tempfile.TemporaryDirectory()
    self.addCleanup(temporary.cleanup)
    self.package = Path(temporary.name)
    self.artifacts = self.package / "artifacts"
    self.artifacts.mkdir()
    self.commands = contract.commands(ROOT)
    for name in self.commands:
      (self.artifacts / name).write_bytes(f"artifact:{name}".encode())
    with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"):
      manifest = contract.record(ROOT, self.artifacts)
    (self.package / "manifest.json").write_text(json.dumps(manifest))

  def verify(self, name):
    with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"):
      return contract.verify(self.package, ROOT, name, self.commands[name])

  def compatible_package(self):
    manifest = json.loads((self.package / "manifest.json").read_text())
    current = contract.source_inventory(ROOT)
    old_hash = "0" * 64
    manifest["source"]["sources"][contract.COMPATIBILITY_SOURCE] = old_hash
    manifest["source_sha256"] = contract._inventory_signature(manifest["source"])
    (self.package / "manifest.json").write_text(json.dumps(manifest))
    evidence_dir = self.package / "compatibility"
    evidence_dir.mkdir()
    (evidence_dir / "qualification.txt").write_text("reviewed paired output evidence\n")
    observations = {name: [{"input": label, "shape": contract.COMPATIBILITY_OUTPUTS[name][0],
                            "dtype": contract.COMPATIBILITY_OUTPUTS[name][1],
                            "build_sha256": str(index + 4) * 64,
                            "runtime_sha256": str(index + 4) * 64}
                           for index, label in enumerate(("A", "B", "A"))]
                    for name in self.commands}
    evidence = {"version": 1, "method": "paired-qcom-output-sha256",
                "build_manifest_sha256": contract._sha(self.package / "manifest.json"),
                "compatible_source_sha256": contract._inventory_signature(current),
                "changed_source": {"path": contract.COMPATIBILITY_SOURCE, "build_sha256": old_hash,
                                   "runtime_sha256": current["sources"][contract.COMPATIBILITY_SOURCE]},
                "artifact_sha256": {name: entry["sha256"] for name, entry in manifest["artifacts"].items()},
                "paired_outputs": observations,
                "evidence_files": {"qualification.txt": contract._sha(evidence_dir / "qualification.txt")}}
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    return evidence, current

  def test_exact_source_and_artifacts_select_expected_target(self):
    name = "dmonitoring_model_tinygrad.pkl"
    self.assertEqual(self.verify(name)[0], self.artifacts / name)
    self.assertEqual(set(json.loads((self.package / "manifest.json").read_text())["artifacts"]), set(self.commands))

  def historical_package(self, compatible=False):
    if compatible:
      evidence, current = self.compatible_package()
    manifest = json.loads((self.package / "manifest.json").read_text())
    command = " ".join((f'{contract.FLAGS} taskset -c 0-7 python3 "{ROOT}/tinygrad_repo/examples/openpilot/compile_onnx.py"',
                        f'"{ROOT / contract.RETIRED_SOURCE}" "{ROOT / contract.MODEL_DIR / contract.RETIRED_TARGET}"',
                        '--out-of-band --benchmark-runs 1'))
    manifest['source']['sources'][contract.RETIRED_SOURCE] = contract.RETIRED_SOURCE_SHA256
    manifest['source']['commands'][contract.RETIRED_TARGET] = contract._tokens(command, ROOT)
    manifest['artifacts'][contract.RETIRED_TARGET] = {'sha256': contract.RETIRED_ARTIFACT_SHA256, 'size': 70473368}
    manifest['source_sha256'] = contract._inventory_signature(manifest['source'])
    (self.package / 'manifest.json').write_text(json.dumps(manifest))
    if compatible:
      current, _ = contract._historical_inventory(manifest, current, self.commands, ROOT)
      evidence['compatible_source_sha256'] = contract._inventory_signature(current)
      evidence['build_manifest_sha256'] = contract._sha(self.package / 'manifest.json')
      evidence['artifact_sha256'][contract.RETIRED_TARGET] = contract.RETIRED_ARTIFACT_SHA256
      shape, dtype = contract.COMPATIBILITY_OUTPUTS[contract.RETIRED_TARGET]
      evidence['paired_outputs'][contract.RETIRED_TARGET] = [
        {'input': label, 'shape': shape, 'dtype': dtype, 'build_sha256': 'a' * 64, 'runtime_sha256': 'a' * 64}
        for label in ('A', 'B', 'A')]
      (self.package / 'compatibility.json').write_text(json.dumps(evidence))

  def test_only_dm_and_warps_are_current_build_inputs(self):
    self.assertEqual(len(self.commands), 5)
    self.assertNotIn(contract.RETIRED_TARGET, self.commands)
    self.assertNotIn(contract.RETIRED_SOURCE, contract.STATIC_SOURCES)
    tree = ast.parse((ROOT / 'openpilot/selfdrive/modeld/SConscript').read_text())
    models = [tuple(arg.value for arg in node.args) for node in ast.walk(tree)
              if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == 'compile_model']
    self.assertEqual(models, [('models/dmonitoring_model.onnx', 'models/dmonitoring_model_tinygrad.pkl')])
    with self.assertRaisesRegex(ValueError, 'command recipe changed'):
      contract.verify(self.package, ROOT, contract.RETIRED_TARGET, 'ignored')

  def test_historical_package_uses_verified_five_subset_without_retired_bytes(self):
    for compatible in (False, True):
      with self.subTest(compatible=compatible):
        self.historical_package(compatible)
        before = (self.package / 'manifest.json').read_bytes()
        for name in self.commands:
          self.assertEqual(self.verify(name)[0], self.artifacts / name)
        self.assertEqual((self.package / 'manifest.json').read_bytes(), before)
        self.assertFalse((self.artifacts / contract.RETIRED_TARGET).exists())

  def test_historical_subset_does_not_hide_unreviewed_source_or_artifact(self):
    self.historical_package(True)
    current = contract.source_inventory(ROOT)
    current['sources']['unreviewed.py'] = 'f' * 64
    with patch.object(contract, 'source_inventory', return_value=current), self.assertRaisesRegex(ValueError, 'outside reviewed'):
      self.verify('dmonitoring_model_tinygrad.pkl')
    (self.artifacts / 'dm_warp_1344x760_tinygrad.pkl').write_bytes(b'changed')
    with self.assertRaisesRegex(ValueError, 'artifact mismatch'):
      self.verify('dmonitoring_model_tinygrad.pkl')

  def test_historical_retirement_requires_exact_reviewed_provenance(self):
    self.historical_package()
    manifest = json.loads((self.package / 'manifest.json').read_text())
    manifest['source']['sources'][contract.RETIRED_SOURCE] = 'f' * 64
    manifest['source_sha256'] = contract._inventory_signature(manifest['source'])
    (self.package / 'manifest.json').write_text(json.dumps(manifest))
    with self.assertRaisesRegex(ValueError, 'Unreviewed retired'):
      self.verify('dmonitoring_model_tinygrad.pkl')

  def test_source_and_command_changes_reject(self):
    name = "dmonitoring_model_tinygrad.pkl"
    with patch.object(contract, "_sources", return_value={"changed.py": "0" * 64}), \
         self.assertRaisesRegex(ValueError, "sources changed"):
      self.verify(name)
    with self.assertRaisesRegex(ValueError, "command recipe changed"):
      contract.verify(self.package, ROOT, name, self.commands[name] + " --different")

  def test_one_artifact_change_or_missing_member_rejects_whole_package(self):
    (self.artifacts / "dm_warp_1344x760_tinygrad.pkl").write_bytes(b"changed")
    with self.assertRaisesRegex(ValueError, "artifact mismatch"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    manifest = json.loads((self.package / "manifest.json").read_text())
    del manifest["artifacts"]["dm_warp_1344x760_tinygrad.pkl"]
    (self.package / "manifest.json").write_text(json.dumps(manifest))
    with self.assertRaisesRegex(ValueError, "source or artifact set"):
      self.verify("dmonitoring_model_tinygrad.pkl")

  def test_wrong_backend_pickle_rejects_even_with_matching_hash(self):
    with self.assertRaises(ValueError):
      contract.verify(self.package, ROOT, "dmonitoring_model_tinygrad.pkl", self.commands["dmonitoring_model_tinygrad.pkl"])

  def test_malformed_manifest_artifacts_reject_cleanly(self):
    manifest = json.loads((self.package / "manifest.json").read_text())
    for malformed in (None, [], "not-a-map"):
      manifest["artifacts"] = malformed
      (self.package / "manifest.json").write_text(json.dumps(manifest))
      with self.subTest(malformed=malformed), self.assertRaisesRegex(ValueError, "artifact set"):
        self.verify("dmonitoring_model_tinygrad.pkl")

  def test_changed_bytes_during_copy_leave_existing_target(self):
    name = "dmonitoring_model_tinygrad.pkl"
    target = self.package / name
    target.write_bytes(b"previous")
    with patch("tools.laptop_device_build.validate_artifacts.require_qcom_pickle"), \
         patch.object(contract.shutil, "copyfileobj", side_effect=lambda source, output: output.write(b"changed")), \
         self.assertRaisesRegex(ValueError, "changed during copy"):
      contract.import_artifact(self.package, ROOT, target, self.commands[name])
    self.assertEqual(target.read_bytes(), b"previous")
    self.assertFalse(list(self.package.glob(".qcom-import-*")))

  def test_exact_compatible_source_and_all_five_artifacts_verify(self):
    self.compatible_package()
    for name in self.commands:
      with self.subTest(name=name):
        self.assertEqual(self.verify(name)[0], self.artifacts / name)

  def test_compatible_source_rejects_unrelated_source_or_recipe_change(self):
    _, current = self.compatible_package()
    changed = copy.deepcopy(current)
    changed["sources"]["unrelated.py"] = "1" * 64
    with patch.object(contract, "source_inventory", return_value=changed), \
         self.assertRaisesRegex(ValueError, "outside reviewed"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    changed = copy.deepcopy(current)
    changed["commands"]["dmonitoring_model_tinygrad.pkl"].append("--new-option")
    with patch.object(contract, "source_inventory", return_value=changed), \
         self.assertRaisesRegex(ValueError, "outside reviewed"):
      self.verify("dmonitoring_model_tinygrad.pkl")

  def test_compatible_source_rejects_artifact_or_paired_output_change(self):
    evidence, _ = self.compatible_package()
    del evidence["paired_outputs"]["dmonitoring_model_tinygrad.pkl"]
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    with self.assertRaisesRegex(ValueError, "lacks a stock target"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    evidence["paired_outputs"]["dmonitoring_model_tinygrad.pkl"] = [
      {"input": label, "shape": contract.COMPATIBILITY_OUTPUTS["dmonitoring_model_tinygrad.pkl"][0],
       "dtype": "float32", "build_sha256": str(index + 4) * 64,
       "runtime_sha256": str(index + 4) * 64}
      for index, label in enumerate(("A", "B", "A"))]
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    (self.artifacts / "dmonitoring_model_tinygrad.pkl").write_bytes(b"changed")
    with self.assertRaisesRegex(ValueError, "artifact mismatch"):
      self.verify("dmonitoring_model_tinygrad.pkl")

  def test_compatible_source_rejects_changed_or_unsafe_evidence(self):
    evidence, _ = self.compatible_package()
    evidence_file = self.package / "compatibility" / "qualification.txt"
    evidence_file.write_text("changed evidence\n")
    with self.assertRaisesRegex(ValueError, "evidence file mismatch"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    evidence_file.write_text("reviewed paired output evidence\n")
    evidence["evidence_files"] = {"../qualification.txt": contract._sha(evidence_file)}
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    with self.assertRaisesRegex(ValueError, "evidence filename"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    evidence["evidence_files"] = {"qualification.txt": contract._sha(evidence_file)}
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    outside = self.package / "outside.txt"
    outside.write_text("reviewed paired output evidence\n")
    evidence_file.unlink()
    evidence_file.symlink_to(outside)
    with self.assertRaisesRegex(ValueError, "evidence file mismatch"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    evidence_file.unlink()
    evidence_file.write_text("reviewed paired output evidence\n")
    evidence_dir = self.package / "compatibility"
    evidence_dir.rename(self.package / "actual_evidence")
    evidence_dir.symlink_to(self.package / "actual_evidence", target_is_directory=True)
    with self.assertRaisesRegex(ValueError, "evidence directory"):
      self.verify("dmonitoring_model_tinygrad.pkl")

  def test_compatible_source_rejects_unequal_outputs_and_stale_source_binding(self):
    evidence, _ = self.compatible_package()
    original = copy.deepcopy(evidence)
    evidence["paired_outputs"]["dmonitoring_model_tinygrad.pkl"][1]["runtime_sha256"] = "a" * 64
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    with self.assertRaisesRegex(ValueError, "Invalid QCOM paired outputs"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    original["compatible_source_sha256"] = "b" * 64
    (self.package / "compatibility.json").write_text(json.dumps(original))
    with self.assertRaisesRegex(ValueError, "not bound"):
      self.verify("dmonitoring_model_tinygrad.pkl")

  def test_compatible_source_rejects_bool_versions_and_duplicate_json_fields(self):
    evidence, _ = self.compatible_package()
    evidence["version"] = True
    (self.package / "compatibility.json").write_text(json.dumps(evidence))
    with self.assertRaisesRegex(ValueError, "not bound"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    (self.package / "compatibility.json").write_text('{"version":1,"version":1}')
    with self.assertRaisesRegex(ValueError, "Duplicate QCOM manifest field"):
      self.verify("dmonitoring_model_tinygrad.pkl")
    manifest = json.loads((self.package / "manifest.json").read_text())
    manifest["version"] = True
    (self.package / "manifest.json").write_text(json.dumps(manifest))
    with self.assertRaisesRegex(ValueError, "source or artifact set"):
      self.verify("dmonitoring_model_tinygrad.pkl")


class TestQcomSupplementalRuntimeCompatibility(unittest.TestCase):
  def setUp(self):
    temporary = tempfile.TemporaryDirectory()
    self.addCleanup(temporary.cleanup)
    self.package = Path(temporary.name)
    self.current = {"sources": {name: hashes[1] for name, hashes in contract.RUNTIME_EXTENSION_CHANGES.items()},
                    "commands": {"driving_tinygrad.pkl": ["DEV=QCOM", "IMAGE=1", "compile"]}}
    self.current["sources"].update(contract.RUNTIME_EXTENSION_ADDITIONS)
    self.current["sources"].update({contract.COMPATIBILITY_SOURCE: "a" * 64, "model.onnx": "b" * 64})
    parent = copy.deepcopy(self.current)
    for name, hashes in contract.RUNTIME_EXTENSION_CHANGES.items():
      parent["sources"][name] = hashes[0]
    for name in contract.RUNTIME_EXTENSION_ADDITIONS:
      del parent["sources"][name]
    saved = copy.deepcopy(parent)
    saved["sources"][contract.COMPATIBILITY_SOURCE] = "c" * 64
    self.manifest = {"source": saved, "source_sha256": contract._inventory_signature(saved),
                     "artifacts": {name: {"sha256": "d" * 64} for name in contract.COMPATIBILITY_OUTPUTS}}
    self.manifest_path = self.package / "manifest.json"
    self.manifest_path.write_text(json.dumps(self.manifest))
    evidence_dir = self.package / "compatibility"
    evidence_dir.mkdir()
    (evidence_dir / "outputs.txt").write_text("A/B/A output hashes\n")
    self.evidence = {"version": 1, "method": "paired-qcom-output-sha256",
                     "build_manifest_sha256": contract._sha(self.manifest_path),
                     "compatible_source_sha256": contract._inventory_signature(parent),
                     "changed_source": {"path": contract.COMPATIBILITY_SOURCE, "build_sha256": "c" * 64, "runtime_sha256": "a" * 64},
                     "artifact_sha256": dict.fromkeys(contract.COMPATIBILITY_OUTPUTS, "d" * 64),
                     "paired_outputs": {name: [{"input": label, "shape": shape, "dtype": dtype,
                                                "build_sha256": "e" * 64, "runtime_sha256": "e" * 64}
                                               for label in ("A", "B", "A")]
                                        for name, (shape, dtype) in contract.COMPATIBILITY_OUTPUTS.items()},
                     "evidence_files": {"outputs.txt": contract._sha(evidence_dir / "outputs.txt")}}
    self.parent_path = self.package / "compatibility.json"
    self.parent_path.write_text(json.dumps(self.evidence))
    self.bind_extension()

  def bind_extension(self):
    extension = {"version": 1, "profile": contract.RUNTIME_EXTENSION_PROFILE,
                 "parent_evidence_sha256": contract._sha(self.parent_path),
                 "compatible_source_sha256": contract._inventory_signature(self.current)}
    (self.package / "runtime-compatibility.json").write_text(json.dumps(extension))

  def verify(self):
    contract._verify_runtime_compatibility(self.package, self.manifest_path, self.manifest,
                                         self.current, dict.fromkeys(contract.COMPATIBILITY_OUTPUTS))

  def test_reviewed_extension_preserves_original_manifest_and_parent_attestation(self):
    before = self.manifest_path.read_bytes(), self.parent_path.read_bytes()
    self.verify()
    self.assertEqual(before, (self.manifest_path.read_bytes(), self.parent_path.read_bytes()))

  def test_empty_arena_loader_extension_preserves_compiled_provenance(self):
    helper = "tinygrad_repo/examples/openpilot/helpers.py"
    self.assertEqual(contract.RUNTIME_EXTENSION_CHANGES[helper], (
      "f4bae8ab81931781dc7134d6450c9fa2f24410b81cbc6ecfcebd6c708eb6d912",
      "6494f23757a1058d8c90db8434004e8f703bb61d55903ac4500c04a7d511c8a3"))
    before = self.manifest_path.read_bytes(), self.parent_path.read_bytes(), copy.deepcopy(self.manifest["artifacts"])
    self.verify()
    self.assertEqual(before, (self.manifest_path.read_bytes(), self.parent_path.read_bytes(), self.manifest["artifacts"]))

  def test_exact_typing_only_helper_preserves_parent_and_artifact_attestation(self):
    before = self.manifest_path.read_bytes(), self.parent_path.read_bytes(), copy.deepcopy(self.manifest["artifacts"])
    for name, source_hash in contract.RUNTIME_EXTENSION_TYPED_ADDITIONS.items():
      self.current["sources"][name] = source_hash
    self.bind_extension()
    self.verify()
    self.assertEqual(before, (self.manifest_path.read_bytes(), self.parent_path.read_bytes(), self.manifest["artifacts"]))
    name = next(iter(contract.RUNTIME_EXTENSION_TYPED_ADDITIONS))
    self.current["sources"][name] = "f" * 64
    self.bind_extension()
    with self.assertRaisesRegex(ValueError, "Unreviewed QCOM runtime addition"):
      self.verify()

  def test_each_reviewed_source_and_added_helper_requires_exact_bytes(self):
    original = copy.deepcopy(self.current)
    for name in (*contract.RUNTIME_EXTENSION_CHANGES, *contract.RUNTIME_EXTENSION_ADDITIONS):
      with self.subTest(name=name):
        self.current = copy.deepcopy(original)
        self.current["sources"][name] = "f" * 64
        self.bind_extension()
        with self.assertRaisesRegex(ValueError, "Unreviewed QCOM runtime"):
          self.verify()

  def test_extra_missing_or_model_changed_source_is_not_normalized_away(self):
    original = copy.deepcopy(self.current)
    for kind in ("extra", "missing", "model"):
      with self.subTest(kind=kind):
        self.current = copy.deepcopy(original)
        if kind == "extra":
          self.current["sources"]["unexpected.py"] = "f" * 64
        elif kind == "missing":
          del self.current["sources"]["model.onnx"]
        else:
          self.current["sources"]["model.onnx"] = "f" * 64
        self.bind_extension()
        with self.assertRaisesRegex(ValueError, "outside reviewed"):
          self.verify()

  def test_compile_command_change_remains_rejected(self):
    self.current["commands"]["driving_tinygrad.pkl"].append("FLOAT16=0")
    self.bind_extension()
    with self.assertRaisesRegex(ValueError, "outside reviewed"):
      self.verify()

  def test_parent_evidence_mutation_and_missing_supplement_reject(self):
    original = self.parent_path.read_bytes()
    self.parent_path.write_bytes(original + b"\n")
    with self.assertRaisesRegex(ValueError, "parent evidence"):
      self.verify()
    self.parent_path.write_bytes(original)
    (self.package / "runtime-compatibility.json").unlink()
    with self.assertRaisesRegex(ValueError, "outside reviewed"):
      self.verify()

  def test_artifact_substitution_and_unequal_paired_output_reject_after_rebinding(self):
    original = copy.deepcopy(self.evidence)
    for kind in ("artifact", "output"):
      with self.subTest(kind=kind):
        self.evidence = copy.deepcopy(original)
        if kind == "artifact":
          self.evidence["artifact_sha256"]["driving_tinygrad.pkl"] = "f" * 64
        else:
          self.evidence["paired_outputs"]["driving_tinygrad.pkl"][0]["runtime_sha256"] = "f" * 64
        self.parent_path.write_text(json.dumps(self.evidence))
        self.bind_extension()
        with self.assertRaises(ValueError):
          self.verify()

  def test_supplement_symlink_unknown_profile_and_extra_fields_reject(self):
    path = self.package / "runtime-compatibility.json"
    good = json.loads(path.read_text())
    for mutation in ({"profile": "unreviewed"}, {"extra": True}, {"version": True}):
      path.write_text(json.dumps(good | mutation))
      with self.subTest(mutation=mutation), self.assertRaises(ValueError):
        self.verify()
    path.unlink()
    path.symlink_to(self.parent_path)
    with self.assertRaisesRegex(ValueError, "Invalid QCOM runtime extension"):
      self.verify()