StarPilot/tools/ci/tests/test_mutation_equivalence.py

from copy import deepcopy
import json
from pathlib import Path
import subprocess
from tempfile import TemporaryDirectory
from unittest import TestCase
from unittest.mock import patch

from opendbc.safety.tests import mutation
from tools.ci import run_safety_qualification as qualification


class TestMutationEquivalence(TestCase):
  @staticmethod
  def fixture(expression, original="250000U", mutated="250001U"):
    source = f"int compare(unsigned int elapsed) {{ return elapsed < ({expression}); }}\n"
    start = source.index(original)
    site = mutation.MutationSite(0, start, start + len(original), start, start + len(original),
                                 1, original, mutated, "boundary", mutation.SAFETY_DIR / "lateral.h", 1)
    return source, [site]

  def test_actual_compiler_proves_same_unsigned_quotient_type_and_value(self):
    source, sites = self.fixture("250000U / 2U")
    proof, = mutation.prove_equivalent_divisions(source, sites, set())
    self.assertEqual((proof["original_expression"], proof["mutated_expression"], proof["c_type"], proof["value"]),
                     ("250000U / 2U", "250001U / 2U", "unsigned int", 125000))
    self.assertEqual(proof["site_id"], 0)
    self.assertEqual(len(proof["compiler_sha256"]), 64)
    self.assertEqual(len(proof["proof_source_sha256"]), 64)
    # Reject a mathematically false claimed quotient using the real compiler.
    false_proof = [{**proof, "value": 125001}]
    result = subprocess.run(["cc", *proof["compiler_flags"]], input=mutation.division_proof_source(false_proof),
                            capture_output=True, text=True, timeout=30)
    self.assertNotEqual(result.returncode, 0)

  def test_narrow_closed_operand_scope_rejects_every_unproved_context(self):
    for expression, original, mutated in (
      ("250000U / 3U", "3U", "4U"),
      ("250000 / 2U", "250000", "250001"),
      ("250000UL / 2U", "250000UL", "250001UL"),
      ("250000U / 0U", "250000U", "250001U"),
      ("250000.0 / 2U", "250000.0", "250001.0"),
      ("(unsigned int)250000U / 2U", "250000U", "250001U"),
      ("250000U / elapsed", "250000U", "250001U"),
      ("250000U / fn()", "250000U", "250001U"),
      ("(250000U / 2U) + elapsed", "250000U", "250001U"),
      ("(250000U / 2U) + 1U", "250000U", "250001U"),
      ("4294967295U / 2U", "4294967295U", "4294967296U"),
    ):
      with self.subTest(expression=expression):
        source, sites = self.fixture(expression, original, mutated)
        self.assertEqual(mutation.equivalent_division_candidates(source, sites, set()), [])
    source, sites = self.fixture("250000U / 2U")
    self.assertEqual(mutation.equivalent_division_candidates(source, sites, {0}), [])

  def test_failed_compiler_proof_is_fatal_not_an_equivalence_allowance(self):
    source, sites = self.fixture("250000U / 2U")
    with patch.object(mutation.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "type mismatch")):
      with self.assertRaisesRegex(RuntimeError, "compiler proof failed"):
        mutation.prove_equivalent_divisions(source, sites, set())

  @staticmethod
  def full_artifact(directory):
    sites, _, excluded, source = mutation.enumerate_sites(mutation.ROOT / mutation.SAFETY_C_REL, directory / "source.i")
    proofs = mutation.prove_equivalent_divisions(source, sites, excluded)
    equivalent_ids = {proof["site_id"] for proof in proofs}
    executed = [site for site in sites if site.site_id not in excluded | equivalent_ids]
    results = [mutation.MutantResult(site, "killed", 0.0, "artifact shape fixture, not a mutation qualification") for site in executed]
    path = directory / "results.json"
    mutation.write_results_json(path, discovered_sites=sites, pruned_ids=excluded, results=results,
                                site_targets={site.site_id: ["fixture.TestShape.test_one"] for site in executed},
                                preprocessed_source=source, baseline_sec=0.0, proven_equivalent=proofs)
    summary = {"candidates": len(sites), "pruned_build_incompatible": len(excluded), "proven_equivalent": len(proofs),
               "total": len(executed), "killed": len(executed), "survived": 0, "infra_error": 0}
    return path, json.loads(path.read_text()), summary

  def test_full_partition_rechecks_actual_context_type_compiler_and_exact_ids(self):
    with TemporaryDirectory() as directory:
      path, original, summary = self.full_artifact(Path(directory))
      self.assertTrue(original["proven_equivalent"])
      self.assertTrue(qualification.complete_mutation_artifact(path, summary))
      for name, change in (
        ("value", lambda artifact: artifact["proven_equivalent"][0].update(value=125001)),
        ("expression", lambda artifact: artifact["proven_equivalent"][0].update(mutated_expression="0U / 2U")),
        ("context", lambda artifact: artifact["proven_equivalent"][0].update(comparison_operator="==")),
        ("compiler", lambda artifact: artifact["proven_equivalent"][0].update(compiler_sha256="f" * 64)),
        ("proof hash", lambda artifact: artifact["proven_equivalent"][0].update(proof_source_sha256="f" * 64)),
        ("preprocessed hash", lambda artifact: artifact.update(preprocessed_source_sha256="f" * 64)),
        ("missing", lambda artifact: artifact["proven_equivalent"].pop()),
        ("duplicate", lambda artifact: artifact["proven_equivalent"].append(artifact["proven_equivalent"][0])),
        ("overlap", lambda artifact: artifact["proven_equivalent"][0].update(site_id=artifact["results"][0]["site_id"])),
        ("out of range", lambda artifact: artifact["proven_equivalent"][0].update(site_id=artifact["discovered"])),
      ):
        with self.subTest(forgery=name):
          artifact = deepcopy(original)
          change(artifact)
          path.write_text(json.dumps(artifact))
          self.assertFalse(qualification.complete_mutation_artifact(path, summary))
      path.write_text(json.dumps(original))
      with patch.object(qualification.subprocess, "run", return_value=subprocess.CompletedProcess([], 1, "", "proof failure")):
        self.assertFalse(qualification.complete_mutation_artifact(path, summary))
      # Even a internally reconciled empty partition cannot conceal a known
      # equivalent candidate by merely relabelling its absence as build pruning.
      artifact = deepcopy(original)
      removed = artifact["proven_equivalent"].pop()
      artifact["pruned_build_incompatible"].append(removed)
      path.write_text(json.dumps(artifact))
      forged_summary = {**summary, "proven_equivalent": summary["proven_equivalent"] - 1,
                        "pruned_build_incompatible": summary["pruned_build_incompatible"] + 1}
      self.assertFalse(qualification.complete_mutation_artifact(path, forged_summary))

  def test_empty_mutated_operator_is_permitted_only_for_negation_removal(self):
    from tools.ci.tests.test_safety_qualification import TestSafetyQualificationRunner
    original = TestSafetyQualificationRunner.mutation_artifact()
    summary = {"candidates": 1, "pruned_build_incompatible": 0, "proven_equivalent": 0,
               "total": 1, "killed": 1, "survived": 0, "infra_error": 0}
    with TemporaryDirectory() as directory:
      path = Path(directory) / "results.json"
      for family, operator, expected in (("remove_negation", "!", True), ("boundary", "!", False),
                                         ("remove_negation", "0", False), ("comparison", "!", False)):
        with self.subTest(family=family, operator=operator):
          artifact = deepcopy(original)
          artifact["results"][0].update(mutator=family, original_op=operator, mutated_op="")
          path.write_text(json.dumps(artifact))
          self.assertEqual(qualification.complete_mutation_artifact(path, summary), expected)

  def test_v1_never_reinterprets_old_survivors_as_equivalent(self):
    from tools.ci.tests.test_safety_qualification import TestSafetyQualificationRunner
    artifact = TestSafetyQualificationRunner.mutation_artifact()
    artifact["results"][0]["outcome"] = "survived"
    summary = {"candidates": 1, "pruned_build_incompatible": 0, "proven_equivalent": 0,
               "total": 1, "killed": 0, "survived": 1, "infra_error": 0}
    with TemporaryDirectory() as directory:
      path = Path(directory) / "results.json"
      path.write_text(json.dumps(artifact))
      self.assertTrue(qualification.complete_mutation_artifact(path, summary))
      artifact["proven_equivalent"] = [{"site_id": 0}]
      path.write_text(json.dumps(artifact))
      self.assertFalse(qualification.complete_mutation_artifact(path, summary))
      log = "\n".join(("Found 1 unique candidates", "  pruned_build_incompatible: 0", "  total: 1",
                       "  killed: 0", "  survived: 1", "  infra_error: 0", ""))
      artifact.pop("proven_equivalent")
      path.write_text(json.dumps(artifact))
      gate = qualification.Gate("mutation-full", Path(directory), qualification.sys.executable)
      consumed_path = Path(directory) / "mutation-results.json"
      path.unlink()
      def fake_run(*args, **kwargs):
        consumed_path.write_text(json.dumps(artifact))
        return 0, log
      with patch.object(gate, "run", side_effect=fake_run):
        # A v1 survived result remains fatal despite a successful subprocess.
        self.assertFalse(gate.execute())
      self.assertTrue(gate.summary["artifact_valid"])