StarPilot/tools/ci/run_safety_qualification.py

#!/usr/bin/env python3
"""Run isolated, source-pinned safety qualification gates without nested uv setup."""

import argparse
import hashlib
import json
import math
import os
from pathlib import Path
import re
import shutil
import signal
import stat
import subprocess
import sys
import time


ROOT = Path(__file__).resolve().parents[2]
SAFETY_TESTS = ROOT / "opendbc_repo/opendbc/safety/tests"
MAX_MUTATION_ARTIFACT_BYTES = 256 * 1024 * 1024
SOURCE_PREFIXES = (
  "opendbc_repo/", "panda/", "openpilot/", "msgq_repo/", "rednose_repo/",
  "teleoprtc_repo/", "tinygrad_repo/", "jetlink_repo/", "tools/", "scripts/",
)
SOURCE_EXACT = {".github/workflows/safety.yaml", "pyproject.toml", "uv.lock", "SConstruct",
                "panda/SConscript", "panda/SConstruct", "opendbc_repo/SConscript", "opendbc_repo/pyproject.toml",
                "openpilot/cereal/SConscript", "openpilot/common/SConscript", "panda/tests/libpanda/SConscript",
                "panda/tests/misra/coverage_table",
                "opendbc_repo/opendbc/safety/tests/misra/coverage_table"}
SOURCE_SUFFIXES = {".c", ".h", ".cc", ".cpp", ".py", ".sh", ".capnp", ".dbc", ".toml", ".yaml", ".txt", ".json",
                   ".s", ".S", ".ld", ".mk", ".pyx", ".pxd", ".hpp"}
SOURCE_NAMES = {"SConscript", "SConstruct", "Makefile", "coverage_table"}
UNITTEST_ACCOUNTING = """
def test_ids(group):
  for item in group:
    if isinstance(item, unittest.TestSuite):
      yield from test_ids(item)
    else:
      yield item.id()
targets = list(test_ids(suite))
print(f'Safety qualification collected {len(targets)} tests', flush=True)
result = unittest.TextTestRunner(verbosity=2).run(suite)
suppressed = set()
for test, _reason in result.skipped:
  skipped_id = test.id()
  match = re.fullmatch(r'setUp(?:Class|Module) \\(([^)]+)\\)', skipped_id)
  if match:
    matched = {index for index, target in enumerate(targets) if target.startswith(match.group(1) + '.')}
    if not matched:
      raise RuntimeError(f'unmatched skip holder: {skipped_id}')
    suppressed.update(matched)
  elif skipped_id not in targets:
    raise RuntimeError(f'unexpected skip holder: {skipped_id}')
print(f'Safety qualification suppressed {len(suppressed)} methods', flush=True)
sys.exit(0 if result.wasSuccessful() else 1)
"""


def source_identity():
  listed = subprocess.check_output(["git", "ls-files", "--cached", "--others", "--exclude-standard", "-z"], cwd=ROOT)
  names = (Path(name.decode()) for name in listed.split(b"\0") if name)
  files = sorted(name for name in names if (name.as_posix() in SOURCE_EXACT or
                 (name.as_posix().startswith(SOURCE_PREFIXES) and (name.suffix in SOURCE_SUFFIXES or name.name in SOURCE_NAMES))) and
                 "obj" not in name.parts and "gen" not in name.parts and (ROOT / name).is_file())
  return {path.as_posix(): hashlib.sha256((ROOT / path).read_bytes()).hexdigest() for path in files}


def generated_identity():
  files = sorted(path for pattern in ("openpilot/cereal/gen/**/*", "opendbc_repo/opendbc/can/*.so",
                                           "panda/tests/libpanda/*.so", "panda/board/obj/**/*.elf")
                 for path in ROOT.glob(pattern) if path.is_file())
  return {path.relative_to(ROOT).as_posix(): hashlib.sha256(path.read_bytes()).hexdigest() for path in files}


def source_head():
  return subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, text=True).strip()


def parse_unittest_summary(output):
  matches = list(re.finditer(r"Ran (\d+) tests? in[^\n]*\n\s*\n(OK|FAILED)(?: \([^\n]*\))?", output))
  match = matches[-1] if matches else None
  status = match.group() if match else ""
  skipped = re.search(r"skipped=(\d+)", status)
  return {"ran": int(match.group(1)) if match else None,
          "skipped": int(skipped.group(1)) if skipped else 0,
          "status": match.group(2) if match else "missing",
          "collected": int(found.group(1)) if (found := re.search(r"^Safety qualification collected (\d+) tests$", output, re.MULTILINE)) else None,
          "suppressed": int(found.group(1)) if (found := re.search(r"^Safety qualification suppressed (\d+) methods$", output, re.MULTILINE)) else None}


def complete_unittest(summary):
  return (summary["status"] == "OK" and summary["collected"] is not None and
          summary["suppressed"] is not None and summary["ran"] + summary["suppressed"] == summary["collected"] and
          summary["ran"] > summary["skipped"])


def parse_mutation_summary(output):
  plain = re.sub(r"\x1b\[[0-9;]*m", "", output)
  summary = {"candidates": int(match.group(1)) if (match := re.search(r"Found (\d+) unique candidates", plain)) else None}
  for label in ("total", "killed", "survived", "infra_error", "pruned_build_incompatible"):
    match = re.search(rf"^  {label}: (\d+)$", plain, re.MULTILINE)
    summary[label] = int(match.group(1)) if match else None
  return summary


def complete_mutation_artifact(path, summary, *, digest_out=None):
  """A recorded candidate set must agree with the subprocess summary."""
  try:
    flags = os.O_RDONLY | os.O_NONBLOCK | getattr(os, "O_NOFOLLOW", 0)
    fd = os.open(path, flags)
    try:
      metadata = os.fstat(fd)
      if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > MAX_MUTATION_ARTIFACT_BYTES:
        return False
      with os.fdopen(fd, "rb", closefd=False) as stream:
        raw = stream.read(MAX_MUTATION_ARTIFACT_BYTES + 1)
      if len(raw) > MAX_MUTATION_ARTIFACT_BYTES:
        return False
    finally:
      os.close(fd)
    artifact = json.loads(raw)
    results = artifact["results"]
    pruned = artifact["pruned_build_incompatible"]
    target_sets = artifact["target_sets"]
    sources = artifact["source_sha256"]
    def digest(value):
      return type(value) is str and re.fullmatch(r"[0-9a-f]{64}", value) is not None
    if (type(artifact["schema_version"]) is not int or artifact["schema_version"] != 1 or
        type(artifact["discovered"]) is not int or artifact["discovered"] != summary["candidates"] or
        type(results) is not list or type(pruned) is not list or type(target_sets) is not list or
        len(pruned) != summary["pruned_build_incompatible"] or
        len(results) != summary["total"] or
        not digest(artifact["preprocessed_source_sha256"]) or
        artifact["safety_input_sha256"] != hashlib.sha256((SAFETY_TESTS / "libsafety/safety.c").read_bytes()).hexdigest() or
        artifact["mutation_runner_sha256"] != hashlib.sha256((SAFETY_TESTS / "mutation.py").read_bytes()).hexdigest() or
        type(artifact["baseline_sec"]) not in (int, float) or not math.isfinite(artifact["baseline_sec"]) or artifact["baseline_sec"] < 0 or
        type(sources) is not dict or not sources or
        any(type(source) is not str or not digest(source_hash) for source, source_hash in sources.items()) or
        any(type(item) is not list or not item or any(type(target) is not str or not target for target in item)
            for item in target_sets)):
      return False
    safety_root = (ROOT / "opendbc_repo/opendbc/safety").resolve()
    for source, claimed_hash in sources.items():
      source_path = (ROOT / "opendbc_repo" / source).resolve()
      if not source_path.is_relative_to(safety_root) or not source_path.is_file() or \
         hashlib.sha256(source_path.read_bytes()).hexdigest() != claimed_hash:
        return False
    ids = [item["site_id"] for item in results]
    pruned_ids = [item["site_id"] for item in pruned]
    if (any(type(site_id) is not int for site_id in ids + pruned_ids) or
        len(ids) != len(set(ids)) or len(pruned_ids) != len(set(pruned_ids)) or
        set(ids) & set(pruned_ids) or len(ids) + len(pruned_ids) != artifact["discovered"] or
        set(ids + pruned_ids) != set(range(artifact["discovered"]))):
      return False
    for label in ("killed", "survived", "infra_error"):
      if sum(item["outcome"] == label for item in results) != summary[label]:
        return False
    valid = (all(type(item["selected_test_set"]) is int and
                0 <= item["selected_test_set"] < len(target_sets) and
                type(item["selected_test_count"]) is int and
                item["selected_test_count"] == len(target_sets[item["selected_test_set"]]) and
                item["source"] in sources and type(item["line"]) is int and item["line"] > 0 and
                item["outcome"] in ("killed", "survived", "infra_error") and
                (item["outcome"] == "infra_error" or (item["failure_kind"] is None and item["exit_code"] is None)) and
                all(type(item[key]) is str and item[key] for key in ("mutator", "original_op", "mutated_op")) and
                all(type(item[key]) is str for key in ("details", "stderr_tail", "stdout_tail", "unittest_output_tail")) and
                (item["failure_kind"] is None or type(item["failure_kind"]) is str) and
                (item["exit_code"] is None or type(item["exit_code"]) is int)
                for item in results) and
            all(item["source"] in sources and type(item["line"]) is int and item["line"] > 0
                for item in pruned))
    if valid and digest_out is not None:
      digest_out.append(hashlib.sha256(raw).hexdigest())
    return valid
  except (OSError, ValueError, TypeError, KeyError, IndexError, RecursionError, OverflowError):
    return False


def check_import_origins(python, env, output):
  script = """
import json
from pathlib import Path
import importlib.util
import opendbc, openpilot, msgq
import opendbc.can.parser, opendbc.can.packer, opendbc.can.dbc
from openpilot.cereal import log as cereal_log
import openpilot.cereal.messaging
import msgq.ipc_pyx
from opendbc.safety.tests.libsafety import libsafety_py
print(json.dumps({name: str(Path(module.__file__).resolve()) for name, module in
                  [('opendbc', opendbc), ('openpilot', openpilot),
                   ('msgq', msgq), ('libsafety_py', libsafety_py),
                   ('can_parser', opendbc.can.parser), ('can_packer', opendbc.can.packer),
                   ('can_dbc', opendbc.can.dbc), ('cereal_log', cereal_log),
                   ('cereal_messaging', openpilot.cereal.messaging), ('msgq_ipc', msgq.ipc_pyx)]} |
                  {'panda': importlib.util.find_spec('panda').origin}))
"""
  result = subprocess.run([python, "-c", script], cwd=output.parent, env=env, capture_output=True, text=True)
  output.write_text(result.stdout + result.stderr)
  if result.returncode:
    return False
  try:
    origins = json.loads(result.stdout)
  except json.JSONDecodeError:
    return False
  expected = {"opendbc": ROOT / "opendbc_repo", "panda": ROOT / "panda", "msgq": ROOT / "msgq_repo",
              "openpilot": ROOT / "openpilot", "libsafety_py": ROOT / "opendbc_repo",
              "can_parser": ROOT / "opendbc_repo", "can_packer": ROOT / "opendbc_repo",
              "can_dbc": ROOT / "opendbc_repo", "cereal_log": ROOT / "openpilot/cereal",
              "cereal_messaging": ROOT / "openpilot/cereal", "msgq_ipc": ROOT / "msgq_repo"}
  return all(isinstance(origins.get(name), str) and Path(origins[name]).is_relative_to(path)
             for name, path in expected.items())


def imported_module_hashes(output):
  origins = json.loads(output.read_text().splitlines()[0])
  return {name: {"path": path, "sha256": hashlib.sha256(Path(path).read_bytes()).hexdigest()}
          for name, path in origins.items()}


class Gate:
  def __init__(self, name, output, python):
    self.name = name
    self.output = output
    self.python = str(python)
    self.env = dict(os.environ)
    self.env.update({"PYTHONPATH": os.pathsep.join(str(ROOT / part) for part in
                                           ("", "opendbc_repo", "msgq_repo", "rednose_repo", "teleoprtc_repo", "tinygrad_repo", "jetlink_repo")),
                     "PARAMS_ROOT": str(output / "params"), "OPENPILOT_PREFIX": f"safety_qual_{os.getpid()}",
                     "PYTHONDONTWRITEBYTECODE": "1", "FUZZ_SEED": "0", "SCALE": "1", "PWD": str(ROOT)})
    for inherited_key in ("HITL", "RELEASE", "CERT", "SKIP_BUILD", "SKIP_TABLES_DIFF",
                          "LIBSAFETY_PREBUILT", "GCOV_PREFIX", "GCOV_PREFIX_STRIP", "CC", "CXX",
                          "CFLAGS", "CXXFLAGS", "LDFLAGS", "LD_LIBRARY_PATH", "DYLD_LIBRARY_PATH"):
      self.env.pop(inherited_key, None)
    self.env["PATH"] = str(Path(self.python).parent) + os.pathsep + self.env.get("PATH", "")
    (output / "params").mkdir()
    self.commands = []
    self.artifacts = {}

  def run(self, label, argv, *, cwd=ROOT, timeout=3600):
    start = time.monotonic()
    try:
      proc = subprocess.Popen(argv, cwd=cwd, env=self.env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                              text=True, start_new_session=True)
      try:
        content, _ = proc.communicate(timeout=timeout)
        code = proc.returncode
      except subprocess.TimeoutExpired:
        os.killpg(proc.pid, signal.SIGKILL)
        content, _ = proc.communicate()
        code = 124
        content += "\nTimed out; process group killed\n"
    except OSError as exc:
      code, content = 127, f"{type(exc).__name__}: {exc}\n"
    log = self.output / f"{label}.log"
    log.write_text(content)
    self.commands.append({"label": label, "argv": [str(arg) for arg in argv], "cwd": str(cwd),
                          "exit_code": code, "seconds": round(time.monotonic() - start, 3), "log": log.name})
    return code, content

  def hash_artifact(self, path):
    path = Path(path)
    if path.is_file():
      self.artifacts[path.relative_to(ROOT).as_posix() if path.is_relative_to(ROOT) else path.name] = hashlib.sha256(path.read_bytes()).hexdigest()

  def cppcheck_dir(self):
    code, output = self.run("cppcheck-location", [self.python, "-c", "import cppcheck; print(cppcheck.DIR)"])
    if code:
      return None
    location = Path(output.strip())
    version, text = self.run("cppcheck-version", [str(location / "cppcheck"), "--version"])
    if version or re.fullmatch(r"Cppcheck 2\.21(?:\.0)?", text.strip()) is None:
      return None
    return location

  def misra(self, panda):
    directory = self.cppcheck_dir()
    if directory is None:
      return False
    misra = directory / "addons/misra.py"
    check_dir = ROOT / ("panda/tests/misra" if panda else "opendbc_repo/opendbc/safety/tests/misra")
    table, text = self.run("misra-table", [self.python, str(misra), "-generate-table"])
    if table or text != (check_dir / "coverage_table").read_text():
      return False
    include = ROOT / ("panda" if panda else "opendbc_repo")
    compiler = "arm-none-eabi-gcc" if panda else "cc"
    status, compiler_include = self.run("compiler-include", [compiler, "-print-file-name=include"])
    if status:
      return False
    common = [str(directory / "cppcheck"), "--inline-suppr", "-I", str(include),
            "-I", compiler_include.strip(), "--suppressions-list=" + str(check_dir / "suppressions.txt"),
            "--error-exitcode=2", "--check-level=exhaustive", "--safety", "--platform=arm32-wchar_t4",
            "--std=c11", "--enable=all", "--addon=misra"]
    if panda:
      common += ["--disable=unusedFunction", "--suppress=*:*inc/*", "--suppress=*:*include/*",
               "-D__GNUC__=9", "-UCMSIS_NVIC_VIRTUAL", "-UCMSIS_VECTAB_VIRTUAL", "-UBOOTSTUB",
               "-DSTM32H7", "-DSTM32H725xx", "-I", str(ROOT / "opendbc_repo"),
               "-I", str(ROOT / "panda/board/stm32h7/inc"),
               str(ROOT / "panda/board/main.c")]
    else:
      common += ["--enable=unusedFunction", "--suppress=missingIncludeSystem", "--suppress=*:*include/*",
               "-D__GNUC__=9", "-D__has_include_next(x)=0",
               str(ROOT / "opendbc_repo/opendbc/safety/tests/misra/main.c")]
    outcomes = {}
    for variant in ("debug", "release"):
      argv = [*common[:-1], "--checkers-report=" + str(self.output / f"checkers-{variant}.txt")]
      if variant == "debug":
        argv.append("-DALLOW_DEBUG")
      argv.append(common[-1])
      code, text = self.run(f"misra-{variant}", argv, timeout=5400)
      outcomes[variant] = code == 0 and re.search(r"misra violation|error|style: ", text) is None
    self.summary = {"variants": outcomes}
    return all(outcomes.values())

  def execute(self):
    if self.name == "coverage":
      version, _ = self.run("gcovr-version", ["gcovr", "--version"])
      if version:
        return False
      build_code = "from opendbc.safety.tests.libsafety import libsafety_py; print(libsafety_py._build_libsafety())"
      built, text = self.run("build-libsafety", [self.python, "-c", build_code])
      if built:
        return False
      native = self.output / "libsafety-debug.so"
      shutil.copy2(Path(text.strip().splitlines()[-1]), native)
      self.hash_artifact(native)
      # The caller supplies a disposable checkout. Only generated coverage files are cleared.
      for path in (SAFETY_TESTS / "libsafety").glob("*.gcda"):
        path.unlink()
      suite_script = """import re, sys, unittest
from opendbc.safety.tests.libsafety import libsafety_py
libsafety_py.load(sys.argv[1])
suite = unittest.TestLoader().discover('.', pattern='test_*.py')
""" + UNITTEST_ACCOUNTING
      code, text = self.run("unittest", [self.python, "-c", suite_script, str(native)], cwd=SAFETY_TESTS)
      summary = parse_unittest_summary(text)
      self.summary = summary
      if sys.platform == "darwin":
        found, llvm_cov = self.run("llvm-cov-path", ["xcrun", "--find", "llvm-cov"])
        if found:
          return False
        gcov_tool = llvm_cov.strip() + " gcov"
      else:
        gcov_tool = "gcov"
      gcovr, _ = self.run("coverage", ["gcovr", "-r", str(SAFETY_TESTS.parent), "--gcov-executable", gcov_tool,
                                         "-d", "--fail-under-line=100", "-e", "^libsafety"], cwd=SAFETY_TESTS)
      return code == 0 and complete_unittest(summary) and gcovr == 0
    if self.name in ("opendbc-misra", "panda-misra"):
      if self.name == "panda-misra":
        built, _ = self.run("build-h7", ["scons", "-j4", "panda/board/obj/panda_h7/main.elf",
                                          "panda/board/obj/panda_h7/bootstub.elf"], timeout=3600)
        if built:
          return False
        self.hash_artifact(ROOT / "panda/board/obj/panda_h7/main.elf")
        self.hash_artifact(ROOT / "panda/board/obj/panda_h7/bootstub.elf")
      return self.misra(self.name == "panda-misra")
    if self.name == "panda-host":
      built, _ = self.run("build", ["scons", "-j4", "panda/tests/libpanda/libpanda.so",
                                      "panda/board/obj/panda_h7/main.elf", "panda/board/obj/panda_h7/bootstub.elf"], timeout=3600)
      if built:
        return False
      self.hash_artifact(ROOT / "panda/tests/libpanda/libpanda.so")
      self.hash_artifact(ROOT / "panda/board/obj/panda_h7/main.elf")
      self.hash_artifact(ROOT / "panda/board/obj/panda_h7/bootstub.elf")
      suite_script = """import re, sys, unittest
suite = unittest.TestLoader().discover('panda/tests', pattern='test_*.py')
""" + UNITTEST_ACCOUNTING
      code, text = self.run("unittest", [self.python, "-W", "error", "-c", suite_script], timeout=5400)
      self.summary = parse_unittest_summary(text)
      return code == 0 and complete_unittest(self.summary)
    if self.name in ("mutation-list", "mutation-full"):
      argv = [self.python, str(SAFETY_TESTS / "mutation.py")]
      artifact = self.output / "mutation-results.json"
      if self.name == "mutation-full" and os.path.lexists(artifact):
        self.summary = {"artifact_valid": False, "error": "mutation result path already exists"}
        return False
      argv += ["--list-only"] if self.name == "mutation-list" else ["-j", "4", "--results-json", str(artifact)]
      code, text = self.run("mutation", argv, cwd=ROOT / "opendbc_repo", timeout=21600)
      self.summary = parse_mutation_summary(text)
      if self.name == "mutation-full":
        artifact_digest = []
        artifact_valid = complete_mutation_artifact(artifact, self.summary, digest_out=artifact_digest)
        self.summary["artifact_valid"] = artifact_valid
        if artifact_valid:
          self.artifacts[artifact.name] = artifact_digest[0]
        counts = self.summary
        reconciled = (all(counts[key] is not None for key in ("candidates", "pruned_build_incompatible", "total", "killed", "survived", "infra_error")) and
                      counts["candidates"] - counts["pruned_build_incompatible"] == counts["total"] ==
                      counts["killed"] + counts["survived"] + counts["infra_error"])
        return code == 0 and reconciled and artifact_valid and counts["survived"] == 0 and counts["infra_error"] == 0
      return code == 0 and self.summary["candidates"] is not None and self.summary["candidates"] > 0
    raise ValueError(self.name)


def main():
  parser = argparse.ArgumentParser(description=__doc__)
  parser.add_argument("--gate", required=True, choices=("coverage", "opendbc-misra", "panda-misra", "panda-host", "mutation-list", "mutation-full"))
  parser.add_argument("--output", required=True, type=Path)
  args = parser.parse_args()
  output = args.output.resolve()
  if output.exists():
    parser.error("Output directory must be new")
  output.mkdir(parents=True)
  python = ROOT / ".venv/bin/python"
  gate = None
  before = None
  after = None
  head_before = None
  head_after = None
  origins_ok = False
  imported_before = {}
  imported_after = {}
  generated_before = {}
  passed = False
  error = None
  try:
    if not python.is_file():
      raise RuntimeError("root locked environment is missing")
    gate = Gate(args.gate, output, python)
    head_before = source_head()
    before = source_identity()
    (output / "source-before.json").write_text(json.dumps(before, indent=2) + "\n")
    generated_before = generated_identity()
    (output / "generated-before.json").write_text(json.dumps(generated_before, indent=2) + "\n")
    extension = ".dylib" if sys.platform == "darwin" else ".so"
    prerequisite = f"openpilot/common/libparams_c{extension}"
    built, _ = gate.run("build-imports", ["scons", "--minimal", "-j4", prerequisite,
                                         "msgq_repo/msgq/ipc_pyx.so"], timeout=3600)
    if built:
      raise RuntimeError("native host import prerequisites failed")
    gate.hash_artifact(ROOT / prerequisite)
    gate.hash_artifact(ROOT / "msgq_repo/msgq/ipc_pyx.so")
    origins_ok = check_import_origins(gate.python, gate.env, output / "imports.log")
    if origins_ok:
      imported_before = imported_module_hashes(output / "imports.log")
    version_codes = [gate.run("python-version", [gate.python, "--version"])[0],
                     gate.run("compiler-version", ["cc", "--version"])[0],
                     gate.run("scons-version", ["scons", "--version"])[0]]
    if args.gate in ("panda-misra", "panda-host"):
      version_codes.append(gate.run("arm-compiler-version", ["arm-none-eabi-gcc", "--version"])[0])
    passed = origins_ok and all(code == 0 for code in version_codes) and gate.execute()
  except Exception as exc:
    error = f"{type(exc).__name__}: {exc}"
  try:
    after = source_identity()
    head_after = source_head()
    (output / "source-after.json").write_text(json.dumps(after, indent=2) + "\n")
  except Exception as exc:
    error = f"source finalization failed: {type(exc).__name__}: {exc}"
  generated = {}
  try:
    generated = generated_identity()
    (output / "generated-inputs.json").write_text(json.dumps(generated, indent=2) + "\n")
  except Exception as exc:
    error = f"generated input finalization failed: {type(exc).__name__}: {exc}"
  try:
    imported_after = {name: {"path": item["path"], "sha256": hashlib.sha256(Path(item["path"]).read_bytes()).hexdigest()}
                      for name, item in imported_before.items()}
  except Exception as exc:
    error = f"imported module finalization failed: {type(exc).__name__}: {exc}"
  report = {"gate": args.gate, "head_before": head_before, "head_after": head_after,
            "source_unchanged": before is not None and before == after, "import_origins_ok": origins_ok,
            "commands": gate.commands if gate else [], "artifacts": gate.artifacts if gate else {},
            "generated_before": len(generated_before), "generated_after": len(generated),
            "imported_modules": imported_before,
            "imported_modules_unchanged": imported_before == imported_after,
            "summary": getattr(gate, "summary", {}), "error": error,
            "passed": passed and error is None and before is not None and before == after and head_before == head_after and imported_before == imported_after}
  (output / "results.json").write_text(json.dumps(report, indent=2) + "\n")
  print(json.dumps({key: report[key] for key in ("gate", "passed", "summary", "source_unchanged", "import_origins_ok")}))
  return int(not report["passed"])


if __name__ == "__main__":
  raise SystemExit(main())