StarPilot/openpilot/common/vendor_manifest.py

"""Validate the source layout without loading native modules or device state."""

import json
import re
import subprocess
import os
import stat
from pathlib import Path, PurePosixPath
from urllib.parse import urlsplit

MANIFEST = "upstream-sync.json"
BASE_DEPENDENCIES = frozenset({"msgq_repo", "opendbc_repo", "panda", "rednose_repo", "teleoprtc_repo", "tinygrad_repo", "mapd_repo"})
EXTENSIONS = frozenset({"jetlink_repo"})
DEPENDENCIES = BASE_DEPENDENCIES | EXTENSIONS
SHA = re.compile(r"[0-9a-f]{40}\Z")


def git(repo: Path | str, *args: str) -> bytes:
  return subprocess.check_output(["git", "-C", str(repo), *args])


def dependency_entries(manifest: dict) -> list[dict]:
  return [*manifest["dependencies"], *manifest.get("vendored_extensions", [])]


def parse_manifest(data: bytes | str) -> dict:
  try:
    value = json.loads(data)
  except (ValueError, UnicodeError) as e:
    raise ValueError("Invalid source manifest JSON") from e
  if not isinstance(value, dict) or type(value.get("schema_version")) is not int or value["schema_version"] != 1:
    raise ValueError("Unsupported source manifest schema")
  upstream = value.get("upstream")
  dependencies = value.get("dependencies")
  extensions = value.get("vendored_extensions", [])
  if not isinstance(upstream, dict) or not isinstance(dependencies, list) or not isinstance(extensions, list):
    raise ValueError("Source manifest must define upstream and dependencies")
  entries = dependency_entries(value)
  paths = []
  for entry in [upstream, *entries]:
    if not isinstance(entry, dict):
      raise ValueError("Invalid source manifest entry")
    if not isinstance(entry.get("commit"), str) or not SHA.fullmatch(entry["commit"]):
      raise ValueError("Source revisions must be full commit hashes")
    url = entry.get("url")
    if not isinstance(url, str) or not url.startswith("https://") or not urlsplit(url).hostname or urlsplit(url).username:
      raise ValueError("Source URLs must use HTTPS")
  for entry in extensions:
    if not isinstance(entry.get("path"), str) or entry["path"] not in EXTENSIONS:
      raise ValueError("Unexpected vendored extension path")
  for entry in entries:
    path = entry.get("path")
    if not isinstance(path, str) or path not in DEPENDENCIES:
      raise ValueError(f"Unexpected dependency path: {path!r}")
    paths.append(path)
    if not isinstance(entry.get("tree"), str) or not SHA.fullmatch(entry["tree"]):
      raise ValueError(f"Missing pristine upstream tree for {path}")
    excluded = entry.get("exclude")
    if not isinstance(excluded, list) or not all(isinstance(name, str) for name in excluded) or len(set(excluded)) != len(excluded):
      raise ValueError(f"Invalid exclusions for {path}")
    included = entry.get("include", [])
    if not isinstance(included, list) or not all(isinstance(name, str) for name in included) or len(set(included)) != len(included):
      raise ValueError(f"Invalid inclusions for {path}")
    for name in [*excluded, *included]:
      invalid_name = not name or name == "." or "\0" in name or name != PurePosixPath(name).as_posix()
      if invalid_name or any(p in ("..", ".git") for p in PurePosixPath(name).parts) or name.startswith("/"):
        raise ValueError(f"Unsafe exclusion for {path}")
  if len(paths) != len(set(paths)) or not BASE_DEPENDENCIES <= set(paths):
    raise ValueError("Source manifest must list each dependency exactly once")
  return value


def tree_entries(repo: Path | str, revision: str) -> dict[str, str]:
  revision = git(repo, "rev-parse", "--verify", f"{revision}^{{commit}}").decode().strip()
  entries = {}
  for record in git(repo, "ls-tree", "-rz", "--full-tree", revision).split(b"\0"):
    if record:
      meta, name = record.split(b"\t", 1)
      entries[name.decode()] = meta.split()[0].decode()
  return entries


def under(name: str, paths: list[str]) -> bool:
  return any(name == path or name.startswith(path + "/") for path in paths)


def validate_layout(manifest: dict, entries: dict[str, str]) -> None:
  for path, mode in entries.items():
    if mode == "160000":
      raise ValueError(f"Source tree contains a gitlink: {path}")
    if PurePosixPath(path).name.lower() in (".gitmodules", "agents.md"):
      raise ValueError(f"Source tree contains excluded metadata: {path}")
  if entries.get(MANIFEST) != "100644":
    raise ValueError("Source manifest must be a tracked regular file")
  dependencies = dependency_entries(manifest)
  declared = {entry["path"] for entry in dependencies}
  for extension in EXTENSIONS - declared:
    if extension in entries or any(name.startswith(extension + "/") for name in entries):
      raise ValueError(f"Missing source provenance for {extension}")
  for dependency in dependencies:
    path = dependency["path"]
    members = [name for name in entries if name.startswith(path + "/")]
    if path in entries or not members:
      raise ValueError(f"Missing tracked dependency folder: {path}")
    if "include" in dependency:
      included = [f"{path}/{name}" for name in dependency["include"]]
      if outside := next((name for name in members if not under(name, included)), None):
        raise ValueError(f"Dependency content outside its include list is tracked: {outside}")
    for excluded in dependency["exclude"]:
      prefix = f"{path}/{excluded}"
      if any(name == prefix or name.startswith(prefix + "/") for name in members):
        raise ValueError(f"Excluded dependency content is tracked: {prefix}")


def validate_revision(repo: Path | str, revision: str) -> dict:
  revision = git(repo, "rev-parse", "--verify", f"{revision}^{{commit}}").decode().strip()
  entries = tree_entries(repo, revision)
  if entries.get(MANIFEST) != "100644":
    raise ValueError("Source manifest must be a tracked regular file")
  manifest = parse_manifest(git(repo, "show", f"{revision}:{MANIFEST}"))
  validate_layout(manifest, entries)
  return manifest


def validate_worktree(repo: Path | str) -> dict:
  repo = Path(repo)
  if (repo / MANIFEST).is_symlink() or not (repo / MANIFEST).is_file():
    raise ValueError("Source manifest must be a regular file")
  manifest = parse_manifest((repo / MANIFEST).read_bytes())
  entries = {}
  for record in git(repo, "ls-files", "--stage", "-z").split(b"\0"):
    if not record:
      continue
    meta, name = record.split(b"\t", 1)
    mode, _, stage = meta.split()
    if stage != b"0":
      raise ValueError(f"Unresolved source conflict: {name.decode()}")
    entries[name.decode()] = mode.decode()
  validate_layout(manifest, entries)
  for dependency in dependency_entries(manifest):
    folder = repo / dependency["path"]
    if not folder.is_dir() or folder.is_symlink():
      raise ValueError(f"Dependency must be an ordinary folder: {folder.name}")
    for current, directories, files in os.walk(folder, followlinks=False):
      if ".git" in directories or ".git" in files:
        raise ValueError(f"Nested Git metadata in dependency: {current}")
    for name, mode in entries.items():
      if name.startswith(folder.name + "/"):
        file = repo / name
        if not file.exists() and not file.is_symlink():
          raise ValueError(f"Missing dependency source: {name}")
        if file.is_symlink() != (mode == "120000"):
          raise ValueError(f"Dependency file type changed: {name}")
        if mode != "120000" and not stat.S_ISREG(file.lstat().st_mode):
          raise ValueError(f"Dependency source is not a regular file: {name}")
        if any(parent.is_symlink() for parent in file.parents if parent != repo and repo in parent.parents):
          raise ValueError(f"Dependency source has a symlink parent: {name}")
  return manifest