diff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yaml index 4be4fac91..f4a8e7cb8 100644 --- a/.github/workflows/docs.yaml +++ b/.github/workflows/docs.yaml @@ -35,8 +35,7 @@ jobs: - name: Build docs run: | git lfs pull - pip install zensical - python scripts/docs.py build + python docs/serve.py --build # Push to docs.comma.ai - uses: actions/checkout@v7 @@ -57,7 +56,7 @@ jobs: git rm -rf . # copy over docs - cp -r ../docs_site/ docs/ + cp -r ../docs/_site/ docs/ # GitHub pages config touch docs/.nojekyll diff --git a/.gitignore b/.gitignore index 5c9956758..48ce8f041 100644 --- a/.gitignore +++ b/.gitignore @@ -54,7 +54,7 @@ compare_runtime*.html openpilot/selfdrive/modeld/models/tg_input_devices.json # build artifacts -docs_site/ +docs/_site/ openpilot/selfdrive/pandad/pandad openpilot/cereal/services.h openpilot/cereal/gen diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md deleted file mode 100644 index e803a3fb8..000000000 --- a/docs/DEVELOPMENT.md +++ /dev/null @@ -1,24 +0,0 @@ -# Docs development - -The `docs/` tree is the source for [docs.comma.ai](https://docs.comma.ai). -The site is updated on pushes to master by this [workflow](../.github/workflows/docs.yaml). - -Those commands must be run in the root directory of openpilot, **not /docs** - -**1. Install the docs dependencies** -``` bash -uv pip install .[docs] -``` - -**2. Build the new site** -``` bash -docs build -``` - -**3. Run the new site locally** -``` bash -docs serve -``` - -References: -* https://zensical.org/docs/ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 000000000..d6a0126b3 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,14 @@ +# Docs development + +The `docs/` tree is the source for [docs.comma.ai](https://docs.comma.ai). +The site is updated on pushes to master by this [workflow](../.github/workflows/docs.yaml). + +**1. Build the site** +``` bash +python docs/serve.py --build +``` + +**2. Run the site locally** (rebuilds on change) +``` bash +python docs/serve.py +``` diff --git a/docs/assets/comma-logo.png b/docs/assets/comma-logo.png index 2838d92bf..19b67d073 120000 --- a/docs/assets/comma-logo.png +++ b/docs/assets/comma-logo.png @@ -1 +1 @@ -../../selfdrive/assets/icons_mici/settings/comma_icon.png \ No newline at end of file +../../openpilot/selfdrive/assets/icons_mici/settings/comma_icon.png \ No newline at end of file diff --git a/docs/assets/favicon.svg b/docs/assets/favicon.svg new file mode 100644 index 000000000..304454837 --- /dev/null +++ b/docs/assets/favicon.svg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e019ed5af500e8820d05934d47e0380728e1b30e01a179f786dc4edb1eccbd7 +size 349 diff --git a/docs/concepts/glossary.md b/docs/concepts/glossary.md deleted file mode 100644 index 4f4dd5475..000000000 --- a/docs/concepts/glossary.md +++ /dev/null @@ -1,3 +0,0 @@ -# openpilot glossary - -{{GLOSSARY_DEFINITIONS}} diff --git a/docs/ext/glossary.py b/docs/ext/glossary.py deleted file mode 100644 index 9bbf3c78d..000000000 --- a/docs/ext/glossary.py +++ /dev/null @@ -1,216 +0,0 @@ -import posixpath -import re -import tomllib -import xml.etree.ElementTree as ET -from pathlib import Path - -from markdown.extensions import Extension -from markdown.preprocessors import Preprocessor -from markdown.treeprocessors import Treeprocessor - -from zensical.extensions.links import LinksTreeprocessor - -GlossaryTerm = tuple[str, re.Pattern[str], str] - -GLOSSARY_FILE = Path(__file__).with_name("glossary.toml") -GLOSSARY_PAGE = "concepts/glossary.md" -GLOSSARY_PLACEHOLDER = "{{GLOSSARY_DEFINITIONS}}" - -SKIP_TAGS = { - "a", - "code", - "h1", - "h2", - "h3", - "h4", - "h5", - "h6", - "kbd", - "pre", - "script", - "style", -} - -def clean_tooltip(description: str) -> str: - text = re.sub(r"\[([^\]]+)]\([^)]+\)", r"\1", description) - text = re.sub(r"`([^`]+)`", r"\1", text) - text = re.sub(r"[*_~]", "", text) - return re.sub(r"\s+", " ", text).strip() - - -def load_glossary() -> tuple[list[GlossaryTerm], str]: - with GLOSSARY_FILE.open("rb") as f: - glossary_data = tomllib.load(f).get("glossary", {}) - - glossary: list[GlossaryTerm] = [] - rendered = [] - for key, value in glossary_data.items(): - label = str(key).strip().replace("_", " ") - description = str(value).strip() - if not description: - continue - - slug = label.replace(" ", "-").replace("_", "-").lower() - glossary.append((slug, re.compile(rf"(?**{label}**: {description}') - - return glossary, "\n".join(rendered) - - -class GlossaryPreprocessor(Preprocessor): - def __init__(self, md, glossary: str): - super().__init__(md) - self.glossary = glossary - - def run(self, lines: list[str]) -> list[str]: - markdown = "\n".join(lines) - if GLOSSARY_PLACEHOLDER not in markdown: - return lines - return markdown.replace(GLOSSARY_PLACEHOLDER, self.glossary).splitlines() - - -class GlossaryTreeprocessor(Treeprocessor): - def __init__(self, md, glossary: list[GlossaryTerm]): - super().__init__(md) - self.glossary = glossary - self.seen: set[str] = set() - - def run(self, root: ET.Element) -> None: - at = self.md.treeprocessors.get_index_for_name("zrelpath") - processor = self.md.treeprocessors[at] - if not isinstance(processor, LinksTreeprocessor): - raise TypeError("Links processor not registered") - if processor.path == GLOSSARY_PAGE: - return - - self.seen.clear() - glossary_href = f"{posixpath.relpath(GLOSSARY_PAGE, posixpath.dirname(processor.path) or '.')}#" - self._walk(root, glossary_href) - - def _walk(self, element: ET.Element, glossary_href: str) -> None: - if element.tag in SKIP_TAGS or element.attrib.get("data-glossary-skip") is not None: - return - - self._replace(element, glossary_href) - - idx = 0 - while idx < len(element): - child = element[idx] - self._walk(child, glossary_href) - idx = self._replace(element, glossary_href, idx) + 1 - - def _replace(self, parent: ET.Element, glossary_href: str, index: int | None = None) -> int: - child = None if index is None else parent[index] - text = parent.text if child is None else child.tail - pieces = self._pieces(text or "", glossary_href) - if not pieces: - return -1 if index is None else index - - if child is None: - parent.text = pieces[0] if isinstance(pieces[0], str) else "" - # Insert replacements for parent.text before the first existing child. - insert_at = -1 - else: - assert index is not None - child.tail = pieces[0] if isinstance(pieces[0], str) else "" - insert_at = index - - start = 1 if isinstance(pieces[0], str) else 0 - previous = child - - for piece in pieces[start:]: - if isinstance(piece, str): - previous.tail = (previous.tail or "") + piece - continue - - insert_at += 1 - parent.insert(insert_at, piece) - previous = piece - - return insert_at - - def _pieces(self, text: str, glossary_href: str) -> list[str | ET.Element]: - if not text.strip(): - return [] - - pieces: list[str | ET.Element] = [] - cursor = 0 - - while True: - best = None - for slug, pattern, tooltip in self.glossary: - if slug in self.seen: - continue - - found = pattern.search(text, cursor) - if found is None: - continue - - candidate = (slug, tooltip, found.start(), found.end()) - if best is None: - best = candidate - continue - - _, _, best_start, best_end = best - _, _, current_start, current_end = candidate - if current_start < best_start: - best = candidate - continue - - if current_start == best_start and current_end - current_start > best_end - best_start: - best = candidate - - if best is None: - break - - slug, tooltip, start, end = best - if start > cursor: - pieces.append(text[cursor:start]) - - link = ET.Element( - "a", - { - "class": "glossary-term", - "data-glossary-term": "", - "href": f"{glossary_href}{slug}", - }, - ) - ET.SubElement(link, "span", {"class": "glossary-term__label"}).text = text[start:end] - ET.SubElement( - link, - "span", - { - "class": "glossary-term__tooltip", - "data-search-exclude": "", - }, - ).text = tooltip - pieces.append(link) - self.seen.add(slug) - cursor = end - - if not pieces: - return [] - if cursor < len(text): - pieces.append(text[cursor:]) - return pieces - - -class GlossaryExtension(Extension): - def extendMarkdown(self, md) -> None: - md.registerExtension(self) - glossary, rendered = load_glossary() - - md.preprocessors.register( - GlossaryPreprocessor(md, rendered), - "docs-ext-glossary-preprocessor", - 27, - ) - md.treeprocessors.register( - GlossaryTreeprocessor(md, glossary), - "docs-ext-glossary-treeprocessor", - 0, - ) - - -def makeExtension(**kwargs) -> GlossaryExtension: - return GlossaryExtension(**kwargs) diff --git a/docs/ext/glossary.toml b/docs/ext/glossary.toml deleted file mode 100644 index 62408d9dd..000000000 --- a/docs/ext/glossary.toml +++ /dev/null @@ -1,8 +0,0 @@ -[glossary] -onroad = "openpilot's system state while ignition is on." -offroad = "openpilot's system state while ignition is off." -route = "A route is a recording of an onroad session." -segment = "Routes are split into one minute chunks called segments." -"comma connect" = "The web viewer for all your routes; check it out at [connect.comma.ai](https://connect.comma.ai)." -panda = "The secondary processor on the device that implements the functional safety and directly talks to the car over CAN. See the [panda repo](https://github.com/commaai/panda)." -"comma four" = "The latest hardware by comma.ai for running openpilot. More info at [comma.ai/shop/comma-four](https://www.comma.ai/shop/comma-four)." diff --git a/docs/serve.py b/docs/serve.py new file mode 100644 index 000000000..dcc78f7b0 --- /dev/null +++ b/docs/serve.py @@ -0,0 +1,444 @@ +import argparse +import functools +import html +import http.server +import json +import posixpath +import re +import shutil +import threading +import time +import urllib.parse +from pathlib import Path + +DOCS_DIR = Path(__file__).resolve().parent +SITE_DIR = DOCS_DIR / "_site" +TEMPLATE_FILE = DOCS_DIR / "template.html" +EXCLUDE_DIRS = {"_site"} + +REPO_URL = "https://github.com/commaai/openpilot/" + +# (title, target) pairs. target is a page path or an absolute URL. +# A None target marks a section header. +NAV: list[tuple[str, str | None]] = [ + ("What is openpilot?", "index.md"), + ("How-to", None), + ("Turn the speed blue", "how-to/turn-the-speed-blue.md"), + ("Connect to a comma 3X or four", "how-to/connect-to-comma.md"), + ("Add support for a car", "how-to/car-port.md"), + ("Concepts", None), + ("Logs", "concepts/logs.md"), + ("Safety", "concepts/safety.md"), + ("Glossary", "concepts/glossary.md"), + ("Contributing", None), + ("Feedback", "contributing/feedback.md"), + ("Roadmap", "contributing/roadmap.md"), + ("Contributing Guide →", "https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md"), + ("Links", None), + ("Blog →", "https://blog.comma.ai"), + ("Bounties →", "https://comma.ai/bounties"), + ("GitHub →", "https://github.com/commaai"), + ("Discord →", "https://discord.comma.ai"), + ("X →", "https://x.com/comma_ai"), +] + +GLOSSARY_DESCRIPTIONS = { + "onroad": "openpilot's system state while ignition is on.", + "offroad": "openpilot's system state while ignition is off.", + "route": "A route is a recording of an onroad session.", + "segment": "Routes are split into one minute chunks called segments.", + "comma connect": "The web viewer for all your routes; check it out at [connect.comma.ai](https://connect.comma.ai).", + "panda": "The secondary processor on the device that implements the functional safety and directly talks to the car over CAN. See the [panda repo](https://github.com/commaai/panda).", + "comma four": "The latest hardware by comma.ai for running openpilot. More info at [comma.ai/shop/comma-four](https://www.comma.ai/shop/comma-four).", +} +GLOSSARY_PAGE = "concepts/glossary.md" +GLOSSARY_ROUTE = GLOSSARY_PAGE.removesuffix(".md") +GLOSSARY_SKIP = frozenset("a code h1 h2 h3 h4 h5 h6 kbd pre script style".split()) + +_ENTITY = re.compile(r"&(?:#x?[0-9a-fA-F]+|[a-zA-Z]+);") +_LIST = re.compile(r"^(\s*)([*+-]|\d+\.)\s+(.*)$") +_HEADING = re.compile(r"^(#{1,6})\s+(.*)$") +_HR = re.compile(r"^(-{3,}|\*{3,}|_{3,})$") +_ATTR_URL = re.compile(r"""(?P
\b(?:href|src)=(?P["']))(?P.*?)(?P=q)""")
+_VOID = frozenset("br img hr meta link input".split())
+
+def page_route(path: str) -> str:
+  path = path.removesuffix(".md")
+  return posixpath.dirname(path) or "." if posixpath.basename(path) == "index" else path
+
+def page_href(current: str, target: str) -> str:
+  route = posixpath.relpath(page_route(target), page_route(current))
+  return ("." if route == "." else route) + "/"
+
+def rewrite_relative_url(value: str, page: str) -> str | None:
+  url = urllib.parse.urlparse(value)
+  if value.startswith(("#", "/")) or url.scheme or url.netloc or not url.path:
+    return None
+  target = posixpath.normpath(posixpath.join(posixpath.dirname(page), url.path))
+  if target == ".." or target.startswith("../"):
+    return None
+  path = page_href(page, target) if target.endswith(".md") else posixpath.relpath(target, page_route(page))
+  return url._replace(path=path).geturl()
+
+def rewrite_html_urls(fragment: str, page: str) -> str:
+  def repl(m: re.Match[str]) -> str:
+    r = rewrite_relative_url(m.group("url"), page)
+    return m.group(0) if r is None else f'{m.group("pre")}{r}{m.group("q")}'
+  return _ATTR_URL.sub(repl, fragment)
+
+def esc(text: str, attr: bool = False) -> str:
+  held: list[str] = []
+  def hold(m: re.Match[str]) -> str:
+    held.append(m.group(0))
+    return f"\0{len(held) - 1}\0"
+  return re.sub(r"\0(\d+)\0", lambda m: held[int(m.group(1))], html.escape(_ENTITY.sub(hold, text), quote=attr))
+
+def clean_tooltip(description: str) -> str:
+  text = re.sub(r"\[([^\]]+)]\([^)]+\)", r"\1", description)
+  return re.sub(r"\s+", " ", re.sub(r"[*_~]", "", re.sub(r"`([^`]+)`", r"\1", text))).strip()
+
+def glossary_slug(label: str) -> str:
+  return label.replace(" ", "-").replace("_", "-").lower()
+
+GLOSSARY_TERMS = [
+  (glossary_slug(l), re.compile(rf"(?**{l}**: {d}' for l, d in GLOSSARY_DESCRIPTIONS.items()
+)
+
+def inject_glossary(body: str, page: str) -> str:
+  if page == GLOSSARY_PAGE:
+    return body
+  route = "." if page == "index.md" else page.removesuffix(".md")
+  base, seen, out, skip, depth = f"{posixpath.relpath(GLOSSARY_ROUTE, route)}/#", set(), [], None, 0
+  for part in re.split(r"(<[^>]+>)", body):
+    if not part:
+      continue
+    if part.startswith("<"):
+      out.append(part)
+      if part.startswith("") or tag in _VOID
+      if closing and skip == tag and depth:
+        depth -= 1
+        skip = None if not depth else skip
+      elif not closing and not void:
+        skip, depth = (tag, 1) if skip is None else (skip, depth + (skip == tag))
+      continue
+    if depth:
+      out.append(part); continue
+    cur, text = 0, part
+    while True:
+      best = None
+      for order, (slug, pat, tip) in enumerate(GLOSSARY_TERMS):
+        if slug in seen or (found := pat.search(text, cur)) is None:
+          continue
+        cand = (found.start(), found.start() - found.end(), order, slug, tip, found.end(), found.group(0))
+        if best is None or cand[:3] < best[:3]:
+          best = cand
+      if best is None:
+        out.append(text[cur:]); break
+      start, _, _, slug, tip, end, matched = best
+      out.append(text[cur:start])
+      out.append(
+        f''
+        f'{matched}'
+        f'{esc(tip)}'
+      )
+      seen.add(slug); cur = end
+  return "".join(out)
+
+def slugify(text: str) -> str:
+  text = html.unescape(re.sub(r"<[^>]+>", "", text)).lower()
+  return re.sub(r"[-\s]+", "-", re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE)).strip("-")
+
+def _parse_link(text: str, start: int) -> tuple[str, str, int] | None:
+  if start >= len(text) or text[start] != "[":
+    return None
+  depth, i = 0, start
+  while i < len(text):
+    depth += (text[i] == "[") - (text[i] == "]")
+    if text[i] == "]" and depth == 0:
+      label = text[start + 1:i]
+      if i + 1 >= len(text) or text[i + 1] != "(":
+        return None
+      j, dp = i + 2, 1
+      while j < len(text) and dp:
+        dp += (text[j] == "(") - (text[j] == ")"); j += 1
+      return None if dp else (label, text[i + 2:j - 1], j)
+    i += 1
+  return None
+
+def render_inline(text: str, page: str) -> str:
+  out, i, n = [], 0, len(text)
+  while i < n:
+    if text[i] == "\n" and i >= 2 and text[i-2:i] == "  " and out and out[-1].endswith("  "):
+      out[-1] = out[-1][:-2]; out.append("
\n"); i += 1; continue
+    if text[i] == "`" and (end := text.find("`", i + 1)) != -1:
+      out.append(f"{esc(text[i+1:end])}"); i = end + 1; continue
+    if text[i] == "!" and i + 1 < n and text[i+1] == "[" and (p := _parse_link(text, i + 1)):
+      label, url, end = p
+      src = rewrite_relative_url(url, page) or url
+      out.append(f'{esc(label, True)}'); i = end; continue
+    if text[i] == "[" and (p := _parse_link(text, i)):
+      label, url, end = p
+      href = rewrite_relative_url(url, page) or url
+      out.append(f'{render_inline(label, page)}'); i = end; continue
+    if text[i] == "<":
+      if text.startswith("", i + 4); end = n if end < 0 else end + 3
+        out.append(rewrite_html_urls(text[i:end], page)); i = end; continue
+      if m := re.match(r"<[^>]+>", text[i:]):
+        out.append(rewrite_html_urls(m.group(0), page)); i += len(m.group(0)); continue
+    if (text.startswith("**", i) or text.startswith("__", i)) and (end := text.find(text[i:i+2], i+2)) != -1:
+      out.append(f"{render_inline(text[i+2:end], page)}"); i = end + 2; continue
+    if text[i] in "*_" and i + 1 < n and text[i+1] not in " \t\n" and (end := text.find(text[i], i+1)) > i + 1:
+      out.append(f"{render_inline(text[i+1:end], page)}"); i = end + 1; continue
+    j = i + 1
+    while j < n and text[j] not in "`[ list[str]:
+  return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")]
+
+def _is_sep(line: str) -> bool:
+  return "|" in line and all(re.fullmatch(r":?-{3,}:?", c.strip()) for c in _trow(line))
+
+def _align(sep: str) -> str:
+  s = sep.strip()
+  if s.startswith(":") and s.endswith(":"): return ' style="text-align: center;"'
+  if s.endswith(":"): return ' style="text-align: right;"'
+  if s.startswith(":"): return ' style="text-align: left;"'
+  return ""
+
+def _list_info(line: str) -> tuple[int, str, str] | None:
+  m = _LIST.match(line)
+  return None if not m else (len(m.group(1)) // 4, "ol" if m.group(2)[-1] == "." else "ul", m.group(3))
+
+def render_markdown(text: str, page: str) -> str:
+  lines, out, i, n = text.splitlines(), [], 0, 0
+  n = len(lines)
+  while i < n:
+    line, s = lines[i], lines[i].strip()
+    if not s:
+      i += 1; continue
+
+    if s.startswith("```"):
+      lang, body = s[3:].strip(), []
+      i += 1
+      while i < n and not lines[i].strip().startswith("```"):
+        body.append(lines[i]); i += 1
+      if i < n: i += 1
+      code, cls = html.escape("\n".join(body) + ("\n" if body else "")), (f' class="language-{html.escape(lang)}"' if lang else "")
+      out.append(f"
{code}
"); continue
+
+    if m := _HEADING.match(s):
+      content, level = m.group(2).rstrip("#").strip(), len(m.group(1))
+      sid = slugify(content)
+      out.append(f'{render_inline(content, page)}#')
+      i += 1; continue
+
+    if _HR.fullmatch(s):
+      out.append("
"); i += 1; continue
+
+    if "|" in line and i + 1 < n and _is_sep(lines[i + 1]):
+      headers, aligns = _trow(line), [_align(c) for c in _trow(lines[i + 1])]
+      i += 2; rows = []
+      while i < n and "|" in lines[i] and lines[i].strip():
+        rows.append(_trow(lines[i])); i += 1
+      parts = ["", "", ""] + [
+        f"{render_inline(h, page)}" for j, h in enumerate(headers)
+      ] + ["", "", ""]
+      for row in rows:
+        parts.append("")
+        for j in range(len(headers)):
+          parts.append(f"{render_inline(row[j] if j < len(row) else '', page)}")
+        parts.append("")
+      out.append("\n".join(parts + ["", "
"])); continue
+
+    if _list_info(line):
+      items: list[tuple[int, str, list[str]]] = []
+      while i < n:
+        if not lines[i].strip():
+          if i + 1 < n and _list_info(lines[i + 1]):
+            i += 1; continue
+          break
+        info = _list_info(lines[i])
+        if not info: break
+        level, kind, body = info
+        chunk = [body]; i += 1
+        while i < n and lines[i].strip() and _list_info(lines[i]) is None:
+          t = lines[i].strip()
+          if t.startswith("```") or _HEADING.match(t) or _HR.fullmatch(t): break
+          chunk.append(lines[i]); i += 1
+        items.append((level, kind, chunk))
+
+      def render_list(start: int, min_level: int) -> tuple[str, int]:
+        if start >= len(items) or items[start][0] < min_level:
+          return "", start
+        kind, chunks, idx = items[start][1], [f"<{items[start][1]}>"], start
+        while idx < len(items) and items[idx][0] >= min_level:
+          level, ikind, body_lines = items[idx]
+          if level > min_level:
+            nested, idx = render_list(idx, level)
+            chunks[-1] = (chunks[-1][:-5] + nested + "") if chunks[-1].endswith("") else chunks[-1] + nested
+            continue
+          if ikind != kind:
+            chunks += [f"", f"<{ikind}>"]; kind = ikind
+          idx += 1
+          body = render_inline("\n".join(body_lines), page)
+          nested = ""
+          if idx < len(items) and items[idx][0] > min_level:
+            nested, idx = render_list(idx, min_level + 1)
+          chunks.append(f"
  • {body}{nested}\n
    • " if nested else f"
  • {body}
    • ")
+        chunks.append(f"")
+        return "\n".join(chunks), idx
+
+      out.append(render_list(0, items[0][0])[0]); continue
+
+    if s.startswith(">"):
+      q = []
+      while i < n and lines[i].strip().startswith(">"):
+        q.append(re.sub(r"^>\s?", "", lines[i].strip())); i += 1
+      out.append(f"
    \n
    {render_inline(chr(10).join(q), page)}
    \n
    "); continue
+
+    if s.startswith("