Files
sunnypilot/docs/serve.py
T
Adeeb Shihadeh 0bf99879d1 docs fixups
2026-07-26 14:34:43 -07:00

556 lines
19 KiB
Python

import argparse
import functools
import html
import http.server
import json
import posixpath
import re
import shutil
import threading
import time
import urllib.parse
from pathlib import Path
DOCS_DIR = Path(__file__).resolve().parent
SITE_DIR = DOCS_DIR / "_site"
TEMPLATE_FILE = DOCS_DIR / "template.html"
EXCLUDE_DIRS = {"_site", "__pycache__"}
REPO_URL = "https://github.com/commaai/openpilot/"
# (title, target) pairs. target is a page path or an absolute URL.
# A None target marks a section header.
NAV: list[tuple[str, str | None]] = [
("What is openpilot?", "index.md"),
("How-to", None),
("Turn the speed blue", "how-to/turn-the-speed-blue.md"),
("Connect to a comma 3X or four", "how-to/connect-to-comma.md"),
("Add support for a car", "how-to/car-port.md"),
("Concepts", None),
("Logs", "concepts/logs.md"),
("Safety", "concepts/safety.md"),
("Glossary", "concepts/glossary.md"),
("Contributing", None),
("Feedback", "contributing/feedback.md"),
("Roadmap", "contributing/roadmap.md"),
("Contributing Guide →", "https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md"),
("Links", None),
("Blog →", "https://blog.comma.ai"),
("Bounties →", "https://comma.ai/bounties"),
("GitHub →", "https://github.com/commaai"),
("Discord →", "https://discord.comma.ai"),
("X →", "https://x.com/comma_ai"),
]
GLOSSARY_DESCRIPTIONS = {
"onroad": "openpilot's system state while ignition is on.",
"offroad": "openpilot's system state while ignition is off.",
"route": "A route is a recording of an onroad session.",
"segment": "Routes are split into one minute chunks called segments.",
"comma connect": "The web viewer for all your routes; check it out at [connect.comma.ai](https://connect.comma.ai).",
"panda": "The secondary processor on the device that implements the functional safety and directly talks to the car over CAN. See the [panda repo](https://github.com/commaai/panda).",
"comma four": "The latest hardware by comma.ai for running openpilot. More info at [comma.ai/shop/comma-four](https://www.comma.ai/shop/comma-four).",
}
GLOSSARY_PAGE = "concepts/glossary.md"
GLOSSARY_ROUTE = GLOSSARY_PAGE.removesuffix(".md")
GLOSSARY_SKIP = frozenset("a code h1 h2 h3 h4 h5 h6 kbd pre script style".split())
_ENTITY = re.compile(r"&(?:#x?[0-9a-fA-F]+|[a-zA-Z]+);")
_LIST = re.compile(r"^(\s*)([*+-]|\d+\.)\s+(.*)$")
_HEADING = re.compile(r"^(#{1,6})\s+(.*)$")
_HR = re.compile(r"^(-{3,}|\*{3,}|_{3,})$")
_ATTR_URL = re.compile(r"""(?P<pre>\b(?:href|src)=(?P<q>["']))(?P<url>.*?)(?P=q)""")
_VOID = frozenset("br img hr meta link input".split())
_URL = re.compile(r"https?://[^\s<>\[\]\"']+")
_ADMONITION = re.compile(r"^\[!(NOTE|TIP|IMPORTANT|WARNING|CAUTION)\]$", re.I)
def page_route(path: str) -> str:
path = path.removesuffix(".md")
return posixpath.dirname(path) or "." if posixpath.basename(path) == "index" else path
def page_href(current: str, target: str) -> str:
route = posixpath.relpath(page_route(target), page_route(current))
return ("." if route == "." else route) + "/"
def rewrite_relative_url(value: str, page: str) -> str | None:
url = urllib.parse.urlparse(value)
if value.startswith(("#", "/")) or url.scheme or url.netloc or not url.path:
return None
target = posixpath.normpath(posixpath.join(posixpath.dirname(page), url.path))
if target == ".." or target.startswith("../"):
return None
path = page_href(page, target) if target.endswith(".md") else posixpath.relpath(target, page_route(page))
return url._replace(path=path).geturl()
def rewrite_html_urls(fragment: str, page: str) -> str:
def repl(m: re.Match[str]) -> str:
r = rewrite_relative_url(m.group("url"), page)
return m.group(0) if r is None else f'{m.group("pre")}{r}{m.group("q")}'
return _ATTR_URL.sub(repl, fragment)
def esc(text: str, attr: bool = False) -> str:
held: list[str] = []
def hold(m: re.Match[str]) -> str:
held.append(m.group(0))
return f"\0{len(held) - 1}\0"
return re.sub(r"\0(\d+)\0", lambda m: held[int(m.group(1))], html.escape(_ENTITY.sub(hold, text), quote=attr))
def clean_tooltip(description: str) -> str:
text = re.sub(r"\[([^\]]+)]\([^)]+\)", r"\1", description)
return re.sub(r"\s+", " ", re.sub(r"[*_~]", "", re.sub(r"`([^`]+)`", r"\1", text))).strip()
def glossary_slug(label: str) -> str:
return label.replace(" ", "-").replace("_", "-").lower()
GLOSSARY_TERMS = [(glossary_slug(l), re.compile(rf"(?<!\w){re.escape(l)}(?!\w)", re.I), clean_tooltip(d)) for l, d in GLOSSARY_DESCRIPTIONS.items()]
GLOSSARY_DEFINITIONS = "\n".join(f'* <span id="{glossary_slug(l)}"></span>**{l}**: {d}' for l, d in GLOSSARY_DESCRIPTIONS.items())
def inject_glossary(body: str, page: str) -> str:
if page == GLOSSARY_PAGE:
return body
route = "." if page == "index.md" else page.removesuffix(".md")
base, seen, out, skip, depth = f"{posixpath.relpath(GLOSSARY_ROUTE, route)}/#", set(), [], None, 0
for part in re.split(r"(<[^>]+>)", body):
if not part:
continue
if part.startswith("<"):
out.append(part)
if part.startswith("<!") or not (m := re.match(r"</?\s*([a-zA-Z0-9]+)", part)):
continue
tag = m.group(1).lower()
if tag not in GLOSSARY_SKIP:
continue
closing, void = part.startswith("</"), part.endswith("/>") or tag in _VOID
if closing and skip == tag and depth:
depth -= 1
skip = None if not depth else skip
elif not closing and not void:
skip, depth = (tag, 1) if skip is None else (skip, depth + (skip == tag))
continue
if depth:
out.append(part)
continue
cur, text = 0, part
while True:
best = None
for order, (slug, pat, tip) in enumerate(GLOSSARY_TERMS):
if slug in seen or (found := pat.search(text, cur)) is None:
continue
cand = (found.start(), found.start() - found.end(), order, slug, tip, found.end(), found.group(0))
if best is None or cand[:3] < best[:3]:
best = cand
if best is None:
out.append(text[cur:])
break
start, _, _, slug, tip, end, matched = best
out.append(text[cur:start])
out.append(
f'<a class="glossary-term" data-glossary-term="" href="{base}{slug}">'
+ f'<span class="glossary-term__label">{matched}</span>'
+ f'<span class="glossary-term__tooltip" data-search-exclude="">{esc(tip)}</span></a>'
)
seen.add(slug)
cur = end
return "".join(out)
def slugify(text: str) -> str:
text = html.unescape(re.sub(r"<[^>]+>", "", text)).lower()
return re.sub(r"[-\s]+", "-", re.sub(r"[^\w\s-]", "", text, flags=re.UNICODE)).strip("-")
def _parse_link(text: str, start: int) -> tuple[str, str, int] | None:
if start >= len(text) or text[start] != "[":
return None
depth, i = 0, start
while i < len(text):
depth += (text[i] == "[") - (text[i] == "]")
if text[i] == "]" and depth == 0:
label = text[start + 1 : i]
if i + 1 >= len(text) or text[i + 1] != "(":
return None
j, dp = i + 2, 1
while j < len(text) and dp:
dp += (text[j] == "(") - (text[j] == ")")
j += 1
return None if dp else (label, text[i + 2 : j - 1], j)
i += 1
return None
def autolink_plain(text: str) -> str:
parts: list[str] = []
last = 0
for m in _URL.finditer(text):
start = m.start()
if start > 0 and text[start - 1].isalnum():
continue
parts.append(esc(text[last:start]))
url = m.group(0).rstrip(".,;:!?)]")
parts.append(f'<a href="{esc(url, True)}">{esc(url)}</a>')
last = start + len(url)
parts.append(esc(text[last:]))
return "".join(parts)
def render_inline(text: str, page: str) -> str:
out, i, n = [], 0, len(text)
while i < n:
if text[i] == "\n" and i >= 2 and text[i - 2 : i] == " " and out and out[-1].endswith(" "):
out[-1] = out[-1][:-2]
out.append("<br>\n")
i += 1
continue
if text[i] == "`" and (end := text.find("`", i + 1)) != -1:
out.append(f"<code>{esc(text[i + 1 : end])}</code>")
i = end + 1
continue
if text[i] == "!" and i + 1 < n and text[i + 1] == "[" and (p := _parse_link(text, i + 1)):
label, url, end = p
src = rewrite_relative_url(url, page) or url
out.append(f'<img alt="{esc(label, True)}" src="{esc(src, True)}">')
i = end
continue
if text[i] == "[" and (p := _parse_link(text, i)):
label, url, end = p
href = rewrite_relative_url(url, page) or url
out.append(f'<a href="{esc(href, True)}">{render_inline(label, page)}</a>')
i = end
continue
if text[i] == "<":
if text.startswith("<!--", i):
end = text.find("-->", i + 4)
end = n if end < 0 else end + 3
out.append(rewrite_html_urls(text[i:end], page))
i = end
continue
if m := re.match(r"<[^>]+>", text[i:]):
out.append(rewrite_html_urls(m.group(0), page))
i += len(m.group(0))
continue
if (text.startswith("**", i) or text.startswith("__", i)) and (end := text.find(text[i : i + 2], i + 2)) != -1:
out.append(f"<strong>{render_inline(text[i + 2 : end], page)}</strong>")
i = end + 2
continue
if text[i] in "*_" and i + 1 < n and text[i + 1] not in " \t\n" and (end := text.find(text[i], i + 1)) > i + 1:
out.append(f"<em>{render_inline(text[i + 1 : end], page)}</em>")
i = end + 1
continue
j = i + 1
while j < n and text[j] not in "`[<!*_":
j += 1
out.append(autolink_plain(text[i:j]))
i = j
return "".join(out)
def _trow(line: str) -> list[str]:
return [c.strip() for c in line.strip().removeprefix("|").removesuffix("|").split("|")]
def _is_sep(line: str) -> bool:
return "|" in line and all(re.fullmatch(r":?-{3,}:?", c.strip()) for c in _trow(line))
def _align(sep: str) -> str:
s = sep.strip()
if s.startswith(":") and s.endswith(":"):
return ' style="text-align: center;"'
if s.endswith(":"):
return ' style="text-align: right;"'
if s.startswith(":"):
return ' style="text-align: left;"'
return ""
def _list_info(line: str) -> tuple[int, str, str] | None:
m = _LIST.match(line)
return None if not m else (len(m.group(1)) // 4, "ol" if m.group(2)[-1] == "." else "ul", m.group(3))
def _render_blocks(text: str, page: str) -> str:
lines, out, i, n = text.splitlines(), [], 0, 0
n = len(lines)
while i < n:
line, s = lines[i], lines[i].strip()
if not s:
i += 1
continue
if s.startswith("```"):
lang, body = s[3:].strip(), []
i += 1
while i < n and not lines[i].strip().startswith("```"):
body.append(lines[i])
i += 1
if i < n:
i += 1
code = html.escape("\n".join(body) + ("\n" if body else ""))
cls = f' class="language-{html.escape(lang)}"' if lang else ""
out.append(f"<pre><code{cls}>{code}</code></pre>")
continue
if m := _HEADING.match(s):
content, level = m.group(2).rstrip("#").strip(), len(m.group(1))
sid = slugify(content)
out.append(f'<h{level} id="{sid}">{render_inline(content, page)}<a class="headerlink" href="#{sid}" title="Permanent link">#</a></h{level}>')
i += 1
continue
if _HR.fullmatch(s):
out.append("<hr>")
i += 1
continue
if "|" in line and i + 1 < n and _is_sep(lines[i + 1]):
headers, aligns = _trow(line), [_align(c) for c in _trow(lines[i + 1])]
i += 2
rows = []
while i < n and "|" in lines[i] and lines[i].strip():
rows.append(_trow(lines[i]))
i += 1
parts = (
["<table>", "<thead>", "<tr>"]
+ [f"<th{aligns[j] if j < len(aligns) else ''}>{render_inline(h, page)}</th>" for j, h in enumerate(headers)]
+ ["</tr>", "</thead>", "<tbody>"]
)
for row in rows:
parts.append("<tr>")
for j in range(len(headers)):
parts.append(f"<td{aligns[j] if j < len(aligns) else ''}>{render_inline(row[j] if j < len(row) else '', page)}</td>")
parts.append("</tr>")
out.append("\n".join(parts + ["</tbody>", "</table>"]))
continue
if _list_info(line):
items: list[tuple[int, str, list[str]]] = []
while i < n:
if not lines[i].strip():
if i + 1 < n and _list_info(lines[i + 1]):
i += 1
continue
break
info = _list_info(lines[i])
if not info:
break
level, kind, body = info
chunk = [body]
i += 1
while i < n and lines[i].strip() and _list_info(lines[i]) is None:
t = lines[i].strip()
if t.startswith(("```", ">")) or _HEADING.match(t) or _HR.fullmatch(t):
break
chunk.append(lines[i])
i += 1
items.append((level, kind, chunk))
def render_list(items: list[tuple[int, str, list[str]]], start: int, min_level: int) -> tuple[str, int]:
if start >= len(items) or items[start][0] < min_level:
return "", start
kind, chunks, idx = items[start][1], [f"<{items[start][1]}>"], start
while idx < len(items) and items[idx][0] >= min_level:
level, ikind, body_lines = items[idx]
if level > min_level:
nested, idx = render_list(items, idx, level)
chunks[-1] = (chunks[-1][:-5] + nested + "</li>") if chunks[-1].endswith("</li>") else chunks[-1] + nested
continue
if ikind != kind:
chunks += [f"</{kind}>", f"<{ikind}>"]
kind = ikind
idx += 1
body = render_inline("\n".join(body_lines), page)
nested = ""
if idx < len(items) and items[idx][0] > min_level:
nested, idx = render_list(items, idx, min_level + 1)
chunks.append(f"<li>{body}{nested}\n</li>" if nested else f"<li>{body}</li>")
chunks.append(f"</{kind}>")
return "\n".join(chunks), idx
out.append(render_list(items, 0, items[0][0])[0])
continue
if s.startswith(">"):
q = []
while i < n and lines[i].strip().startswith(">"):
q.append(re.sub(r"^>\s?", "", lines[i].strip()))
i += 1
m = _ADMONITION.match(q[0].strip()) if q else None
if m:
kind = m.group(1).lower()
title = m.group(1).capitalize()
body = _render_blocks("\n".join(q[1:]), page)
out.append(f'<div class="admonition {kind}">\n<p class="admonition-title">{title}</p>\n{body}\n</div>')
else:
out.append(f"<blockquote>\n<p>{render_inline(chr(10).join(q), page)}</p>\n</blockquote>")
continue
if s.startswith("<!--"):
out.append(rewrite_html_urls(line, page))
i += 1
# Preserve a blank line after HTML comments (python-markdown does).
if i < n and not lines[i].strip():
out.append("")
while i < n and not lines[i].strip():
i += 1
continue
buf = [line]
i += 1
while i < n and lines[i].strip():
t = lines[i].strip()
if t.startswith(("```", ">")) or _HEADING.match(t) or _HR.fullmatch(t):
break
if "|" in lines[i] and i + 1 < n and _is_sep(lines[i + 1]):
break
buf.append(lines[i])
i += 1
out.append(f"<p>{render_inline(chr(10).join(buf), page)}</p>")
return "\n".join(out)
def render_markdown(text: str, page: str) -> str:
return inject_glossary(_render_blocks(text, page), page)
def page_title(source: str) -> str:
for line in source.splitlines():
if line.startswith("# "):
return line[2:].strip()
return "openpilot docs"
def write_html_redirect(rel: Path) -> None:
if rel.name == "index.md":
return
target = f"{rel.stem}/"
out = SITE_DIR / rel.with_suffix(".html")
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(
"\n".join(
[
"<!doctype html>",
f'<meta http-equiv="refresh" content="0; url={html.escape(target)}">',
f'<link rel="canonical" href="{html.escape(target)}">',
f"<script>location.replace({json.dumps(target)} + location.search + location.hash)</script>",
]
)
)
def copy_assets() -> None:
for src in DOCS_DIR.rglob("*"):
if not src.is_file():
continue
rel = src.relative_to(DOCS_DIR)
if any(part in EXCLUDE_DIRS for part in rel.parts):
continue
if src.suffix == ".md" or src in (Path(__file__).resolve(), TEMPLATE_FILE):
continue
dest = SITE_DIR / rel
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src, dest)
def render_nav_html(current_page: str) -> str:
parts: list[str] = []
for title, target in NAV:
if target is None:
parts.append(f'<div class="nav-section">{html.escape(title)}</div>')
elif target.startswith(("http://", "https://")):
parts.append(f'<a href="{html.escape(target)}">{html.escape(title)}</a>')
else:
active = ' class="active"' if target == current_page else ""
parts.append(f'<a href="{html.escape(page_href(current_page, target))}"{active}>{html.escape(title)}</a>')
return "\n".join(parts)
def build() -> None:
template = TEMPLATE_FILE.read_text()
pages = [
(path.relative_to(DOCS_DIR), path.read_text())
for path in sorted(DOCS_DIR.rglob("*.md"))
if path != DOCS_DIR / "README.md" and not any(part in EXCLUDE_DIRS for part in path.relative_to(DOCS_DIR).parts)
]
pages.append((Path(GLOSSARY_PAGE), f"# openpilot glossary\n\n{GLOSSARY_DEFINITIONS}"))
pages.sort()
if SITE_DIR.exists():
shutil.rmtree(SITE_DIR)
SITE_DIR.mkdir(parents=True)
copy_assets()
for rel_path, source in pages:
rel = rel_path.as_posix()
body = render_markdown(source, rel)
title = page_title(source)
route = page_route(rel)
root = "../" * (0 if route == "." else len(route.split("/")))
edit_path = "serve.py" if rel == GLOSSARY_PAGE else rel
page_html = template
for name, value in {
"TITLE": html.escape(title),
"ROOT": root,
"HOME_HREF": page_href(rel, "index.md"),
"NAV": render_nav_html(rel),
"BODY": body,
"EDIT_URL": html.escape(f"{REPO_URL}blob/master/docs/{edit_path}"),
}.items():
page_html = page_html.replace(f"{{{{{name}}}}}", value)
out = SITE_DIR / ("" if route == "." else route) / "index.html"
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(page_html)
write_html_redirect(rel_path)
print(f"docs: built {len(pages)} pages into {SITE_DIR}")
def _watched_files() -> list[Path]:
return [p for p in DOCS_DIR.rglob("*") if p.is_file() and not any(part in EXCLUDE_DIRS for part in p.relative_to(DOCS_DIR).parts)]
def serve() -> None:
build()
mtimes = {p: p.stat().st_mtime for p in _watched_files()}
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(SITE_DIR))
httpd = http.server.ThreadingHTTPServer(("", 0), handler)
print(f"docs: serving on http://localhost:{httpd.server_port}/ (watching for changes)")
try:
threading.Thread(target=httpd.serve_forever, daemon=True).start()
while True:
time.sleep(0.5)
new_mtimes = {p: p.stat().st_mtime for p in _watched_files()}
if new_mtimes != mtimes:
mtimes = new_mtimes
print("docs: change detected, rebuilding...")
try:
build()
except Exception as e:
print(f"docs: build failed: {e}")
except KeyboardInterrupt:
pass
finally:
httpd.shutdown()
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Build or serve the openpilot documentation site.")
parser.add_argument("--build", action="store_true", help="Build the site and exit.")
args = parser.parse_args()
if args.build:
build()
else:
serve()