Update opendbc_repo

* modeld_v2: artifact parity with upstream tg

* m

* m

* Update helpers.py

* ew

* Update convert_upstream_pkl.py

* Update test_convert_upstream_pkl.py

---------

Co-authored-by: Jason Wen <haibin.wen3@gmail.com>

.github/workflows/test_models.yml

@@ -46,6 +46,23 @@ jobs:
           CHECK_OOB: 0
           GMMU: 0
         run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
+      - name: Resolve commaai's tinygrad pin, which their precompiled chestnut model is built and converted with
+        id: upstream-tg
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          read -r sha url < <(gh api repos/commaai/openpilot/contents/tinygrad_repo --jq '.sha + " " + .submodule_git_url')
+          echo "commaai/openpilot master pins $url@$sha, ours is $(git rev-parse HEAD:tinygrad_repo)"
+          dir="$RUNNER_TEMP/upstream_tinygrad"
+          git init -q "$dir" && git -C "$dir" fetch -q --depth 1 "$url" "$sha" && git -C "$dir" checkout -q FETCH_HEAD
+          echo "path=$dir" >> "$GITHUB_OUTPUT"
+      - name: Convert a pickled USB+AMD jit under commaai's tinygrad, as the chestnut model build does
+        env:
+          DEV: MOCKUSB+AMD
+          CHECK_OOB: 0
+          GMMU: 0
+          PYTHONPATH: ${{ steps.upstream-tg.outputs.path }}:${{ github.workspace }}
+        run: tools/op.sh test -v openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py
       - name: Where tinygrad moved, for the helper or failure point named above
         if: failure() && github.event_name == 'pull_request'
         run: |

openpilot/sunnypilot/modeld_v2/convert_upstream_pkl.py

@@ -66,47 +66,75 @@ def _patched(buf, topo) -> tuple[bytearray, dict]:
   return blob, rows
 
 
-def _evaluate(u, env):
+def _packed_args(program) -> dict[int, int]:
+  from tinygrad.device import TinyELF
+  params = getattr(program.arg, "params", None) or ()
+  return {off: p.arg.slot for (off, _), p in zip(TinyELF.iter_sig(tuple((None, p.addrspace, p.dtype, ()) for p in params)), params, strict=True)}
+
+
+def _outer_slot(u, packed: dict[int, int]) -> int | None:
+  from tinygrad.uop.ops import Ops
+  if u.op is Ops.PARAM:
+    return None if packed else u.arg.slot
+  if u.op.name == "REFPARAM":
+    return u.arg.slot
+  if packed and u.op is Ops.LOAD and u.src[0].op is Ops.INDEX and _root(u.src[0].src[0]).op is Ops.PARAM:
+    index = _evaluate(u.src[0].src[1], {}, packed)
+    return None if index is None else packed.get(index * u.dtype.itemsize)
+  return None
+
+
+def _evaluate(u, env, packed: dict[int, int]):
   from tinygrad.uop.ops import GroupOp, Ops, exec_alu
   if u.op in (Ops.AFTER, Ops.CAST, Ops.BITCAST):
-    return _evaluate(u.src[0], env)
+    return _evaluate(u.src[0], env, packed)
   if u.op is Ops.CONST:
     return u.arg
-  if u.op in (Ops.PARAM, Ops.RANGE):
-    return env.get(u.arg.slot if u.op is Ops.PARAM else u)
+  if u.op is Ops.RANGE:
+    return env.get(u)
+  if (slot := _outer_slot(u, packed)) is not None:
+    return env.get(slot)
   if u.op is Ops.LOAD:
-    table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env)
+    table, index = _root(u.src[0].src[0]), _evaluate(u.src[0].src[1], env, packed)
     return struct.unpack_from(u.dtype.fmt, table.arg, index * u.dtype.itemsize)[0] if table.op is Ops.BINARY and index is not None else None
-  if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env) for x in u.src]):
+  if u.op in GroupOp.ALU and None not in (values := [_evaluate(x, env, packed) for x in u.src]):
     return exec_alu(u.op, u.dtype, values, *([False] if "truncate_output" in inspect.signature(exec_alu).parameters else []))
   return None
 
 
 def _input_pokes(submit, kernargs) -> dict[int, tuple[int, int]]:
   from tinygrad.uop.ops import Ops
-  outer = dict(enumerate(submit.src[1:]))
+  outer, packed = dict(enumerate(submit.src[1:])), _packed_args(submit.src[0])
   kernargs_params = {i for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]) is kernargs}
   inputs = {i: _root(a.src[0]).arg.slot for i, a in outer.items() if a.op is Ops.GETADDR and _root(a.src[0]).op is Ops.PARAM}
   pokes = {}
   for call in (u for u in submit.src[0].src[0].toposort() if u.op is Ops.CALL and len(u.src) > 3):
     _, address, value, *_ = call.src[1:]
-    used = list({x.arg.slot for x in value.toposort() if x.op is Ops.PARAM} & inputs.keys())
-    if len(used) != 1 or not kernargs_params & {x.arg.slot for x in address.toposort() if x.op is Ops.PARAM}:
+    used = list({slot for x in value.toposort() if (slot := _outer_slot(x, packed)) is not None} & inputs.keys())
+    if len(used) != 1 or not kernargs_params & {_outer_slot(x, packed) for x in address.toposort()}:
       continue
     ranges = list(dict.fromkeys(x for x in (*address.toposort(), *value.toposort()) if x.op is Ops.RANGE))
     for trip in itertools.product(*[range(int(r.vmax) + 1) for r in ranges]):
       env = dict.fromkeys(outer, 0) | dict(zip(ranges, trip, strict=True))
-      if (offset := _evaluate(address, env)) % 8 == 0:
-        pokes[offset] = (inputs[used[0]], _evaluate(value, env))
+      if (offset := _evaluate(address, env, packed)) % 8 == 0:
+        pokes[offset] = (inputs[used[0]], _evaluate(value, env, packed))
   return pokes
 
 
-def _elf(image: bytes, rodata: int) -> bytes:
-  names, offset, data, headers = b"\0.text\0.rodata\0.shstrtab\0", 64, b"", []
-  for name, kind, address, content in [(0, 0, 0, b""), (1, 1, ALIGNMENT, image[:rodata]), (7, 1, ALIGNMENT + rodata, image[rodata:]), (15, 3, 0, names)]:
-    headers.append(struct.pack("<IIQQQQIIQQ", name, kind, 2 if kind == 1 else 0, address, offset, len(content), 0, 0, 1, 0))
+def _elf(image: bytes, rodata: int, entry: int) -> bytes:
+  spans = [(".text", 0, rodata), (".rodata", rodata, len(image))] if entry < rodata else \
+          [(".data", 0, rodata), (".rodata", rodata, entry), (".text", entry, len(image))]
+  sections, names = [(0, 0, 0, 0, b"")], b"\0"
+  for name, start, end in (span for span in spans if span[1] < span[2]):
+    sections.append((len(names), 1, 6 if name == ".text" else 2, ALIGNMENT + start, image[start:end]))
+    names += name.encode() + b"\0"
+  sections.append((len(names), 3, 0, 0, names + b".shstrtab\0"))
+  offset, data, headers = 64, b"", []
+  for name, kind, flags, address, content in sections:
+    headers.append(struct.pack("<IIQQQQIIQQ", name, kind, flags, address, offset, len(content), 0, 0, 1, 0))
     data, offset = data + content, offset + len(content)
-  return b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, 4, 3) + data + b"".join(headers)
+  header = struct.pack("<HHIQQQIHHHHHH", 1, 0xe0, 1, 0, 0, offset, 0, 64, 0, 0, 64, len(sections), len(sections) - 1)
+  return b"\x7fELF\x02\x01\x01" + bytes(9) + header + data + b"".join(headers)
 
 
 def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
@@ -155,7 +183,7 @@ def convert(jit, kwargs: dict, frame: dict, target) -> tuple[dict, memoryview]:
     if image_alloc not in programs:
       image = bytes(_patched(image_alloc, topo)[0])
       rodata = next(r for r in range(0, len(image) - 24, 64) if r + struct.unpack_from("<q", image, r + 16)[0] == entry)
-      programs[image_alloc] = blob.add(_elf(image, rodata))
+      programs[image_alloc] = blob.add(_elf(image, rodata, entry))
     start, size = _view(arguments.src[0])
     args = []
     for slot in range(start, start + unwrap(size), 8):

openpilot/sunnypilot/modeld_v2/helpers.py

@@ -264,8 +264,8 @@ def _program_uop(call: dict, lib: bytes):
     parameter = UOp.param(call["globals"][compact_slot], _dtype(dtype), math.prod(shape))
     image = (shape[0], shape[1]) if len(shape) == 3 and shape[2] == 4 else None
     parameters.append(parameter.replace(arg=_fields(parameter.arg, name=name, image=image)))
-  info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), globals=tuple(call["globals"]),
-                 outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
+  info = _fields(ProgramInfo, global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), params=tuple(parameters),
+                 globals=tuple(call["globals"]), outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=_fields(Target, **call["target"]))
   return UOp(Ops.PROGRAM, src=(UOp(Ops.SINK, arg=KernelInfo(name=call["name"])), UOp(Ops.LINEAR, src=tuple(parameters)),
                                UOp(Ops.SOURCE, arg=""), UOp(Ops.BINARY, arg=lib)), arg=info)
 

openpilot/sunnypilot/modeld_v2/tests/test_convert_upstream_pkl.py

@@ -6,6 +6,7 @@ See the LICENSE.md file in the root directory for more details.
 """
 import os
 import re
+import struct
 import sys
 import tempfile
 import unittest
@@ -25,8 +26,8 @@ if sys.platform == "darwin" and os.environ.get("DEV", "").startswith("MOCKUSB"):
 
 from openpilot.common.test import OpenpilotTestCase
 from openpilot.sunnypilot.modeld_v2 import convert_upstream_pkl
-from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import convert
-from openpilot.sunnypilot.modeld_v2.helpers import PlainJit, load_model
+from openpilot.sunnypilot.modeld_v2.convert_upstream_pkl import _elf, convert
+from openpilot.sunnypilot.modeld_v2.helpers import ALIGNMENT, PlainJit, load_model
 from openpilot.sunnypilot.modeld_v2.plain_artifact_compiler import capture_frame
 
 IMG, DESIRE, STATE, OUT = (6, 8, 8), (8,), (4, 16), (1, 32)
@@ -119,3 +120,18 @@ class TestConvertUpstreamPkl(OpenpilotTestCase):
     bundle = load_model(out, cache=False, verify=True)
     assert bundle["metadata"] == {"name": "tiny"} and bundle["input_specs"] == self.input_specs and bundle["output_specs"] == self.output_specs
     self.assert_matches_eager(bundle["run"], 6)
+
+
+class TestKernelElf(OpenpilotTestCase):
+  def test_sections_follow_the_code(self):
+    from tinygrad.runtime.support.elf import elf_loader
+    code = bytes(range(1, 129))
+    for rodata, entry in ((len(code), 0), (0, 256)):
+      image = bytearray(code + bytes(64) if entry < rodata else bytes(256) + code)
+      struct.pack_into("<q", image, rodata + 16, entry - rodata)
+      loaded, sections, _ = elf_loader(_elf(bytes(image), rodata, entry))
+      headers = {section.name: section.header for section in sections}
+      assert bytes(loaded[ALIGNMENT:]) == bytes(image), "the uploaded program differs from the pickle's"
+      assert headers[".rodata"].sh_addr == ALIGNMENT + rodata, "the loader reads the kernel descriptor at .rodata"
+      text = headers[".text"]
+      assert text.sh_addr <= ALIGNMENT + entry < text.sh_addr + text.sh_size, "the amd mock sizes the code from the entry to the end of .text"

release/ci/model_generator.py

@@ -13,26 +13,41 @@ from pathlib import Path
 from datetime import datetime, UTC
 
 
+_VERSION_REGEX = re.compile(r'^(?:\d+[a-zA-Z]+|\d+[vVbB]\d+|[vVbB]\d+|\d{4})$')
+_DECIMAL_VERSION_REGEX = re.compile(r'^([vVbB]?)(\d+(?:\.\d+)+)$')
+
+
+def _version(word: str) -> str | None:
+  match = _DECIMAL_VERSION_REGEX.match(word.strip('()[]{}.,;:'))
+  if match:
+    return match.group(1).upper() + match.group(2)
+  stripped = re.sub(r'[^a-zA-Z0-9]', '', word)
+  return stripped.upper() if _VERSION_REGEX.match(stripped) else None
+
+
+def _truncate(short_name: str) -> str:
+  return short_name[:8].rstrip('.')
+
+
 def create_short_name(full_name: str) -> str:
-  # Remove parentheses and extract alphanumeric words
+  # Remove parentheses and extract alphanumeric words, keeping decimals for versioning
   clean_name = re.sub(r'\([^)]*\)', '', full_name)
-  words = [re.sub(r'[^a-zA-Z0-9]', '', word) for word in clean_name.split() if re.sub(r'[^a-zA-Z0-9]', '', word)]
+  words = [re.sub(r'[^a-zA-Z0-9.]', '', word) for word in clean_name.split()]
+  words = [word for word in words if word]
 
   if len(words) == 1:
-    return words[0][:8].upper()
+    return _truncate((_version(words[0]) or words[0]).upper())
 
-  # Handle special case: Name + Version (e.g., "Word A1" -> "WordA1")
-  if len(words) == 2 and re.match(r'^[A-Za-z]\d+$', words[1]):
-    return (words[0] + words[1])[:8].upper()
+  # Handle special case: Name + Version (e.g., "Word A1" -> "WordA1", "Model v1.1" -> "ModelV1.1")
+  if len(words) == 2 and re.match(r'^[A-Za-z]?\d+$', words[1]):
+    return _truncate((words[0] + (_version(words[1]) or words[1])).upper())
 
   result = ""
   for word in words:
     # Version or number patterns
-    if (re.match(r'^\d+[a-zA-Z]+$', word) or
-        re.match(r'^\d+[vVbB]\d+$', word) or
-        re.match(r'^[vVbB]\d+$', word) or
-        re.match(r'^\d{4}$', word)):
-      result += word.upper()
+    version = _version(word)
+    if version:
+      result += version
     # All uppercase abbreviations (2-3 letters)
     elif re.match(r'^[A-Z]{2,3}$', word):
       result += word
@@ -45,7 +60,7 @@ def create_short_name(full_name: str) -> str:
       result += word
     else:
       result += word[0].upper()
-  return result[:8]
+  return _truncate(result)
 
 
 def create_pkl_name(full_name: str) -> str: