Merge branch 'master' into validate-fusion

* ci: lfs pointers survive gc

* comma got gc'ed

* format

.github/workflows/lfs-maintenance.yaml

@@ -33,7 +33,7 @@ jobs:
 
       - name: LFS Fetch
         run: |
-          git lfs fetch
+          git lfs fetch || echo "::warning::some objects are missing on commaai/openpilot-lfs; mirroring the rest"
 
       - name: Set up Git
         run: |
@@ -68,6 +68,7 @@ jobs:
           git config lfs.activitytimeout 300
           git config lfs.transfer.maxretries 5
           git config lfs.concurrenttransfers 4
+          git config lfs.allowincompletepush true
 
       - name: Push LFS
         id: sync-and-commit
@@ -81,3 +82,80 @@ jobs:
           done
           echo "All push attempts failed"
           exit 1
+
+      - name: Checkout sunnypilot
+        uses: actions/checkout@v4
+        with:
+          repository: 'sunnypilot/sunnypilot'
+          path: sunnypilot
+          fetch-depth: 0
+          filter: blob:limit=1k
+          lfs: false
+
+      - name: Reference objects on sunnypilot-lfs
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          set -euo pipefail
+          GIT_LFS_SKIP_SMUDGE=1 git clone --quiet https://huggingface.co/sunnypilot/sunnypilot-lfs hf-lfs
+          mkdir -p hf-lfs/all
+          cat > scan_pointers.py <<'EOF'
+          import re, subprocess, sys, threading
+          repo = sys.argv[1]
+          chk = subprocess.run(["git", "-C", repo, "cat-file", "--batch-all-objects",
+                                "--batch-check=%(objectname) %(objecttype) %(objectsize)"],
+                               capture_output=True, text=True, check=True).stdout
+          small = [f[0] for f in (l.split() for l in chk.splitlines()) if f[1] == "blob" and int(f[2]) < 1024]
+          p = subprocess.Popen(["git", "-C", repo, "cat-file", "--batch=%(objectsize)"],
+                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+          def feed():
+            for sha in small:
+              p.stdin.write(sha.encode() + b"\n")
+            p.stdin.close()
+          threading.Thread(target=feed, daemon=True).start()
+          rx = re.compile(rb"^version https://git-lfs\.github\.com/spec/v1\noid sha256:([0-9a-f]{64})\nsize ([0-9]+)\n")
+          seen = {}
+          for _ in small:
+            n = int(p.stdout.readline()); body = p.stdout.read(n); p.stdout.read(1)
+            m = rx.match(body)
+            if m:
+              seen.setdefault(m.group(1).decode(), m.group(2).decode())
+          for oid, size in sorted(seen.items()):
+            print(oid, size)
+          EOF
+          for repo in . sunnypilot; do
+            python3 scan_pointers.py "$repo"
+          done | sort -u | while read -r oid size; do
+            [ -e "hf-lfs/all/$oid" ] && continue
+            printf 'version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %s\n' "$oid" "$size" > "hf-lfs/all/$oid"
+            echo "$oid"
+          done > new_oids.txt
+          echo "new pointer files: $(wc -l < new_oids.txt)"
+
+          dropped=0
+          while mapfile -t -n 100 chunk && ((${#chunk[@]})); do
+            body=$(for oid in "${chunk[@]}"; do
+                     size=$(awk '/^size /{print $2}' "hf-lfs/all/$oid")
+                     printf '{"oid":"%s","size":%s}\n' "$oid" "$size"
+                   done | jq -s '{operation:"download",transfers:["basic"],objects:.}')
+            for attempt in 1 2 3 4 5; do
+              resp=$(curl -sS -u "user:${HF_TOKEN}" -w '\n%{http_code}' \
+                       -H 'Accept: application/vnd.git-lfs+json' -H 'Content-Type: application/vnd.git-lfs+json' \
+                       --data "$body" "${LFS_URL}/objects/batch") && [ "$(tail -n1 <<<"$resp")" = 200 ] && break
+              echo "batch check attempt $attempt failed ($(tail -n1 <<<"$resp")), retrying in 60s"; sleep 60
+            done
+            [ "$(tail -n1 <<<"$resp")" = 200 ] || { echo "batch check failed"; exit 1; }
+            for oid in $(sed '$d' <<<"$resp" | jq -r '.objects[] | select(.error) | .oid'); do
+              rm -f "hf-lfs/all/$oid"; dropped=$((dropped+1))
+            done
+          done < new_oids.txt
+          echo "dropped (not on HF): $dropped"
+
+          cd hf-lfs
+          git add -A all
+          if git diff --cached --quiet; then
+            echo "nothing new to reference"
+            exit 0
+          fi
+          git commit -q -m "Reference $(git diff --cached --name-only | wc -l | tr -d ' ') new LFS objects"
+          git push origin HEAD:main

openpilot/sunnypilot/modeld_v2/helpers.py

@@ -10,6 +10,7 @@ import builtins
 import contextlib
 import dataclasses
 import functools
+import gc
 import hashlib
 import io
 import itertools
@@ -111,17 +112,21 @@ def read_parts(path: str) -> tuple[dict, dict]:
   return parts, _MetadataUnpickler(io.BytesIO(bytes(blob[slice(*header["extras"])]))).load()
 
 
-def _extended(path: str, extend) -> tuple[dict, dict, str]:
+def _extended(path: str, extend) -> tuple[dict, dict, str, dict]:
   parts, extras = read_parts(path)
   added = extend(parts, extras) if extend else {}
-  return parts | added, extras, _parts_digest(added) if added else ""
+  return parts | added, extras, _parts_digest(added) if added else "", {key: parts[key] for key in added if key in parts}
 
 
 def load_model(path: str, device_map: dict | None = None, verify: bool = True, cache: bool = True, extend=None) -> dict:
-  parts, extras, extra = _extended(path, extend)
+  parts, extras, extra, replaced = _extended(path, extend)
   if cache and (cached := _read_cache(path, parts, device_map, extra)) is not None:
     return extras | cached
   shared: dict = {}
+  if verify and replaced:
+    for header, blob in replaced.values():
+      PlainJit(header, blob, device_map, verify, shared)
+    gc.collect()
   jits = {key: PlainJit(header, blob, device_map, verify, shared) for key, (header, blob) in parts.items()}
   if cache:
     _write_cache(path, jits, device_map, extra)