ci: lfs pointers survive gc (#2066)

* ci: lfs pointers survive gc

* comma got gc'ed

* format

.github/workflows/lfs-maintenance.yaml

@@ -33,7 +33,7 @@ jobs:
 
       - name: LFS Fetch
         run: |
-          git lfs fetch
+          git lfs fetch || echo "::warning::some objects are missing on commaai/openpilot-lfs; mirroring the rest"
 
       - name: Set up Git
         run: |
@@ -68,6 +68,7 @@ jobs:
           git config lfs.activitytimeout 300
           git config lfs.transfer.maxretries 5
           git config lfs.concurrenttransfers 4
+          git config lfs.allowincompletepush true
 
       - name: Push LFS
         id: sync-and-commit
@@ -81,3 +82,80 @@ jobs:
           done
           echo "All push attempts failed"
           exit 1
+
+      - name: Checkout sunnypilot
+        uses: actions/checkout@v4
+        with:
+          repository: 'sunnypilot/sunnypilot'
+          path: sunnypilot
+          fetch-depth: 0
+          filter: blob:limit=1k
+          lfs: false
+
+      - name: Reference objects on sunnypilot-lfs
+        env:
+          HF_TOKEN: ${{ secrets.HF_TOKEN }}
+        run: |
+          set -euo pipefail
+          GIT_LFS_SKIP_SMUDGE=1 git clone --quiet https://huggingface.co/sunnypilot/sunnypilot-lfs hf-lfs
+          mkdir -p hf-lfs/all
+          cat > scan_pointers.py <<'EOF'
+          import re, subprocess, sys, threading
+          repo = sys.argv[1]
+          chk = subprocess.run(["git", "-C", repo, "cat-file", "--batch-all-objects",
+                                "--batch-check=%(objectname) %(objecttype) %(objectsize)"],
+                               capture_output=True, text=True, check=True).stdout
+          small = [f[0] for f in (l.split() for l in chk.splitlines()) if f[1] == "blob" and int(f[2]) < 1024]
+          p = subprocess.Popen(["git", "-C", repo, "cat-file", "--batch=%(objectsize)"],
+                               stdin=subprocess.PIPE, stdout=subprocess.PIPE)
+          def feed():
+            for sha in small:
+              p.stdin.write(sha.encode() + b"\n")
+            p.stdin.close()
+          threading.Thread(target=feed, daemon=True).start()
+          rx = re.compile(rb"^version https://git-lfs\.github\.com/spec/v1\noid sha256:([0-9a-f]{64})\nsize ([0-9]+)\n")
+          seen = {}
+          for _ in small:
+            n = int(p.stdout.readline()); body = p.stdout.read(n); p.stdout.read(1)
+            m = rx.match(body)
+            if m:
+              seen.setdefault(m.group(1).decode(), m.group(2).decode())
+          for oid, size in sorted(seen.items()):
+            print(oid, size)
+          EOF
+          for repo in . sunnypilot; do
+            python3 scan_pointers.py "$repo"
+          done | sort -u | while read -r oid size; do
+            [ -e "hf-lfs/all/$oid" ] && continue
+            printf 'version https://git-lfs.github.com/spec/v1\noid sha256:%s\nsize %s\n' "$oid" "$size" > "hf-lfs/all/$oid"
+            echo "$oid"
+          done > new_oids.txt
+          echo "new pointer files: $(wc -l < new_oids.txt)"
+
+          dropped=0
+          while mapfile -t -n 100 chunk && ((${#chunk[@]})); do
+            body=$(for oid in "${chunk[@]}"; do
+                     size=$(awk '/^size /{print $2}' "hf-lfs/all/$oid")
+                     printf '{"oid":"%s","size":%s}\n' "$oid" "$size"
+                   done | jq -s '{operation:"download",transfers:["basic"],objects:.}')
+            for attempt in 1 2 3 4 5; do
+              resp=$(curl -sS -u "user:${HF_TOKEN}" -w '\n%{http_code}' \
+                       -H 'Accept: application/vnd.git-lfs+json' -H 'Content-Type: application/vnd.git-lfs+json' \
+                       --data "$body" "${LFS_URL}/objects/batch") && [ "$(tail -n1 <<<"$resp")" = 200 ] && break
+              echo "batch check attempt $attempt failed ($(tail -n1 <<<"$resp")), retrying in 60s"; sleep 60
+            done
+            [ "$(tail -n1 <<<"$resp")" = 200 ] || { echo "batch check failed"; exit 1; }
+            for oid in $(sed '$d' <<<"$resp" | jq -r '.objects[] | select(.error) | .oid'); do
+              rm -f "hf-lfs/all/$oid"; dropped=$((dropped+1))
+            done
+          done < new_oids.txt
+          echo "dropped (not on HF): $dropped"
+
+          cd hf-lfs
+          git add -A all
+          if git diff --cached --quiet; then
+            echo "nothing new to reference"
+            exit 0
+          fi
+          git commit -q -m "Reference $(git diff --cached --name-only | wc -l | tr -d ' ') new LFS objects"
+          git push origin HEAD:main

openpilot/sunnypilot/modeld_v2/helpers.py

@@ -258,7 +258,7 @@ def _program_uop(call: dict, lib: bytes):
     parameter = UOp.param(call["globals"][compact_slot], _dtype(dtype), math.prod(shape))
     image = (shape[0], shape[1]) if len(shape) == 3 and shape[2] == 4 else None
     parameters.append(parameter.replace(arg=dataclasses.replace(parameter.arg, name=name, image=image)))
-  info = ProgramInfo(global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), globals=tuple(call["globals"]),
+  info = ProgramInfo(global_size=tuple(call["global_size"]), local_size=tuple(call["local_size"]), vars=(), globals=tuple(call["globals"]),
                      outs=tuple(call["outs"]), ins=tuple(call["ins"]), target=Target(**call["target"]))
   return UOp(Ops.PROGRAM, src=(UOp(Ops.SINK, arg=KernelInfo(name=call["name"])), UOp(Ops.LINEAR, src=tuple(parameters)),
                                UOp(Ops.SOURCE, arg=""), UOp(Ops.BINARY, arg=lib)), arg=info)

openpilot/sunnypilot/modeld_v2/plain_artifact_compiler.py

@@ -39,7 +39,6 @@ def capture_frame(jit, kwargs: dict, output_container: str | None = None):
 
 
 def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict, memoryview]:
-  from tinygrad.dtype import AddrSpace
   from tinygrad.uop.ops import Ops
   sink = tempfile.TemporaryFile(dir=".")
   blob, bases, input_specs = Blob(sink), {}, {}
@@ -61,12 +60,10 @@ def serialize_jit(jit, kernel_linear, check: dict | None = None) -> tuple[dict,
   for call in kernel_linear.src:
     ast, arguments = call.src[0], [describe(argument) for argument in call.src[1:]]
     if ast.op is Ops.PROGRAM:
-      elf, info, compact = ast.to_elf(), ast.arg, {slot: i for i, slot in enumerate(ast.arg.globals)}
-      if any(p.addrspace is AddrSpace.ALU for p in ast.kernel_params):
-        raise ValueError(f"{elf.name} takes a variable, which plain artifacts don't support")
+      elf, info = ast.to_elf(), ast.arg
       global_size, local_size = info.launch_dims({})
       calls.append({"kind": "program", "name": elf.name, "lib": blob.add(bytes(elf.lib)), "target": dataclasses.asdict(elf.target),
-                    "signature": [[p.arg.name, compact[p.arg.slot], repr(p.dtype), [int(x) for x in p._shape]] for p in ast.kernel_params],
+                    "signature": [[name, slot, repr(dtype), [int(x) for x in shape]] for name, slot, dtype, shape in elf.signature],
                     "global_size": [int(x) for x in global_size], "local_size": [int(x) for x in local_size],
                     "globals": list(info.globals), "outs": list(info.outs), "ins": list(info.ins), "args": arguments})
       outs, ins = info.outs, info.ins