CommaCarSegments: public database of segments for each platform (#31114)

* comma car segments

* comma car segments for test_models

* oneliner

Co-authored-by: Shane Smiskol <shane@smiskol.com>

* better name

* not used here

* sort

* remove print

* better comment

---------

Co-authored-by: Shane Smiskol <shane@smiskol.com>
old-commit-hash: caa9153974c21fd827d0e991f1789cf3e81b5c3b

selfdrive/car/tests/test_models.py

@@ -23,9 +23,8 @@ from openpilot.selfdrive.car.tests.routes import non_tested_cars, routes, CarTes
 from openpilot.selfdrive.controls.controlsd import Controls
 from openpilot.selfdrive.test.helpers import read_segment_list
 from openpilot.system.hardware.hw import DEFAULT_DOWNLOAD_CACHE_ROOT
-from openpilot.tools.lib.openpilotci import get_url
+from openpilot.tools.lib.comma_car_segments import get_url
 from openpilot.tools.lib.logreader import LogReader
-from openpilot.tools.lib.sanitizer import sanitize
 from openpilot.tools.lib.route import Route, SegmentName, RouteName
 
 from panda.tests.libpanda import libpanda_py
@@ -86,7 +85,6 @@ class TestCarModelBase(unittest.TestCase):
 
   @classmethod
   def get_testing_data_from_logreader(cls, lr):
-    lr = sanitize(lr)
     car_fw = []
     can_msgs = []
     cls.elm_frame = None

tools/lib/comma_car_segments.py

@@ -0,0 +1,82 @@
+import os
+import requests
+
+# Forks with additional car support can fork the commaCarSegments repo on huggingface or host the LFS files themselves
+COMMA_CAR_SEGMENTS_REPO = os.environ.get("COMMA_CAR_SEGMENTS_REPO", "https://huggingface.co/datasets/commaai/commaCarSegments")
+COMMA_CAR_SEGMENTS_BRANCH = os.environ.get("COMMA_CAR_SEGMENTS_BRANCH", "main")
+
+COMMA_CAR_SEGMENTS_LFS_INSTANCE = os.environ.get("COMMA_CAR_SEGMENTS_LFS_INSTANCE", "https://huggingface.co/datasets/commaai/commaCarSegments")
+
+def get_comma_car_segments_database():
+  return requests.get(get_repo_raw_url("database.json")).json()
+
+
+# Helpers related to interfacing with the openpilot-data repository, which contains a collection of public segments for users to perform validation on.
+
+def parse_lfs_pointer(text):
+  header, lfs_version = text.splitlines()[0].split(" ")
+  assert header == "version"
+  assert lfs_version == "https://git-lfs.github.com/spec/v1"
+
+  header, oid_raw = text.splitlines()[1].split(" ")
+  assert header == "oid"
+  header, oid = oid_raw.split(":")
+  assert header == "sha256"
+
+  header, size = text.splitlines()[2].split(" ")
+  assert header == "size"
+
+  return oid, size
+
+def get_lfs_file_url(oid, size):
+  data = {
+    "operation": "download",
+    "transfers": [ "basic" ],
+    "objects": [
+      {
+        "oid": oid,
+        "size": int(size)
+      }
+    ],
+    "hash_algo": "sha256"
+  }
+
+  headers = {
+    "Accept": "application/vnd.git-lfs+json",
+    "Content-Type": "application/vnd.git-lfs+json"
+  }
+
+  response = requests.post(f"{COMMA_CAR_SEGMENTS_LFS_INSTANCE}.git/info/lfs/objects/batch", json=data, headers=headers)
+
+  assert response.ok
+
+  obj = response.json()["objects"][0]
+
+  assert "error" not in obj, obj
+
+  return obj["actions"]["download"]["href"]
+
+def get_repo_raw_url(path):
+  if "huggingface" in COMMA_CAR_SEGMENTS_REPO:
+    return f"{COMMA_CAR_SEGMENTS_REPO}/raw/{COMMA_CAR_SEGMENTS_BRANCH}/{path}"
+
+def get_repo_url(path):
+  # Automatically switch to LFS if we are requesting a file that is stored in LFS
+
+  response = requests.head(get_repo_raw_url(path))
+
+  if "text/plain" in response.headers.get("content-type"):
+    # This is an LFS pointer, so download the raw data from lfs
+    response = requests.get(get_repo_raw_url(path))
+    assert response.status_code == 200
+    oid, size = parse_lfs_pointer(response.text)
+
+    return get_lfs_file_url(oid, size)
+  else:
+    # File has not been uploaded to LFS yet
+    # (either we are on a fork where the data hasn't been pushed to LFS yet, or the CI job to push hasn't finished)
+    return get_repo_raw_url(path)
+
+
+def get_url(route, segment, file="rlog.bz2"):
+  return get_repo_url(f"segments/{route.replace('|', '/')}/{segment}/{file}")

tools/lib/logreader.py

@@ -209,6 +209,10 @@ class LogReader:
     return _LogFileReader("", dat=dat)
 
 
+def get_first_message(lr: LogIterable, msg_type):
+  return next(filter(lambda m: m.which() == msg_type, lr), None)
+
+
 if __name__ == "__main__":
   import codecs
   # capnproto <= 0.8.0 throws errors converting byte data to string

tools/lib/tests/test_comma_car_segments.py

@@ -0,0 +1,41 @@
+
+
+import unittest
+
+import requests
+from openpilot.tools.lib.comma_car_segments import get_comma_car_segments_database, get_url
+from openpilot.tools.lib.logreader import LogReader, get_first_message
+from openpilot.tools.lib.route import SegmentRange
+
+
+class TestCommaCarSegments(unittest.TestCase):
+  def test_database(self):
+    database = get_comma_car_segments_database()
+
+    platforms = database.keys()
+
+    assert len(platforms) > 100
+
+  def test_download_segment(self):
+    database = get_comma_car_segments_database()
+
+    fp = "SUBARU FORESTER 2019"
+
+    segment = database[fp][0]
+
+    sr = SegmentRange(segment)
+
+    url = get_url(sr.route_name, sr._slice)
+
+    resp = requests.get(url)
+    self.assertEqual(resp.status_code, 200)
+
+    lr = LogReader(url)
+
+    CP = get_first_message(lr, "carParams").carParams
+
+    self.assertEqual(CP.carFingerprint, fp)
+
+
+if __name__ == "__main__":
+  unittest.main()