Konik Tooling

tools/auto_source.py

@@ -1,5 +1,19 @@
-#!/usr/bin/env python3
+#!/bin/sh
+""":"
+REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)"
+PYTHON_BIN="$REPO_ROOT/.venv/bin/python"
+
+if [ -x "$PYTHON_BIN" ]; then
+  exec "$PYTHON_BIN" "$0" "$@"
+fi
+
+exec python3 "$0" "$@"
+":"""
 import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
 from openpilot.tools.lib.logreader import LogReader, ReadMode
 
 

tools/cabana/utils/api.cc

@@ -9,6 +9,7 @@
 #include <QDebug>
 #include <QJsonDocument>
 #include <QNetworkRequest>
+#include <QUrl>
 
 #include <memory>
 #include <string>
@@ -39,6 +40,36 @@ std::optional<QString> getDongleId() {
 
 namespace CommaApi {
 
+const QString COMMA_API_HOST = "https://api.commadotai.com";
+const QString COMMA_API_HOST_ALIAS = "https://api.comma.ai";
+
+QString normalizeApiHost(const QString &host) {
+  QUrl url(host);
+  if (url.scheme().isEmpty()) {
+    url = QUrl("https://" + host);
+  }
+  QString normalized = QString("%1://%2").arg(url.scheme().toLower(), url.authority().toLower());
+  while (normalized.endsWith('/')) {
+    normalized.chop(1);
+  }
+  return normalized == COMMA_API_HOST_ALIAS ? COMMA_API_HOST : normalized;
+}
+
+QString authTokenForHost(const QString &api_host) {
+  QString token_json = QString::fromStdString(util::read_file(util::getenv("HOME") + "/.comma/auth.json"));
+  QJsonDocument json_d = QJsonDocument::fromJson(token_json.toUtf8());
+  QJsonObject auth = json_d.object();
+
+  const QString normalized_host = normalizeApiHost(api_host);
+  const QString token = auth["tokens"].toObject()[normalized_host].toString();
+  if (!token.isEmpty()) {
+    return token;
+  }
+
+  const QString env_host = normalizeApiHost(QString::fromStdString(util::getenv("API_HOST", COMMA_API_HOST.toStdString())));
+  return (normalized_host == COMMA_API_HOST || normalized_host == env_host) ? auth["access_token"].toString() : "";
+}
+
 EVP_PKEY *get_private_key() {
   static std::unique_ptr<EVP_PKEY, decltype(&EVP_PKEY_free)> pkey(nullptr, EVP_PKEY_free);
   if (!pkey) {
@@ -117,9 +148,7 @@ void HttpRequest::sendRequest(const QString &requestURL, const HttpRequest::Meth
   if (create_jwt) {
     token = CommaApi::create_jwt();
   } else {
-    QString token_json = QString::fromStdString(util::read_file(util::getenv("HOME") + "/.comma/auth.json"));
-    QJsonDocument json_d = QJsonDocument::fromJson(token_json.toUtf8());
-    token = json_d["access_token"].toString();
+    token = CommaApi::authTokenForHost(requestURL);
   }
 
   QNetworkRequest request;

tools/cabana/utils/api.h

@@ -10,6 +10,8 @@
 namespace CommaApi {
 
 const QString BASE_URL = util::getenv("API_HOST", "https://api.commadotai.com").c_str();
+QString normalizeApiHost(const QString &host);
+QString authTokenForHost(const QString &api_host);
 QByteArray rsa_sign(const QByteArray &data);
 QString create_jwt(const QJsonObject &payloads = {}, int expiry = 3600);
 

tools/lib/api.py

@@ -2,6 +2,7 @@ import os
 import requests
 
 from openpilot.common.params import Params
+from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, normalize_api_host
 
 
 def _use_konik_server():
@@ -11,27 +12,40 @@ def _use_konik_server():
     return False
 
 
-API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if _use_konik_server() else 'commadotai.com'}")
+API_HOST = normalize_api_host(os.getenv('API_HOST', KONIK_API_HOST if _use_konik_server() else DEFAULT_API_HOST))
+
+
+def route_api_hosts() -> list[str]:
+  if os.getenv("API_HOST"):
+    return [API_HOST]
+  return [DEFAULT_API_HOST, KONIK_API_HOST]
 
 # TODO: this should be merged into common.api
 
 class CommaApi:
-  def __init__(self, token=None):
+  def __init__(self, token=None, host: str | None = None):
+    self.host = normalize_api_host(host or API_HOST)
     self.session = requests.Session()
     self.session.headers['User-agent'] = 'OpenpilotTools'
     if token:
       self.session.headers['Authorization'] = 'JWT ' + token
 
   def request(self, method, endpoint, **kwargs):
-    with self.session.request(method, API_HOST + '/' + endpoint, **kwargs) as resp:
-      resp_json = resp.json()
-      if isinstance(resp_json, dict) and resp_json.get('error'):
-        if resp.status_code in [401, 403]:
-          raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py')
+    with self.session.request(method, self.host + '/' + endpoint.lstrip('/'), **kwargs) as resp:
+      try:
+        resp_json = resp.json()
+      except ValueError:
+        resp_json = None
 
-        e = APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error'])))
-        e.status_code = resp.status_code
-        raise e
+      if resp.status_code in [401, 403]:
+        raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py', resp.status_code)
+
+      if resp.status_code >= 400:
+        description = resp.text.strip() if resp.text else resp.reason
+        raise APIError(f"{resp.status_code}:{description}", resp.status_code)
+
+      if isinstance(resp_json, dict) and resp_json.get('error'):
+        raise APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error'])), resp.status_code)
       return resp_json
 
   def get(self, endpoint, **kwargs):
@@ -41,7 +55,10 @@ class CommaApi:
     return self.request('POST', endpoint, **kwargs)
 
 class APIError(Exception):
-  pass
+  def __init__(self, message, status_code=None):
+    super().__init__(message)
+    self.status_code = status_code
 
-class UnauthorizedError(Exception):
-  pass
+class UnauthorizedError(APIError):
+  def __init__(self, message, status_code=None):
+    super().__init__(message, status_code)

tools/lib/auth.py

@@ -1,4 +1,14 @@
-#!/usr/bin/env python3
+#!/bin/sh
+""":"
+REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)"
+PYTHON_BIN="$REPO_ROOT/.venv/bin/python"
+
+if [ -x "$PYTHON_BIN" ]; then
+  exec "$PYTHON_BIN" "$0" "$@"
+fi
+
+exec python3 "$0" "$@"
+":"""
 """
 Usage::
 
@@ -27,12 +37,15 @@ import pprint
 import sys
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
+from pathlib import Path
 from typing import Any
 from urllib.parse import parse_qs, urlencode
 
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
+
 from openpilot.common.params import Params
 from openpilot.tools.lib.api import APIError, CommaApi, UnauthorizedError
-from openpilot.tools.lib.auth_config import set_token, get_token
+from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, get_token, normalize_api_host, set_token
 
 
 def _use_konik_server():
@@ -42,7 +55,7 @@ def _use_konik_server():
     return False
 
 
-API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if _use_konik_server() else 'comma.ai'}")
+API_HOST = normalize_api_host(os.getenv('API_HOST', KONIK_API_HOST if _use_konik_server() else DEFAULT_API_HOST))
 PORT = 3000
 
 
@@ -69,7 +82,19 @@ class ClientRedirectHandler(BaseHTTPRequestHandler):
     pass  # this prevent http server from dumping messages to stdout
 
 
-def auth_redirect_link(method):
+def resolve_api_host(host: str) -> str:
+  if host in ("comma", "commaai", "commadotai"):
+    return DEFAULT_API_HOST
+  if host == "konik":
+    return KONIK_API_HOST
+  return normalize_api_host(host)
+
+
+def auth_redirect_api_host(api_host: str) -> str:
+  return "https://api.comma.ai" if normalize_api_host(api_host) == DEFAULT_API_HOST else normalize_api_host(api_host)
+
+
+def auth_redirect_link(method, api_host=API_HOST):
   provider_id = {
     'google': 'g',
     'apple': 'a',
@@ -77,7 +102,7 @@ def auth_redirect_link(method):
   }[method]
 
   params = {
-    'redirect_uri': f"{API_HOST}/v2/auth/{provider_id}/redirect/",
+    'redirect_uri': f"{auth_redirect_api_host(api_host)}/v2/auth/{provider_id}/redirect/",
     'state': f'service,localhost:{PORT}',
   }
 
@@ -92,7 +117,7 @@ def auth_redirect_link(method):
     return 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params)
   elif method == 'github':
     params.update({
-      'client_id': '28c4ecb54bb7272cb5a4',
+      'client_id': 'Ov23liy0AI1YCd15pypf' if normalize_api_host(api_host) == KONIK_API_HOST else '28c4ecb54bb7272cb5a4',
       'scope': 'read:user',
     })
     return 'https://github.com/login/oauth/authorize?' + urlencode(params)
@@ -108,8 +133,8 @@ def auth_redirect_link(method):
     raise NotImplementedError(f"no redirect implemented for method {method}")
 
 
-def login(method):
-  oauth_uri = auth_redirect_link(method)
+def login(method, api_host=API_HOST):
+  oauth_uri = auth_redirect_link(method, api_host)
 
   web_server = ClientRedirectServer(('localhost', PORT), ClientRedirectHandler)
   print(f'To sign in, use your browser and navigate to {oauth_uri}')
@@ -126,29 +151,35 @@ def login(method):
       break
 
   try:
-    auth_resp = CommaApi().post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']})
-    set_token(auth_resp['access_token'])
+    auth_resp = CommaApi(host=api_host).post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']})
+    return auth_resp['access_token']
   except APIError as e:
     print(f'Authentication Error: {e}', file=sys.stderr)
+    return None
 
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser(description='Login to your comma account')
+  parser.add_argument('--host', default=API_HOST, help='API host to authenticate with: comma, konik, or a URL')
   parser.add_argument('method', default='google', const='google', nargs='?', choices=['google', 'apple', 'github', 'jwt'])
   parser.add_argument('jwt', nargs='?')
 
   args = parser.parse_args()
+  api_host = resolve_api_host(args.host)
   if args.method == 'jwt':
     if args.jwt is None:
       print("method JWT selected, but no JWT was provided")
       exit(1)
 
-    set_token(args.jwt)
+    token = args.jwt
   else:
-    login(args.method)
+    token = login(args.method, api_host)
+    if token is None:
+      exit(1)
 
   try:
-    me = CommaApi(token=get_token()).get('/v1/me')
+    me = CommaApi(token=token, host=api_host).get('/v1/me')
+    set_token(token, api_host)
     print("Authenticated!")
     pprint.pprint(me)
   except UnauthorizedError:

tools/lib/auth_config.py

@@ -1,5 +1,7 @@
 import json
 import os
+from urllib.parse import urlparse, urlunparse
+
 from openpilot.system.hardware.hw import Paths
 
 
@@ -7,23 +9,88 @@ class MissingAuthConfigError(Exception):
   pass
 
 
-def get_token():
+DEFAULT_API_HOST = "https://api.commadotai.com"
+KONIK_API_HOST = "https://api.konik.ai"
+COMMA_API_HOST_ALIASES = {"https://api.comma.ai", DEFAULT_API_HOST}
+
+
+def normalize_api_host(host: str | None) -> str:
+  if not host:
+    host = DEFAULT_API_HOST
+
+  parsed = urlparse(host)
+  if not parsed.scheme:
+    parsed = urlparse(f"https://{host}")
+
+  normalized = urlunparse((parsed.scheme.lower(), parsed.netloc.lower(), "", "", "", "")).rstrip("/")
+  if normalized in COMMA_API_HOST_ALIASES:
+    return DEFAULT_API_HOST
+  return normalized
+
+
+def _auth_path() -> str:
+  return os.path.join(Paths.config_root(), 'auth.json')
+
+
+def _read_auth() -> dict:
   try:
-    with open(os.path.join(Paths.config_root(), 'auth.json')) as f:
-      auth = json.load(f)
-      return auth['access_token']
+    with open(_auth_path()) as f:
+      return json.load(f)
   except Exception:
+    return {}
+
+
+def get_token(host: str | None = None):
+  auth = _read_auth()
+  if not auth:
     return None
 
+  if host is not None:
+    normalized_host = normalize_api_host(host)
+    token = auth.get('tokens', {}).get(normalized_host)
+    if token is not None:
+      return token
+
+    env_host = normalize_api_host(os.getenv("API_HOST", DEFAULT_API_HOST))
+    if normalized_host == DEFAULT_API_HOST or normalized_host == env_host:
+      return auth.get('access_token')
+    return None
+
+  return auth.get('access_token')
+
+
+def set_token(token, host: str | None = None):
+  normalized_host = normalize_api_host(host or os.getenv("API_HOST", DEFAULT_API_HOST))
+  auth = _read_auth()
+  tokens = auth.setdefault('tokens', {})
+  tokens[normalized_host] = token
+
+  # Preserve the legacy field for older tooling. Avoid replacing a comma token
+  # when adding a Konik token unless this is an old-style hostless write.
+  if host is None or normalized_host == DEFAULT_API_HOST or 'access_token' not in auth:
+    auth['access_token'] = token
 
-def set_token(token):
   os.makedirs(Paths.config_root(), exist_ok=True)
-  with open(os.path.join(Paths.config_root(), 'auth.json'), 'w') as f:
-    json.dump({'access_token': token}, f)
+  with open(_auth_path(), 'w') as f:
+    json.dump(auth, f)
 
 
-def clear_token():
-  try:
-    os.unlink(os.path.join(Paths.config_root(), 'auth.json'))
-  except FileNotFoundError:
-    pass
+def clear_token(host: str | None = None):
+  if host is None:
+    try:
+      os.unlink(_auth_path())
+    except FileNotFoundError:
+      pass
+    return
+
+  auth = _read_auth()
+  if not auth:
+    return
+
+  normalized_host = normalize_api_host(host)
+  token = auth.get('tokens', {}).pop(normalized_host, None)
+  if normalized_host == DEFAULT_API_HOST and auth.get('access_token') == token:
+    auth.pop('access_token', None)
+
+  with open(_auth_path(), 'w') as f:
+    json.dump(auth, f)

tools/lib/logreader.py

@@ -204,11 +204,12 @@ def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode)
 
 
 def parse_indirect(identifier: str) -> str:
-  if "useradmin.comma.ai" in identifier:
+  parsed = urlparse(identifier)
+  if parsed.netloc in ("useradmin.comma.ai", "useradmin.konik.ai"):
     query = parse_qs(urlparse(identifier).query)
     identifier = query["onebox"][0]
-  elif "connect.comma.ai" in identifier:
-    path = urlparse(identifier).path.strip("/").split("/")
+  elif parsed.netloc in ("connect.comma.ai", "connect.konik.ai", "stable.konik.ai"):
+    path = parsed.path.strip("/").split("/")
     path = ['/'.join(path[:2]), *path[2:]]  # recombine log id
 
     identifier = path[0]

tools/lib/route.py

@@ -7,7 +7,7 @@ from collections import defaultdict
 from itertools import chain
 
 from openpilot.tools.lib.auth_config import get_token
-from openpilot.tools.lib.api import APIError, CommaApi
+from openpilot.tools.lib.api import APIError, CommaApi, route_api_hosts
 from openpilot.tools.lib.helpers import RE
 
 
@@ -23,6 +23,7 @@ class FileName:
 
 class Route:
   def __init__(self, name, data_dir=None):
+    self._api_host = None
     self._metadata = None
     self._name = RouteName(name)
     self.files = None
@@ -35,8 +36,7 @@ class Route:
   @property
   def metadata(self):
     if not self._metadata:
-      api = CommaApi(get_token())
-      self._metadata = api.get('v1/route/' + self.name.canonical_name)
+      self._metadata = self._get_route_metadata()
     return self._metadata
 
   @property
@@ -73,9 +73,9 @@ class Route:
 
   # TODO: refactor this, it's super repetitive
   def _get_segments_remote(self):
-    api = CommaApi(get_token())
-    route_files = api.get('v1/route/' + self.name.canonical_name + '/files')
+    route_files = self._get_route_files()
     self.files = list(chain.from_iterable(route_files.values()))
+    metadata_url = self.metadata['url']
 
     segments = {}
     for url in self.files:
@@ -90,7 +90,7 @@ class Route:
           url if fn in FileName.DCAMERA else segments[segment_name].dcamera_path,
           url if fn in FileName.ECAMERA else segments[segment_name].ecamera_path,
           url if fn in FileName.QCAMERA else segments[segment_name].qcamera_path,
-          self.metadata['url'],
+          metadata_url,
         )
       else:
         segments[segment_name] = Segment(
@@ -101,11 +101,35 @@ class Route:
           url if fn in FileName.DCAMERA else None,
           url if fn in FileName.ECAMERA else None,
           url if fn in FileName.QCAMERA else None,
-          self.metadata['url'],
+          metadata_url,
         )
 
     return sorted(segments.values(), key=lambda seg: seg.name.segment_num)
 
+  def _get_route_metadata(self):
+    return self._get_route_endpoint('v1/route/' + self.name.canonical_name)
+
+  def _get_route_files(self):
+    return self._get_route_endpoint('v1/route/' + self.name.canonical_name + '/files')
+
+  def _get_route_endpoint(self, endpoint: str):
+    hosts = [self._api_host] if self._api_host is not None else route_api_hosts()
+    errors = {}
+
+    for host in hosts:
+      try:
+        api = CommaApi(get_token(host), host=host)
+        response = api.get(endpoint)
+        self._api_host = host
+        return response
+      except APIError as e:
+        errors[host] = e
+        if e.status_code == 404 and self._api_host is None:
+          continue
+        raise
+
+    raise APIError(f"route {self.name.canonical_name} not found on: {', '.join(errors.keys())}", 404)
+
   def _get_segments_local(self, data_dir):
     files = os.listdir(data_dir)
     segment_files = defaultdict(list)
@@ -306,13 +330,20 @@ class SegmentName:
 
 @cache
 def get_max_seg_number_cached(sr: 'SegmentRange') -> int:
-  try:
-    api = CommaApi(get_token())
-    max_seg_number = api.get("/v1/route/" + sr.route_name.replace("/", "|"))["maxqlog"]
-    assert isinstance(max_seg_number, int)
-    return max_seg_number
-  except Exception as e:
-    raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from e
+  errors = {}
+  for host in route_api_hosts():
+    try:
+      api = CommaApi(get_token(host), host=host)
+      max_seg_number = api.get("/v1/route/" + sr.route_name.replace("/", "|"))["maxqlog"]
+      assert isinstance(max_seg_number, int)
+      return max_seg_number
+    except APIError as e:
+      errors[host] = e
+      if e.status_code == 404:
+        continue
+      raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from e
+
+  raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from next(iter(errors.values()), None)
 
 
 class SegmentRange:

tools/lib/tests/test_auth_config.py

@@ -0,0 +1,12 @@
+from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, get_token, set_token
+
+
+def test_set_token_preserves_comma_token_when_adding_konik(mocker, tmp_path):
+  mocker.patch("openpilot.tools.lib.auth_config.Paths.config_root", return_value=str(tmp_path))
+
+  set_token("comma-token", DEFAULT_API_HOST)
+  set_token("konik-token", KONIK_API_HOST)
+
+  assert get_token(DEFAULT_API_HOST) == "comma-token"
+  assert get_token(KONIK_API_HOST) == "konik-token"
+  assert get_token() == "comma-token"

tools/lib/tests/test_logreader.py

@@ -95,10 +95,15 @@ class TestLogReader:
     (f"{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"),
     (f"https://connect.comma.ai/{TEST_ROUTE}/13/14", f"{TEST_ROUTE}/0:1"),
     (f"https://connect.comma.ai/{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"),
+    (f"https://connect.konik.ai/{TEST_ROUTE}/13/14", f"{TEST_ROUTE}/0:1"),
+    (f"https://stable.konik.ai/{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"),
   ])
   def test_parse_indirect_accepts_second_window_route_style(self, identifier, expected):
     assert parse_indirect(identifier) == expected
 
+  def test_parse_indirect_accepts_konik_useradmin(self):
+    assert parse_indirect(f"https://useradmin.konik.ai/?onebox={TEST_ROUTE}") == TEST_ROUTE
+
   @pytest.mark.parametrize("cache_enabled", [True, False])
   def test_direct_parsing(self, mocker, cache_enabled):
     file_exists_mock = mocker.patch("openpilot.tools.lib.filereader.file_exists")

tools/lib/tests/test_route_library.py

@@ -1,6 +1,10 @@
 from collections import namedtuple
 
-from openpilot.tools.lib.route import SegmentName
+import pytest
+
+from openpilot.tools.lib.api import APIError, UnauthorizedError
+from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST
+from openpilot.tools.lib.route import Route, SegmentName
 
 class TestRouteLibrary:
   def test_segment_name_formats(self):
@@ -25,3 +29,52 @@ class TestRouteLibrary:
 
     for case in cases:
       _validate(case)
+
+  def test_route_falls_back_to_konik_when_comma_route_missing(self, mocker):
+    route_name = "59679e5e40b60ce0/0000091b--316e931f07"
+    file_url = "https://konik.example/59679e5e40b60ce0/0000091b--316e931f07/0/qlog.zst"
+    calls = []
+
+    class FakeApi:
+      def __init__(self, token=None, host=None):
+        self.host = host
+
+      def get(self, endpoint):
+        calls.append((self.host, endpoint))
+        if self.host == DEFAULT_API_HOST:
+          raise APIError("404:not found", 404)
+        if endpoint.endswith("/files"):
+          return {"qlogs": [file_url]}
+        return {"url": "https://connect.konik.ai/59679e5e40b60ce0/0000091b--316e931f07"}
+
+    mocker.patch("openpilot.tools.lib.route.route_api_hosts", return_value=[DEFAULT_API_HOST, KONIK_API_HOST])
+    mocker.patch("openpilot.tools.lib.route.get_token", return_value=None)
+    mocker.patch("openpilot.tools.lib.route.CommaApi", FakeApi)
+
+    route = Route(route_name)
+
+    assert route.qlog_paths() == [file_url]
+    assert calls[0] == (DEFAULT_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files")
+    assert calls[1] == (KONIK_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files")
+    assert calls[2] == (KONIK_API_HOST, f"v1/route/{route_name.replace('/', '|')}")
+
+  def test_route_does_not_fall_back_to_konik_when_comma_unauthorized(self, mocker):
+    route_name = "59679e5e40b60ce0/0000091b--316e931f07"
+    calls = []
+
+    class FakeApi:
+      def __init__(self, token=None, host=None):
+        self.host = host
+
+      def get(self, endpoint):
+        calls.append((self.host, endpoint))
+        raise UnauthorizedError("unauthorized", 401)
+
+    mocker.patch("openpilot.tools.lib.route.route_api_hosts", return_value=[DEFAULT_API_HOST, KONIK_API_HOST])
+    mocker.patch("openpilot.tools.lib.route.get_token", return_value=None)
+    mocker.patch("openpilot.tools.lib.route.CommaApi", FakeApi)
+
+    with pytest.raises(UnauthorizedError):
+      Route(route_name)
+
+    assert calls == [(DEFAULT_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files")]

tools/replay/api.cc

@@ -7,7 +7,10 @@
 #include <openssl/sha.h>
 
 #include <cassert>
+#include <algorithm>
+#include <cctype>
 #include <chrono>
+#include <cstdlib>
 #include <iostream>
 
 #include "common/params.h"
@@ -16,6 +19,30 @@
 
 namespace CommaApi2 {
 
+const std::string COMMA_API_HOST = "https://api.commadotai.com";
+const std::string COMMA_API_HOST_ALIAS = "https://api.comma.ai";
+const std::string KONIK_API_HOST = "https://api.konik.ai";
+
+std::string normalize_api_host(const std::string &host) {
+  std::string normalized = host.empty() ? COMMA_API_HOST : host;
+  while (!normalized.empty() && normalized.back() == '/') {
+    normalized.pop_back();
+  }
+  if (normalized.find("://") == std::string::npos) {
+    normalized = "https://" + normalized;
+  }
+  std::transform(normalized.begin(), normalized.end(), normalized.begin(), [](unsigned char c) { return std::tolower(c); });
+  return normalized == COMMA_API_HOST_ALIAS ? COMMA_API_HOST : normalized;
+}
+
+std::vector<std::string> route_api_hosts() {
+  const char *api_host = std::getenv("API_HOST");
+  if (api_host != nullptr && api_host[0] != '\0') {
+    return {normalize_api_host(api_host)};
+  }
+  return {COMMA_API_HOST, KONIK_API_HOST};
+}
+
 // Base64 URL-safe character set (uses '-' and '_' instead of '+' and '/')
 static const std::string base64url_chars =
     "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
@@ -104,7 +131,25 @@ std::string create_jwt(const json11::Json &extra, int exp_time) {
   return jwt + "." + base64url_encode(signature);
 }
 
-std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry) {
+std::string auth_token_for_host(const json11::Json &json, const std::string &api_host) {
+  const std::string normalized_host = normalize_api_host(api_host);
+
+  const json11::Json tokens = json["tokens"];
+  if (tokens.is_object()) {
+    const std::string token = tokens[normalized_host].string_value();
+    if (!token.empty()) {
+      return token;
+    }
+  }
+
+  const std::string env_host = normalize_api_host(util::getenv("API_HOST", COMMA_API_HOST));
+  if (normalized_host == COMMA_API_HOST || normalized_host == env_host) {
+    return json["access_token"].string_value();
+  }
+  return "";
+}
+
+std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry, const std::string &api_host) {
   if (use_jwt) {
     return create_jwt(payloads, expiry);
   }
@@ -116,15 +161,15 @@ std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry)
     std::cerr << "Error parsing auth.json " << err << std::endl;
     return "";
   }
-  return json["access_token"].string_value();
+  return auth_token_for_host(json, api_host);
 }
 
-std::string httpGet(const std::string &url, long *response_code) {
+std::string httpGet(const std::string &url, long *response_code, const std::string &api_host) {
   CURL *curl = curl_easy_init();
   assert(curl);
 
   std::string readBuffer;
-  const std::string token = CommaApi2::create_token(!Hardware::PC());
+  const std::string token = CommaApi2::create_token(!Hardware::PC(), {}, 3600, api_host);
 
   // Set up the lambda for the write callback
   // The '+' makes the lambda non-capturing, allowing it to be used as a C function pointer

tools/replay/api.h

@@ -2,6 +2,7 @@
 
 #include <curl/curl.h>
 #include <string>
+#include <vector>
 
 #include "common/util.h"
 #include "third_party/json11/json11.hpp"
@@ -9,7 +10,9 @@
 namespace CommaApi2 {
 
 const std::string BASE_URL = util::getenv("API_HOST", "https://api.commadotai.com").c_str();
-std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600);
-std::string httpGet(const std::string &url, long *response_code = nullptr);
+std::string normalize_api_host(const std::string &host);
+std::vector<std::string> route_api_hosts();
+std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600, const std::string &api_host = BASE_URL);
+std::string httpGet(const std::string &url, long *response_code = nullptr, const std::string &api_host = BASE_URL);
 
 }  // namespace CommaApi2

tools/replay/route.cc

@@ -15,7 +15,7 @@ Route::Route(const std::string &route, const std::string &data_dir, bool auto_so
 
 RouteIdentifier Route::parseRoute(const std::string &str) {
   RouteIdentifier identifier = {};
-  static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)");
+  static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?(/[qra])?$)");
   std::smatch match;
 
   if (std::regex_match(str, match, pattern)) {
@@ -104,30 +104,38 @@ bool Route::loadFromAutoSource() {
 }
 
 bool Route::loadFromServer(int retries) {
-  const std::string url = CommaApi2::BASE_URL + "/v1/route/" + route_.str + "/files";
-  for (int i = 1; i <= retries; ++i) {
-    long response_code = 0;
-    std::string result = CommaApi2::httpGet(url, &response_code);
-    if (response_code == 200) {
-      return loadFromJson(result);
+  const auto api_hosts = CommaApi2::route_api_hosts();
+  for (const auto &api_host : api_hosts) {
+    const std::string url = api_host + "/v1/route/" + route_.str + "/files";
+    for (int i = 1; i <= retries; ++i) {
+      long response_code = 0;
+      std::string result = CommaApi2::httpGet(url, &response_code, api_host);
+      if (response_code == 200) {
+        return loadFromJson(result);
+      }
+
+      if (response_code == 401 || response_code == 403) {
+        rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<");
+        err_ = RouteLoadError::Unauthorized;
+        return false;
+      }
+      if (response_code == 404) {
+        err_ = RouteLoadError::FileNotFound;
+        break;
+      }
+
+      err_ = RouteLoadError::NetworkError;
+      rWarning("Retrying %d/%d", i, retries);
+      util::sleep_for(3000);
     }
 
-    if (response_code == 401 || response_code == 403) {
-      rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<");
-      err_ = RouteLoadError::Unauthorized;
-      break;
+    if (err_ == RouteLoadError::NetworkError) {
+      return false;
     }
-    if (response_code == 404) {
-      rWarning("The specified route could not be found on the server.");
-      err_ = RouteLoadError::FileNotFound;
-      break;
-    }
-
-    err_ = RouteLoadError::NetworkError;
-    rWarning("Retrying %d/%d", i, retries);
-    util::sleep_for(3000);
   }
 
+  rWarning(api_hosts.size() > 1 ? "The specified route could not be found on comma or Konik." : "The specified route could not be found on the server.");
+  err_ = RouteLoadError::FileNotFound;
   return false;
 }
 

tools/scripts/fetch_image_from_route.py

@@ -1,7 +1,18 @@
-#!/usr/bin/env python3
-import sys
+#!/bin/sh
+""":"
+REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)"
+PYTHON_BIN="$REPO_ROOT/.venv/bin/python"
 
-from openpilot.starpilot.common.starpilot_utilities import use_konik_server
+if [ -x "$PYTHON_BIN" ]; then
+  exec "$PYTHON_BIN" "$0" "$@"
+fi
+
+exec python3 "$0" "$@"
+":"""
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
 
 if len(sys.argv) < 4:
   print(f"{sys.argv[0]} <route> <segment> <frame number> [front|wide|driver]")
@@ -14,25 +25,22 @@ cameras = {
   "driver": "dcameras"
 }
 
-import requests
 from PIL import Image
-from openpilot.tools.lib.auth_config import get_token
 from openpilot.tools.lib.framereader import FrameReader
-
-jwt = get_token()
+from openpilot.tools.lib.route import Route
 
 route = sys.argv[1]
 segment = int(sys.argv[2])
 frame = int(sys.argv[3])
 camera = cameras[sys.argv[4]] if len(sys.argv) > 4 and sys.argv[4] in cameras else "cameras"
 
-url = f"https://api.{'konik.ai' if use_konik_server() else 'commadotai.com'}/v1/route/{route}/files"
-r = requests.get(url, headers={"Authorization": f"JWT {jwt}"}, timeout=10)
-assert r.status_code == 200
-print("got api response")
-
-segments = r.json()[camera]
-if segment >= len(segments):
+route_files = Route(route)
+segments = {
+  "cameras": route_files.camera_paths(),
+  "ecameras": route_files.ecamera_paths(),
+  "dcameras": route_files.dcamera_paths(),
+}[camera]
+if segment >= len(segments) or segments[segment] is None:
   raise Exception(f"segment {segment} not found, got {len(segments)} segments")
 
 fr = FrameReader(segments[segment])