diff --git a/logs b/logs index a232a5caf..1d73c3989 100755 --- a/logs +++ b/logs @@ -34,9 +34,11 @@ import requests import zstandard from cereal import log +from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, get_token, normalize_api_host -API_HOST = os.getenv("COMMA_API_HOST", "https://api.commadotai.com").rstrip("/") +API_HOST = normalize_api_host(os.getenv("COMMA_API_HOST") or os.getenv("API_HOST") or DEFAULT_API_HOST) +API_HOSTS = [API_HOST] if os.getenv("COMMA_API_HOST") or os.getenv("API_HOST") else [DEFAULT_API_HOST, KONIK_API_HOST] ROUTE_ID_RE = re.compile(r"([0-9a-f]{16})/([^/]+)") @@ -146,12 +148,12 @@ def parse_route_id(raw: str) -> RouteId: return RouteId(dongle_id=dongle_id, log_id=log_id) -def route_url(route: RouteId) -> str: - return f"{API_HOST}/v1/route/{quote(route.canonical_name, safe='')}/" +def route_url(route: RouteId, api_host: str) -> str: + return f"{api_host}/v1/route/{quote(route.canonical_name, safe='')}/" -def route_files_url(route: RouteId) -> str: - return f"{API_HOST}/v1/route/{quote(route.canonical_name, safe='')}/files" +def route_files_url(route: RouteId, api_host: str) -> str: + return f"{api_host}/v1/route/{quote(route.canonical_name, safe='')}/files" def format_segments(segments: list[int]) -> str: @@ -184,8 +186,13 @@ def relative_posix(path: Path, root: Path) -> str: return path.relative_to(root).as_posix() -def fetch_json(session: requests.Session, url: str, timeout: float) -> Any: - response = session.get(url, timeout=timeout, allow_redirects=True) +def api_headers(api_host: str) -> dict[str, str] | None: + token = get_token(api_host) + return {"Authorization": f"JWT {token}"} if token else None + + +def fetch_json(session: requests.Session, url: str, timeout: float, api_host: str | None = None) -> Any: + response = session.get(url, timeout=timeout, allow_redirects=True, headers=api_headers(api_host) if api_host else None) response.raise_for_status() return response.json() @@ -312,24 +319,42 @@ def filename_from_url(url: str) -> str: def validate_route(route: RouteId, session: requests.Session, timeout: float) -> dict[str, Any]: - try: - route_meta = fetch_json(session, route_url(route), timeout) - except requests.HTTPError as exc: - status_code = exc.response.status_code if exc.response is not None else "unknown" - raise ValidationError(route, [f"route is not publicly accessible from comma connect (HTTP {status_code})."]) from exc + route_meta = None + files_payload = None + api_host = None + not_found_hosts = [] + + for candidate_host in API_HOSTS: + try: + route_meta = fetch_json(session, route_url(route, candidate_host), timeout, candidate_host) + except requests.HTTPError as exc: + status_code = exc.response.status_code if exc.response is not None else "unknown" + if status_code == 404 and len(API_HOSTS) > 1: + not_found_hosts.append(candidate_host) + continue + raise ValidationError(route, [f"route is not publicly accessible from {candidate_host} (HTTP {status_code})."]) from exc + + try: + files_payload = fetch_json(session, route_files_url(route, candidate_host), timeout, candidate_host) + except requests.HTTPError as exc: + status_code = exc.response.status_code if exc.response is not None else "unknown" + if status_code == 404 and len(API_HOSTS) > 1: + not_found_hosts.append(candidate_host) + continue + raise ValidationError(route, [f"public route files could not be fetched from {candidate_host} (HTTP {status_code})."]) from exc + + api_host = candidate_host + break + + if route_meta is None or files_payload is None or api_host is None: + raise ValidationError(route, [f"route was not found on: {', '.join(not_found_hosts) or ', '.join(API_HOSTS)}."]) if not route_meta.get("is_public", False): raise ValidationError(route, ["route metadata loaded, but `is_public` was false."]) - try: - files_payload = fetch_json(session, route_files_url(route), timeout) - except requests.HTTPError as exc: - status_code = exc.response.status_code if exc.response is not None else "unknown" - raise ValidationError(route, [f"public route files could not be fetched from comma connect (HTTP {status_code})."]) from exc - expected_segments = expected_segment_count(route_meta, files_payload) if expected_segments <= 0: - raise ValidationError(route, ["could not determine any route segments from comma connect."]) + raise ValidationError(route, ["could not determine any route segments from the route API."]) failures: list[str] = [] stream_urls: dict[str, list[Any]] = {} @@ -384,6 +409,7 @@ def validate_route(route: RouteId, session: requests.Session, timeout: float) -> raise ValidationError(route, failures) return { + "api_host": api_host, "route_meta": route_meta, "files_payload": files_payload, "expected_segments": expected_segments, @@ -532,6 +558,7 @@ def print_validation_summary(route: RouteId, validation: dict[str, Any]) -> None params = validation["params"] map_tiles: MapTileSummary = validation["map_tiles"] print(f"Validated {route.cli_name}") + print(f" route API: {validation['api_host']}") print(f" public route: yes") print(f" segments: {validation['expected_segments']}") print(f" VisionSpeedLimitDetection: {params.get('VisionSpeedLimitDetection', '')}") diff --git a/tools/auto_source.py b/tools/auto_source.py index bef6a43e5..f8202da20 100755 --- a/tools/auto_source.py +++ b/tools/auto_source.py @@ -1,5 +1,19 @@ -#!/usr/bin/env python3 +#!/bin/sh +""":" +REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/.." && pwd)" +PYTHON_BIN="$REPO_ROOT/.venv/bin/python" + +if [ -x "$PYTHON_BIN" ]; then + exec "$PYTHON_BIN" "$0" "$@" +fi + +exec python3 "$0" "$@" +":""" import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + from openpilot.tools.lib.logreader import LogReader, ReadMode diff --git a/tools/cabana/utils/api.cc b/tools/cabana/utils/api.cc index 89379a843..3ca46e993 100644 --- a/tools/cabana/utils/api.cc +++ b/tools/cabana/utils/api.cc @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -39,6 +40,36 @@ std::optional getDongleId() { namespace CommaApi { +const QString COMMA_API_HOST = "https://api.commadotai.com"; +const QString COMMA_API_HOST_ALIAS = "https://api.comma.ai"; + +QString normalizeApiHost(const QString &host) { + QUrl url(host); + if (url.scheme().isEmpty()) { + url = QUrl("https://" + host); + } + QString normalized = QString("%1://%2").arg(url.scheme().toLower(), url.authority().toLower()); + while (normalized.endsWith('/')) { + normalized.chop(1); + } + return normalized == COMMA_API_HOST_ALIAS ? COMMA_API_HOST : normalized; +} + +QString authTokenForHost(const QString &api_host) { + QString token_json = QString::fromStdString(util::read_file(util::getenv("HOME") + "/.comma/auth.json")); + QJsonDocument json_d = QJsonDocument::fromJson(token_json.toUtf8()); + QJsonObject auth = json_d.object(); + + const QString normalized_host = normalizeApiHost(api_host); + const QString token = auth["tokens"].toObject()[normalized_host].toString(); + if (!token.isEmpty()) { + return token; + } + + const QString env_host = normalizeApiHost(QString::fromStdString(util::getenv("API_HOST", COMMA_API_HOST.toStdString()))); + return (normalized_host == COMMA_API_HOST || normalized_host == env_host) ? auth["access_token"].toString() : ""; +} + EVP_PKEY *get_private_key() { static std::unique_ptr pkey(nullptr, EVP_PKEY_free); if (!pkey) { @@ -117,9 +148,7 @@ void HttpRequest::sendRequest(const QString &requestURL, const HttpRequest::Meth if (create_jwt) { token = CommaApi::create_jwt(); } else { - QString token_json = QString::fromStdString(util::read_file(util::getenv("HOME") + "/.comma/auth.json")); - QJsonDocument json_d = QJsonDocument::fromJson(token_json.toUtf8()); - token = json_d["access_token"].toString(); + token = CommaApi::authTokenForHost(requestURL); } QNetworkRequest request; diff --git a/tools/cabana/utils/api.h b/tools/cabana/utils/api.h index ad64d7e72..061e3baf2 100644 --- a/tools/cabana/utils/api.h +++ b/tools/cabana/utils/api.h @@ -10,6 +10,8 @@ namespace CommaApi { const QString BASE_URL = util::getenv("API_HOST", "https://api.commadotai.com").c_str(); +QString normalizeApiHost(const QString &host); +QString authTokenForHost(const QString &api_host); QByteArray rsa_sign(const QByteArray &data); QString create_jwt(const QJsonObject &payloads = {}, int expiry = 3600); diff --git a/tools/lib/api.py b/tools/lib/api.py index 285854c68..99948280e 100644 --- a/tools/lib/api.py +++ b/tools/lib/api.py @@ -2,6 +2,7 @@ import os import requests from openpilot.common.params import Params +from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, normalize_api_host def _use_konik_server(): @@ -11,27 +12,40 @@ def _use_konik_server(): return False -API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if _use_konik_server() else 'commadotai.com'}") +API_HOST = normalize_api_host(os.getenv('API_HOST', KONIK_API_HOST if _use_konik_server() else DEFAULT_API_HOST)) + + +def route_api_hosts() -> list[str]: + if os.getenv("API_HOST"): + return [API_HOST] + return [DEFAULT_API_HOST, KONIK_API_HOST] # TODO: this should be merged into common.api class CommaApi: - def __init__(self, token=None): + def __init__(self, token=None, host: str | None = None): + self.host = normalize_api_host(host or API_HOST) self.session = requests.Session() self.session.headers['User-agent'] = 'OpenpilotTools' if token: self.session.headers['Authorization'] = 'JWT ' + token def request(self, method, endpoint, **kwargs): - with self.session.request(method, API_HOST + '/' + endpoint, **kwargs) as resp: - resp_json = resp.json() - if isinstance(resp_json, dict) and resp_json.get('error'): - if resp.status_code in [401, 403]: - raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py') + with self.session.request(method, self.host + '/' + endpoint.lstrip('/'), **kwargs) as resp: + try: + resp_json = resp.json() + except ValueError: + resp_json = None - e = APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error']))) - e.status_code = resp.status_code - raise e + if resp.status_code in [401, 403]: + raise UnauthorizedError('Unauthorized. Authenticate with tools/lib/auth.py', resp.status_code) + + if resp.status_code >= 400: + description = resp.text.strip() if resp.text else resp.reason + raise APIError(f"{resp.status_code}:{description}", resp.status_code) + + if isinstance(resp_json, dict) and resp_json.get('error'): + raise APIError(str(resp.status_code) + ":" + resp_json.get('description', str(resp_json['error'])), resp.status_code) return resp_json def get(self, endpoint, **kwargs): @@ -41,7 +55,10 @@ class CommaApi: return self.request('POST', endpoint, **kwargs) class APIError(Exception): - pass + def __init__(self, message, status_code=None): + super().__init__(message) + self.status_code = status_code -class UnauthorizedError(Exception): - pass +class UnauthorizedError(APIError): + def __init__(self, message, status_code=None): + super().__init__(message, status_code) diff --git a/tools/lib/auth.py b/tools/lib/auth.py index d84b41c12..ed9a05a96 100755 --- a/tools/lib/auth.py +++ b/tools/lib/auth.py @@ -1,4 +1,14 @@ -#!/usr/bin/env python3 +#!/bin/sh +""":" +REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +PYTHON_BIN="$REPO_ROOT/.venv/bin/python" + +if [ -x "$PYTHON_BIN" ]; then + exec "$PYTHON_BIN" "$0" "$@" +fi + +exec python3 "$0" "$@" +":""" """ Usage:: @@ -27,12 +37,15 @@ import pprint import sys import webbrowser from http.server import BaseHTTPRequestHandler, HTTPServer +from pathlib import Path from typing import Any from urllib.parse import parse_qs, urlencode +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) + from openpilot.common.params import Params from openpilot.tools.lib.api import APIError, CommaApi, UnauthorizedError -from openpilot.tools.lib.auth_config import set_token, get_token +from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, get_token, normalize_api_host, set_token def _use_konik_server(): @@ -42,7 +55,7 @@ def _use_konik_server(): return False -API_HOST = os.getenv('API_HOST', f"https://api.{'konik.ai' if _use_konik_server() else 'comma.ai'}") +API_HOST = normalize_api_host(os.getenv('API_HOST', KONIK_API_HOST if _use_konik_server() else DEFAULT_API_HOST)) PORT = 3000 @@ -69,7 +82,19 @@ class ClientRedirectHandler(BaseHTTPRequestHandler): pass # this prevent http server from dumping messages to stdout -def auth_redirect_link(method): +def resolve_api_host(host: str) -> str: + if host in ("comma", "commaai", "commadotai"): + return DEFAULT_API_HOST + if host == "konik": + return KONIK_API_HOST + return normalize_api_host(host) + + +def auth_redirect_api_host(api_host: str) -> str: + return "https://api.comma.ai" if normalize_api_host(api_host) == DEFAULT_API_HOST else normalize_api_host(api_host) + + +def auth_redirect_link(method, api_host=API_HOST): provider_id = { 'google': 'g', 'apple': 'a', @@ -77,7 +102,7 @@ def auth_redirect_link(method): }[method] params = { - 'redirect_uri': f"{API_HOST}/v2/auth/{provider_id}/redirect/", + 'redirect_uri': f"{auth_redirect_api_host(api_host)}/v2/auth/{provider_id}/redirect/", 'state': f'service,localhost:{PORT}', } @@ -92,7 +117,7 @@ def auth_redirect_link(method): return 'https://accounts.google.com/o/oauth2/auth?' + urlencode(params) elif method == 'github': params.update({ - 'client_id': '28c4ecb54bb7272cb5a4', + 'client_id': 'Ov23liy0AI1YCd15pypf' if normalize_api_host(api_host) == KONIK_API_HOST else '28c4ecb54bb7272cb5a4', 'scope': 'read:user', }) return 'https://github.com/login/oauth/authorize?' + urlencode(params) @@ -108,8 +133,8 @@ def auth_redirect_link(method): raise NotImplementedError(f"no redirect implemented for method {method}") -def login(method): - oauth_uri = auth_redirect_link(method) +def login(method, api_host=API_HOST): + oauth_uri = auth_redirect_link(method, api_host) web_server = ClientRedirectServer(('localhost', PORT), ClientRedirectHandler) print(f'To sign in, use your browser and navigate to {oauth_uri}') @@ -126,29 +151,35 @@ def login(method): break try: - auth_resp = CommaApi().post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']}) - set_token(auth_resp['access_token']) + auth_resp = CommaApi(host=api_host).post('v2/auth/', data={'code': web_server.query_params['code'], 'provider': web_server.query_params['provider']}) + return auth_resp['access_token'] except APIError as e: print(f'Authentication Error: {e}', file=sys.stderr) + return None if __name__ == '__main__': parser = argparse.ArgumentParser(description='Login to your comma account') + parser.add_argument('--host', default=API_HOST, help='API host to authenticate with: comma, konik, or a URL') parser.add_argument('method', default='google', const='google', nargs='?', choices=['google', 'apple', 'github', 'jwt']) parser.add_argument('jwt', nargs='?') args = parser.parse_args() + api_host = resolve_api_host(args.host) if args.method == 'jwt': if args.jwt is None: print("method JWT selected, but no JWT was provided") exit(1) - set_token(args.jwt) + token = args.jwt else: - login(args.method) + token = login(args.method, api_host) + if token is None: + exit(1) try: - me = CommaApi(token=get_token()).get('/v1/me') + me = CommaApi(token=token, host=api_host).get('/v1/me') + set_token(token, api_host) print("Authenticated!") pprint.pprint(me) except UnauthorizedError: diff --git a/tools/lib/auth_config.py b/tools/lib/auth_config.py index c4e7b6261..761dbe594 100644 --- a/tools/lib/auth_config.py +++ b/tools/lib/auth_config.py @@ -1,5 +1,7 @@ import json import os +from urllib.parse import urlparse, urlunparse + from openpilot.system.hardware.hw import Paths @@ -7,23 +9,88 @@ class MissingAuthConfigError(Exception): pass -def get_token(): +DEFAULT_API_HOST = "https://api.commadotai.com" +KONIK_API_HOST = "https://api.konik.ai" +COMMA_API_HOST_ALIASES = {"https://api.comma.ai", DEFAULT_API_HOST} + + +def normalize_api_host(host: str | None) -> str: + if not host: + host = DEFAULT_API_HOST + + parsed = urlparse(host) + if not parsed.scheme: + parsed = urlparse(f"https://{host}") + + normalized = urlunparse((parsed.scheme.lower(), parsed.netloc.lower(), "", "", "", "")).rstrip("/") + if normalized in COMMA_API_HOST_ALIASES: + return DEFAULT_API_HOST + return normalized + + +def _auth_path() -> str: + return os.path.join(Paths.config_root(), 'auth.json') + + +def _read_auth() -> dict: try: - with open(os.path.join(Paths.config_root(), 'auth.json')) as f: - auth = json.load(f) - return auth['access_token'] + with open(_auth_path()) as f: + return json.load(f) except Exception: + return {} + + +def get_token(host: str | None = None): + auth = _read_auth() + if not auth: return None + if host is not None: + normalized_host = normalize_api_host(host) + token = auth.get('tokens', {}).get(normalized_host) + if token is not None: + return token + + env_host = normalize_api_host(os.getenv("API_HOST", DEFAULT_API_HOST)) + if normalized_host == DEFAULT_API_HOST or normalized_host == env_host: + return auth.get('access_token') + return None + + return auth.get('access_token') + + +def set_token(token, host: str | None = None): + normalized_host = normalize_api_host(host or os.getenv("API_HOST", DEFAULT_API_HOST)) + auth = _read_auth() + tokens = auth.setdefault('tokens', {}) + tokens[normalized_host] = token + + # Preserve the legacy field for older tooling. Avoid replacing a comma token + # when adding a Konik token unless this is an old-style hostless write. + if host is None or normalized_host == DEFAULT_API_HOST or 'access_token' not in auth: + auth['access_token'] = token -def set_token(token): os.makedirs(Paths.config_root(), exist_ok=True) - with open(os.path.join(Paths.config_root(), 'auth.json'), 'w') as f: - json.dump({'access_token': token}, f) + with open(_auth_path(), 'w') as f: + json.dump(auth, f) -def clear_token(): - try: - os.unlink(os.path.join(Paths.config_root(), 'auth.json')) - except FileNotFoundError: - pass +def clear_token(host: str | None = None): + if host is None: + try: + os.unlink(_auth_path()) + except FileNotFoundError: + pass + return + + auth = _read_auth() + if not auth: + return + + normalized_host = normalize_api_host(host) + token = auth.get('tokens', {}).pop(normalized_host, None) + if normalized_host == DEFAULT_API_HOST and auth.get('access_token') == token: + auth.pop('access_token', None) + + with open(_auth_path(), 'w') as f: + json.dump(auth, f) diff --git a/tools/lib/logreader.py b/tools/lib/logreader.py index 9bd992d4f..d2974a501 100755 --- a/tools/lib/logreader.py +++ b/tools/lib/logreader.py @@ -204,11 +204,12 @@ def auto_source(identifier: str, sources: list[Source], default_mode: ReadMode) def parse_indirect(identifier: str) -> str: - if "useradmin.comma.ai" in identifier: + parsed = urlparse(identifier) + if parsed.netloc in ("useradmin.comma.ai", "useradmin.konik.ai"): query = parse_qs(urlparse(identifier).query) identifier = query["onebox"][0] - elif "connect.comma.ai" in identifier: - path = urlparse(identifier).path.strip("/").split("/") + elif parsed.netloc in ("connect.comma.ai", "connect.konik.ai", "stable.konik.ai"): + path = parsed.path.strip("/").split("/") path = ['/'.join(path[:2]), *path[2:]] # recombine log id identifier = path[0] diff --git a/tools/lib/route.py b/tools/lib/route.py index 1fc26fb99..eefe10990 100644 --- a/tools/lib/route.py +++ b/tools/lib/route.py @@ -7,7 +7,7 @@ from collections import defaultdict from itertools import chain from openpilot.tools.lib.auth_config import get_token -from openpilot.tools.lib.api import APIError, CommaApi +from openpilot.tools.lib.api import APIError, CommaApi, route_api_hosts from openpilot.tools.lib.helpers import RE @@ -23,6 +23,7 @@ class FileName: class Route: def __init__(self, name, data_dir=None): + self._api_host = None self._metadata = None self._name = RouteName(name) self.files = None @@ -35,8 +36,7 @@ class Route: @property def metadata(self): if not self._metadata: - api = CommaApi(get_token()) - self._metadata = api.get('v1/route/' + self.name.canonical_name) + self._metadata = self._get_route_metadata() return self._metadata @property @@ -73,9 +73,9 @@ class Route: # TODO: refactor this, it's super repetitive def _get_segments_remote(self): - api = CommaApi(get_token()) - route_files = api.get('v1/route/' + self.name.canonical_name + '/files') + route_files = self._get_route_files() self.files = list(chain.from_iterable(route_files.values())) + metadata_url = self.metadata['url'] segments = {} for url in self.files: @@ -90,7 +90,7 @@ class Route: url if fn in FileName.DCAMERA else segments[segment_name].dcamera_path, url if fn in FileName.ECAMERA else segments[segment_name].ecamera_path, url if fn in FileName.QCAMERA else segments[segment_name].qcamera_path, - self.metadata['url'], + metadata_url, ) else: segments[segment_name] = Segment( @@ -101,11 +101,35 @@ class Route: url if fn in FileName.DCAMERA else None, url if fn in FileName.ECAMERA else None, url if fn in FileName.QCAMERA else None, - self.metadata['url'], + metadata_url, ) return sorted(segments.values(), key=lambda seg: seg.name.segment_num) + def _get_route_metadata(self): + return self._get_route_endpoint('v1/route/' + self.name.canonical_name) + + def _get_route_files(self): + return self._get_route_endpoint('v1/route/' + self.name.canonical_name + '/files') + + def _get_route_endpoint(self, endpoint: str): + hosts = [self._api_host] if self._api_host is not None else route_api_hosts() + errors = {} + + for host in hosts: + try: + api = CommaApi(get_token(host), host=host) + response = api.get(endpoint) + self._api_host = host + return response + except APIError as e: + errors[host] = e + if e.status_code == 404 and self._api_host is None: + continue + raise + + raise APIError(f"route {self.name.canonical_name} not found on: {', '.join(errors.keys())}", 404) + def _get_segments_local(self, data_dir): files = os.listdir(data_dir) segment_files = defaultdict(list) @@ -306,13 +330,20 @@ class SegmentName: @cache def get_max_seg_number_cached(sr: 'SegmentRange') -> int: - try: - api = CommaApi(get_token()) - max_seg_number = api.get("/v1/route/" + sr.route_name.replace("/", "|"))["maxqlog"] - assert isinstance(max_seg_number, int) - return max_seg_number - except Exception as e: - raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from e + errors = {} + for host in route_api_hosts(): + try: + api = CommaApi(get_token(host), host=host) + max_seg_number = api.get("/v1/route/" + sr.route_name.replace("/", "|"))["maxqlog"] + assert isinstance(max_seg_number, int) + return max_seg_number + except APIError as e: + errors[host] = e + if e.status_code == 404: + continue + raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from e + + raise Exception("unable to get max_segment_number. ensure you have access to this route or the route is public.") from next(iter(errors.values()), None) class SegmentRange: diff --git a/tools/lib/tests/test_auth_config.py b/tools/lib/tests/test_auth_config.py new file mode 100644 index 000000000..b8d5e0110 --- /dev/null +++ b/tools/lib/tests/test_auth_config.py @@ -0,0 +1,12 @@ +from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST, get_token, set_token + + +def test_set_token_preserves_comma_token_when_adding_konik(mocker, tmp_path): + mocker.patch("openpilot.tools.lib.auth_config.Paths.config_root", return_value=str(tmp_path)) + + set_token("comma-token", DEFAULT_API_HOST) + set_token("konik-token", KONIK_API_HOST) + + assert get_token(DEFAULT_API_HOST) == "comma-token" + assert get_token(KONIK_API_HOST) == "konik-token" + assert get_token() == "comma-token" diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index b87d1fc5d..2dca6ec45 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -95,10 +95,15 @@ class TestLogReader: (f"{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"), (f"https://connect.comma.ai/{TEST_ROUTE}/13/14", f"{TEST_ROUTE}/0:1"), (f"https://connect.comma.ai/{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"), + (f"https://connect.konik.ai/{TEST_ROUTE}/13/14", f"{TEST_ROUTE}/0:1"), + (f"https://stable.konik.ai/{TEST_ROUTE}/13/14/a", f"{TEST_ROUTE}/0:1/a"), ]) def test_parse_indirect_accepts_second_window_route_style(self, identifier, expected): assert parse_indirect(identifier) == expected + def test_parse_indirect_accepts_konik_useradmin(self): + assert parse_indirect(f"https://useradmin.konik.ai/?onebox={TEST_ROUTE}") == TEST_ROUTE + @pytest.mark.parametrize("cache_enabled", [True, False]) def test_direct_parsing(self, mocker, cache_enabled): file_exists_mock = mocker.patch("openpilot.tools.lib.filereader.file_exists") diff --git a/tools/lib/tests/test_route_library.py b/tools/lib/tests/test_route_library.py index 491bb8132..f888ba678 100644 --- a/tools/lib/tests/test_route_library.py +++ b/tools/lib/tests/test_route_library.py @@ -1,6 +1,10 @@ from collections import namedtuple -from openpilot.tools.lib.route import SegmentName +import pytest + +from openpilot.tools.lib.api import APIError, UnauthorizedError +from openpilot.tools.lib.auth_config import DEFAULT_API_HOST, KONIK_API_HOST +from openpilot.tools.lib.route import Route, SegmentName class TestRouteLibrary: def test_segment_name_formats(self): @@ -25,3 +29,52 @@ class TestRouteLibrary: for case in cases: _validate(case) + + def test_route_falls_back_to_konik_when_comma_route_missing(self, mocker): + route_name = "59679e5e40b60ce0/0000091b--316e931f07" + file_url = "https://konik.example/59679e5e40b60ce0/0000091b--316e931f07/0/qlog.zst" + calls = [] + + class FakeApi: + def __init__(self, token=None, host=None): + self.host = host + + def get(self, endpoint): + calls.append((self.host, endpoint)) + if self.host == DEFAULT_API_HOST: + raise APIError("404:not found", 404) + if endpoint.endswith("/files"): + return {"qlogs": [file_url]} + return {"url": "https://connect.konik.ai/59679e5e40b60ce0/0000091b--316e931f07"} + + mocker.patch("openpilot.tools.lib.route.route_api_hosts", return_value=[DEFAULT_API_HOST, KONIK_API_HOST]) + mocker.patch("openpilot.tools.lib.route.get_token", return_value=None) + mocker.patch("openpilot.tools.lib.route.CommaApi", FakeApi) + + route = Route(route_name) + + assert route.qlog_paths() == [file_url] + assert calls[0] == (DEFAULT_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files") + assert calls[1] == (KONIK_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files") + assert calls[2] == (KONIK_API_HOST, f"v1/route/{route_name.replace('/', '|')}") + + def test_route_does_not_fall_back_to_konik_when_comma_unauthorized(self, mocker): + route_name = "59679e5e40b60ce0/0000091b--316e931f07" + calls = [] + + class FakeApi: + def __init__(self, token=None, host=None): + self.host = host + + def get(self, endpoint): + calls.append((self.host, endpoint)) + raise UnauthorizedError("unauthorized", 401) + + mocker.patch("openpilot.tools.lib.route.route_api_hosts", return_value=[DEFAULT_API_HOST, KONIK_API_HOST]) + mocker.patch("openpilot.tools.lib.route.get_token", return_value=None) + mocker.patch("openpilot.tools.lib.route.CommaApi", FakeApi) + + with pytest.raises(UnauthorizedError): + Route(route_name) + + assert calls == [(DEFAULT_API_HOST, f"v1/route/{route_name.replace('/', '|')}/files")] diff --git a/tools/replay/api.cc b/tools/replay/api.cc index 85e4e52b2..cf2e8daa0 100644 --- a/tools/replay/api.cc +++ b/tools/replay/api.cc @@ -7,7 +7,10 @@ #include #include +#include +#include #include +#include #include #include "common/params.h" @@ -16,6 +19,30 @@ namespace CommaApi2 { +const std::string COMMA_API_HOST = "https://api.commadotai.com"; +const std::string COMMA_API_HOST_ALIAS = "https://api.comma.ai"; +const std::string KONIK_API_HOST = "https://api.konik.ai"; + +std::string normalize_api_host(const std::string &host) { + std::string normalized = host.empty() ? COMMA_API_HOST : host; + while (!normalized.empty() && normalized.back() == '/') { + normalized.pop_back(); + } + if (normalized.find("://") == std::string::npos) { + normalized = "https://" + normalized; + } + std::transform(normalized.begin(), normalized.end(), normalized.begin(), [](unsigned char c) { return std::tolower(c); }); + return normalized == COMMA_API_HOST_ALIAS ? COMMA_API_HOST : normalized; +} + +std::vector route_api_hosts() { + const char *api_host = std::getenv("API_HOST"); + if (api_host != nullptr && api_host[0] != '\0') { + return {normalize_api_host(api_host)}; + } + return {COMMA_API_HOST, KONIK_API_HOST}; +} + // Base64 URL-safe character set (uses '-' and '_' instead of '+' and '/') static const std::string base64url_chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" @@ -104,7 +131,25 @@ std::string create_jwt(const json11::Json &extra, int exp_time) { return jwt + "." + base64url_encode(signature); } -std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry) { +std::string auth_token_for_host(const json11::Json &json, const std::string &api_host) { + const std::string normalized_host = normalize_api_host(api_host); + + const json11::Json tokens = json["tokens"]; + if (tokens.is_object()) { + const std::string token = tokens[normalized_host].string_value(); + if (!token.empty()) { + return token; + } + } + + const std::string env_host = normalize_api_host(util::getenv("API_HOST", COMMA_API_HOST)); + if (normalized_host == COMMA_API_HOST || normalized_host == env_host) { + return json["access_token"].string_value(); + } + return ""; +} + +std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry, const std::string &api_host) { if (use_jwt) { return create_jwt(payloads, expiry); } @@ -116,15 +161,15 @@ std::string create_token(bool use_jwt, const json11::Json &payloads, int expiry) std::cerr << "Error parsing auth.json " << err << std::endl; return ""; } - return json["access_token"].string_value(); + return auth_token_for_host(json, api_host); } -std::string httpGet(const std::string &url, long *response_code) { +std::string httpGet(const std::string &url, long *response_code, const std::string &api_host) { CURL *curl = curl_easy_init(); assert(curl); std::string readBuffer; - const std::string token = CommaApi2::create_token(!Hardware::PC()); + const std::string token = CommaApi2::create_token(!Hardware::PC(), {}, 3600, api_host); // Set up the lambda for the write callback // The '+' makes the lambda non-capturing, allowing it to be used as a C function pointer diff --git a/tools/replay/api.h b/tools/replay/api.h index dff59c065..9d41f38e6 100644 --- a/tools/replay/api.h +++ b/tools/replay/api.h @@ -2,6 +2,7 @@ #include #include +#include #include "common/util.h" #include "third_party/json11/json11.hpp" @@ -9,7 +10,9 @@ namespace CommaApi2 { const std::string BASE_URL = util::getenv("API_HOST", "https://api.commadotai.com").c_str(); -std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600); -std::string httpGet(const std::string &url, long *response_code = nullptr); +std::string normalize_api_host(const std::string &host); +std::vector route_api_hosts(); +std::string create_token(bool use_jwt, const json11::Json& payloads = {}, int expiry = 3600, const std::string &api_host = BASE_URL); +std::string httpGet(const std::string &url, long *response_code = nullptr, const std::string &api_host = BASE_URL); } // namespace CommaApi2 diff --git a/tools/replay/route.cc b/tools/replay/route.cc index ba0082826..fd08867d0 100644 --- a/tools/replay/route.cc +++ b/tools/replay/route.cc @@ -15,7 +15,7 @@ Route::Route(const std::string &route, const std::string &data_dir, bool auto_so RouteIdentifier Route::parseRoute(const std::string &str) { RouteIdentifier identifier = {}; - static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?$)"); + static const std::regex pattern(R"(^(([a-z0-9]{16})[|_/])?(.{20})((--|/)((-?\d+(:(-?\d+)?)?)|(:-?\d+)))?(/[qra])?$)"); std::smatch match; if (std::regex_match(str, match, pattern)) { @@ -104,30 +104,38 @@ bool Route::loadFromAutoSource() { } bool Route::loadFromServer(int retries) { - const std::string url = CommaApi2::BASE_URL + "/v1/route/" + route_.str + "/files"; - for (int i = 1; i <= retries; ++i) { - long response_code = 0; - std::string result = CommaApi2::httpGet(url, &response_code); - if (response_code == 200) { - return loadFromJson(result); + const auto api_hosts = CommaApi2::route_api_hosts(); + for (const auto &api_host : api_hosts) { + const std::string url = api_host + "/v1/route/" + route_.str + "/files"; + for (int i = 1; i <= retries; ++i) { + long response_code = 0; + std::string result = CommaApi2::httpGet(url, &response_code, api_host); + if (response_code == 200) { + return loadFromJson(result); + } + + if (response_code == 401 || response_code == 403) { + rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<"); + err_ = RouteLoadError::Unauthorized; + return false; + } + if (response_code == 404) { + err_ = RouteLoadError::FileNotFound; + break; + } + + err_ = RouteLoadError::NetworkError; + rWarning("Retrying %d/%d", i, retries); + util::sleep_for(3000); } - if (response_code == 401 || response_code == 403) { - rWarning(">> Unauthorized. Authenticate with tools/lib/auth.py <<"); - err_ = RouteLoadError::Unauthorized; - break; + if (err_ == RouteLoadError::NetworkError) { + return false; } - if (response_code == 404) { - rWarning("The specified route could not be found on the server."); - err_ = RouteLoadError::FileNotFound; - break; - } - - err_ = RouteLoadError::NetworkError; - rWarning("Retrying %d/%d", i, retries); - util::sleep_for(3000); } + rWarning(api_hosts.size() > 1 ? "The specified route could not be found on comma or Konik." : "The specified route could not be found on the server."); + err_ = RouteLoadError::FileNotFound; return false; } diff --git a/tools/scripts/fetch_image_from_route.py b/tools/scripts/fetch_image_from_route.py index d02f75fdf..4fbf9b398 100755 --- a/tools/scripts/fetch_image_from_route.py +++ b/tools/scripts/fetch_image_from_route.py @@ -1,7 +1,18 @@ -#!/usr/bin/env python3 -import sys +#!/bin/sh +""":" +REPO_ROOT="$(CDPATH= cd -- "$(dirname "$0")/../.." && pwd)" +PYTHON_BIN="$REPO_ROOT/.venv/bin/python" -from openpilot.starpilot.common.starpilot_utilities import use_konik_server +if [ -x "$PYTHON_BIN" ]; then + exec "$PYTHON_BIN" "$0" "$@" +fi + +exec python3 "$0" "$@" +":""" +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) if len(sys.argv) < 4: print(f"{sys.argv[0]} [front|wide|driver]") @@ -14,25 +25,22 @@ cameras = { "driver": "dcameras" } -import requests from PIL import Image -from openpilot.tools.lib.auth_config import get_token from openpilot.tools.lib.framereader import FrameReader - -jwt = get_token() +from openpilot.tools.lib.route import Route route = sys.argv[1] segment = int(sys.argv[2]) frame = int(sys.argv[3]) camera = cameras[sys.argv[4]] if len(sys.argv) > 4 and sys.argv[4] in cameras else "cameras" -url = f"https://api.{'konik.ai' if use_konik_server() else 'commadotai.com'}/v1/route/{route}/files" -r = requests.get(url, headers={"Authorization": f"JWT {jwt}"}, timeout=10) -assert r.status_code == 200 -print("got api response") - -segments = r.json()[camera] -if segment >= len(segments): +route_files = Route(route) +segments = { + "cameras": route_files.camera_paths(), + "ecameras": route_files.ecamera_paths(), + "dcameras": route_files.dcamera_paths(), +}[camera] +if segment >= len(segments) or segments[segment] is None: raise Exception(f"segment {segment} not found, got {len(segments)} segments") fr = FrameReader(segments[segment])