From c35df583a5cb287304a88701b2a9040f74788ecc Mon Sep 17 00:00:00 2001 From: Adeeb Shihadeh Date: Sat, 31 Jan 2026 15:52:50 -0800 Subject: [PATCH] tools: enable log caching by default (#36962) --- .github/workflows/repo-maintenance.yaml | 2 +- .github/workflows/tests.yaml | 2 +- selfdrive/car/tests/big_cars_test.sh | 1 - selfdrive/test/process_replay/README.md | 2 +- tools/lib/tests/test_caching.py | 13 ++++++++----- tools/lib/tests/test_logreader.py | 10 ++++++++-- tools/lib/url_file.py | 4 ++-- tools/replay/can_replay.py | 2 -- tools/tuning/measure_steering_accuracy.py | 4 ---- 9 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/repo-maintenance.yaml b/.github/workflows/repo-maintenance.yaml index 7bb91c0ca..b8b29e602 100644 --- a/.github/workflows/repo-maintenance.yaml +++ b/.github/workflows/repo-maintenance.yaml @@ -8,7 +8,7 @@ on: env: BASE_IMAGE: openpilot-base BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c jobs: update_translations: diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index c5802b5cb..c7da61945 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -25,7 +25,7 @@ env: DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }} BUILD: selfdrive/test/docker_build.sh base - RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c + RUN: docker run --shm-size 2G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PYTHONWARNINGS=error -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c PYTEST: pytest --continue-on-collection-errors --durations=0 -n logical diff --git a/selfdrive/car/tests/big_cars_test.sh b/selfdrive/car/tests/big_cars_test.sh index 863b8bead..bb6e82dd0 100755 --- a/selfdrive/car/tests/big_cars_test.sh +++ b/selfdrive/car/tests/big_cars_test.sh @@ -6,7 +6,6 @@ cd $BASEDIR export MAX_EXAMPLES=300 export INTERNAL_SEG_CNT=300 -export FILEREADER_CACHE=1 export INTERNAL_SEG_LIST=selfdrive/car/tests/test_models_segs.txt cd selfdrive/car/tests && pytest test_models.py test_car_interfaces.py diff --git a/selfdrive/test/process_replay/README.md b/selfdrive/test/process_replay/README.md index dc801e428..8e279c71c 100644 --- a/selfdrive/test/process_replay/README.md +++ b/selfdrive/test/process_replay/README.md @@ -5,7 +5,7 @@ Process replay is a regression test designed to identify any changes in the outp If the test fails, make sure that you didn't unintentionally change anything. If there are intentional changes, the reference logs will be updated. Use `test_processes.py` to run the test locally. -Use `FILEREADER_CACHE='1' test_processes.py` to cache log files. +Log files are cached by default. Use `DISABLE_FILEREADER_CACHE='1' test_processes.py` to disable caching. Currently the following processes are tested: diff --git a/tools/lib/tests/test_caching.py b/tools/lib/tests/test_caching.py index 6e70ef90b..cb14098e6 100644 --- a/tools/lib/tests/test_caching.py +++ b/tools/lib/tests/test_caching.py @@ -56,13 +56,13 @@ class TestFileDownload: for k, v in retry_defaults.items(): assert getattr(URLFile.pool_manager().connection_pool_kw["retries"], k) == v - # ensure caching off by default and cache dir doesn't get created - os.environ.pop("FILEREADER_CACHE", None) + # ensure caching on by default and cache dir gets created + os.environ.pop("DISABLE_FILEREADER_CACHE", None) if os.path.exists(Paths.download_cache_root()): shutil.rmtree(Paths.download_cache_root()) URLFile(f"{host}/test.txt").get_length() URLFile(f"{host}/test.txt").read() - assert not os.path.exists(Paths.download_cache_root()) + assert os.path.exists(Paths.download_cache_root()) def compare_loads(self, url, start=0, length=None): """Compares range between cached and non cached version""" @@ -90,7 +90,7 @@ class TestFileDownload: def test_small_file(self): # Make sure we don't force cache - os.environ["FILEREADER_CACHE"] = "0" + os.environ.pop("DISABLE_FILEREADER_CACHE", None) small_file_url = "https://raw.githubusercontent.com/commaai/openpilot/master/docs/SAFETY.md" # If you want large file to be larger than a chunk # large_file_url = "https://commadataci.blob.core.windows.net/openpilotci/0375fdf7b1ce594d/2019-06-13--08-32-25/3/fcamera.hevc" @@ -119,7 +119,10 @@ class TestFileDownload: @pytest.mark.parametrize("cache_enabled", [True, False]) def test_recover_from_missing_file(self, host, cache_enabled): - os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0" + if cache_enabled: + os.environ.pop("DISABLE_FILEREADER_CACHE", None) + else: + os.environ["DISABLE_FILEREADER_CACHE"] = "1" file_url = f"{host}/test.png" diff --git a/tools/lib/tests/test_logreader.py b/tools/lib/tests/test_logreader.py index ee75a8b1c..0151940c4 100644 --- a/tools/lib/tests/test_logreader.py +++ b/tools/lib/tests/test_logreader.py @@ -93,7 +93,10 @@ class TestLogReader: @pytest.mark.parametrize("cache_enabled", [True, False]) def test_direct_parsing(self, mocker, cache_enabled): file_exists_mock = mocker.patch("openpilot.tools.lib.filereader.file_exists") - os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0" + if cache_enabled: + os.environ.pop("DISABLE_FILEREADER_CACHE", None) + else: + os.environ["DISABLE_FILEREADER_CACHE"] = "1" qlog = tempfile.NamedTemporaryFile(mode='wb', delete=False) with requests.get(QLOG_FILE, stream=True) as r: @@ -181,7 +184,10 @@ class TestLogReader: @parameterized.expand([(True,), (False,)]) @pytest.mark.slow def test_run_across_segments(self, cache_enabled): - os.environ["FILEREADER_CACHE"] = "1" if cache_enabled else "0" + if cache_enabled: + os.environ.pop("DISABLE_FILEREADER_CACHE", None) + else: + os.environ["DISABLE_FILEREADER_CACHE"] = "1" lr = LogReader(f"{TEST_ROUTE}/0:4") assert len(lr.run_across_segments(4, noop)) == len(list(lr)) diff --git a/tools/lib/url_file.py b/tools/lib/url_file.py index 8e2f0a922..de1207046 100644 --- a/tools/lib/url_file.py +++ b/tools/lib/url_file.py @@ -74,8 +74,8 @@ class URLFile: self._timeout = Timeout(connect=timeout, read=timeout) self._pos = 0 self._length: int | None = None - # True by default, false if FILEREADER_CACHE is defined, but can be overwritten by the cache input - self._force_download = not int(os.environ.get("FILEREADER_CACHE", "0")) + # Caching enabled by default, can be disabled with DISABLE_FILEREADER_CACHE=1, or overwritten by the cache input + self._force_download = int(os.environ.get("DISABLE_FILEREADER_CACHE", "0")) == 1 if cache is not None: self._force_download = not cache diff --git a/tools/replay/can_replay.py b/tools/replay/can_replay.py index 13c30a62a..071732007 100755 --- a/tools/replay/can_replay.py +++ b/tools/replay/can_replay.py @@ -5,8 +5,6 @@ import time import usb1 import threading -os.environ['FILEREADER_CACHE'] = '1' - from openpilot.common.realtime import config_realtime_process, Ratekeeper, DT_CTRL from openpilot.selfdrive.pandad import can_capnp_to_list from openpilot.tools.lib.logreader import LogReader diff --git a/tools/tuning/measure_steering_accuracy.py b/tools/tuning/measure_steering_accuracy.py index e4aef0ba1..ae3344c2e 100755 --- a/tools/tuning/measure_steering_accuracy.py +++ b/tools/tuning/measure_steering_accuracy.py @@ -117,12 +117,8 @@ if __name__ == "__main__": parser.add_argument('--route', help="route name") parser.add_argument('--addr', default='127.0.0.1', help="IP address for optional ZMQ listener, default to msgq") parser.add_argument('--group', default='all', help="speed group to display, [crawl|slow|medium|fast|veryfast|germany|all], default to all") - parser.add_argument('--cache', default=False, action='store_true', help="use cached data, default to False") args = parser.parse_args() - if args.cache: - os.environ['FILEREADER_CACHE'] = '1' - tool = SteeringAccuracyTool(args) if args.route is not None: