mawei/StarPilot
1
0
Fork 0
mirror of https://github.com/firestar5683/StarPilot.git synced 2026-09-28 18:33:45 +08:00
Code Issues Packages Projects Releases Wiki Activity

ci: fix retry in test_camerad (#34167)

Browse Source 
* flaky was flaky

* delay
This commit is contained in:
Maxime Desroches
2024-12-06 19:29:32 -08:00
committed by GitHub
parent c7889a16be
commit 0902527e27
3 changed files with 85 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pyproject.toml
-1
View File
@@ -101,7 +101,6 @@ dev = [
"azure-identity",
"azure-storage-blob",
"dictdiffer",
"flaky",
"lru-dict",
"matplotlib",
"parameterized >=0.8, <0.9",
system/camerad/test/test_camerad.py
+37 -35
View File
@@ -1,12 +1,12 @@
import pytest
import time
import numpy as np
from flaky import flaky
from collections import defaultdict
import cereal.messaging as messaging
from cereal import log
from cereal.services import SERVICE_LIST
from openpilot.common.retry import retry
from openpilot.system.manager.process_config import managed_processes
TEST_TIMESPAN = 10
@@ -17,44 +17,46 @@ FRAME_DELTA_TOLERANCE = {log.FrameData.ImageSensor.ar0231: 1.0,
CAMERAS = ('roadCameraState', 'driverCameraState', 'wideRoadCameraState')
# TODO: this shouldn't be needed
@flaky(max_runs=3)
@retry(attempts=3, delay=5.0)
def setup_camerad(cls):
# run camerad and record logs
managed_processes['camerad'].start()
time.sleep(3)
socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS}
cls.logs = defaultdict(list)
start_time = time.monotonic()
while time.monotonic()- start_time < TEST_TIMESPAN:
for cam, s in socks.items():
cls.logs[cam] += messaging.drain_sock(s)
time.sleep(0.2)
managed_processes['camerad'].stop()
cls.log_by_frame_id = defaultdict(list)
cls.sensor_type = None
for cam, msgs in cls.logs.items():
if cls.sensor_type is None:
cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw
expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN
assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}"
dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency)
assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
for m in msgs:
cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m)
# strip beginning and end
for _ in range(3):
mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys())
del cls.log_by_frame_id[mn]
del cls.log_by_frame_id[mx]
@pytest.mark.tici
class TestCamerad:
@classmethod
def setup_class(cls):
# run camerad and record logs
managed_processes['camerad'].start()
time.sleep(3)
socks = {c: messaging.sub_sock(c, conflate=False, timeout=100) for c in CAMERAS}
cls.logs = defaultdict(list)
start_time = time.monotonic()
while time.monotonic()- start_time < TEST_TIMESPAN:
for cam, s in socks.items():
cls.logs[cam] += messaging.drain_sock(s)
time.sleep(0.2)
managed_processes['camerad'].stop()
cls.log_by_frame_id = defaultdict(list)
cls.sensor_type = None
for cam, msgs in cls.logs.items():
if cls.sensor_type is None:
cls.sensor_type = getattr(msgs[0], msgs[0].which()).sensor.raw
expected_frames = SERVICE_LIST[cam].frequency * TEST_TIMESPAN
assert expected_frames*0.95 < len(msgs) < expected_frames*1.05, f"unexpected frame count {cam}: {expected_frames=}, got {len(msgs)}"
dts = np.abs(np.diff([getattr(m, m.which()).timestampSof/1e6 for m in msgs]) - 1000/SERVICE_LIST[cam].frequency)
assert (dts < FRAME_DELTA_TOLERANCE[cls.sensor_type]).all(), f"{cam} dts(ms) out of spec: max diff {dts.max()}, 99 percentile {np.percentile(dts, 99)}"
for m in msgs:
cls.log_by_frame_id[getattr(m, m.which()).frameId].append(m)
# strip beginning and end
for _ in range(3):
mn, mx = min(cls.log_by_frame_id.keys()), max(cls.log_by_frame_id.keys())
del cls.log_by_frame_id[mn]
del cls.log_by_frame_id[mx]
setup_camerad(cls)
def test_frame_skips(self):
skips = {}
uv.lock
Generated
+48 -59
View File
@@ -338,7 +338,7 @@ name = "coloredlogs"
version = "15.0.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "humanfriendly" },
{ name = "humanfriendly", marker = "platform_system != 'Darwin'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/c7/eed8f27100517e8c0e6b923d5f0845d0cb99763da6fdee00478f91db7325/coloredlogs-15.0.1.tar.gz", hash = "sha256:7c991aa71a4577af2f82600d8f8f3a89f936baeaf9b50a9c197da014e5bf16b0", size = 278520 }
wheels = [
@@ -535,15 +535,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/b9/f8/feced7779d755758a52d1f6635d990b8d98dc0a29fa568bbe0625f18fdf3/filelock-3.16.1-py3-none-any.whl", hash = "sha256:2082e5703d51fbf98ea75855d9d5527e33d8ff23099bec374a134febee6946b0", size = 16163 },
]
[[package]]
name = "flaky"
version = "3.8.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/5b/c5/ef69119a01427204ff2db5fc8f98001087bcce719bbb94749dcd7b191365/flaky-3.8.1.tar.gz", hash = "sha256:47204a81ec905f3d5acfbd61daeabcada8f9d4031616d9bcb0618461729699f5", size = 25248 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/7f/b8/b830fc43663246c3f3dd1ae7dca4847b96ed992537e85311e27fa41ac40e/flaky-3.8.1-py2.py3-none-any.whl", hash = "sha256:194ccf4f0d3a22b2de7130f4b62e45e977ac1b5ccad74d4d48f3005dcc38815e", size = 19139 },
]
[[package]]
name = "flatbuffers"
version = "24.3.25"
@@ -661,10 +652,10 @@ name = "gymnasium"
version = "1.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "cloudpickle" },
{ name = "farama-notifications" },
{ name = "numpy" },
{ name = "typing-extensions" },
{ name = "cloudpickle", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "farama-notifications", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4e/12/1047b8fdbfcdce74022048d916e844ad7e6e1114d81d26a7aed657e3a76d/gymnasium-1.0.0.tar.gz", hash = "sha256:9d2b66f30c1b34fe3c2ce7fae65ecf365d0e9982d2b3d860235e773328a3b403", size = 821389 }
wheels = [
@@ -676,7 +667,7 @@ name = "humanfriendly"
version = "10.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyreadline3", marker = "sys_platform == 'win32'" },
{ name = "pyreadline3", marker = "platform_system != 'Darwin' and sys_platform == 'win32'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/cc/3f/2c29224acb2e2df4d2046e4c73ee2662023c58ff5b113c4c1adac0886c43/humanfriendly-10.0.tar.gz", hash = "sha256:6b0b831ce8f15f7300721aa49829fc4e83921a9a301cc7f606be6686a2288ddc", size = 360702 }
wheels = [
@@ -971,22 +962,22 @@ name = "metadrive-simulator"
version = "0.4.2.3"
source = { url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl" }
dependencies = [
{ name = "filelock" },
{ name = "gymnasium" },
{ name = "lxml" },
{ name = "matplotlib" },
{ name = "numpy" },
{ name = "opencv-python-headless" },
{ name = "panda3d" },
{ name = "panda3d-gltf" },
{ name = "pillow" },
{ name = "progressbar" },
{ name = "psutil" },
{ name = "pygments" },
{ name = "requests" },
{ name = "shapely" },
{ name = "tqdm" },
{ name = "yapf" },
{ name = "filelock", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "gymnasium", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "lxml", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "matplotlib", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "opencv-python-headless", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d-gltf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pillow", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "progressbar", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "psutil", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pygments", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "requests", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "shapely", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "tqdm", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "yapf", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
wheels = [
{ url = "https://github.com/commaai/metadrive/releases/download/MetaDrive-minimal/metadrive_simulator-0.4.2.3-py3-none-any.whl", hash = "sha256:6242d4e37e6c592d5eb1cadf497637540d3b754b89813a88c50a93c7fc88b02d" },
@@ -1235,12 +1226,12 @@ name = "onnxruntime"
version = "1.20.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coloredlogs" },
{ name = "flatbuffers" },
{ name = "numpy" },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "sympy" },
{ name = "coloredlogs", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "flatbuffers", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "numpy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "packaging", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "protobuf", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
{ name = "sympy", marker = "platform_machine == 'aarch64' and platform_system == 'Linux'" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/a5/da/c44bf9bd66cd6d9018a921f053f28d819445c4d84b4dd4777271b0fe52a2/onnxruntime-1.20.1-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f6243e34d74423bdd1edf0ae9596dd61023b260f546ee17d701723915f06a9f7", size = 11955227 },
@@ -1254,12 +1245,12 @@ name = "onnxruntime-gpu"
version = "1.20.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "coloredlogs" },
{ name = "flatbuffers" },
{ name = "numpy" },
{ name = "packaging" },
{ name = "protobuf" },
{ name = "sympy" },
{ name = "coloredlogs", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "flatbuffers", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "numpy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "packaging", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "protobuf", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
{ name = "sympy", marker = "(platform_machine != 'aarch64' and platform_system != 'Darwin') or (platform_system != 'Darwin' and platform_system != 'Linux')" },
]
wheels = [
{ url = "https://files.pythonhosted.org/packages/e0/a5/5c2287d61f359c7342e9d59d1e3dd728a982dea85f846c7af305a801c3ca/onnxruntime_gpu-1.20.1-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1795e8bc6f9a1488a4d51d242edc4232a5ae60ec44ab4d4b0a7c65b3d17fcbff", size = 291519550 },
@@ -1271,7 +1262,7 @@ name = "opencv-python-headless"
version = "4.10.0.84"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/2f/7e/d20f68a5f1487adf19d74378d349932a386b1ece3be9be9915e5986db468/opencv-python-headless-4.10.0.84.tar.gz", hash = "sha256:f2017c6101d7c2ef8d7bc3b414c37ff7f54d64413a1847d89970b6b7069b4e1a", size = 95117755 }
wheels = [
@@ -1330,7 +1321,6 @@ dev = [
{ name = "azure-identity" },
{ name = "azure-storage-blob" },
{ name = "dictdiffer" },
{ name = "flaky" },
{ name = "lru-dict" },
{ name = "matplotlib" },
{ name = "parameterized" },
@@ -1386,7 +1376,6 @@ requires-dist = [
{ name = "crcmod" },
{ name = "cython" },
{ name = "dictdiffer", marker = "extra == 'dev'" },
{ name = "flaky", marker = "extra == 'dev'" },
{ name = "future-fstrings" },
{ name = "hypothesis", marker = "extra == 'testing'", specifier = "==6.47.*" },
{ name = "inputs" },
@@ -1480,8 +1469,8 @@ name = "panda3d-gltf"
version = "0.13"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "panda3d" },
{ name = "panda3d-simplepbr" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "panda3d-simplepbr", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/07/7f/9f18fc3fa843a080acb891af6bcc12262e7bdf1d194a530f7042bebfc81f/panda3d-gltf-0.13.tar.gz", hash = "sha256:d06d373bdd91cf530909b669f43080e599463bbf6d3ef00c3558bad6c6b19675", size = 25573 }
wheels = [
@@ -1493,8 +1482,8 @@ name = "panda3d-simplepbr"
version = "0.12.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "panda3d" },
{ name = "typing-extensions" },
{ name = "panda3d", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b1/af/505608eef09d7f9b822e69dc7631cd14102650b8fe1b6f60d9562d2788d9/panda3d-simplepbr-0.12.0.tar.gz", hash = "sha256:c71d490afeeb3a90455dcfde1d30c41f321a38742a97d18834e5c31016331ed5", size = 1929980 }
wheels = [
@@ -4365,9 +4354,9 @@ name = "pyopencl"
version = "2024.3"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "platformdirs" },
{ name = "pytools" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "pytools", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ec/28/4679ea08b84532a67fd2d270c8f87aec64dab9ab99e618927b6a26ea063e/pyopencl-2024.3.tar.gz", hash = "sha256:d5d08de9b0a6d85695caba1769aceae4e7661f06951c507bd1ce8fb7a89e2413", size = 422604 }
wheels = [
@@ -4421,7 +4410,7 @@ name = "pyqt5"
version = "5.15.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "pyqt5-sip" },
{ name = "pyqt5-sip", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/28/6c/640e3f5c734c296a7193079a86842a789edb7988dca39eab44579088a1d1/PyQt5-5.15.2.tar.gz", hash = "sha256:372b08dc9321d1201e4690182697c5e7ffb2e0770e6b4a45519025134b12e4fc", size = 3265445 }
wheels = [
@@ -4639,9 +4628,9 @@ name = "pytools"
version = "2024.1.10"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs" },
{ name = "siphash24" },
{ name = "typing-extensions" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "siphash24", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
{ name = "typing-extensions", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ee/0f/56e109c0307f831b5d598ad73976aaaa84b4d0e98da29a642e797eaa940c/pytools-2024.1.10.tar.gz", hash = "sha256:9af6f4b045212c49be32bb31fe19606c478ee4b09631886d05a32459f4ce0a12", size = 81741 }
wheels = [
@@ -4944,7 +4933,7 @@ name = "shapely"
version = "2.0.6"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "numpy" },
{ name = "numpy", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/4a/89/0d20bac88016be35ff7d3c0c2ae64b477908f1b1dfa540c5d69ac7af07fe/shapely-2.0.6.tar.gz", hash = "sha256:997f6159b1484059ec239cacaa53467fd8b5564dabe186cd84ac2944663b0bf6", size = 282361 }
wheels = [
@@ -5169,7 +5158,7 @@ name = "yapf"
version = "0.43.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "platformdirs" },
{ name = "platformdirs", marker = "platform_machine != 'aarch64' or platform_system != 'Linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/23/97/b6f296d1e9cc1ec25c7604178b48532fa5901f721bcf1b8d8148b13e5588/yapf-0.43.0.tar.gz", hash = "sha256:00d3aa24bfedff9420b2e0d5d9f5ab6d9d4268e72afbf59bb3fa542781d5218e", size = 254907 }
wheels = [