diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0e68bf796..6846991aa 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -49,7 +49,6 @@ jobs: - name: Run mutation tests run: | source setup.sh - scons -j8 python opendbc/safety/tests/mutation.py car_diff: @@ -61,10 +60,6 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Build opendbc - run: | - source setup.sh - scons -j8 - name: Test car diff if: github.event_name == 'pull_request' run: source setup.sh && python opendbc/car/tests/car_diff.py | tee diff.txt diff --git a/.github/workflows/update-cars-docs.yml b/.github/workflows/update-cars-docs.yml index 1d2adefc6..57466137e 100644 --- a/.github/workflows/update-cars-docs.yml +++ b/.github/workflows/update-cars-docs.yml @@ -15,7 +15,6 @@ jobs: - name: Generate Car Docs run: | pip install -e . - scons -c && scons -j$(nproc) python -m pip install jinja2==3.1.4 python opendbc/car/docs.py - uses: stefanzweifel/git-auto-commit-action@8621497c8c39c72f3e2a999a26b4ca1b5058a842 diff --git a/.gitignore b/.gitignore index 7aebff785..e76e92219 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,6 @@ *.dylib .*.swp .DS_Store -.sconsign.dblite .hypothesis *.egg-info/ *.html @@ -21,14 +20,10 @@ .vscode/ __pycache__/ *.profraw -.sconf_temp/ - opendbc/can/build/ opendbc/can/obj/ -opendbc/dbc/*_generated.dbc cppcheck-addon-ctu-file-list opendbc/safety/tests/coverage-out -compile_commands.json .mull/ diff --git a/SConscript b/SConscript deleted file mode 100644 index 6e191d710..000000000 --- a/SConscript +++ /dev/null @@ -1,5 +0,0 @@ -SConscript(['opendbc/dbc/SConscript']) - -# test files -if GetOption('extras'): - SConscript('opendbc/safety/tests/libsafety/SConscript') diff --git a/SConstruct b/SConstruct deleted file mode 100644 index 15289e752..000000000 --- a/SConstruct +++ /dev/null @@ -1,7 +0,0 @@ -AddOption('--minimal', - action='store_false', - dest='extras', - default=True, - help='the minimum build. no tests, tools, etc.') - -SConscript(['SConscript']) diff --git a/opendbc/__init__.py b/opendbc/__init__.py index 9f4ee6588..16b2824bb 100644 --- a/opendbc/__init__.py +++ b/opendbc/__init__.py @@ -4,3 +4,14 @@ DBC_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'dbc') # -I include path for e.g. "#include " INCLUDE_PATH = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), "../")) + +_generated_dbc_cache: dict[str, str] | None = None + +def get_generated_dbcs() -> dict[str, str]: + """Lazily generate all *_generated DBC content in memory. + Returns {name: content} where name has no .dbc extension.""" + global _generated_dbc_cache + if _generated_dbc_cache is None: + from opendbc.dbc.generator.generator import generate_all + _generated_dbc_cache = generate_all() + return _generated_dbc_cache diff --git a/opendbc/can/dbc.py b/opendbc/can/dbc.py index 4642232a3..cb8b59bbe 100644 --- a/opendbc/can/dbc.py +++ b/opendbc/can/dbc.py @@ -4,7 +4,7 @@ from collections.abc import Callable from dataclasses import dataclass from functools import cache -from opendbc import DBC_PATH +from opendbc import DBC_PATH, get_generated_dbcs # TODO: these should just be passed in along with the DBC file from opendbc.car.honda.hondacan import honda_checksum @@ -77,16 +77,32 @@ VAL_SPLIT_RE = re.compile(r'["]+') @cache class DBC: def __init__(self, name: str): - dbc_path = name - if not os.path.exists(dbc_path): + if os.path.exists(name): + self._parse_file(name) + else: dbc_path = os.path.join(DBC_PATH, name + ".dbc") + if os.path.exists(dbc_path): + self._parse_file(dbc_path) + else: + # try in-memory generated DBC + generated = get_generated_dbcs() + content = generated.get(name) + if content is None: + raise FileNotFoundError(f"DBC not found: {name}") + self._parse_content(name, content) - self._parse(dbc_path) - - def _parse(self, path: str): + def _parse_file(self, path: str): self.name = os.path.basename(path).replace(".dbc", "") with open(path) as f: lines = f.readlines() + self._parse_lines(lines) + + def _parse_content(self, name: str, content: str): + self.name = name + lines = content.splitlines(keepends=True) + self._parse_lines(lines) + + def _parse_lines(self, lines: list[str]): checksum_state = get_checksum_state(self.name) be_bits = [j + i * 8 for i in range(64) for j in range(7, -1, -1)] diff --git a/opendbc/can/tests/__init__.py b/opendbc/can/tests/__init__.py index 3bf02fdae..1dd625863 100644 --- a/opendbc/can/tests/__init__.py +++ b/opendbc/can/tests/__init__.py @@ -1,8 +1,9 @@ import glob import os -from opendbc import DBC_PATH +from opendbc import DBC_PATH, get_generated_dbcs -ALL_DBCS = [os.path.basename(dbc).split('.')[0] for dbc in - glob.glob(f"{DBC_PATH}/*.dbc")] +static_dbcs = [os.path.basename(dbc).split('.')[0] for dbc in + glob.glob(f"{DBC_PATH}/*.dbc")] +ALL_DBCS = sorted(set(static_dbcs + list(get_generated_dbcs().keys()))) TEST_DBC = os.path.abspath(os.path.join(os.path.dirname(__file__), "test.dbc")) diff --git a/opendbc/dbc/SConscript b/opendbc/dbc/SConscript deleted file mode 100644 index 481e024ba..000000000 --- a/opendbc/dbc/SConscript +++ /dev/null @@ -1,31 +0,0 @@ -import os -from pathlib import Path - -env = Environment(ENV=os.environ) - -generator = File("generator/generator.py") - -source_files = [ - File(str(f)) - for f in Path("generator").rglob("*") - if f.is_file() and f.suffix in {".py", ".dbc"} -] - -output_files = [ - f.name.replace(".dbc", "_generated.dbc") - for f in Path("generator").rglob("*.dbc") - if not f.name.startswith("_") -] - -# include DBCs generated by python scripts -output_files += [ - f.name.replace(".py", "_generated.dbc") - for f in Path("generator").rglob("*.py") - if not f.name.startswith(("_", "test_")) and f.name != "generator.py" -] - -generated = env.Command( - target=list(set(output_files)), - source=[generator] + source_files, - action="python3 ${SOURCES[0]}", -) diff --git a/opendbc/dbc/generator/chrysler/_stellantis_common_ram.py b/opendbc/dbc/generator/chrysler/_stellantis_common_ram.py index fa408f131..2744e7fee 100755 --- a/opendbc/dbc/generator/chrysler/_stellantis_common_ram.py +++ b/opendbc/dbc/generator/chrysler/_stellantis_common_ram.py @@ -31,13 +31,15 @@ chrysler_to_ram = { }, } -if __name__ == "__main__": + +def generate(): src = '_stellantis_common.dbc' chrysler_path = os.path.dirname(os.path.realpath(__file__)) + result = {} for out, addr_lookup in chrysler_to_ram.items(): - with open(os.path.join(chrysler_path, src), encoding='utf-8') as in_f, open(os.path.join(chrysler_path, out), 'w', encoding='utf-8') as out_f: - out_f.write(f'CM_ "Generated from {src}"\n\n') + with open(os.path.join(chrysler_path, src), encoding='utf-8') as in_f: + parts = [f'CM_ "Generated from {src}"\n\n'] wrote_addrs = set() for line in in_f.readlines(): @@ -45,10 +47,13 @@ if __name__ == "__main__": sl = line.split(' ') addr = int(sl[1]) wrote_addrs.add(addr) - sl[1] = str(addr_lookup.get(addr, addr)) line = ' '.join(sl) - out_f.write(line) + parts.append(line) missing_addrs = set(addr_lookup.keys()) - wrote_addrs assert len(missing_addrs) == 0, f"Missing addrs from {src}: {missing_addrs}" + + result[out] = ''.join(parts) + + return result diff --git a/opendbc/dbc/generator/generator.py b/opendbc/dbc/generator/generator.py index 187eabf01..f8286eac1 100755 --- a/opendbc/dbc/generator/generator.py +++ b/opendbc/dbc/generator/generator.py @@ -1,65 +1,92 @@ #!/usr/bin/env python3 +import importlib import os import re -import glob -import subprocess +from pathlib import Path generator_path = os.path.dirname(os.path.realpath(__file__)) -opendbc_root = os.path.join(generator_path, '../') include_pattern = re.compile(r'CM_ "IMPORT (.*?)";\n') -generated_suffix = '_generated.dbc' -def read_dbc(src_dir: str, filename: str) -> str: +def _read_dbc(src_dir: str, filename: str, extra_files: dict[str, str] | None = None) -> str: + if extra_files and filename in extra_files: + return extra_files[filename] with open(os.path.join(src_dir, filename), encoding='utf-8') as file_in: return file_in.read() -def create_dbc(src_dir: str, filename: str, output_path: str): - dbc_file_in = read_dbc(src_dir, filename) - +def _create_dbc_content(src_dir: str, filename: str, extra_files: dict[str, str] | None = None) -> str: + dbc_file_in = _read_dbc(src_dir, filename, extra_files) includes = include_pattern.findall(dbc_file_in) - output_filename = filename.replace('.dbc', generated_suffix) - output_file_location = os.path.join(output_path, output_filename) + parts = ['CM_ "AUTOGENERATED FILE, DO NOT EDIT";\n'] + for include_filename in includes: + parts.append(f'\n\nCM_ "Imported file {include_filename} starts here";\n') + parts.append(_read_dbc(src_dir, include_filename, extra_files)) - with open(output_file_location, 'w', encoding='utf-8') as dbc_file_out: - dbc_file_out.write('CM_ "AUTOGENERATED FILE, DO NOT EDIT";\n') + parts.append(f'\nCM_ "{filename} starts here";\n') + core_dbc = include_pattern.sub('', dbc_file_in) + parts.append(core_dbc) - for include_filename in includes: - include_file_header = f'\n\nCM_ "Imported file {include_filename} starts here";\n' - dbc_file_out.write(include_file_header) - - include_file = read_dbc(src_dir, include_filename) - dbc_file_out.write(include_file) - - dbc_file_out.write(f'\nCM_ "{filename} starts here";\n') - - core_dbc = include_pattern.sub('', dbc_file_in) - dbc_file_out.write(core_dbc) + return ''.join(parts) -def create_all(output_path: str): - # clear out old DBCs - for f in glob.glob(f"{output_path}/*{generated_suffix}"): - os.remove(f) +def _collect_script_outputs() -> dict[str, dict[str, str]]: + """Import and call generate() from each sub-generator script. + Returns {dir_name: {filename: content}}.""" + outputs: dict[str, dict[str, str]] = {} - # run python generator scripts first - for f in glob.glob(f"{generator_path}/*/*.py"): - subprocess.check_call(f) + for py_file in sorted(Path(generator_path).rglob("*.py")): + if py_file.name.startswith("test_") or py_file.name == "generator.py": + continue + dir_name = py_file.parent.name + module_name = f"opendbc.dbc.generator.{dir_name}.{py_file.stem}" + mod = importlib.import_module(module_name) + if hasattr(mod, 'generate'): + outputs.setdefault(dir_name, {}).update(mod.generate()) + + return outputs + + +def generate_all() -> dict[str, str]: + """Generate all DBC content in memory. Returns {name: content} where name has no .dbc extension.""" + script_outputs = _collect_script_outputs() + + result = {} for src_dir, _, filenames in os.walk(generator_path): if src_dir == generator_path: continue - #print(src_dir) - for filename in filenames: - if filename.startswith('_') or not filename.endswith('.dbc'): - continue + dir_name = os.path.basename(src_dir) + extra = script_outputs.get(dir_name, {}) - #print(filename) - create_dbc(src_dir, filename, output_path) + # all non-_ .dbc files: on-disk templates + script-generated + all_dbc_files = {f for f in filenames if f.endswith('.dbc') and not f.startswith('_')} + all_dbc_files |= {f for f in extra if not f.startswith('_')} + + for filename in sorted(all_dbc_files): + output_name = filename.replace('.dbc', '_generated') + content = _create_dbc_content(src_dir, filename, extra) + result[output_name] = content + + return result + + +def create_all(output_path: str): + """Generate all DBC files and write them to output_path (for backward compatibility).""" + import glob + generated_suffix = '_generated.dbc' + + # clear out old generated DBCs + for f in glob.glob(os.path.join(output_path, f"*{generated_suffix}")): + os.remove(f) + + for name, content in generate_all().items(): + with open(os.path.join(output_path, name + '.dbc'), 'w', encoding='utf-8') as f: + f.write(content) if __name__ == "__main__": + opendbc_root = os.path.join(generator_path, '../') create_all(opendbc_root) diff --git a/opendbc/dbc/generator/hyundai/hyundai_kia_mando_corner_radar.py b/opendbc/dbc/generator/hyundai/hyundai_kia_mando_corner_radar.py index aad417e32..e8d723193 100755 --- a/opendbc/dbc/generator/hyundai/hyundai_kia_mando_corner_radar.py +++ b/opendbc/dbc/generator/hyundai/hyundai_kia_mando_corner_radar.py @@ -1,12 +1,10 @@ #!/usr/bin/env python3 from collections import namedtuple -import os -if __name__ == "__main__": - dbc_name = os.path.basename(__file__).replace(".py", ".dbc") - hyundai_path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f: - f.write(""" + +def generate(): + parts = [] + parts.append(""" VERSION "" @@ -45,8 +43,8 @@ BS_: BU_: XXX """) - for a in [0x100, 0x200]: - f.write(f""" + for a in [0x100, 0x200]: + parts.append(f""" BO_ {a} RADAR_POINTS_METADATA_0x{a:x}: 64 RADAR SG_ SIGNAL_1 : 0|32@1+ (1,0) [0|255] "" XXX SG_ SIGNAL_2 : 32|32@1+ (1,0) [0|65535] "" XXX @@ -108,63 +106,65 @@ BO_ {a} RADAR_POINTS_METADATA_0x{a:x}: 64 RADAR SG_ SIGNAL_58 : 418|1@1+ (1,0) [0|3] "" XXX """) - # radar points are sent at 20 Hz in groups of 1 to 13 messages - # each message has 5 radar points for a total of 65 points max - # each radar point is 101 bits so the alignment is not consistent - RadarPointSignal = namedtuple("RadarPointSignal", ["name", "start", "length", "scale", "offset"]) - radar_point_signals = ( - RadarPointSignal("DISTANCE", 7, 14, 1/64, 0), - RadarPointSignal("", 21, 2, 1, 0), - RadarPointSignal("", 23, 8, 1/512, -127/512), - RadarPointSignal("REL_VELOCITY", 31, 13, 1/32, -66), - RadarPointSignal("", 44, 2, 1, 0), - RadarPointSignal("", 46, 2, 1, 0), - RadarPointSignal("AZIMUTH", 48, 12, 1/512, -2047/512), - RadarPointSignal("", 60, 2, 1, 0), - RadarPointSignal("", 62, 1, 1, 0), - RadarPointSignal("", 63, 7, 1, 0), - RadarPointSignal("", 70, 1, 1, 0), - RadarPointSignal("", 71, 6, 1, 0), - RadarPointSignal("", 77, 2, 1, 0), - RadarPointSignal("", 79, 8, 1/512, -127/512), - RadarPointSignal("", 87, 1, 1, 0), - RadarPointSignal("", 88, 2, 1, 0), - RadarPointSignal("", 90, 3, 1, 0), - # last 15 bits are controlled by LAYOUT_ID (seems to always zero, so below is layout 0) - RadarPointSignal("", 93, 6, 1, 0), - RadarPointSignal("", 99, 8, 1, 0), - RadarPointSignal("", 107, 1, 1, 0), - ) - radar_point_bit_count = sum([s.length for s in radar_point_signals]) + # radar points are sent at 20 Hz in groups of 1 to 13 messages + # each message has 5 radar points for a total of 65 points max + # each radar point is 101 bits so the alignment is not consistent + RadarPointSignal = namedtuple("RadarPointSignal", ["name", "start", "length", "scale", "offset"]) + radar_point_signals = ( + RadarPointSignal("DISTANCE", 7, 14, 1/64, 0), + RadarPointSignal("", 21, 2, 1, 0), + RadarPointSignal("", 23, 8, 1/512, -127/512), + RadarPointSignal("REL_VELOCITY", 31, 13, 1/32, -66), + RadarPointSignal("", 44, 2, 1, 0), + RadarPointSignal("", 46, 2, 1, 0), + RadarPointSignal("AZIMUTH", 48, 12, 1/512, -2047/512), + RadarPointSignal("", 60, 2, 1, 0), + RadarPointSignal("", 62, 1, 1, 0), + RadarPointSignal("", 63, 7, 1, 0), + RadarPointSignal("", 70, 1, 1, 0), + RadarPointSignal("", 71, 6, 1, 0), + RadarPointSignal("", 77, 2, 1, 0), + RadarPointSignal("", 79, 8, 1/512, -127/512), + RadarPointSignal("", 87, 1, 1, 0), + RadarPointSignal("", 88, 2, 1, 0), + RadarPointSignal("", 90, 3, 1, 0), + # last 15 bits are controlled by LAYOUT_ID (seems to always zero, so below is layout 0) + RadarPointSignal("", 93, 6, 1, 0), + RadarPointSignal("", 99, 8, 1, 0), + RadarPointSignal("", 107, 1, 1, 0), + ) + radar_point_bit_count = sum([s.length for s in radar_point_signals]) - for a in [0x101, 0x201]: - f.write(f""" + for a in [0x101, 0x201]: + parts.append(f""" BO_ {a} RADAR_POINTS_0x{a:x}: 64 RADAR SG_ MESSAGE_ID : 0|5@1+ (1,0) [0|31] "" XXX SG_ LAYOUT_ID : 5|2@1+ (1,0) [0|3] "" XXX """) - bit_idx = radar_point_signals[0].start - for i in range(5): - signal_idx = 1 - for sig in radar_point_signals: - if sig.name: - sig_name = f"POINT_{i+1}_{sig.name}" - else: - sig_name = f"POINT_{i+1}_SIGNAL_{signal_idx}" - signal_idx += 1 + bit_idx = radar_point_signals[0].start + for i in range(5): + signal_idx = 1 + for sig in radar_point_signals: + if sig.name: + sig_name = f"POINT_{i+1}_{sig.name}" + else: + sig_name = f"POINT_{i+1}_SIGNAL_{signal_idx}" + signal_idx += 1 - sig_start_idx = i * radar_point_bit_count + sig.start - assert bit_idx == sig_start_idx, f"signal overlap or gap!!! {bit_idx} != {sig_start_idx}" - min_val = round(sig.offset, 10) - max_val = round((2**sig.length - 1) * sig.scale + sig.offset, 10) + sig_start_idx = i * radar_point_bit_count + sig.start + assert bit_idx == sig_start_idx, f"signal overlap or gap!!! {bit_idx} != {sig_start_idx}" + min_val = round(sig.offset, 10) + max_val = round((2**sig.length - 1) * sig.scale + sig.offset, 10) - f.write(f" SG_ {sig_name} : {sig_start_idx}|{sig.length}@1+ ({sig.scale},{sig.offset}) [{min_val}|{max_val}] \"\" XXX\n") - bit_idx += sig.length + parts.append(f" SG_ {sig_name} : {sig_start_idx}|{sig.length}@1+ ({sig.scale},{sig.offset}) [{min_val}|{max_val}] \"\" XXX\n") + bit_idx += sig.length - # checksum is across a group of 0x100/200 and 0x101/201 messages (no checksums inside the other messages) - # ccitt_crc16 = mkCrcFun(0x11021, initCrc=0xffff, xorOut=0x0000, rev=False) - for a in [0x104, 0x204]: - f.write(f""" + # checksum is across a group of 0x100/200 and 0x101/201 messages (no checksums inside the other messages) + # ccitt_crc16 = mkCrcFun(0x11021, initCrc=0xffff, xorOut=0x0000, rev=False) + for a in [0x104, 0x204]: + parts.append(f""" BO_ {a} RADAR_POINTS_CHECKSUM_0x{a:x}: 3 RADAR SG_ CRC16 : 0|16@1+ (1,0) [0|65535] "" XXX """) + + return {"hyundai_kia_mando_corner_radar.dbc": "".join(parts)} diff --git a/opendbc/dbc/generator/hyundai/hyundai_kia_mando_front_radar.py b/opendbc/dbc/generator/hyundai/hyundai_kia_mando_front_radar.py index ee8dde64d..78c5410ca 100755 --- a/opendbc/dbc/generator/hyundai/hyundai_kia_mando_front_radar.py +++ b/opendbc/dbc/generator/hyundai/hyundai_kia_mando_front_radar.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -import os -if __name__ == "__main__": - dbc_name = os.path.basename(__file__).replace(".py", ".dbc") - hyundai_path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(hyundai_path, dbc_name), "w", encoding='utf-8') as f: - f.write(""" + +def generate(): + parts = [] + parts.append(""" VERSION "" @@ -44,9 +42,9 @@ BS_: BU_: XXX """) - # note: 0x501/0x502 seem to be special in 0x5XX range - for a in range(0x500, 0x500 + 32): - f.write(f""" + # note: 0x501/0x502 seem to be special in 0x5XX range + for a in range(0x500, 0x500 + 32): + parts.append(f""" BO_ {a} RADAR_TRACK_{a:x}: 8 RADAR SG_ UNKNOWN_1 : 7|8@0- (1,0) [-128|127] "" XXX SG_ AZIMUTH : 12|10@0- (0.2,0) [-102.4|102.2] "" XXX @@ -59,3 +57,5 @@ BO_ {a} RADAR_TRACK_{a:x}: 8 RADAR SG_ REL_SPEED : 53|14@0- (0.01,0) [-81.92|81.92] "" XXX SG_ STATE_2 : 55|2@0+ (1,0) [0|3] "" XXX """) + + return {"hyundai_kia_mando_front_radar.dbc": "".join(parts)} diff --git a/opendbc/dbc/generator/rivian/rivian_mando_front_radar.py b/opendbc/dbc/generator/rivian/rivian_mando_front_radar.py index d9a89752f..5848d26fe 100755 --- a/opendbc/dbc/generator/rivian/rivian_mando_front_radar.py +++ b/opendbc/dbc/generator/rivian/rivian_mando_front_radar.py @@ -1,11 +1,9 @@ #!/usr/bin/env python3 -import os -if __name__ == "__main__": - dbc_name = os.path.basename(__file__).replace(".py", ".dbc") - rivian_path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(rivian_path, dbc_name), "w", encoding='utf-8') as f: - f.write(""" + +def generate(): + parts = [] + parts.append(""" VERSION "" @@ -44,8 +42,8 @@ BS_: BU_: XXX """) - for a in range(0x500, 0x500 + 32): - f.write(f""" + for a in range(0x500, 0x500 + 32): + parts.append(f""" BO_ {a} RADAR_TRACK_{a:x}: 8 RADAR SG_ CHECKSUM : 0|8@1+ (1,0) [0|255] "" XXX SG_ COUNTER : 11|4@0+ (1,0) [0|15] "" XXX @@ -57,8 +55,10 @@ BO_ {a} RADAR_TRACK_{a:x}: 8 RADAR SG_ REL_SPEED : 53|14@0- (0.01,0) [-81.92|81.92] "m/s" XXX """) - for a in range(0x500, 0x500 + 32): - f.write(f""" + for a in range(0x500, 0x500 + 32): + parts.append(f""" VAL_ {a} STATE 0 "Empty" 1 "New" 2 "New_updated" 3 "Updated" 4 "Coasting" 7 "New_coasting" ; VAL_ {a} MODE 0 "None" 1 "SRR" 2 "LRR" 3 "SRR_and_LRR" ; """) + + return {"rivian_mando_front_radar.dbc": "".join(parts)} diff --git a/opendbc/dbc/generator/tesla/tesla_radar_bosch.py b/opendbc/dbc/generator/tesla/tesla_radar_bosch.py index 6586655ec..b84458325 100755 --- a/opendbc/dbc/generator/tesla/tesla_radar_bosch.py +++ b/opendbc/dbc/generator/tesla/tesla_radar_bosch.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -import os from opendbc.dbc.generator.tesla._radar_common import get_radar_point_definition, get_val_definition -if __name__ == "__main__": - dbc_name = os.path.basename(__file__).replace(".py", ".dbc") - tesla_path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(tesla_path, dbc_name), "w", encoding='utf-8') as f: - f.write(""" + +def generate(): + parts = [] + parts.append(""" VERSION "" NS_ : @@ -133,15 +131,15 @@ BO_ 1281 TeslaRadarAlertMatrix: 8 Radar SG_ unused62 : 62|2@1+ (1,0) [0|3] "" Autopilot """) - M_RANGE = range(0x310, 0x36D + 1, 3) - for i, base_id in enumerate(M_RANGE): - f.write(get_radar_point_definition(base_id, f"RadarPoint{i}")) + M_RANGE = range(0x310, 0x36D + 1, 3) + for i, base_id in enumerate(M_RANGE): + parts.append(get_radar_point_definition(base_id, f"RadarPoint{i}")) - L_RANGE = range(0x371, 0x37D + 1, 3) - for i, base_id in enumerate(L_RANGE): - f.write(get_radar_point_definition(base_id, f"ProcessedRadarPoint{i+1}")) + L_RANGE = range(0x371, 0x37D + 1, 3) + for i, base_id in enumerate(L_RANGE): + parts.append(get_radar_point_definition(base_id, f"ProcessedRadarPoint{i+1}")) - f.write(""" + parts.append(""" BO_ 697 VIN_VIP_405HS: 8 Autopilot SG_ VIN_MuxID M : 0|8@1+ (1,0) [0|0] "" Radar SG_ VIN_Part1 m16 : 47|24@0+ (1,0) [0|16777215] "" Radar @@ -278,5 +276,7 @@ BA_ "GenMsgCycleTime" BO_ 729 1000; VAL_ 681 Msg2A9_FourWheelDrive 3 "SNA" 2 "UNUSED" 1 "4WD" 0 "2WD" ;""") - for base_id in list(M_RANGE) + list(L_RANGE): - f.write(get_val_definition(base_id)) + for base_id in list(M_RANGE) + list(L_RANGE): + parts.append(get_val_definition(base_id)) + + return {"tesla_radar_bosch.dbc": "".join(parts)} diff --git a/opendbc/dbc/generator/tesla/tesla_radar_continental.py b/opendbc/dbc/generator/tesla/tesla_radar_continental.py index 36355d4a0..e4468b61b 100755 --- a/opendbc/dbc/generator/tesla/tesla_radar_continental.py +++ b/opendbc/dbc/generator/tesla/tesla_radar_continental.py @@ -1,13 +1,11 @@ #!/usr/bin/env python3 -import os from opendbc.dbc.generator.tesla._radar_common import get_radar_point_definition, get_val_definition -if __name__ == "__main__": - dbc_name = os.path.basename(__file__).replace(".py", ".dbc") - tesla_path = os.path.dirname(os.path.realpath(__file__)) - with open(os.path.join(tesla_path, dbc_name), "w", encoding='utf-8') as f: - f.write(""" + +def generate(): + parts = [] + parts.append(""" VERSION "" NS_ : @@ -66,12 +64,14 @@ BO_ 1601 UDS_radcRequest: 8 Diag SG_ UDS_radcRequestData : 7|64@0+ (1,0) [0|1.84467e+19] "" Radar """) - POINT_RANGE = range(0x410, 0x45E + 1, 2) - for i, base_id in enumerate(POINT_RANGE): - f.write(get_radar_point_definition(base_id, f"RadarPoint{i}")) + POINT_RANGE = range(0x410, 0x45E + 1, 2) + for i, base_id in enumerate(POINT_RANGE): + parts.append(get_radar_point_definition(base_id, f"RadarPoint{i}")) - f.write(""" + parts.append(""" VAL_ 1025 lowPowerMode 1 "COMMANDED_LOW_POWER" 0 "DEFAULT_LOW_POWER" 2 "NORMAL_POWER" 3 "SNA";""") - for base_id in list(POINT_RANGE): - f.write(get_val_definition(base_id)) + for base_id in list(POINT_RANGE): + parts.append(get_val_definition(base_id)) + + return {"tesla_radar_continental.dbc": "".join(parts)} diff --git a/opendbc/safety/tests/libsafety/SConscript b/opendbc/safety/tests/libsafety/SConscript deleted file mode 100644 index 3117d985f..000000000 --- a/opendbc/safety/tests/libsafety/SConscript +++ /dev/null @@ -1,45 +0,0 @@ -import os -import platform - -system = platform.system() - -env = Environment( - CFLAGS=[ - '-Wall', - "-Wextra", - '-Werror', - '-nostdlib', - '-fno-builtin', - '-std=gnu11', - '-Wfatal-errors', - '-Wno-pointer-to-int-cast', - '-g', - '-O0', - '-fno-omit-frame-pointer', - '-DALLOW_DEBUG', - ], - LINKFLAGS=[ - '-fsanitize=undefined', - '-fno-sanitize-recover=undefined', - ], - CPPPATH=["#"], - tools=["default", "compilation_db"], -) - -# add coverage if available -# Use TryCompile (not TryLink) because -nostdlib in CFLAGS breaks the link probe. -conf = Configure(env, log_file=os.devnull) -prev = env['CFLAGS'][:] -env.Append(CFLAGS=['-fprofile-arcs', '-ftest-coverage']) -has_coverage = conf.TryCompile('int x;\n', '.c') -env['CFLAGS'] = prev -if has_coverage: - env.Append(CFLAGS=['-fprofile-arcs', '-ftest-coverage']) - env.Append(LINKFLAGS=['-fprofile-arcs', '-ftest-coverage']) -env = conf.Finish() - -safety = env.SharedObject("safety.os", "safety.c") -libsafety = env.SharedLibrary("libsafety.so", [safety]) - -# GCC-style note file is generated by compiler, allow scons to clean it up -env.SideEffect("safety.gcno", safety) diff --git a/opendbc/safety/tests/libsafety/libsafety_py.py b/opendbc/safety/tests/libsafety/libsafety_py.py index be98fdab2..244c0ae66 100644 --- a/opendbc/safety/tests/libsafety/libsafety_py.py +++ b/opendbc/safety/tests/libsafety/libsafety_py.py @@ -1,10 +1,40 @@ import os +import subprocess +import tempfile +from pathlib import Path + from cffi import FFI from opendbc.safety import LEN_TO_DLC libsafety_dir = os.path.dirname(os.path.abspath(__file__)) + +def _build_libsafety() -> str: + """Compile libsafety.so to a temp file and return its path.""" + root = str(Path(libsafety_dir).parents[3]) + safety_c = os.path.join(libsafety_dir, "safety.c") + safety_os = os.path.join(libsafety_dir, "safety.os") + + cflags = [ + '-Wall', '-Wextra', '-Werror', '-nostdlib', '-fno-builtin', + '-std=gnu11', '-Wfatal-errors', '-Wno-pointer-to-int-cast', + '-g', '-O0', '-fno-omit-frame-pointer', '-DALLOW_DEBUG', + '-fprofile-arcs', '-ftest-coverage', + ] + ldflags = [ + '-fsanitize=undefined', '-fno-sanitize-recover=undefined', + '-fprofile-arcs', '-ftest-coverage', + ] + + fd, libsafety_so = tempfile.mkstemp(suffix='.so') + os.close(fd) + + subprocess.check_call(['cc', '-fPIC', *cflags, '-I', root, '-c', safety_c, '-o', safety_os]) + subprocess.check_call(['cc', '-shared', safety_os, '-o', libsafety_so, *ldflags]) + return libsafety_so + + ffi = FFI() ffi.cdef(""" @@ -83,12 +113,18 @@ int mutation_get_active_mutant(void); class LibSafety: pass -libsafety: LibSafety = ffi.dlopen(os.path.join(libsafety_dir, "libsafety.so")) +libsafety: LibSafety def load(path): global libsafety libsafety = ffi.dlopen(str(path)) +def __getattr__(name): + if name == "libsafety": + load(_build_libsafety()) + return libsafety + raise AttributeError(name) + def make_CANPacket(addr: int, bus: int, dat): ret = ffi.new('CANPacket_t *') ret[0].extended = 1 if addr >= 0x800 else 0 diff --git a/opendbc/safety/tests/test.sh b/opendbc/safety/tests/test.sh index 83a8fb97f..3df7eee9a 100755 --- a/opendbc/safety/tests/test.sh +++ b/opendbc/safety/tests/test.sh @@ -6,9 +6,8 @@ cd $DIR source ../../../setup.sh -# reset coverage data and generate gcc note file +# reset coverage data rm -f ./libsafety/*.gcda -scons -j$(nproc) -D # run safety tests and generate coverage data python -m unittest discover -s . -p 'test_*.py' -t ../../../ diff --git a/pyproject.toml b/pyproject.toml index 36d5f12c3..02e733790 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,6 @@ requires-python = ">=3.11,<3.13" # pycapnp doesn't work with 3.13 urls = { "homepage" = "https://github.com/commaai/opendbc" } dependencies = [ - "scons", "numpy", "tqdm", "pycapnp==2.1.0", @@ -90,9 +89,6 @@ ignore = [ ] flake8-implicit-str-concat.allow-multiline=false -[tool.ruff.lint.per-file-ignores] -"site_scons/*" = ["ALL"] - [tool.ruff.lint.flake8-tidy-imports.banned-api] "numpy.mean".msg = "Sum and divide. np.mean is slow" diff --git a/test.sh b/test.sh index e81841b65..d7fd72b2a 100755 --- a/test.sh +++ b/test.sh @@ -9,13 +9,10 @@ source ./setup.sh # *** uv lockfile check *** uv lock --check -# *** build *** -scons -j8 - # *** lint + test *** lefthook run test # *** all done *** GREEN='\033[0;32m' NC='\033[0m' -printf "\n${GREEN}All good!${NC} Finished build, lint, and test in ${SECONDS}s\n" +printf "\n${GREEN}All good!${NC} Finished lint and test in ${SECONDS}s\n" diff --git a/uv.lock b/uv.lock index d92da4bf9..5b013cdd4 100644 --- a/uv.lock +++ b/uv.lock @@ -384,7 +384,6 @@ dependencies = [ { name = "numpy" }, { name = "pycapnp" }, { name = "pycryptodome" }, - { name = "scons" }, { name = "tqdm" }, ] @@ -428,7 +427,6 @@ requires-dist = [ { name = "pycapnp", specifier = "==2.1.0" }, { name = "pycryptodome" }, { name = "ruff", marker = "extra == 'testing'" }, - { name = "scons" }, { name = "tqdm" }, { name = "tree-sitter", marker = "extra == 'testing'" }, { name = "tree-sitter-c", marker = "extra == 'testing'" }, @@ -547,15 +545,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6d/78/097c0798b1dab9f8affe73da9642bb4500e098cb27fd8dc9724816ac747b/ruff-0.15.2-py3-none-win_arm64.whl", hash = "sha256:cabddc5822acdc8f7b5527b36ceac55cc51eec7b1946e60181de8fe83ca8876e", size = 10941649, upload-time = "2026-02-19T22:32:18.108Z" }, ] -[[package]] -name = "scons" -version = "4.10.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7d/c9/2f430bb39e4eccba32ce8008df4a3206df651276422204e177a09e12b30b/scons-4.10.1.tar.gz", hash = "sha256:99c0e94a42a2c1182fa6859b0be697953db07ba936ecc9817ae0d218ced20b15", size = 3258403, upload-time = "2025-11-16T22:43:39.258Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/bf/931fb9fbb87234c32b8b1b1c15fba23472a10777c12043336675633809a7/scons-4.10.1-py3-none-any.whl", hash = "sha256:bd9d1c52f908d874eba92a8c0c0a8dcf2ed9f3b88ab956d0fce1da479c4e7126", size = 4136069, upload-time = "2025-11-16T22:43:35.933Z" }, -] - [[package]] name = "sortedcontainers" version = "2.4.0"