mirror of
https://github.com/sunnypilot/sunnypilot.git
synced 2026-06-11 09:24:38 +08:00
modeld_v2: safe model validation (#1855)
* modeld_v2: safe model validation * fix string * numpy * dumb * god use full attribute names please --------- Co-authored-by: Jason Wen <haibin.wen3@gmail.com>
This commit is contained in:
James Vecellio-Grant
committed by
GitHub
GitHub
parent
dfb21bd53e
commit
066ba92e77
@@ -188,7 +188,7 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device):
|
||||
n_frames = img_shape[1] // 6
|
||||
img_buf_shape = (frame_skip * (n_frames - 1) + 1, 6, img_shape[2], img_shape[3])
|
||||
|
||||
npy_keys = {}
|
||||
numpy_keys = {}
|
||||
queue_keys = {}
|
||||
|
||||
for key, shape in input_shapes.items():
|
||||
@@ -196,7 +196,7 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device):
|
||||
continue
|
||||
if len(shape) == 3 and shape[1] > 1:
|
||||
if key.startswith('desire'):
|
||||
npy_keys[key] = np.zeros(shape[2], dtype=np.float32)
|
||||
numpy_keys[key] = np.zeros(shape[2], dtype=np.float32)
|
||||
queue_keys[f'{key}_q'] = Tensor(
|
||||
np.zeros((frame_skip * shape[1], shape[0], shape[2]), dtype=np.float32),
|
||||
device=device).contiguous().realize()
|
||||
@@ -205,24 +205,24 @@ def make_supercombo_input_queues(input_shapes, frame_skip, device):
|
||||
np.zeros((frame_skip * (shape[1] - 1) + 1, shape[0], shape[2]), dtype=np.float32),
|
||||
device=device).contiguous().realize()
|
||||
else:
|
||||
npy_keys[key] = np.zeros(shape, dtype=np.float32)
|
||||
numpy_keys[key] = np.zeros(shape, dtype=np.float32)
|
||||
elif len(shape) == 2:
|
||||
npy_keys[key] = np.zeros(shape, dtype=np.float32)
|
||||
numpy_keys[key] = np.zeros(shape, dtype=np.float32)
|
||||
|
||||
if 'traffic_convention' not in npy_keys:
|
||||
if 'traffic_convention' not in numpy_keys:
|
||||
tc_shape = input_shapes.get('traffic_convention', (1, 2))
|
||||
npy_keys['traffic_convention'] = np.zeros(tc_shape, dtype=np.float32)
|
||||
numpy_keys['traffic_convention'] = np.zeros(tc_shape, dtype=np.float32)
|
||||
|
||||
npy_keys['tfm'] = np.zeros((3, 3), dtype=np.float32)
|
||||
npy_keys['big_tfm'] = np.zeros((3, 3), dtype=np.float32)
|
||||
numpy_keys['tfm'] = np.zeros((3, 3), dtype=np.float32)
|
||||
numpy_keys['big_tfm'] = np.zeros((3, 3), dtype=np.float32)
|
||||
|
||||
input_queues = {
|
||||
'img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
|
||||
'big_img_q': Tensor(np.zeros(img_buf_shape, dtype=np.uint8), device=device).contiguous().realize(),
|
||||
**queue_keys,
|
||||
**{k: Tensor(v, device='NPY').realize() for k, v in npy_keys.items()},
|
||||
**{k: Tensor(v, device='NPY').realize() for k, v in numpy_keys.items()},
|
||||
}
|
||||
return input_queues, npy_keys
|
||||
return input_queues, numpy_keys
|
||||
|
||||
|
||||
def make_run_supercombo(model_runner, nv12: NV12Frame, model_w, model_h,
|
||||
|
||||
@@ -62,11 +62,6 @@ def _find_driving_pkl(bundle):
|
||||
if _pkl_exists(pkl_path):
|
||||
return pkl_path
|
||||
|
||||
fallback = os.path.join(model_root, 'driving_tinygrad.pkl')
|
||||
if _pkl_exists(fallback):
|
||||
return fallback
|
||||
return None
|
||||
|
||||
|
||||
class FrameMeta:
|
||||
frame_id: int = 0
|
||||
@@ -125,7 +120,7 @@ class ModelState(ModelStateBase):
|
||||
self._vision_input_names = [k for k in model_metadata['input_shapes'] if 'img' in k]
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import make_supercombo_input_queues
|
||||
frame_skip = derive_frame_skip({}, model_metadata['input_shapes'])
|
||||
self.input_queues, self.npy = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.DEV)
|
||||
self.input_queues, self.numpy_inputs = make_supercombo_input_queues(model_metadata['input_shapes'], frame_skip, device=self.DEV)
|
||||
else:
|
||||
vision_metadata = metadata['vision']
|
||||
policy_keys = [k for k in metadata if k != 'vision']
|
||||
@@ -143,7 +138,7 @@ class ModelState(ModelStateBase):
|
||||
policy_input_shapes = first_policy_metadata['input_shapes']
|
||||
self._vision_input_names = [k for k in vision_input_shapes if 'img' in k]
|
||||
frame_skip = derive_frame_skip(vision_input_shapes, policy_input_shapes)
|
||||
self.input_queues, self.npy = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.DEV)
|
||||
self.input_queues, self.numpy_inputs = make_split_input_queues(vision_input_shapes, policy_input_shapes, frame_skip, device=self.DEV)
|
||||
|
||||
from openpilot.sunnypilot.modeld_v2.parse_model_outputs_split import Parser as SplitParser
|
||||
from openpilot.sunnypilot.modeld_v2.parse_model_outputs import Parser as CombinedParser
|
||||
@@ -183,7 +178,7 @@ class ModelState(ModelStateBase):
|
||||
|
||||
@property
|
||||
def desire_key(self) -> str:
|
||||
return next(k for k in self.npy if k.startswith('desire'))
|
||||
return next(k for k in self.numpy_inputs if k.startswith('desire'))
|
||||
|
||||
def run(self, bufs: dict[str, VisionBuf], transforms: dict[str, np.ndarray],
|
||||
inputs: dict[str, np.ndarray], prepare_only: bool) -> dict[str, np.ndarray] | None:
|
||||
@@ -199,16 +194,16 @@ class ModelState(ModelStateBase):
|
||||
|
||||
desire_key = self.desire_key
|
||||
inputs[desire_key][0] = 0
|
||||
self.npy[desire_key][:] = np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0)
|
||||
self.numpy_inputs[desire_key][:] = np.where(inputs[desire_key] - self.prev_desire > .99, inputs[desire_key], 0)
|
||||
self.prev_desire[:] = inputs[desire_key]
|
||||
for key in ('traffic_convention', 'lateral_control_params'):
|
||||
if key in self.npy and key in inputs:
|
||||
self.npy[key][:] = inputs[key]
|
||||
if key in self.numpy_inputs and key in inputs:
|
||||
self.numpy_inputs[key][:] = inputs[key]
|
||||
|
||||
road_key = next(n for n in bufs if 'big' not in n)
|
||||
wide_key = next(n for n in bufs if 'big' in n)
|
||||
self.npy['tfm'][:, :] = transforms[road_key].reshape(3, 3)
|
||||
self.npy['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3)
|
||||
self.numpy_inputs['tfm'][:, :] = transforms[road_key].reshape(3, 3)
|
||||
self.numpy_inputs['big_tfm'][:, :] = transforms[wide_key].reshape(3, 3)
|
||||
|
||||
if prepare_only:
|
||||
self._warp_enqueue(**self.input_queues, frame=self.full_frames[road_key], big_frame=self.full_frames[wide_key])
|
||||
@@ -236,8 +231,8 @@ class ModelState(ModelStateBase):
|
||||
if 'planplus' in outputs and 'plan' in outputs:
|
||||
outputs['plan'] = outputs['plan'] + outputs['planplus']
|
||||
|
||||
if 'desired_curvature' in outputs and 'prev_desired_curv' in self.npy:
|
||||
buf = self.npy['prev_desired_curv']
|
||||
if 'desired_curvature' in outputs and 'prev_desired_curv' in self.numpy_inputs:
|
||||
buf = self.numpy_inputs['prev_desired_curv']
|
||||
buf[0, :-1] = buf[0, 1:]
|
||||
buf[0, -1, :] = outputs['desired_curvature'][0, :] if not self.mlsim else 0
|
||||
|
||||
@@ -409,7 +404,7 @@ def main(demo=False):
|
||||
'traffic_convention': traffic_convention,
|
||||
}
|
||||
|
||||
if 'lateral_control_params' in model.npy:
|
||||
if 'lateral_control_params' in model.numpy_inputs:
|
||||
inputs['lateral_control_params'] = np.array([v_ego, lat_delay], dtype=np.float32)
|
||||
|
||||
mt1 = time.perf_counter()
|
||||
|
||||
@@ -1,62 +0,0 @@
|
||||
## Neural networks in openpilot
|
||||
To view the architecture of the ONNX networks, you can use [netron](https://netron.app/)
|
||||
|
||||
## Supercombo
|
||||
### Supercombo input format (Full size: 799906 x float32)
|
||||
* **image stream**
|
||||
* Two consecutive images (256 * 512 * 3 in RGB) recorded at 20 Hz : 393216 = 2 * 6 * 128 * 256
|
||||
* Each 256 * 512 image is represented in YUV420 with 6 channels : 6 * 128 * 256
|
||||
* Channels 0,1,2,3 represent the full-res Y channel and are represented in numpy as Y[::2, ::2], Y[::2, 1::2], Y[1::2, ::2], and Y[1::2, 1::2]
|
||||
* Channel 4 represents the half-res U channel
|
||||
* Channel 5 represents the half-res V channel
|
||||
* **wide image stream**
|
||||
* Two consecutive images (256 * 512 * 3 in RGB) recorded at 20 Hz : 393216 = 2 * 6 * 128 * 256
|
||||
* Each 256 * 512 image is represented in YUV420 with 6 channels : 6 * 128 * 256
|
||||
* Channels 0,1,2,3 represent the full-res Y channel and are represented in numpy as Y[::2, ::2], Y[::2, 1::2], Y[1::2, ::2], and Y[1::2, 1::2]
|
||||
* Channel 4 represents the half-res U channel
|
||||
* Channel 5 represents the half-res V channel
|
||||
* **desire**
|
||||
* one-hot encoded buffer to command model to execute certain actions, bit needs to be sent for the past 5 seconds (at 20FPS) : 100 * 8
|
||||
* **traffic convention**
|
||||
* one-hot encoded vector to tell model whether traffic is right-hand or left-hand traffic : 2
|
||||
* **feature buffer**
|
||||
* A buffer of intermediate features that gets appended to the current feature to form a 5 seconds temporal context (at 20FPS) : 99 * 512
|
||||
|
||||
|
||||
### Supercombo output format (Full size: XXX x float32)
|
||||
Read [here](https://github.com/commaai/openpilot/blob/90af436a121164a51da9fa48d093c29f738adf6a/selfdrive/modeld/models/driving.h#L236) for more.
|
||||
|
||||
|
||||
## Driver Monitoring Model
|
||||
* .onnx model can be run with onnx runtimes
|
||||
* .dlc file is a pre-quantized model and only runs on qualcomm DSPs
|
||||
|
||||
### input format
|
||||
* single image W = 1440 H = 960 luminance channel (Y) from the planar YUV420 format:
|
||||
* full input size is 1440 * 960 = 1382400
|
||||
* normalized ranging from 0.0 to 1.0 in float32 (onnx runner) or ranging from 0 to 255 in uint8 (snpe runner)
|
||||
* camera calibration angles (roll, pitch, yaw) from liveCalibration: 3 x float32 inputs
|
||||
|
||||
### output format
|
||||
* 84 x float32 outputs = 2 + 41 * 2 ([parsing example](https://github.com/commaai/openpilot/blob/22ce4e17ba0d3bfcf37f8255a4dd1dc683fe0c38/selfdrive/modeld/models/dmonitoring.cc#L33))
|
||||
* for each person in the front seats (2 * 41)
|
||||
* face pose: 12 = 6 + 6
|
||||
* face orientation [pitch, yaw, roll] in camera frame: 3
|
||||
* face position [dx, dy] relative to image center: 2
|
||||
* normalized face size: 1
|
||||
* standard deviations for above outputs: 6
|
||||
* face visible probability: 1
|
||||
* eyes: 20 = (8 + 1) + (8 + 1) + 1 + 1
|
||||
* eye position and size, and their standard deviations: 8
|
||||
* eye visible probability: 1
|
||||
* eye closed probability: 1
|
||||
* wearing sunglasses probability: 1
|
||||
* face occluded probability: 1
|
||||
* touching wheel probability: 1
|
||||
* paying attention probability: 1
|
||||
* (deprecated) distracted probabilities: 2
|
||||
* using phone probability: 1
|
||||
* distracted probability: 1
|
||||
* common outputs 2
|
||||
* poor camera vision probability: 1
|
||||
* left hand drive probability: 1
|
||||
@@ -1,101 +0,0 @@
|
||||
// clang++ -O2 repro.cc && ./a.out
|
||||
|
||||
#include <sched.h>
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include <cstdint>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <ctime>
|
||||
|
||||
static inline double millis_since_boot() {
|
||||
struct timespec t;
|
||||
clock_gettime(CLOCK_BOOTTIME, &t);
|
||||
return t.tv_sec * 1000.0 + t.tv_nsec * 1e-6;
|
||||
}
|
||||
|
||||
#define MODEL_WIDTH 320
|
||||
#define MODEL_HEIGHT 640
|
||||
|
||||
// null function still breaks it
|
||||
#define input_lambda(x) x
|
||||
|
||||
// this is copied from models/dmonitoring.cc, and is the code that triggers the issue
|
||||
void inner(uint8_t *resized_buf, float *net_input_buf) {
|
||||
int resized_width = MODEL_WIDTH;
|
||||
int resized_height = MODEL_HEIGHT;
|
||||
|
||||
// one shot conversion, O(n) anyway
|
||||
// yuvframe2tensor, normalize
|
||||
for (int r = 0; r < MODEL_HEIGHT/2; r++) {
|
||||
for (int c = 0; c < MODEL_WIDTH/2; c++) {
|
||||
// Y_ul
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r] = input_lambda(resized_buf[(2*r*resized_width) + (2*c)]);
|
||||
// Y_ur
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r + (2*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width) + (2*c+1)]);
|
||||
// Y_dl
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r + ((MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c)]);
|
||||
// Y_dr
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r + (3*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(2*r*resized_width+1) + (2*c+1)]);
|
||||
// U
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r + (4*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + (r*resized_width/2) + c]);
|
||||
// V
|
||||
net_input_buf[(c*MODEL_HEIGHT/2) + r + (5*(MODEL_WIDTH/2)*(MODEL_HEIGHT/2))] = input_lambda(resized_buf[(resized_width*resized_height) + ((resized_width/2)*(resized_height/2)) + (r*resized_width/2) + c]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
float trial() {
|
||||
int resized_width = MODEL_WIDTH;
|
||||
int resized_height = MODEL_HEIGHT;
|
||||
|
||||
int yuv_buf_len = (MODEL_WIDTH/2) * (MODEL_HEIGHT/2) * 6; // Y|u|v -> y|y|y|y|u|v
|
||||
|
||||
// allocate the buffers
|
||||
uint8_t *resized_buf = (uint8_t*)malloc(resized_width*resized_height*3/2);
|
||||
float *net_input_buf = (float*)malloc(yuv_buf_len*sizeof(float));
|
||||
printf("allocate -- %p 0x%x -- %p 0x%lx\n", resized_buf, resized_width*resized_height*3/2, net_input_buf, yuv_buf_len*sizeof(float));
|
||||
|
||||
// test for bad buffers
|
||||
static int CNT = 20;
|
||||
float avg = 0.0;
|
||||
for (int i = 0; i < CNT; i++) {
|
||||
double s4 = millis_since_boot();
|
||||
inner(resized_buf, net_input_buf);
|
||||
double s5 = millis_since_boot();
|
||||
avg += s5-s4;
|
||||
}
|
||||
avg /= CNT;
|
||||
|
||||
// once it's bad, it's reliably bad
|
||||
if (avg > 10) {
|
||||
printf("HIT %f\n", avg);
|
||||
printf("BAD\n");
|
||||
|
||||
for (int i = 0; i < 200; i++) {
|
||||
double s4 = millis_since_boot();
|
||||
inner(resized_buf, net_input_buf);
|
||||
double s5 = millis_since_boot();
|
||||
printf("%.2f ", s5-s4);
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
exit(0);
|
||||
}
|
||||
|
||||
// don't free so we get a different buffer each time
|
||||
//free(resized_buf);
|
||||
//free(net_input_buf);
|
||||
|
||||
return avg;
|
||||
}
|
||||
|
||||
int main() {
|
||||
while (true) {
|
||||
float ret = trial();
|
||||
printf("got %f\n", ret);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,16 +46,6 @@ class TestFindDrivingPkl:
|
||||
assert result is not None
|
||||
assert 'driving_fof_tinygrad.pkl' in result
|
||||
|
||||
def test_finds_fallback_driving_tinygrad(self, tmp_path, monkeypatch):
|
||||
(tmp_path / 'driving_tinygrad.pkl').write_bytes(b'fake')
|
||||
from openpilot.system.hardware import hw
|
||||
monkeypatch.setattr(hw.Paths, 'model_root', staticmethod(lambda: str(tmp_path)))
|
||||
|
||||
bundle = DummyBundle(models=[DummyModel('vision', 'nonexistent.pkl')])
|
||||
result = _find_driving_pkl(bundle)
|
||||
assert result is not None
|
||||
assert 'driving_tinygrad.pkl' in result
|
||||
|
||||
|
||||
# Init — assertion guard
|
||||
|
||||
@@ -84,8 +74,8 @@ class TestStockEquivalence:
|
||||
skip_keys = {'action_t'}
|
||||
assert set(state.input_queues.keys()) == set(stock_queues.keys()) - skip_keys, \
|
||||
f"Queue keys differ: v2={set(state.input_queues.keys())}, stock={set(stock_queues.keys())}"
|
||||
assert set(state.npy.keys()) == set(stock_npy.keys()) - skip_keys, \
|
||||
f"Npy keys differ: v2={set(state.npy.keys())}, stock={set(stock_npy.keys())}"
|
||||
assert set(state.numpy_inputs.keys()) == set(stock_npy.keys()) - skip_keys, \
|
||||
f"Npy keys differ: v2={set(state.numpy_inputs.keys())}, stock={set(stock_npy.keys())}"
|
||||
|
||||
def test_split_queue_keys_work_with_desire_key(self, model_state_factory):
|
||||
from openpilot.sunnypilot.modeld_v2.compile_modeld import derive_frame_skip, make_split_input_queues
|
||||
@@ -188,16 +178,16 @@ class TestInputQueueCreation:
|
||||
def test_npy_contains_transforms(self, archetype_name, model_state_factory):
|
||||
arch = ARCHETYPES[archetype_name]
|
||||
state = model_state_factory(arch)
|
||||
assert 'tfm' in state.npy, f"{arch.name}: 'tfm' missing from npy"
|
||||
assert 'big_tfm' in state.npy, f"{arch.name}: 'big_tfm' missing from npy"
|
||||
assert state.npy['tfm'].shape == (3, 3)
|
||||
assert state.npy['big_tfm'].shape == (3, 3)
|
||||
assert 'tfm' in state.numpy_inputs, f"{arch.name}: 'tfm' missing from npy"
|
||||
assert 'big_tfm' in state.numpy_inputs, f"{arch.name}: 'big_tfm' missing from npy"
|
||||
assert state.numpy_inputs['tfm'].shape == (3, 3)
|
||||
assert state.numpy_inputs['big_tfm'].shape == (3, 3)
|
||||
|
||||
@pytest.mark.parametrize("archetype_name", ARCHETYPE_NAMES)
|
||||
def test_npy_contains_desire(self, archetype_name, model_state_factory):
|
||||
arch = ARCHETYPES[archetype_name]
|
||||
state = model_state_factory(arch)
|
||||
assert arch.expected_desire_key in state.npy, \
|
||||
assert arch.expected_desire_key in state.numpy_inputs, \
|
||||
f"{arch.name}: '{arch.expected_desire_key}' missing from npy"
|
||||
|
||||
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
clang++ -I /home/batman/one/external/tensorflow/include/ -L /home/batman/one/external/tensorflow/lib -Wl,-rpath=/home/batman/one/external/tensorflow/lib main.cc -ltensorflow
|
||||
@@ -1,69 +0,0 @@
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstdlib>
|
||||
#include "tensorflow/c/c_api.h"
|
||||
|
||||
void* read_file(const char* path, size_t* out_len) {
|
||||
FILE* f = fopen(path, "r");
|
||||
if (!f) {
|
||||
return NULL;
|
||||
}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long f_len = ftell(f);
|
||||
rewind(f);
|
||||
|
||||
char* buf = (char*)calloc(f_len, 1);
|
||||
assert(buf);
|
||||
|
||||
size_t num_read = fread(buf, f_len, 1, f);
|
||||
fclose(f);
|
||||
|
||||
if (num_read != 1) {
|
||||
free(buf);
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (out_len) {
|
||||
*out_len = f_len;
|
||||
}
|
||||
|
||||
return buf;
|
||||
}
|
||||
|
||||
static void DeallocateBuffer(void* data, size_t) {
|
||||
free(data);
|
||||
}
|
||||
|
||||
int main(int argc, char* argv[]) {
|
||||
TF_Buffer* buf;
|
||||
TF_Graph* graph;
|
||||
TF_Status* status;
|
||||
char *path = argv[1];
|
||||
|
||||
// load model
|
||||
{
|
||||
size_t model_size;
|
||||
char tmp[1024];
|
||||
snprintf(tmp, sizeof(tmp), "%s.pb", path);
|
||||
printf("loading model %s\n", tmp);
|
||||
uint8_t *model_data = (uint8_t *)read_file(tmp, &model_size);
|
||||
buf = TF_NewBuffer();
|
||||
buf->data = model_data;
|
||||
buf->length = model_size;
|
||||
buf->data_deallocator = DeallocateBuffer;
|
||||
printf("loaded model of size %d\n", model_size);
|
||||
}
|
||||
|
||||
// import graph
|
||||
status = TF_NewStatus();
|
||||
graph = TF_NewGraph();
|
||||
TF_ImportGraphDefOptions *opts = TF_NewImportGraphDefOptions();
|
||||
TF_GraphImportGraphDef(graph, buf, opts, status);
|
||||
TF_DeleteImportGraphDefOptions(opts);
|
||||
TF_DeleteBuffer(buf);
|
||||
if (TF_GetCode(status) != TF_OK) {
|
||||
printf("FAIL: %s\n", TF_Message(status));
|
||||
} else {
|
||||
printf("SUCCESS\n");
|
||||
}
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
import sys
|
||||
import tensorflow as tf
|
||||
|
||||
with open(sys.argv[1], "rb") as f:
|
||||
graph_def = tf.compat.v1.GraphDef()
|
||||
graph_def.ParseFromString(f.read())
|
||||
#tf.io.write_graph(graph_def, '', sys.argv[1]+".try")
|
||||
@@ -1,38 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import os
|
||||
import time
|
||||
import numpy as np
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from openpilot.system.manager.process_config import managed_processes
|
||||
|
||||
|
||||
N = int(os.getenv("N", "5"))
|
||||
TIME = int(os.getenv("TIME", "30"))
|
||||
|
||||
if __name__ == "__main__":
|
||||
sock = messaging.sub_sock('modelV2', conflate=False, timeout=1000)
|
||||
|
||||
execution_times = []
|
||||
|
||||
for _ in range(N):
|
||||
os.environ['LOGPRINT'] = 'debug'
|
||||
managed_processes['modeld'].start()
|
||||
time.sleep(5)
|
||||
|
||||
t = []
|
||||
start = time.monotonic()
|
||||
while time.monotonic() - start < TIME:
|
||||
msgs = messaging.drain_sock(sock, wait_for_one=True)
|
||||
for m in msgs:
|
||||
t.append(m.modelV2.modelExecutionTime)
|
||||
|
||||
execution_times.append(np.array(t[10:]) * 1000)
|
||||
managed_processes['modeld'].stop()
|
||||
|
||||
print("\n\n")
|
||||
print(f"ran modeld {N} times for {TIME}s each")
|
||||
for _, t in enumerate(execution_times):
|
||||
print(f"\tavg: {sum(t)/len(t):0.2f}ms, min: {min(t):0.2f}ms, max: {max(t):0.2f}ms")
|
||||
print("\n\n")
|
||||
@@ -6,80 +6,138 @@ See the LICENSE.md file in the root directory for more details.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import os
|
||||
import pickle
|
||||
from pathlib import Path
|
||||
import numpy as np
|
||||
|
||||
from openpilot.common.params import Params
|
||||
from cereal import custom
|
||||
from openpilot.sunnypilot.models.constants import Meta, MetaTombRaider, MetaSimPose
|
||||
from openpilot.common.params import Params
|
||||
from openpilot.common.swaglog import cloudlog
|
||||
from openpilot.sunnypilot.models.constants import Meta, MetaSimPose, MetaTombRaider
|
||||
from openpilot.system.hardware.hw import Paths
|
||||
from pathlib import Path
|
||||
|
||||
# see the README.md for more details on the model selector versioning
|
||||
CURRENT_SELECTOR_VERSION = 15
|
||||
REQUIRED_MIN_SELECTOR_VERSION = 14
|
||||
|
||||
# SET ME TO THE EXACT JSON VERSION WE SET IN SUNNYPILOT_MODELS REPO
|
||||
REQUIRED_JSON_VERSION = 15
|
||||
|
||||
CUSTOM_MODEL_PATH = Paths.model_root()
|
||||
METADATA_PATH = Path(__file__).parent / '../models/supercombo_metadata.pkl'
|
||||
|
||||
ModelManager = custom.ModelManagerSP
|
||||
_LAST_VALIDATED_RAW = None
|
||||
|
||||
|
||||
def _compute_hash(file_path: str) -> str | None:
|
||||
from openpilot.common.file_chunker import read_file_chunked
|
||||
try:
|
||||
return hashlib.sha256(read_file_chunked(file_path)).hexdigest().lower()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
|
||||
|
||||
async def verify_file(file_path: str, expected_hash: str) -> bool:
|
||||
from openpilot.common.file_chunker import read_file_chunked
|
||||
try:
|
||||
data = read_file_chunked(file_path)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
return hashlib.sha256(data).hexdigest().lower() == expected_hash.lower()
|
||||
file_hash = _compute_hash(file_path)
|
||||
return file_hash == expected_hash.lower() if file_hash else False
|
||||
|
||||
|
||||
def _verify_file(file_path: str, expected_hash: str) -> bool:
|
||||
file_hash = _compute_hash(file_path)
|
||||
return file_hash == expected_hash.lower() if file_hash else False
|
||||
|
||||
|
||||
def is_bundle_version_compatible(bundle: dict) -> bool:
|
||||
"""
|
||||
Checks whether the model bundle is compatible with the current selector version constraints.
|
||||
|
||||
The bundle specifies a `minimum_selector_version`, which defines the minimum selector version
|
||||
The bundle parsed from the json specifies a `minimum_selector_version`, which defines the minimum selector version
|
||||
required to load the model. This function ensures that:
|
||||
|
||||
1. The model is not too old: the bundle must require at least `REQUIRED_MIN_SELECTOR_VERSION`.
|
||||
2. The model is not too new: it must support the current selector version (`CURRENT_SELECTOR_VERSION`).
|
||||
|
||||
This allows the selector to enforce both a minimum and maximum range of supported models,
|
||||
even if a model would otherwise be compatible.
|
||||
|
||||
:param bundle: Dictionary containing `minimum_selector_version`, as defined by the model bundle.
|
||||
:type bundle: Dict
|
||||
:return: True if the selector version is within the accepted range for the bundle; otherwise False.
|
||||
:rtype: Bool
|
||||
the bundle MUST match the `REQUIRED_JSON_VERSION` set here in helpers.
|
||||
"""
|
||||
return bool(REQUIRED_MIN_SELECTOR_VERSION <= bundle.get("minimumSelectorVersion", 0) <= CURRENT_SELECTOR_VERSION)
|
||||
return bundle.get("minimumSelectorVersion", 0) == REQUIRED_JSON_VERSION
|
||||
|
||||
|
||||
def get_active_bundle(params: Params = None) -> custom.ModelManagerSP.ModelBundle:
|
||||
"""Gets the active model bundle from cache"""
|
||||
if params is None:
|
||||
params = Params()
|
||||
def _bundle_artifacts(bundle: custom.ModelManagerSP.ModelBundle) -> list[tuple[str, str]]:
|
||||
artifacts = []
|
||||
for model in getattr(bundle, 'models', []) or []:
|
||||
for artifact in (getattr(model, 'artifact', None), getattr(model, 'metadata', None)):
|
||||
if artifact and getattr(artifact, 'fileName', None) and getattr(artifact, 'downloadUri', None):
|
||||
sha256 = getattr(artifact.downloadUri, 'sha256', None)
|
||||
if sha256:
|
||||
artifacts.append((artifact.fileName, sha256))
|
||||
return artifacts
|
||||
|
||||
|
||||
def _bundle_is_valid_locally(bundle: custom.ModelManagerSP.ModelBundle) -> bool:
|
||||
model_root = Paths.model_root()
|
||||
return all(_verify_file(os.path.join(model_root, file_name), expected_hash)
|
||||
for file_name, expected_hash in _bundle_artifacts(bundle))
|
||||
|
||||
|
||||
def _bundle_needs_reset(active_bundle: custom.ModelManagerSP.ModelBundle, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None) -> bool:
|
||||
if active_bundle is None:
|
||||
return False
|
||||
|
||||
if available_bundles is not None:
|
||||
matching_bundle = None
|
||||
for bundle in available_bundles:
|
||||
if getattr(active_bundle, 'ref', None) and getattr(bundle, 'ref', None):
|
||||
if active_bundle.ref == bundle.ref:
|
||||
matching_bundle = bundle
|
||||
break
|
||||
elif getattr(active_bundle, 'internalName', None) == getattr(bundle, 'internalName', None):
|
||||
matching_bundle = bundle
|
||||
break
|
||||
|
||||
if matching_bundle is None:
|
||||
return True
|
||||
if active_bundle.minimumSelectorVersion != matching_bundle.minimumSelectorVersion:
|
||||
return True
|
||||
|
||||
active_runner = getattr(active_bundle, 'runner', None)
|
||||
matching_runner = getattr(matching_bundle, 'runner', None)
|
||||
if active_runner is not None and matching_runner is not None:
|
||||
if getattr(active_runner, 'raw', active_runner) != getattr(matching_runner, 'raw', matching_runner):
|
||||
return True
|
||||
if set(_bundle_artifacts(active_bundle)) != set(_bundle_artifacts(matching_bundle)):
|
||||
return True
|
||||
|
||||
return not _bundle_is_valid_locally(active_bundle)
|
||||
|
||||
|
||||
def validate_active_bundle(params: Params, available_bundles: list[custom.ModelManagerSP.ModelBundle] | None = None) -> None:
|
||||
global _LAST_VALIDATED_RAW
|
||||
|
||||
raw_bundle = params.get("ModelManager_ActiveBundle")
|
||||
if not raw_bundle:
|
||||
return
|
||||
|
||||
if raw_bundle == _LAST_VALIDATED_RAW:
|
||||
return
|
||||
|
||||
active_bundle = get_active_bundle(params, raw_bundle_dict=raw_bundle)
|
||||
if active_bundle is None or _bundle_needs_reset(active_bundle, available_bundles):
|
||||
cloudlog.warning("Active model bundle invalid; resetting to default")
|
||||
params.remove("ModelManager_ActiveBundle")
|
||||
params.put("ModelRunnerTypeCache", int(custom.ModelManagerSP.Runner.stock), block=True)
|
||||
_LAST_VALIDATED_RAW = None
|
||||
else:
|
||||
_LAST_VALIDATED_RAW = raw_bundle
|
||||
|
||||
|
||||
def get_active_bundle(params: Params | None = None, raw_bundle_dict: dict | bytes | None = None) -> "custom.ModelManagerSP.ModelBundle | None":
|
||||
params = params or Params()
|
||||
try:
|
||||
if (active_bundle := params.get("ModelManager_ActiveBundle") or {}) and is_bundle_version_compatible(active_bundle):
|
||||
return custom.ModelManagerSP.ModelBundle(**active_bundle)
|
||||
active_bundle_dict = raw_bundle_dict if raw_bundle_dict is not None else (params.get("ModelManager_ActiveBundle") or {})
|
||||
if active_bundle_dict and is_bundle_version_compatible(active_bundle_dict):
|
||||
return custom.ModelManagerSP.ModelBundle(**active_bundle_dict)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def get_active_model_runner(params: Params = None, force_check=False) -> int:
|
||||
if params is None:
|
||||
params = Params()
|
||||
|
||||
def get_active_model_runner(params: Params | None = None, force_check: bool = False) -> int:
|
||||
params = params or Params()
|
||||
cached_runner_type = params.get("ModelRunnerTypeCache")
|
||||
if cached_runner_type is not None and not force_check:
|
||||
return cached_runner_type
|
||||
|
||||
runner_type = custom.ModelManagerSP.Runner.stock
|
||||
|
||||
if active_bundle := get_active_bundle(params):
|
||||
runner_type = active_bundle.runner.raw
|
||||
|
||||
@@ -88,66 +146,40 @@ def get_active_model_runner(params: Params = None, force_check=False) -> int:
|
||||
|
||||
return runner_type
|
||||
|
||||
|
||||
def _get_model():
|
||||
if bundle := get_active_bundle():
|
||||
drive_model = next(model for model in bundle.models if model.type == ModelManager.Model.Type.supercombo)
|
||||
return drive_model
|
||||
|
||||
return None
|
||||
|
||||
def load_metadata():
|
||||
metadata_path = METADATA_PATH
|
||||
|
||||
if model := _get_model():
|
||||
metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}"
|
||||
def load_metadata():
|
||||
model = _get_model()
|
||||
metadata_path = f"{CUSTOM_MODEL_PATH}/{model.metadata.fileName}" if model else METADATA_PATH
|
||||
|
||||
with open(metadata_path, 'rb') as f:
|
||||
return pickle.load(f)
|
||||
|
||||
|
||||
def prepare_inputs(model_metadata) -> dict[str, np.ndarray]:
|
||||
# img buffers are managed in openCL transform code so we don't pass them as inputs
|
||||
inputs = {
|
||||
k: np.zeros(v, dtype=np.float32).flatten()
|
||||
for k, v in model_metadata['input_shapes'].items()
|
||||
if 'img' not in k
|
||||
def prepare_inputs(model_metadata: dict) -> dict[str, np.ndarray]:
|
||||
return {
|
||||
key: np.zeros(shape, dtype=np.float32).flatten()
|
||||
for key, shape in model_metadata['input_shapes'].items()
|
||||
if 'img' not in key
|
||||
}
|
||||
|
||||
return inputs
|
||||
|
||||
def load_meta_constants(model_metadata: dict):
|
||||
""" Loads the appropriate meta model class based on key shapes"""
|
||||
if 'sim_pose' in model_metadata['input_shapes']:
|
||||
return MetaSimPose
|
||||
|
||||
def load_meta_constants(model_metadata):
|
||||
"""
|
||||
Determines and loads the appropriate meta model class based on the metadata provided. The function checks
|
||||
specific keys and conditions within the provided metadata dictionary to identify the corresponding meta
|
||||
model class to return.
|
||||
meta_slice = model_metadata['output_slices']['meta']
|
||||
if (meta_slice.start, meta_slice.stop, meta_slice.step) == (5868, 5921, None):
|
||||
return MetaTombRaider
|
||||
|
||||
:param model_metadata: Dictionary containing metadata about the model. It includes
|
||||
details such as input shapes, output slices, and other configurations for identifying
|
||||
metadata-dependent meta model classes.
|
||||
:type model_metadata: dict
|
||||
:return: The appropriate meta model class (Meta, MetaSimPose, or MetaTombRaider)
|
||||
based on the conditions and metadata provided.
|
||||
:rtype: type
|
||||
"""
|
||||
meta = Meta # Default Meta
|
||||
|
||||
if 'sim_pose' in model_metadata['input_shapes'].keys():
|
||||
# Meta for models with sim_pose input
|
||||
meta = MetaSimPose
|
||||
else:
|
||||
# Meta for Tomb Raider, it does not include sim_pose input but has the same meta slice as previous models
|
||||
meta_slice = model_metadata['output_slices']['meta']
|
||||
meta_tf_slice = slice(5868, 5921, None)
|
||||
|
||||
if (
|
||||
meta_slice.start == meta_tf_slice.start and
|
||||
meta_slice.stop == meta_tf_slice.stop and
|
||||
meta_slice.step == meta_tf_slice.step
|
||||
):
|
||||
meta = MetaTombRaider
|
||||
|
||||
return meta
|
||||
return Meta
|
||||
|
||||
|
||||
# The following method(s) are modeld helper methods
|
||||
|
||||
@@ -17,7 +17,7 @@ from openpilot.system.hardware.hw import Paths
|
||||
|
||||
from cereal import messaging, custom
|
||||
from openpilot.sunnypilot.models.fetcher import ModelFetcher
|
||||
from openpilot.sunnypilot.models.helpers import verify_file, get_active_bundle
|
||||
from openpilot.sunnypilot.models.helpers import get_active_bundle, validate_active_bundle, verify_file
|
||||
|
||||
|
||||
class ModelManagerSP:
|
||||
@@ -239,6 +239,7 @@ class ModelManagerSP:
|
||||
while True:
|
||||
try:
|
||||
self.available_models = self.model_fetcher.get_available_bundles()
|
||||
validate_active_bundle(self.params, self.available_models)
|
||||
self.active_bundle = get_active_bundle(self.params)
|
||||
|
||||
if (index_to_download := self.params.get("ModelManager_DownloadIndex")) is not None:
|
||||
@@ -252,8 +253,8 @@ class ModelManagerSP:
|
||||
self.selected_bundle = None
|
||||
|
||||
if self.params.get("ModelManager_ClearCache"):
|
||||
self.clear_model_cache()
|
||||
self.params.remove("ModelManager_ClearCache")
|
||||
self.clear_model_cache()
|
||||
self.params.remove("ModelManager_ClearCache")
|
||||
|
||||
self._report_status()
|
||||
rk.keep_time()
|
||||
|
||||
Reference in New Issue
Block a user