dragonpilot beta3

date: 2023-10-09T10:55:55
commit: 91b6e3aecd7170f24bccacb10c515ec281c30295

system/camerad/snapshot/snapshot.py

@@ -7,11 +7,11 @@ from PIL import Image
 
 import cereal.messaging as messaging
 from cereal.visionipc import VisionIpcClient, VisionStreamType
-from common.params import Params
-from common.realtime import DT_MDL
-from system.hardware import PC
-from selfdrive.controls.lib.alertmanager import set_offroad_alert
-from selfdrive.manager.process_config import managed_processes
+from openpilot.common.params import Params
+from openpilot.common.realtime import DT_MDL
+from openpilot.system.hardware import PC
+from openpilot.selfdrive.controls.lib.alertmanager import set_offroad_alert
+from openpilot.selfdrive.manager.process_config import managed_processes
 
 LM_THRESH = 120  # defined in system/camerad/imgproc/utils.h
 

system/hardware/__init__.py

@@ -1,9 +1,9 @@
 import os
 from typing import cast
 
-from system.hardware.base import HardwareBase
-from system.hardware.tici.hardware import Tici
-from system.hardware.pc.hardware import Pc
+from openpilot.system.hardware.base import HardwareBase
+from openpilot.system.hardware.tici.hardware import Tici
+from openpilot.system.hardware.pc.hardware import Pc
 
 TICI = os.path.isfile('/TICI')
 AGNOS = os.path.isfile('/AGNOS')

system/hardware/base.h

@@ -2,6 +2,9 @@
 
 #include <cstdlib>
 #include <fstream>
+#include <map>
+#include <string>
+
 #include "cereal/messaging/messaging.h"
 
 // no-op base hw class
@@ -11,10 +14,10 @@ public:
   static constexpr float MIN_VOLUME = 0.2;
 
   static std::string get_os_version() { return ""; }
-  static std::string get_name() { return ""; };
-  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; };
-  static int get_voltage() { return 0; };
-  static int get_current() { return 0; };
+  static std::string get_name() { return ""; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::UNKNOWN; }
+  static int get_voltage() { return 0; }
+  static int get_current() { return 0; }
 
   static std::string get_serial() { return "cccccc"; }
 

system/hardware/hw.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <string>
+
 #include "system/hardware/base.h"
 #include "common/util.h"
 
@@ -12,16 +14,37 @@
 #endif
 
 namespace Path {
-inline std::string log_root() {
-  if (const char *env = getenv("LOG_ROOT")) {
-    return env;
+  inline std::string openpilot_prefix() {
+    return util::getenv("OPENPILOT_PREFIX", "");
+  }
+
+  inline std::string comma_home() {
+    return util::getenv("HOME") + "/.comma" + Path::openpilot_prefix();
+  }
+
+  inline std::string log_root() {
+    if (const char *env = getenv("LOG_ROOT")) {
+      return env;
+    }
+    return Hardware::PC() ? Path::comma_home() + "/media/0/realdata" : "/data/media/0/realdata";
+  }
+
+  inline std::string params() {
+    return Hardware::PC() ? util::getenv("PARAMS_ROOT", Path::comma_home() + "/params") : "/data/params";
+  }
+
+  inline std::string rsa_file() {
+    return Hardware::PC() ? Path::comma_home() + "/persist/comma/id_rsa" : "/persist/comma/id_rsa";
+  }
+  
+  inline std::string swaglog_ipc() {
+    return "ipc:///tmp/logmessage" + Path::openpilot_prefix();
+  }
+
+  inline std::string download_cache_root() {
+    if (const char *env = getenv("COMMA_CACHE")) {
+      return env;
+    }
+    return "/tmp/comma_download_cache" + Path::openpilot_prefix() + "/";
   }
-  return Hardware::PC() ? util::getenv("HOME") + "/.comma/media/0/realdata" : "/data/media/0/realdata";
-}
-inline std::string params() {
-  return Hardware::PC() ? util::getenv("PARAMS_ROOT", util::getenv("HOME") + "/.comma/params") : "/data/params";
-}
-inline std::string rsa_file() {
-  return Hardware::PC() ? util::getenv("HOME") + "/.comma/persist/comma/id_rsa" : "/persist/comma/id_rsa";
-}
 }  // namespace Path

system/hardware/hw.py

@@ -0,0 +1,35 @@
+import os
+from pathlib import Path
+
+from openpilot.system.hardware import PC
+
+class Paths:
+  @staticmethod
+  def comma_home() -> str:
+    return os.path.join(str(Path.home()), ".comma" + os.environ.get("OPENPILOT_PREFIX", ""))
+
+  @staticmethod
+  def log_root() -> str:
+    if os.environ.get('LOG_ROOT', False):
+      return os.environ['LOG_ROOT']
+    elif PC:
+      return str(Path(Paths.comma_home()) / "media" / "0" / "realdata")
+    else:
+      return '/data/media/0/realdata/'
+
+  @staticmethod
+  def swaglog_root() -> str:
+    if PC:
+      return os.path.join(Paths.comma_home(), "log")
+    else:
+      return "/data/log/"
+
+  @staticmethod
+  def swaglog_ipc() -> str:
+    return "ipc:///tmp/logmessage" + os.environ.get("OPENPILOT_PREFIX", "")
+
+  @staticmethod
+  def download_cache_root() -> str:
+    if os.environ.get('COMMA_CACHE', False):
+      return os.environ['COMMA_CACHE']
+    return "/tmp/comma_download_cache" + os.environ.get("OPENPILOT_PREFIX", "") + "/"

system/hardware/pc/hardware.h

@@ -1,12 +1,14 @@
 #pragma once
 
+#include <string>
+
 #include "system/hardware/base.h"
 
 class HardwarePC : public HardwareNone {
 public:
   static std::string get_os_version() { return "openpilot for PC"; }
-  static std::string get_name() { return "pc"; };
-  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; };
+  static std::string get_name() { return "pc"; }
+  static cereal::InitData::DeviceType get_device_type() { return cereal::InitData::DeviceType::PC; }
   static bool PC() { return true; }
   static bool TICI() { return util::getenv("TICI", 0) == 1; }
   static bool AGNOS() { return util::getenv("TICI", 0) == 1; }

system/hardware/pc/hardware.py

@@ -1,7 +1,7 @@
 import random
 
 from cereal import log
-from system.hardware.base import HardwareBase, ThermalConfig
+from openpilot.system.hardware.base import HardwareBase, ThermalConfig
 
 NetworkType = log.DeviceState.NetworkType
 NetworkStrength = log.DeviceState.NetworkStrength

system/hardware/tici/agnos.json

@@ -67,6 +67,10 @@
     "size": 10737418240,
     "sparse": true,
     "full_check": false,
-    "has_ab": true
+    "has_ab": true,
+    "alt": {
+      "hash": "256442a55fcb9e8f72969f003a4db91598dee1136f8dda85b553a557d36b93d8",
+      "url": "https://commadist.azureedge.net/agnosupdate/system-skip-chunks-e1fa3018bce9bad01c6967e5e21f1141cf5c8f02d2edfaed51c738f74a32a432.img.xz"
+    }
   }
 ]

system/hardware/tici/agnos.py

@@ -10,7 +10,7 @@ from typing import Dict, Generator, List, Tuple, Union
 
 import requests
 
-import system.hardware.tici.casync as casync
+import openpilot.system.hardware.tici.casync as casync
 
 SPARSE_CHUNK_FMT = struct.Struct('H2xI4x')
 CAIBX_URL = "https://commadist.azureedge.net/agnosupdate/"

system/hardware/tici/amplifier.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import time
 from smbus2 import SMBus
 from collections import namedtuple

system/hardware/tici/hardware.h

@@ -2,6 +2,8 @@
 
 #include <cstdlib>
 #include <fstream>
+#include <map>
+#include <string>
 
 #include "common/params.h"
 #include "common/util.h"
@@ -15,19 +17,19 @@ public:
   static bool AGNOS() { return true; }
   static std::string get_os_version() {
     return "AGNOS " + util::read_file("/VERSION");
-  };
+  }
 
   static std::string get_name() {
     std::string devicetree_model = util::read_file("/sys/firmware/devicetree/base/model");
     return (devicetree_model.find("tizi") != std::string::npos) ? "tizi" : "tici";
-  };
+  }
 
   static cereal::InitData::DeviceType get_device_type() {
     return (get_name() == "tizi") ? cereal::InitData::DeviceType::TIZI : cereal::InitData::DeviceType::TICI;
-  };
+  }
 
-  static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); };
-  static int get_current() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/curr1_input").c_str()); };
+  static int get_voltage() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/in1_input").c_str()); }
+  static int get_current() { return std::atoi(util::read_file("/sys/class/hwmon/hwmon1/curr1_input").c_str()); }
 
   static std::string get_serial() {
     static std::string serial("");
@@ -47,8 +49,8 @@ public:
     return serial;
   }
 
-  static void reboot() { std::system("sudo reboot"); };
-  static void poweroff() { std::system("sudo poweroff"); };
+  static void reboot() { std::system("sudo reboot"); }
+  static void poweroff() { std::system("sudo poweroff"); }
   static void set_brightness(int percent) {
     std::string max = util::read_file("/sys/class/backlight/panel0-backlight/max_brightness");
 
@@ -57,14 +59,14 @@ public:
       brightness_control << (int)(percent * (std::stof(max)/100.)) << "\n";
       brightness_control.close();
     }
-  };
+  }
   static void set_display_power(bool on) {
     std::ofstream bl_power_control("/sys/class/backlight/panel0-backlight/bl_power");
     if (bl_power_control.is_open()) {
       bl_power_control << (on ? "0" : "4") << "\n";
       bl_power_control.close();
     }
-  };
+  }
   static void set_volume(float volume) {
     volume = util::map_val(volume, 0.f, 1.f, MIN_VOLUME, MAX_VOLUME);
 
@@ -99,6 +101,6 @@ public:
     return ret;
   }
 
-  static bool get_ssh_enabled() { return Params().getBool("SshEnabled"); };
-  static void set_ssh_enabled(bool enabled) { Params().putBool("SshEnabled", enabled); };
+  static bool get_ssh_enabled() { return Params().getBool("SshEnabled"); }
+  static void set_ssh_enabled(bool enabled) { Params().putBool("SshEnabled", enabled); }
 };

system/hardware/tici/hardware.py

@@ -8,11 +8,11 @@ from functools import cached_property, lru_cache
 from pathlib import Path
 
 from cereal import log
-from common.gpio import gpio_set, gpio_init, get_irqs_for_action
-from system.hardware.base import HardwareBase, ThermalConfig
-from system.hardware.tici import iwlist
-from system.hardware.tici.pins import GPIO
-from system.hardware.tici.amplifier import Amplifier
+from openpilot.common.gpio import gpio_set, gpio_init, get_irqs_for_action
+from openpilot.system.hardware.base import HardwareBase, ThermalConfig
+from openpilot.system.hardware.tici import iwlist
+from openpilot.system.hardware.tici.pins import GPIO
+from openpilot.system.hardware.tici.amplifier import Amplifier
 
 NM = 'org.freedesktop.NetworkManager'
 NM_CON_ACT = NM + '.Connection.Active'
@@ -97,7 +97,7 @@ def get_device_type():
 class Tici(HardwareBase):
   @cached_property
   def bus(self):
-    import dbus  # pylint: disable=import-error
+    import dbus
     return dbus.SystemBus()
 
   @cached_property
@@ -586,7 +586,7 @@ class Tici(HardwareBase):
     gpio_init(GPIO.STM_RST_N, True)
 
     gpio_set(GPIO.STM_RST_N, 1)
-    time.sleep(2)
+    time.sleep(1)
     gpio_set(GPIO.STM_RST_N, 0)
 
   def recover_internal_panda(self):
@@ -595,9 +595,9 @@ class Tici(HardwareBase):
 
     gpio_set(GPIO.STM_RST_N, 1)
     gpio_set(GPIO.STM_BOOT0, 1)
-    time.sleep(1)
+    time.sleep(0.5)
     gpio_set(GPIO.STM_RST_N, 0)
-    time.sleep(1)
+    time.sleep(0.5)
     gpio_set(GPIO.STM_BOOT0, 0)
 
 

system/hardware/tici/pins.py

@@ -6,7 +6,7 @@ class GPIO:
   HUB_RST_N = 30
   UBLOX_RST_N = 32
   UBLOX_SAFEBOOT_N = 33
-  UBLOX_PWR_EN = 34
+  GNSS_PWR_EN = 34 # SCHEMATIC LABEL: GPIO_UBLOX_PWR_EN
   STM_RST_N = 124
   STM_BOOT0 = 134
 

system/loggerd/config.py

@@ -1,13 +1,7 @@
 import os
 from pathlib import Path
-from system.hardware import PC
-
-if os.environ.get('LOG_ROOT', False):
-  ROOT = os.environ['LOG_ROOT']
-elif PC:
-  ROOT = str(Path.home() / ".comma" / "media" / "0" / "realdata")
-else:
-  ROOT = '/data/media/0/realdata/'
+from openpilot.system.hardware import PC
+from openpilot.system.hardware.hw import Paths
 
 
 CAMERA_FPS = 20
@@ -23,7 +17,7 @@ STATS_FLUSH_TIME_S = 60
 
 def get_available_percent(default=None):
   try:
-    statvfs = os.statvfs(ROOT)
+    statvfs = os.statvfs(Paths.log_root())
     available_percent = 100.0 * statvfs.f_bavail / statvfs.f_blocks
   except OSError:
     available_percent = default
@@ -33,7 +27,7 @@ def get_available_percent(default=None):
 
 def get_available_bytes(default=None):
   try:
-    statvfs = os.statvfs(ROOT)
+    statvfs = os.statvfs(Paths.log_root())
     available_bytes = statvfs.f_bavail * statvfs.f_frsize
   except OSError:
     available_bytes = default

system/loggerd/deleter.py

@@ -3,11 +3,11 @@ import os
 import shutil
 import threading
 from typing import List
-
-from system.swaglog import cloudlog
-from system.loggerd.config import ROOT, get_available_bytes, get_available_percent
-from system.loggerd.uploader import listdir_by_creation
-from system.loggerd.xattr_cache import getxattr
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.swaglog import cloudlog
+from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
+from openpilot.system.loggerd.uploader import listdir_by_creation
+from openpilot.system.loggerd.xattr_cache import getxattr
 
 MIN_BYTES = 5 * 1024 * 1024 * 1024
 MIN_PERCENT = 10
@@ -20,7 +20,7 @@ PRESERVE_COUNT = 5
 
 
 def has_preserve_xattr(d: str) -> bool:
-  return getxattr(os.path.join(ROOT, d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
+  return getxattr(os.path.join(Paths.log_root(), d), PRESERVE_ATTR_NAME) == PRESERVE_ATTR_VALUE
 
 
 def get_preserved_segments(dirs_by_creation: List[str]) -> List[str]:
@@ -51,14 +51,14 @@ def deleter_thread(exit_event):
     out_of_percent = get_available_percent(default=MIN_PERCENT + 1) < MIN_PERCENT
 
     if out_of_percent or out_of_bytes:
-      dirs = listdir_by_creation(ROOT)
+      dirs = listdir_by_creation(Paths.log_root())
 
       # skip deleting most recent N preserved segments (and their prior segment)
       preserved_dirs = get_preserved_segments(dirs)
 
       # remove the earliest directory we can
       for delete_dir in sorted(dirs, key=lambda d: (d in DELETE_LAST, d in preserved_dirs)):
-        delete_path = os.path.join(ROOT, delete_dir)
+        delete_path = os.path.join(Paths.log_root(), delete_dir)
 
         if any(name.endswith(".lock") for name in os.listdir(delete_path)):
           continue

system/loggerd/encoder/encoder.h

@@ -2,6 +2,7 @@
 
 #include <cassert>
 #include <cstdint>
+#include <memory>
 #include <thread>
 
 #include "cereal/messaging/messaging.h"
@@ -15,7 +16,7 @@
 class VideoEncoder {
 public:
   VideoEncoder(const EncoderInfo &encoder_info, int in_width, int in_height);
-  virtual ~VideoEncoder() {};
+  virtual ~VideoEncoder() {}
   virtual int encode_frame(VisionBuf* buf, VisionIpcBufExtra *extra) = 0;
   virtual void encoder_open(const char* path) = 0;
   virtual void encoder_close() = 0;

system/loggerd/logger.h

@@ -1,11 +1,12 @@
 #pragma once
 
-#include <cassert>
 #include <pthread.h>
 
+#include <cassert>
 #include <cstdint>
 #include <cstdio>
 #include <memory>
+#include <string>
 
 #include <capnp/serialize.h>
 #include <kj/array.h>
@@ -15,8 +16,6 @@
 #include "common/swaglog.h"
 #include "system/hardware/hw.h"
 
-const std::string LOG_ROOT = Path::log_root();
-
 #define LOGGER_MAX_HANDLES 16
 
 class RawFile {

system/loggerd/loggerd.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vector>
+
 #include "cereal/messaging/messaging.h"
 #include "cereal/services.h"
 #include "cereal/visionipc/visionipc_client.h"

system/loggerd/uploader.py

@@ -13,13 +13,13 @@ from typing import BinaryIO, Iterator, List, Optional, Tuple, Union
 
 from cereal import log
 import cereal.messaging as messaging
-from common.api import Api
-from common.params import Params
-from common.realtime import set_core_affinity
-from system.hardware import TICI
-from system.loggerd.xattr_cache import getxattr, setxattr
-from system.loggerd.config import ROOT
-from system.swaglog import cloudlog
+from openpilot.common.api import Api
+from openpilot.common.params import Params
+from openpilot.common.realtime import set_core_affinity
+from openpilot.system.hardware import TICI
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
+from openpilot.system.swaglog import cloudlog
 
 NetworkType = log.DeviceState.NetworkType
 UPLOAD_ATTR_NAME = 'user.upload'
@@ -244,7 +244,7 @@ def uploader_fn(exit_event: threading.Event) -> None:
   except Exception:
     cloudlog.exception("failed to set core affinity")
 
-  clear_locks(ROOT)
+  clear_locks(Paths.log_root())
 
   params = Params()
   dongle_id = params.get("DongleId", encoding='utf8')
@@ -258,7 +258,7 @@ def uploader_fn(exit_event: threading.Event) -> None:
 
   sm = messaging.SubMaster(['deviceState'])
   pm = messaging.PubMaster(['uploaderState'])
-  uploader = Uploader(dongle_id, ROOT)
+  uploader = Uploader(dongle_id, Paths.log_root())
 
   backoff = 0.1
   while not exit_event.is_set():

system/loggerd/video_writer.h

@@ -16,11 +16,10 @@ public:
   ~VideoWriter();
 private:
   std::string vid_path, lock_path;
-
   FILE *of = nullptr;
 
   AVCodecContext *codec_ctx;
   AVFormatContext *ofmt_ctx;
   AVStream *out_stream;
-  bool remuxing, raw;
-};
+  bool remuxing;
+};

system/logmessaged.py

@@ -3,8 +3,9 @@ import zmq
 from typing import NoReturn
 
 import cereal.messaging as messaging
-from common.logging_extra import SwagLogFileFormatter
-from system.swaglog import get_file_handler
+from openpilot.common.logging_extra import SwagLogFileFormatter
+from openpilot.system.hardware.hw import Paths
+from openpilot.system.swaglog import get_file_handler
 
 
 def main() -> NoReturn:
@@ -14,7 +15,7 @@ def main() -> NoReturn:
 
   ctx = zmq.Context.instance()
   sock = ctx.socket(zmq.PULL)
-  sock.bind("ipc:///tmp/logmessage")
+  sock.bind(Paths.swaglog_ipc())
 
   # and we publish them
   log_message_sock = messaging.pub_sock('logMessage')

system/micd.py

@@ -2,9 +2,9 @@
 import numpy as np
 
 from cereal import messaging
-from common.filter_simple import FirstOrderFilter
-from common.realtime import Ratekeeper
-from system.swaglog import cloudlog
+from openpilot.common.filter_simple import FirstOrderFilter
+from openpilot.common.realtime import Ratekeeper
+from openpilot.system.swaglog import cloudlog
 
 RATE = 10
 FFT_SAMPLES = 4096
@@ -85,7 +85,7 @@ class Mic:
 
   def micd_thread(self):
     # sounddevice must be imported after forking processes
-    import sounddevice as sd  # pylint: disable=import-outside-toplevel
+    import sounddevice as sd
 
     with sd.InputStream(channels=1, samplerate=SAMPLE_RATE, callback=self.callback) as stream:
       cloudlog.info(f"micd stream started: {stream.samplerate=} {stream.channels=} {stream.dtype=} {stream.device=}")

system/sensord/pigeond.py

@@ -10,11 +10,11 @@ from datetime import datetime
 from typing import List, Optional, Tuple
 
 from cereal import messaging
-from common.params import Params
-from system.swaglog import cloudlog
-from system.hardware import TICI
-from common.gpio import gpio_init, gpio_set
-from system.hardware.tici.pins import GPIO
+from openpilot.common.params import Params
+from openpilot.system.swaglog import cloudlog
+from openpilot.system.hardware import TICI
+from openpilot.common.gpio import gpio_init, gpio_set
+from openpilot.system.hardware.tici.pins import GPIO
 
 UBLOX_TTY = "/dev/ttyHS0"
 
@@ -27,11 +27,11 @@ UBLOX_ASSIST_ACK = b"\xb5\x62\x13\x60\x08\x00"
 
 def set_power(enabled: bool) -> None:
   gpio_init(GPIO.UBLOX_SAFEBOOT_N, True)
-  gpio_init(GPIO.UBLOX_PWR_EN, True)
+  gpio_init(GPIO.GNSS_PWR_EN, True)
   gpio_init(GPIO.UBLOX_RST_N, True)
 
   gpio_set(GPIO.UBLOX_SAFEBOOT_N, True)
-  gpio_set(GPIO.UBLOX_PWR_EN, enabled)
+  gpio_set(GPIO.GNSS_PWR_EN, enabled)
   gpio_set(GPIO.UBLOX_RST_N, enabled)
 
 def add_ubx_checksum(msg: bytes) -> bytes:
@@ -164,12 +164,12 @@ def initialize_pigeon(pigeon: TTYPigeon) -> bool:
       pigeon.send_with_ack(b"\xB5\x62\x06\x08\x06\x00\x64\x00\x01\x00\x00\x00\x79\x10")
 
       # UBX-CFG-NAV5 (0x06 0x24)
-      pigeon.send_with_ack(b"\xB5\x62\x06\x24\x24\x00\x05\x00\x04\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x5A\x63") # noqa: E501
+      pigeon.send_with_ack(b"\xB5\x62\x06\x24\x24\x00\x05\x00\x04\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x5A\x63")
 
       # UBX-CFG-ODO (0x06 0x1E)
       pigeon.send_with_ack(b"\xB5\x62\x06\x1E\x14\x00\x00\x00\x00\x00\x01\x03\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x3C\x37")
       pigeon.send_with_ack(b"\xB5\x62\x06\x39\x08\x00\xFF\xAD\x62\xAD\x1E\x63\x00\x00\x83\x0C")
-      pigeon.send_with_ack(b"\xB5\x62\x06\x23\x28\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x56\x24") # noqa: E501
+      pigeon.send_with_ack(b"\xB5\x62\x06\x23\x28\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x56\x24")
 
       # UBX-CFG-NAV5 (0x06 0x24)
       pigeon.send_with_ack(b"\xB5\x62\x06\x24\x00\x00\x2A\x84")

system/sensord/rawgps/nmeaport.py

@@ -0,0 +1,169 @@
+import os
+import sys
+from dataclasses import dataclass, fields
+from subprocess import check_output, CalledProcessError
+from time import sleep
+from typing import NoReturn
+
+DEBUG = int(os.environ.get("DEBUG", "0"))
+
+@dataclass
+class GnssClockNmeaPort:
+  # flags bit mask:
+  # 0x01 = leap_seconds valid
+  # 0x02 = time_uncertainty_ns valid
+  # 0x04 = full_bias_ns valid
+  # 0x08 = bias_ns valid
+  # 0x10 = bias_uncertainty_ns valid
+  # 0x20 = drift_nsps valid
+  # 0x40 = drift_uncertainty_nsps valid
+  flags: int
+  leap_seconds: int
+  time_ns: int
+  time_uncertainty_ns: int # 1-sigma
+  full_bias_ns: int
+  bias_ns: float
+  bias_uncertainty_ns: float # 1-sigma
+  drift_nsps: float
+  drift_uncertainty_nsps: float # 1-sigma
+
+  def __post_init__(self):
+    for field in fields(self):
+      val = getattr(self, field.name)
+      setattr(self, field.name, field.type(val) if val else None)
+
+@dataclass
+class GnssMeasNmeaPort:
+  messageCount: int
+  messageNum: int
+  svCount: int
+  # constellation enum:
+  # 1 = GPS
+  # 2 = SBAS
+  # 3 = GLONASS
+  # 4 = QZSS
+  # 5 = BEIDOU
+  # 6 = GALILEO
+  constellation: int
+  svId: int
+  flags: int # always zero
+  time_offset_ns: int
+  # state bit mask:
+  # 0x0001 = CODE LOCK
+  # 0x0002 = BIT SYNC
+  # 0x0004 = SUBFRAME SYNC
+  # 0x0008 = TIME OF WEEK DECODED
+  # 0x0010 = MSEC AMBIGUOUS
+  # 0x0020 = SYMBOL SYNC
+  # 0x0040 = GLONASS STRING SYNC
+  # 0x0080 = GLONASS TIME OF DAY DECODED
+  # 0x0100 = BEIDOU D2 BIT SYNC
+  # 0x0200 = BEIDOU D2 SUBFRAME SYNC
+  # 0x0400 = GALILEO E1BC CODE LOCK
+  # 0x0800 = GALILEO E1C 2ND CODE LOCK
+  # 0x1000 = GALILEO E1B PAGE SYNC
+  # 0x2000 = GALILEO E1B PAGE SYNC
+  state: int
+  time_of_week_ns: int
+  time_of_week_uncertainty_ns: int # 1-sigma
+  carrier_to_noise_ratio: float
+  pseudorange_rate: float
+  pseudorange_rate_uncertainty: float # 1-sigma
+
+  def __post_init__(self):
+    for field in fields(self):
+      val = getattr(self, field.name)
+      setattr(self, field.name, field.type(val) if val else None)
+
+def nmea_checksum_ok(s):
+  checksum = 0
+  for i, c in enumerate(s[1:]):
+    if c == "*":
+      if i != len(s) - 4: # should be 3rd to last character
+        print("ERROR: NMEA string does not have checksum delimiter in correct location:", s)
+        return False
+      break
+    checksum ^= ord(c)
+  else:
+    print("ERROR: NMEA string does not have checksum delimiter:", s)
+    return False
+
+  return True
+
+def process_nmea_port_messages(device:str="/dev/ttyUSB1") -> NoReturn:
+  while True:
+    try:
+      with open(device, "r") as nmeaport:
+        for line in nmeaport:
+          line = line.strip()
+          if DEBUG:
+            print(line)
+          if not line.startswith("$"): # all NMEA messages start with $
+            continue
+          if not nmea_checksum_ok(line):
+            continue
+
+          fields = line.split(",")
+          match fields[0]:
+            case "$GNCLK":
+              # fields at end are reserved (not used)
+              gnss_clock = GnssClockNmeaPort(*fields[1:10]) # type: ignore[arg-type]
+              print(gnss_clock)
+            case "$GNMEAS":
+              # fields at end are reserved (not used)
+              gnss_meas = GnssMeasNmeaPort(*fields[1:14]) # type: ignore[arg-type]
+              print(gnss_meas)
+    except Exception as e:
+      print(e)
+      sleep(1)
+
+def main() -> NoReturn:
+  from openpilot.common.gpio import gpio_init, gpio_set
+  from openpilot.system.hardware.tici.pins import GPIO
+  from openpilot.system.sensord.rawgps.rawgpsd import at_cmd
+
+  try:
+    check_output(["pidof", "rawgpsd"])
+    print("rawgpsd is running, please kill openpilot before running this script! (aborted)")
+    sys.exit(1)
+  except CalledProcessError as e:
+    if e.returncode != 1: # 1 == no process found (boardd not running)
+      raise e
+
+  print("power up antenna ...")
+  gpio_init(GPIO.GNSS_PWR_EN, True)
+  gpio_set(GPIO.GNSS_PWR_EN, True)
+
+  if b"+QGPS: 0" not in (at_cmd("AT+QGPS?") or b""):
+    print("stop location tracking ...")
+    at_cmd("AT+QGPSEND")
+
+  if b'+QGPSCFG: "outport",usbnmea' not in (at_cmd('AT+QGPSCFG="outport"') or b""):
+    print("configure outport ...")
+    at_cmd('AT+QGPSCFG="outport","usbnmea"') # usbnmea = /dev/ttyUSB1
+
+  if b'+QGPSCFG: "gnssrawdata",3,0' not in (at_cmd('AT+QGPSCFG="gnssrawdata"') or b""):
+    print("configure gnssrawdata ...")
+    # AT+QGPSCFG="gnssrawdata",<constellation-mask>,<port>'
+    # <constellation-mask> values:
+    # 0x01 = GPS
+    # 0x02 = GLONASS
+    # 0x04 = BEIDOU
+    # 0x08 = GALILEO
+    # 0x10 = QZSS
+    # <port> values:
+    # 0 = NMEA port
+    # 1 = AT port
+    at_cmd('AT+QGPSCFG="gnssrawdata",3,0') # enable all constellations, output data to NMEA port
+    print("rebooting ...")
+    at_cmd('AT+CFUN=1,1')
+    print("re-run this script when it is back up")
+    sys.exit(2)
+
+  print("starting location tracking ...")
+  at_cmd("AT+QGPS=1")
+
+  process_nmea_port_messages()
+
+if __name__ == "__main__":
+  main()

system/sensord/rawgps/rawgpsd.py

@@ -15,14 +15,14 @@ from struct import unpack_from, calcsize, pack
 
 from cereal import log
 import cereal.messaging as messaging
-from common.gpio import gpio_init, gpio_set
+from openpilot.common.gpio import gpio_init, gpio_set
 from laika.gps_time import GPSTime, utc_to_gpst, get_leap_seconds
 from laika.helpers import get_prn_from_nmea_id
 from laika.constants import SECS_IN_HR, SECS_IN_DAY, SECS_IN_WEEK
-from system.hardware.tici.pins import GPIO
-from system.swaglog import cloudlog
-from system.sensord.rawgps.modemdiag import ModemDiag, DIAG_LOG_F, setup_logs, send_recv
-from system.sensord.rawgps.structs import (dict_unpacker, position_report, relist,
+from openpilot.system.hardware.tici.pins import GPIO
+from openpilot.system.swaglog import cloudlog
+from openpilot.system.sensord.rawgps.modemdiag import ModemDiag, DIAG_LOG_F, setup_logs, send_recv
+from openpilot.system.sensord.rawgps.structs import (dict_unpacker, position_report, relist,
                                               gps_measurement_report, gps_measurement_report_sv,
                                               glonass_measurement_report, glonass_measurement_report_sv,
                                               oemdre_measurement_report, oemdre_measurement_report_sv, oemdre_svpoly_report,
@@ -271,7 +271,7 @@ def main() -> NoReturn:
   def cleanup(sig, frame):
     cloudlog.warning("caught sig disabling quectel gps")
 
-    gpio_set(GPIO.UBLOX_PWR_EN, False)
+    gpio_set(GPIO.GNSS_PWR_EN, False)
     teardown_quectel(diag)
     cloudlog.warning("quectel cleanup done")
 
@@ -289,8 +289,8 @@ def main() -> NoReturn:
   want_assistance = not r
   current_gps_time = utc_to_gpst(GPSTime.from_datetime(datetime.utcnow()))
   cloudlog.warning("quectel setup done")
-  gpio_init(GPIO.UBLOX_PWR_EN, True)
-  gpio_set(GPIO.UBLOX_PWR_EN, True)
+  gpio_init(GPIO.GNSS_PWR_EN, True)
+  gpio_set(GPIO.GNSS_PWR_EN, True)
 
   pm = messaging.PubMaster(['qcomGnss', 'gpsLocation'])
 

system/sensord/rawgps/test_rawgps.py

@@ -8,10 +8,10 @@ import subprocess
 import numpy as np
 
 import cereal.messaging as messaging
-from system.hardware import TICI
-from system.sensord.rawgps.rawgpsd import at_cmd, wait_for_modem
-from selfdrive.manager.process_config import managed_processes
-from common.transformations.coordinates import ecef_from_geodetic
+from openpilot.system.hardware import TICI
+from openpilot.system.sensord.rawgps.rawgpsd import at_cmd, wait_for_modem
+from openpilot.selfdrive.manager.process_config import managed_processes
+from openpilot.common.transformations.coordinates import ecef_from_geodetic
 
 GOOD_SIGNAL = bool(int(os.getenv("GOOD_SIGNAL", '0')))
 

system/sensord/sensors/file_sensor.h

@@ -1,19 +0,0 @@
-#pragma once
-
-#include <fstream>
-#include <string>
-
-#include "cereal/gen/cpp/log.capnp.h"
-#include "system/sensord/sensors/sensor.h"
-
-class FileSensor : public Sensor {
-protected:
-  std::ifstream file;
-
-public:
-  FileSensor(std::string filename);
-  ~FileSensor();
-  int init();
-  bool has_interrupt_enabled();
-  virtual bool get_event(MessageBuilder &msg, uint64_t ts = 0) = 0;
-};

system/sensord/sensors/i2c_sensor.h

@@ -2,6 +2,7 @@
 
 #include <cstdint>
 #include <unistd.h>
+#include <vector>
 #include "cereal/gen/cpp/log.capnp.h"
 
 #include "common/i2c.h"

system/sensord/sensors/light_sensor.h

@@ -1,9 +0,0 @@
-#pragma once
-#include "file_sensor.h"
-
-class LightSensor : public FileSensor {
-public:
-  LightSensor(std::string filename);
-  bool get_event(MessageBuilder &msg, uint64_t ts = 0);
-  int shutdown() { return 0; }
-};

system/sensord/sensors/mmc5603nj_magn.h

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <vector>
+
 #include "system/sensord/sensors/i2c_sensor.h"
 
 // Address of the chip on the bus
@@ -19,9 +21,14 @@
 #define MMC5603NJ_AUTO_SR_EN     (1 << 5)
 #define MMC5603NJ_CMM_EN         (1 << 4)
 #define MMC5603NJ_EN_PRD_SET     (1 << 3)
+#define MMC5603NJ_SET            (1 << 3)
+#define MMC5603NJ_RESET          (1 << 4)
 
 class MMC5603NJ_Magn : public I2CSensor {
+private:
   uint8_t get_device_address() {return MMC5603NJ_I2C_ADDR;}
+  void start_measurement();
+  std::vector<float> read_measurement();
 public:
   MMC5603NJ_Magn(I2CBus *bus);
   int init();

system/sensord/sensors/sensor.h

@@ -5,14 +5,18 @@
 class Sensor {
 public:
   int gpio_fd = -1;
+  uint64_t start_ts = 0;
   uint64_t init_delay = 500e6; // default dealy 500ms
-  virtual ~Sensor() {};
+  virtual ~Sensor() {}
   virtual int init() = 0;
   virtual bool get_event(MessageBuilder &msg, uint64_t ts = 0) = 0;
   virtual bool has_interrupt_enabled() = 0;
   virtual int shutdown() = 0;
 
-  virtual bool is_data_valid(uint64_t st, uint64_t ct) {
-    return (ct - st) > init_delay;
+  virtual bool is_data_valid(uint64_t current_ts) {
+    if (start_ts == 0) {
+      start_ts = current_ts;
+    }
+    return (current_ts - start_ts) > init_delay;
   }
 };

system/swaglog.py

@@ -7,17 +7,13 @@ from logging.handlers import BaseRotatingHandler
 
 import zmq
 
-from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from system.hardware import PC
+from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
+from openpilot.system.hardware.hw import Paths
 
-if PC:
-  SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
-else:
-  SWAGLOG_DIR = "/data/log/"
 
 def get_file_handler():
-  Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
-  base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
+  Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
+  base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
   handler = SwaglogRotatingFileHandler(base_filename)
   return handler
 
@@ -77,6 +73,9 @@ class UnixDomainSocketHandler(logging.Handler):
     self.sock = None
 
   def __del__(self):
+    self.close()
+
+  def close(self):
     if self.sock is not None:
       self.sock.close()
     if self.zctx is not None:
@@ -86,7 +85,7 @@ class UnixDomainSocketHandler(logging.Handler):
     self.zctx = zmq.Context()
     self.sock = self.zctx.socket(zmq.PUSH)
     self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect("ipc:///tmp/logmessage")
+    self.sock.connect(Paths.swaglog_ipc())
     self.pid = os.getpid()
 
   def emit(self, record):
@@ -129,6 +128,8 @@ elif print_level == 'info':
 elif print_level == 'warning':
   outhandler.setLevel(logging.WARNING)
 
+ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
+
 log.addHandler(outhandler)
 # logs are sent through IPC before writing to disk to prevent disk I/O blocking
-log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))
+log.addHandler(ipchandler)

system/timezoned.py

@@ -8,9 +8,12 @@ from typing import NoReturn
 import requests
 from timezonefinder import TimezoneFinder
 
-from common.params import Params
-from system.hardware import AGNOS
-from system.swaglog import cloudlog
+from openpilot.common.params import Params
+from openpilot.system.hardware import AGNOS
+from openpilot.system.swaglog import cloudlog
+from openpilot.system.version import get_version
+
+REQUEST_HEADERS = {'User-Agent': "openpilot-" + get_version()}
 
 
 def set_timezone(valid_timezones, timezone):
@@ -18,7 +21,7 @@ def set_timezone(valid_timezones, timezone):
     cloudlog.error(f"Timezone not supported {timezone}")
     return
 
-  cloudlog.debug(f"Setting timezone to {timezone}")
+  cloudlog.info(f"Setting timezone to {timezone}")
   try:
     if AGNOS:
       tzpath = os.path.join("/usr/share/zoneinfo/", timezone)
@@ -58,7 +61,7 @@ def main() -> NoReturn:
     if location is None:
       cloudlog.debug("Setting timezone based on IP lookup")
       try:
-        r = requests.get("https://ipapi.co/timezone", timeout=10)
+        r = requests.get("https://ipapi.co/timezone", headers=REQUEST_HEADERS, timeout=10)
         if r.status_code == 200:
           set_timezone(valid_timezones, r.text)
         else:

system/ubloxd/ublox_msg.h

@@ -2,10 +2,11 @@
 
 #include <cassert>
 #include <cstdint>
+#include <ctime>
 #include <memory>
 #include <string>
 #include <unordered_map>
-#include <ctime>
+#include <utility>
 
 #include "cereal/messaging/messaging.h"
 #include "common/util.h"
@@ -51,7 +52,7 @@ namespace ublox {
     assert(msg.size() > 2);
 
     uint8_t ck_a = 0, ck_b = 0;
-    for(int i = 2; i < msg.size(); i++) {
+    for (int i = 2; i < msg.size(); i++) {
       ck_a = (ck_a + msg[i]) & 0xFF;
       ck_b = (ck_b + ck_a) & 0xFF;
     }

system/version.py

@@ -4,8 +4,8 @@ import subprocess
 from typing import List, Optional
 from functools import lru_cache
 
-from common.basedir import BASEDIR
-from system.swaglog import cloudlog
+from openpilot.common.basedir import BASEDIR
+from openpilot.system.swaglog import cloudlog
 
 RELEASE_BRANCHES = ['release3-staging', 'dashcam3-staging', 'release3', 'dashcam3', 'nightly']
 TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging']
@@ -127,7 +127,7 @@ def is_dirty() -> bool:
 
 
 if __name__ == "__main__":
-  from common.params import Params
+  from openpilot.common.params import Params
 
   params = Params()
   params.put("TermsVersion", terms_version)