Move a bunch of stuff to system/ part 3 (#24829)

* move swaglog.py

* timezoned

* logmessaged

* version.py

* fix linter

selfdrive/athena/athenad.py

@@ -36,8 +36,8 @@ from system.hardware import HARDWARE, PC, AGNOS
 from selfdrive.loggerd.config import ROOT
 from selfdrive.loggerd.xattr_cache import getxattr, setxattr
 from selfdrive.statsd import STATS_DIR
-from selfdrive.swaglog import SWAGLOG_DIR, cloudlog
-from selfdrive.version import get_commit, get_origin, get_short_branch, get_version
+from system.swaglog import SWAGLOG_DIR, cloudlog
+from system.version import get_commit, get_origin, get_short_branch, get_version
 
 ATHENA_HOST = os.getenv('ATHENA_HOST', 'wss://athena.comma.ai')
 HANDLER_THREADS = int(os.getenv('HANDLER_THREADS', "4"))

selfdrive/athena/manage_athenad.py

@@ -5,8 +5,8 @@ from multiprocessing import Process
 
 from common.params import Params
 from selfdrive.manager.process import launcher
-from selfdrive.swaglog import cloudlog
-from selfdrive.version import get_version, is_dirty
+from system.swaglog import cloudlog
+from system.version import get_version, is_dirty
 
 ATHENA_MGR_PID_PARAM = "AthenadPid"
 

selfdrive/athena/registration.py

@@ -12,7 +12,7 @@ from common.spinner import Spinner
 from common.basedir import PERSIST
 from selfdrive.controls.lib.alertmanager import set_offroad_alert
 from system.hardware import HARDWARE, PC
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 UNREGISTERED_DONGLE_ID = "UnregisteredDevice"

selfdrive/athena/tests/test_athenad.py

@@ -16,7 +16,7 @@ from unittest import mock
 from websocket import ABNF
 from websocket._exceptions import WebSocketConnectionClosedException
 
-from selfdrive import swaglog
+from system import swaglog
 from selfdrive.athena import athenad
 from selfdrive.athena.athenad import MAX_RETRY_COUNT, dispatcher
 from selfdrive.athena.tests.helpers import MockWebsocket, MockParams, MockApi, EchoSocket, with_http_server

selfdrive/boardd/pandad.py

@@ -11,7 +11,7 @@ from panda import DEFAULT_FW_FN, DEFAULT_H7_FW_FN, MCU_TYPE_H7, Panda, PandaDFU
 from common.basedir import BASEDIR
 from common.params import Params
 from system.hardware import HARDWARE
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 def get_expected_signature(panda: Panda) -> bytes:

selfdrive/boardd/tests/boardd_old.py

@@ -13,7 +13,7 @@ import time
 
 import cereal.messaging as messaging
 from common.realtime import Ratekeeper
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.boardd.boardd import can_capnp_to_can_list
 from cereal import car
 

selfdrive/car/car_helpers.py

@@ -4,12 +4,12 @@ from typing import Dict, List
 from cereal import car
 from common.params import Params
 from common.basedir import BASEDIR
-from selfdrive.version import is_comma_remote, is_tested_branch
+from system.version import is_comma_remote, is_tested_branch
 from selfdrive.car.interfaces import get_interface_attr
 from selfdrive.car.fingerprints import eliminate_incompatible_cars, all_legacy_fingerprint_cars
 from selfdrive.car.vin import get_vin, VIN_UNKNOWN
 from selfdrive.car.fw_versions import get_fw_versions, match_fw_to_car
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint
 

selfdrive/car/disable_ecu.py

@@ -1,5 +1,5 @@
 from selfdrive.car.isotp_parallel_query import IsoTpParallelQuery
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 EXT_DIAG_REQUEST = b'\x10\x03'
 EXT_DIAG_RESPONSE = b'\x50\x03'

selfdrive/car/fw_versions.py

@@ -12,7 +12,7 @@ from selfdrive.car.interfaces import get_interface_attr
 from selfdrive.car.fingerprints import FW_VERSIONS
 from selfdrive.car.isotp_parallel_query import IsoTpParallelQuery
 from selfdrive.car.toyota.values import CAR as TOYOTA
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 Ecu = car.CarParams.Ecu
 

selfdrive/car/isotp_parallel_query.py

@@ -4,7 +4,7 @@ from functools import partial
 from typing import Optional
 
 import cereal.messaging as messaging
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from panda.python.uds import CanClient, IsoTpMessage, FUNCTIONAL_ADDRS, get_rx_addr_for_tx_addr
 

selfdrive/car/mock/interface.py

@@ -2,7 +2,7 @@
 import math
 from cereal import car
 from common.conversions import Conversions as CV
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 import cereal.messaging as messaging
 from selfdrive.car import gen_empty_fingerprint, get_safety_config
 from selfdrive.car.interfaces import CarInterfaceBase

selfdrive/car/vin.py

@@ -6,7 +6,7 @@ import cereal.messaging as messaging
 import panda.python.uds as uds
 from panda.python.uds import FUNCTIONAL_ADDRS
 from selfdrive.car.isotp_parallel_query import IsoTpParallelQuery
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 OBD_VIN_REQUEST = b'\x09\x02'
 OBD_VIN_RESPONSE = b'\x49\x02\x01'

selfdrive/controls/controlsd.py

@@ -11,7 +11,7 @@ from common.params import Params, put_nonblocking
 import cereal.messaging as messaging
 from common.conversions import Conversions as CV
 from panda import ALTERNATIVE_EXPERIENCE
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.boardd.boardd import can_list_to_can_capnp
 from selfdrive.car.car_helpers import get_car, get_startup_event, get_one_can
 from selfdrive.controls.lib.lane_planner import CAMERA_OFFSET

selfdrive/controls/lib/events.py

@@ -8,7 +8,7 @@ import cereal.messaging as messaging
 from common.conversions import Conversions as CV
 from common.realtime import DT_CTRL
 from selfdrive.locationd.calibrationd import MIN_SPEED_FILTER
-from selfdrive.version import get_short_branch
+from system.version import get_short_branch
 
 AlertSize = log.ControlsState.AlertSize
 AlertStatus = log.ControlsState.AlertStatus

selfdrive/controls/lib/lane_planner.py

@@ -3,7 +3,7 @@ from cereal import log
 from common.filter_simple import FirstOrderFilter
 from common.numpy_fast import interp
 from common.realtime import DT_MDL
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 TRAJECTORY_SIZE = 33

selfdrive/controls/lib/lateral_planner.py

@@ -1,7 +1,7 @@
 import numpy as np
 from common.realtime import sec_since_boot, DT_MDL
 from common.numpy_fast import interp
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.controls.lib.lateral_mpc_lib.lat_mpc import LateralMpc
 from selfdrive.controls.lib.drive_helpers import CONTROL_N, MPC_COST_LAT, LAT_MPC_N, CAR_ROTATION_RADIUS
 from selfdrive.controls.lib.lane_planner import LanePlanner, TRAJECTORY_SIZE

selfdrive/controls/lib/longitudinal_mpc_lib/long_mpc.py

@@ -4,7 +4,7 @@ import numpy as np
 
 from common.realtime import sec_since_boot
 from common.numpy_fast import clip, interp
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.modeld.constants import index_function
 from selfdrive.controls.lib.radar_helpers import _LEAD_ACCEL_TAU
 

selfdrive/controls/lib/longitudinal_planner.py

@@ -12,7 +12,7 @@ from selfdrive.controls.lib.longcontrol import LongCtrlState
 from selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import LongitudinalMpc
 from selfdrive.controls.lib.longitudinal_mpc_lib.long_mpc import T_IDXS as T_IDXS_MPC
 from selfdrive.controls.lib.drive_helpers import V_CRUISE_MAX, CONTROL_N
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 LON_MPC_STEP = 0.2  # first step is 0.2s
 AWARENESS_DECEL = -0.2  # car smoothly decel at .2m/s^2 when user is distracted

selfdrive/controls/plannerd.py

@@ -2,7 +2,7 @@
 from cereal import car
 from common.params import Params
 from common.realtime import Priority, config_realtime_process
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.controls.lib.longitudinal_planner import Planner
 from selfdrive.controls.lib.lateral_planner import LateralPlanner
 import cereal.messaging as messaging

selfdrive/controls/radard.py

@@ -10,7 +10,7 @@ from common.params import Params
 from common.realtime import Ratekeeper, Priority, config_realtime_process
 from selfdrive.controls.lib.cluster.fastcluster_py import cluster_points_centroid
 from selfdrive.controls.lib.radar_helpers import Cluster, Track, RADAR_TO_CAMERA
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 class KalmanParams():

selfdrive/debug/disable_ecu.py

@@ -3,7 +3,7 @@ import traceback
 
 import cereal.messaging as messaging
 from selfdrive.car.isotp_parallel_query import IsoTpParallelQuery
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 EXT_DIAG_REQUEST = b'\x10\x03'
 EXT_DIAG_RESPONSE = b'\x50\x03'

selfdrive/locationd/calibrationd.py

@@ -20,7 +20,7 @@ from common.realtime import set_realtime_priority
 from common.transformations.model import model_height
 from common.transformations.camera import get_view_frame_from_road_frame
 from common.transformations.orientation import rot_from_euler, euler_from_rot
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 MIN_SPEED_FILTER = 15 * CV.MPH_TO_MS
 MAX_VEL_ANGLE_STD = np.radians(0.25)

selfdrive/locationd/laikad.py

@@ -17,7 +17,7 @@ from selfdrive.locationd.models.constants import GENERATED_DIR, ObservationKind
 from selfdrive.locationd.models.gnss_kf import GNSSKalman
 from selfdrive.locationd.models.gnss_kf import States as GStates
 import common.transformations.coordinates as coord
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 MAX_TIME_GAP = 10
 

selfdrive/locationd/models/car_kf.py

@@ -7,7 +7,7 @@ import numpy as np
 
 from selfdrive.controls.lib.vehicle_model import ACCELERATION_DUE_TO_GRAVITY
 from selfdrive.locationd.models.constants import ObservationKind
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 from rednose.helpers.kalmanfilter import KalmanFilter
 

selfdrive/locationd/paramsd.py

@@ -10,7 +10,7 @@ from common.realtime import config_realtime_process, DT_MDL
 from common.numpy_fast import clip
 from selfdrive.locationd.models.car_kf import CarKalman, ObservationKind, States
 from selfdrive.locationd.models.constants import GENERATED_DIR
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 
 MAX_ANGLE_OFFSET_DELTA = 20 * DT_MDL  # Max 20 deg/s

selfdrive/loggerd/deleter.py

@@ -2,7 +2,7 @@
 import os
 import shutil
 import threading
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.loggerd.config import ROOT, get_available_bytes, get_available_percent
 from selfdrive.loggerd.uploader import listdir_by_creation
 

selfdrive/loggerd/tests/test_loggerd.py

@@ -17,7 +17,7 @@ from common.params import Params
 from common.timeout import Timeout
 from selfdrive.loggerd.config import ROOT
 from selfdrive.manager.process_config import managed_processes
-from selfdrive.version import get_version
+from system.version import get_version
 from tools.lib.logreader import LogReader
 from cereal.visionipc import VisionIpcServer, VisionStreamType
 from common.transformations.camera import tici_f_frame_size, tici_d_frame_size, tici_e_frame_size

selfdrive/loggerd/tests/test_uploader.py

@@ -6,7 +6,7 @@ import unittest
 import logging
 import json
 
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 import selfdrive.loggerd.uploader as uploader
 
 from selfdrive.loggerd.tests.loggerd_tests_common import UploaderTestCase

selfdrive/loggerd/uploader.py

@@ -18,7 +18,7 @@ from common.realtime import set_core_affinity
 from system.hardware import TICI
 from selfdrive.loggerd.xattr_cache import getxattr, setxattr
 from selfdrive.loggerd.config import ROOT
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 NetworkType = log.DeviceState.NetworkType
 UPLOAD_ATTR_NAME = 'user.upload'

selfdrive/logmessaged.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python3
-import zmq
-from typing import NoReturn
-
-import cereal.messaging as messaging
-from common.logging_extra import SwagLogFileFormatter
-from selfdrive.swaglog import get_file_handler
-
-
-def main() -> NoReturn:
-  log_handler = get_file_handler()
-  log_handler.setFormatter(SwagLogFileFormatter(None))
-  log_level = 20  # logging.INFO
-
-  ctx = zmq.Context().instance()
-  sock = ctx.socket(zmq.PULL)
-  sock.bind("ipc:///tmp/logmessage")
-
-  # and we publish them
-  log_message_sock = messaging.pub_sock('logMessage')
-  error_log_message_sock = messaging.pub_sock('errorLogMessage')
-
-  while True:
-    dat = b''.join(sock.recv_multipart())
-    level = dat[0]
-    record = dat[1:].decode("utf-8")
-    if level >= log_level:
-      log_handler.emit(record)
-
-    # then we publish them
-    msg = messaging.new_message()
-    msg.logMessage = record
-    log_message_sock.send(msg.to_bytes())
-
-    if level >= 40:  # logging.ERROR
-      msg = messaging.new_message()
-      msg.errorLogMessage = record
-      error_log_message_sock.send(msg.to_bytes())
-
-
-if __name__ == "__main__":
-  main()

selfdrive/manager/build.py

@@ -11,8 +11,8 @@ from common.basedir import BASEDIR
 from common.spinner import Spinner
 from common.text_window import TextWindow
 from system.hardware import AGNOS
-from selfdrive.swaglog import cloudlog, add_file_handler
-from selfdrive.version import is_dirty
+from system.swaglog import cloudlog, add_file_handler
+from system.version import is_dirty
 
 MAX_CACHE_SIZE = 4e9 if "CI" in os.environ else 2e9
 CACHE_DIR = Path("/data/scons_cache" if AGNOS else "/tmp/scons_cache")

selfdrive/manager/manager.py

@@ -18,8 +18,8 @@ from selfdrive.manager.helpers import unblock_stdout
 from selfdrive.manager.process import ensure_running
 from selfdrive.manager.process_config import managed_processes
 from selfdrive.athena.registration import register, UNREGISTERED_DONGLE_ID
-from selfdrive.swaglog import cloudlog, add_file_handler
-from selfdrive.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \
+from system.swaglog import cloudlog, add_file_handler
+from system.version import is_dirty, get_commit, get_version, get_origin, get_short_branch, \
                               terms_version, training_version
 
 

selfdrive/manager/process.py

@@ -16,7 +16,7 @@ from cereal import car
 from common.basedir import BASEDIR
 from common.params import Params
 from common.realtime import sec_since_boot
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from system.hardware import HARDWARE
 from cereal import log
 

selfdrive/manager/process_config.py

@@ -18,16 +18,19 @@ def logging(started, params, CP: car.CarParams) -> bool:
   return started and run
 
 procs = [
+  NativeProcess("clocksd", "system/clocksd", ["./clocksd"]),
+  NativeProcess("logcatd", "system/logcatd", ["./logcatd"]),
+  NativeProcess("proclogd", "system/proclogd", ["./proclogd"]),
+  PythonProcess("logmessaged", "system.logmessaged", offroad=True),
+  PythonProcess("timezoned", "system.timezoned", enabled=not PC, offroad=True),
+
   DaemonProcess("manage_athenad", "selfdrive.athena.manage_athenad", "AthenadPid"),
   # due to qualcomm kernel bugs SIGKILLing camerad sometimes causes page table corruption
   NativeProcess("camerad", "selfdrive/camerad", ["./camerad"], unkillable=True, callback=driverview),
-  NativeProcess("clocksd", "system/clocksd", ["./clocksd"]),
   NativeProcess("dmonitoringmodeld", "selfdrive/modeld", ["./dmonitoringmodeld"], enabled=(not PC or WEBCAM), callback=driverview),
-  NativeProcess("logcatd", "system/logcatd", ["./logcatd"]),
   NativeProcess("encoderd", "selfdrive/loggerd", ["./encoderd"]),
   NativeProcess("loggerd", "selfdrive/loggerd", ["./loggerd"], onroad=False, callback=logging),
   NativeProcess("modeld", "selfdrive/modeld", ["./modeld"]),
-  NativeProcess("proclogd", "system/proclogd", ["./proclogd"]),
   NativeProcess("sensord", "selfdrive/sensord", ["./sensord"], enabled=not PC),
   NativeProcess("ubloxd", "selfdrive/locationd", ["./ubloxd"], enabled=(not PC or WEBCAM)),
   NativeProcess("ui", "selfdrive/ui", ["./ui"], offroad=True, watchdog_max_dt=(5 if not PC else None)),
@@ -38,14 +41,12 @@ procs = [
   PythonProcess("controlsd", "selfdrive.controls.controlsd"),
   PythonProcess("deleter", "selfdrive.loggerd.deleter", offroad=True),
   PythonProcess("dmonitoringd", "selfdrive.monitoring.dmonitoringd", enabled=(not PC or WEBCAM), callback=driverview),
-  PythonProcess("logmessaged", "selfdrive.logmessaged", offroad=True),
   PythonProcess("navd", "selfdrive.navd.navd"),
   PythonProcess("pandad", "selfdrive.boardd.pandad", offroad=True),
   PythonProcess("paramsd", "selfdrive.locationd.paramsd"),
   PythonProcess("plannerd", "selfdrive.controls.plannerd"),
   PythonProcess("radard", "selfdrive.controls.radard"),
   PythonProcess("thermald", "selfdrive.thermald.thermald", offroad=True),
-  PythonProcess("timezoned", "selfdrive.timezoned", enabled=not PC, offroad=True),
   PythonProcess("tombstoned", "selfdrive.tombstoned", enabled=not PC, offroad=True),
   PythonProcess("updated", "selfdrive.updated", enabled=not PC, onroad=False, offroad=True),
   PythonProcess("uploader", "selfdrive.loggerd.uploader", offroad=True),

selfdrive/navd/navd.py

@@ -14,7 +14,7 @@ from selfdrive.navd.helpers import (Coordinate, coordinate_from_param,
                                     distance_along_geometry, maxspeed_to_ms,
                                     minimum_distance,
                                     parse_banner_instructions)
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 REROUTE_DISTANCE = 25
 MANEUVER_TRANSITION_THRESHOLD = 10

selfdrive/sensord/rawgps/rawgpsd.py

@@ -10,7 +10,7 @@ from struct import unpack_from, calcsize, pack
 
 import cereal.messaging as messaging
 from cereal import log
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 
 from selfdrive.sensord.rawgps.modemdiag import ModemDiag, DIAG_LOG_F, setup_logs, send_recv
 from selfdrive.sensord.rawgps.structs import dict_unpacker

selfdrive/sentry.py

@@ -6,8 +6,8 @@ from sentry_sdk.integrations.threading import ThreadingIntegration
 from common.params import Params
 from selfdrive.athena.registration import is_registered_device
 from system.hardware import HARDWARE, PC
-from selfdrive.swaglog import cloudlog
-from selfdrive.version import get_branch, get_commit, get_origin, get_version, \
+from system.swaglog import cloudlog
+from system.version import get_branch, get_commit, get_origin, get_version, \
                               is_comma_remote, is_dirty, is_tested_branch
 
 

selfdrive/statsd.py

@@ -9,10 +9,10 @@ from typing import NoReturn, Union, List, Dict
 
 from common.params import Params
 from cereal.messaging import SubMaster
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from system.hardware import HARDWARE
 from common.file_helpers import atomic_write_in_dir
-from selfdrive.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty
+from system.version import get_normalized_origin, get_short_branch, get_short_version, is_dirty
 from selfdrive.loggerd.config import STATS_DIR, STATS_DIR_FILE_LIMIT, STATS_SOCKET, STATS_FLUSH_TIME_S
 
 

selfdrive/swaglog.py

@@ -1,122 +0,0 @@
-import logging
-import os
-import time
-from pathlib import Path
-from logging.handlers import BaseRotatingHandler
-
-import zmq
-
-from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from system.hardware import PC
-
-if PC:
-  SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
-else:
-  SWAGLOG_DIR = "/data/log/"
-
-def get_file_handler():
-  Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
-  base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
-  handler = SwaglogRotatingFileHandler(base_filename)
-  return handler
-
-class SwaglogRotatingFileHandler(BaseRotatingHandler):
-  def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
-    super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
-    self.base_filename = base_filename
-    self.interval = interval # seconds
-    self.max_bytes = max_bytes
-    self.backup_count = backup_count
-    self.log_files = self.get_existing_logfiles()
-    log_indexes = [f.split(".")[-1] for f in self.log_files]
-    self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
-    self.last_rollover = None
-    self.doRollover()
-
-  def _open(self):
-    self.last_rollover = time.monotonic()
-    self.last_file_idx += 1
-    next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
-    stream = open(next_filename, self.mode, encoding=self.encoding)
-    self.log_files.insert(0, next_filename)
-    return stream
-
-  def get_existing_logfiles(self):
-    log_files = list()
-    base_dir = os.path.dirname(self.base_filename)
-    for fn in os.listdir(base_dir):
-      fp = os.path.join(base_dir, fn)
-      if fp.startswith(self.base_filename) and os.path.isfile(fp):
-        log_files.append(fp)
-    return sorted(log_files)
-
-  def shouldRollover(self, record):
-    size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
-    time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
-    return size_exceeded or time_exceeded
-
-  def doRollover(self):
-    if self.stream:
-      self.stream.close()
-    self.stream = self._open()
-
-    if self.backup_count > 0:
-      while len(self.log_files) > self.backup_count:
-        to_delete = self.log_files.pop()
-        if os.path.exists(to_delete): # just being safe, should always exist
-          os.remove(to_delete)
-
-class UnixDomainSocketHandler(logging.Handler):
-  def __init__(self, formatter):
-    logging.Handler.__init__(self)
-    self.setFormatter(formatter)
-    self.pid = None
-
-  def connect(self):
-    self.zctx = zmq.Context()
-    self.sock = self.zctx.socket(zmq.PUSH)
-    self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect("ipc:///tmp/logmessage")
-    self.pid = os.getpid()
-
-  def emit(self, record):
-    if os.getpid() != self.pid:
-      self.connect()
-
-    msg = self.format(record).rstrip('\n')
-    # print("SEND".format(repr(msg)))
-    try:
-      s = chr(record.levelno)+msg
-      self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
-    except zmq.error.Again:
-      # drop :/
-      pass
-
-
-def add_file_handler(log):
-  """
-  Function to add the file log handler to swaglog.
-  This can be used to store logs when logmessaged is not running.
-  """
-  handler = get_file_handler()
-  handler.setFormatter(SwagLogFileFormatter(log))
-  log.addHandler(handler)
-
-
-cloudlog = log = SwagLogger()
-log.setLevel(logging.DEBUG)
-
-
-outhandler = logging.StreamHandler()
-
-print_level = os.environ.get('LOGPRINT', 'warning')
-if print_level == 'debug':
-  outhandler.setLevel(logging.DEBUG)
-elif print_level == 'info':
-  outhandler.setLevel(logging.INFO)
-elif print_level == 'warning':
-  outhandler.setLevel(logging.WARNING)
-
-log.addHandler(outhandler)
-# logs are sent through IPC before writing to disk to prevent disk I/O blocking
-log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))

selfdrive/test/helpers.py

@@ -4,7 +4,7 @@ from functools import wraps
 
 from system.hardware import PC
 from selfdrive.manager.process_config import managed_processes
-from selfdrive.version import training_version, terms_version
+from system.version import training_version, terms_version
 
 
 def set_params_enabled():

selfdrive/test/process_replay/model_replay.py

@@ -16,7 +16,7 @@ from selfdrive.manager.process_config import managed_processes
 from selfdrive.test.openpilotci import BASE_URL, get_url
 from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
 from selfdrive.test.process_replay.test_processes import format_diff
-from selfdrive.version import get_commit
+from system.version import get_commit
 from tools.lib.framereader import FrameReader
 from tools.lib.logreader import LogReader
 

selfdrive/test/process_replay/test_debayer.py

@@ -9,7 +9,7 @@ import pyopencl as cl  # install with `PYOPENCL_CL_PRETEND_VERSION=2.0 pip insta
 from system.hardware import PC, TICI
 from common.basedir import BASEDIR
 from selfdrive.test.openpilotci import BASE_URL, get_url
-from selfdrive.version import get_commit
+from system.version import get_commit
 from selfdrive.camerad.snapshot.snapshot import yuv_to_rgb
 from tools.lib.logreader import LogReader
 from tools.lib.filereader import FileReader

selfdrive/test/process_replay/test_processes.py

@@ -11,7 +11,7 @@ from selfdrive.car.car_helpers import interface_names
 from selfdrive.test.openpilotci import get_url, upload_file
 from selfdrive.test.process_replay.compare_logs import compare_logs, save_log
 from selfdrive.test.process_replay.process_replay import CONFIGS, PROC_REPLAY_DIR, FAKEDATA, check_enabled, replay_process
-from selfdrive.version import get_commit
+from system.version import get_commit
 from tools.lib.filereader import FileReader
 from tools.lib.logreader import LogReader
 

selfdrive/test/test_onroad.py

@@ -39,7 +39,7 @@ PROCS = {
   "./_soundd": 1.0,
   "selfdrive.monitoring.dmonitoringd": 1.90,
   "./proclogd": 1.54,
-  "selfdrive.logmessaged": 0.2,
+  "system.logmessaged": 0.2,
   "./clocksd": 0.02,
   "./ubloxd": 0.02,
   "selfdrive.tombstoned": 0,

selfdrive/thermald/fan_controller.py

@@ -3,7 +3,7 @@ from abc import ABC, abstractmethod
 
 from common.realtime import DT_TRML
 from common.numpy_fast import interp
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.controls.lib.pid import PIDController
 
 class BaseFanController(ABC):

selfdrive/thermald/power_monitoring.py

@@ -5,7 +5,7 @@ from cereal import log
 from common.params import Params, put_nonblocking
 from common.realtime import sec_since_boot
 from system.hardware import HARDWARE
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.statsd import statlog
 
 CAR_VOLTAGE_LOW_PASS_K = 0.091 # LPF gain for 5s tau (dt/tau / (dt/tau + 1))

selfdrive/thermald/thermald.py

@@ -20,10 +20,10 @@ from selfdrive.controls.lib.alertmanager import set_offroad_alert
 from system.hardware import HARDWARE, TICI, AGNOS
 from selfdrive.loggerd.config import get_available_percent
 from selfdrive.statsd import statlog
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.thermald.power_monitoring import PowerMonitoring
 from selfdrive.thermald.fan_controller import TiciFanController
-from selfdrive.version import terms_version, training_version
+from system.version import terms_version, training_version
 
 ThermalStatus = log.DeviceState.ThermalStatus
 NetworkType = log.DeviceState.NetworkType

selfdrive/timezoned.py

@@ -1,89 +0,0 @@
-#!/usr/bin/env python3
-import json
-import os
-import time
-import subprocess
-from typing import NoReturn
-
-import requests
-from timezonefinder import TimezoneFinder
-
-from common.params import Params
-from system.hardware import AGNOS
-from selfdrive.swaglog import cloudlog
-
-
-def set_timezone(valid_timezones, timezone):
-  if timezone not in valid_timezones:
-    cloudlog.error(f"Timezone not supported {timezone}")
-    return
-
-  cloudlog.debug(f"Setting timezone to {timezone}")
-  try:
-    if AGNOS:
-      tzpath = os.path.join("/usr/share/zoneinfo/", timezone)
-      subprocess.check_call(f'sudo su -c "ln -snf {tzpath} /data/etc/tmptime && \
-                              mv /data/etc/tmptime /data/etc/localtime"', shell=True)
-      subprocess.check_call(f'sudo su -c "echo \"{timezone}\" > /data/etc/timezone"', shell=True)
-    else:
-      subprocess.check_call(f'sudo timedatectl set-timezone {timezone}', shell=True)
-  except subprocess.CalledProcessError:
-    cloudlog.exception(f"Error setting timezone to {timezone}")
-
-
-def main() -> NoReturn:
-  params = Params()
-  tf = TimezoneFinder()
-
-  # Get allowed timezones
-  valid_timezones = subprocess.check_output('timedatectl list-timezones', shell=True, encoding='utf8').strip().split('\n')
-
-  while True:
-    time.sleep(60)
-
-    is_onroad = not params.get_bool("IsOffroad")
-    if is_onroad:
-      continue
-
-    # Set based on param
-    timezone = params.get("Timezone", encoding='utf8')
-    if timezone is not None:
-      cloudlog.debug("Setting timezone based on param")
-      set_timezone(valid_timezones, timezone)
-      continue
-
-    location = params.get("LastGPSPosition", encoding='utf8')
-
-    # Find timezone based on IP geolocation if no gps location is available
-    if location is None:
-      cloudlog.debug("Setting timezone based on IP lookup")
-      try:
-        r = requests.get("https://ipapi.co/timezone", timeout=10)
-        if r.status_code == 200:
-          set_timezone(valid_timezones, r.text)
-        else:
-          cloudlog.error(f"Unexpected status code from api {r.status_code}")
-
-        time.sleep(3600)  # Don't make too many API requests
-      except requests.exceptions.RequestException:
-        cloudlog.exception("Error getting timezone based on IP")
-        continue
-
-    # Find timezone by reverse geocoding the last known gps location
-    else:
-      cloudlog.debug("Setting timezone based on GPS location")
-      try:
-        location = json.loads(location)
-      except Exception:
-        cloudlog.exception("Error parsing location")
-        continue
-
-      timezone = tf.timezone_at(lng=location['longitude'], lat=location['latitude'])
-      if timezone is None:
-        cloudlog.error(f"No timezone found based on location, {location}")
-        continue
-      set_timezone(valid_timezones, timezone)
-
-
-if __name__ == "__main__":
-  main()

selfdrive/tombstoned.py

@@ -12,8 +12,8 @@ from typing import NoReturn
 from common.file_helpers import mkdirs_exists_ok
 from selfdrive.loggerd.config import ROOT
 import selfdrive.sentry as sentry
-from selfdrive.swaglog import cloudlog
-from selfdrive.version import get_commit
+from system.swaglog import cloudlog
+from system.version import get_commit
 
 MAX_SIZE = 1_000_000 * 100  # allow up to 100M
 MAX_TOMBSTONE_FN_LEN = 62  # 85 - 23 ("<dongle id>/crash/")

selfdrive/updated.py

@@ -38,9 +38,9 @@ from markdown_it import MarkdownIt
 from common.basedir import BASEDIR
 from common.params import Params
 from system.hardware import AGNOS, HARDWARE
-from selfdrive.swaglog import cloudlog
+from system.swaglog import cloudlog
 from selfdrive.controls.lib.alertmanager import set_offroad_alert
-from selfdrive.version import is_tested_branch
+from system.version import is_tested_branch
 
 LOCK_FILE = os.getenv("UPDATER_LOCK_FILE", "/tmp/safe_staging_overlay.lock")
 STAGING_ROOT = os.getenv("UPDATER_STAGING_ROOT", "/data/safe_staging")

selfdrive/version.py

@@ -1,139 +0,0 @@
-#!/usr/bin/env python3
-import os
-import subprocess
-from typing import List, Optional
-from functools import lru_cache
-
-from common.basedir import BASEDIR
-from selfdrive.swaglog import cloudlog
-
-TESTED_BRANCHES = ['devel', 'release3-staging', 'dashcam3-staging', 'release3', 'dashcam3']
-
-training_version: bytes = b"0.2.0"
-terms_version: bytes = b"2"
-
-
-def cache(user_function, /):
-  return lru_cache(maxsize=None)(user_function)
-
-
-def run_cmd(cmd: List[str]) -> str:
-  return subprocess.check_output(cmd, encoding='utf8').strip()
-
-
-def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[str]:
-  try:
-    return run_cmd(cmd)
-  except subprocess.CalledProcessError:
-    return default
-
-
-@cache
-def get_commit(branch: str = "HEAD", default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", branch], default=default)
-
-
-@cache
-def get_short_branch(default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], default=default)
-
-
-@cache
-def get_branch(default: Optional[str] = None) -> Optional[str]:
-  return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], default=default)
-
-
-@cache
-def get_origin(default: Optional[str] = None) -> Optional[str]:
-  try:
-    local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"])
-    tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"])
-    return run_cmd(["git", "config", "remote." + tracking_remote + ".url"])
-  except subprocess.CalledProcessError:  # Not on a branch, fallback
-    return run_cmd_default(["git", "config", "--get", "remote.origin.url"], default=default)
-
-
-@cache
-def get_normalized_origin(default: Optional[str] = None) -> Optional[str]:
-  origin: Optional[str] = get_origin()
-
-  if origin is None:
-    return default
-
-  return origin.replace("git@", "", 1) \
-               .replace(".git", "", 1) \
-               .replace("https://", "", 1) \
-               .replace(":", "/", 1)
-
-
-@cache
-def get_version() -> str:
-  with open(os.path.join(BASEDIR, "common", "version.h")) as _versionf:
-    version = _versionf.read().split('"')[1]
-  return version
-
-@cache
-def get_short_version() -> str:
-  return get_version().split('-')[0]  # type: ignore
-
-@cache
-def is_prebuilt() -> bool:
-  return os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
-
-
-@cache
-def is_comma_remote() -> bool:
-  # note to fork maintainers, this is used for release metrics. please do not
-  # touch this to get rid of the orange startup alert. there's better ways to do that
-  origin: Optional[str] = get_origin()
-  if origin is None:
-    return False
-
-  return origin.startswith('git@github.com:commaai') or origin.startswith('https://github.com/commaai')
-
-
-@cache
-def is_tested_branch() -> bool:
-  return get_short_branch() in TESTED_BRANCHES
-
-
-@cache
-def is_dirty() -> bool:
-  origin = get_origin()
-  branch = get_branch()
-  if (origin is None) or (branch is None):
-    return True
-
-  dirty = False
-  try:
-    # Actually check dirty files
-    if not is_prebuilt():
-      # This is needed otherwise touched files might show up as modified
-      try:
-        subprocess.check_call(["git", "update-index", "--refresh"])
-      except subprocess.CalledProcessError:
-        pass
-
-      dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0)
-  except subprocess.CalledProcessError:
-    cloudlog.exception("git subprocess failed while checking dirty")
-    dirty = True
-
-  return dirty
-
-
-if __name__ == "__main__":
-  from common.params import Params
-
-  params = Params()
-  params.put("TermsVersion", terms_version)
-  params.put("TrainingVersion", training_version)
-
-  print(f"Dirty: {is_dirty()}")
-  print(f"Version: {get_version()}")
-  print(f"Short version: {get_short_version()}")
-  print(f"Origin: {get_origin()}")
-  print(f"Normalized origin: {get_normalized_origin()}")
-  print(f"Branch: {get_branch()}")
-  print(f"Short branch: {get_short_branch()}")
-  print(f"Prebuilt: {is_prebuilt()}")