swaglog.py goes in common/ (#30631)

* swaglog.py goes in common/

* all of these go in release

* we'll even include the pyx

system/loggerd/deleter.py

@@ -4,7 +4,7 @@ import shutil
 import threading
 from typing import List
 from openpilot.system.hardware.hw import Paths
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 from openpilot.system.loggerd.config import get_available_bytes, get_available_percent
 from openpilot.system.loggerd.uploader import listdir_by_creation
 from openpilot.system.loggerd.xattr_cache import getxattr

system/loggerd/tests/test_uploader.py

@@ -9,7 +9,7 @@ from pathlib import Path
 from typing import List, Optional
 from openpilot.system.hardware.hw import Paths
 
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 from openpilot.system.loggerd.uploader import uploader_fn, UPLOAD_ATTR_NAME, UPLOAD_ATTR_VALUE
 
 from openpilot.system.loggerd.tests.loggerd_tests_common import UploaderTestCase

system/loggerd/uploader.py

@@ -19,7 +19,7 @@ from openpilot.common.realtime import set_core_affinity
 from openpilot.system.hardware import TICI
 from openpilot.system.hardware.hw import Paths
 from openpilot.system.loggerd.xattr_cache import getxattr, setxattr
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 
 NetworkType = log.DeviceState.NetworkType
 UPLOAD_ATTR_NAME = 'user.upload'

system/logmessaged.py

@@ -5,7 +5,7 @@ from typing import NoReturn
 import cereal.messaging as messaging
 from openpilot.common.logging_extra import SwagLogFileFormatter
 from openpilot.system.hardware.hw import Paths
-from openpilot.system.swaglog import get_file_handler
+from openpilot.common.swaglog import get_file_handler
 
 
 def main() -> NoReturn:

system/micd.py

@@ -3,7 +3,7 @@ import numpy as np
 
 from cereal import messaging
 from openpilot.common.realtime import Ratekeeper
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 
 RATE = 10
 FFT_SAMPLES = 4096

system/qcomgpsd/qcomgpsd.py

@@ -17,7 +17,7 @@ from cereal import log
 import cereal.messaging as messaging
 from openpilot.common.gpio import gpio_init, gpio_set
 from openpilot.system.hardware.tici.pins import GPIO
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 from openpilot.system.qcomgpsd.modemdiag import ModemDiag, DIAG_LOG_F, setup_logs, send_recv
 from openpilot.system.qcomgpsd.structs import (dict_unpacker, position_report, relist,
                                               gps_measurement_report, gps_measurement_report_sv,

system/sensord/pigeond.py

@@ -11,7 +11,7 @@ from typing import List, Optional, Tuple
 
 from cereal import messaging
 from openpilot.common.params import Params
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 from openpilot.system.hardware import TICI
 from openpilot.common.gpio import gpio_init, gpio_set
 from openpilot.system.hardware.tici.pins import GPIO

system/swaglog.py

@@ -1,135 +0,0 @@
-import logging
-import os
-import time
-import warnings
-from pathlib import Path
-from logging.handlers import BaseRotatingHandler
-
-import zmq
-
-from openpilot.common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
-from openpilot.system.hardware.hw import Paths
-
-
-def get_file_handler():
-  Path(Paths.swaglog_root()).mkdir(parents=True, exist_ok=True)
-  base_filename = os.path.join(Paths.swaglog_root(), "swaglog")
-  handler = SwaglogRotatingFileHandler(base_filename)
-  return handler
-
-class SwaglogRotatingFileHandler(BaseRotatingHandler):
-  def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
-    super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
-    self.base_filename = base_filename
-    self.interval = interval # seconds
-    self.max_bytes = max_bytes
-    self.backup_count = backup_count
-    self.log_files = self.get_existing_logfiles()
-    log_indexes = [f.split(".")[-1] for f in self.log_files]
-    self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
-    self.last_rollover = None
-    self.doRollover()
-
-  def _open(self):
-    self.last_rollover = time.monotonic()
-    self.last_file_idx += 1
-    next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
-    stream = open(next_filename, self.mode, encoding=self.encoding)
-    self.log_files.insert(0, next_filename)
-    return stream
-
-  def get_existing_logfiles(self):
-    log_files = list()
-    base_dir = os.path.dirname(self.base_filename)
-    for fn in os.listdir(base_dir):
-      fp = os.path.join(base_dir, fn)
-      if fp.startswith(self.base_filename) and os.path.isfile(fp):
-        log_files.append(fp)
-    return sorted(log_files)
-
-  def shouldRollover(self, record):
-    size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
-    time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
-    return size_exceeded or time_exceeded
-
-  def doRollover(self):
-    if self.stream:
-      self.stream.close()
-    self.stream = self._open()
-
-    if self.backup_count > 0:
-      while len(self.log_files) > self.backup_count:
-        to_delete = self.log_files.pop()
-        if os.path.exists(to_delete): # just being safe, should always exist
-          os.remove(to_delete)
-
-class UnixDomainSocketHandler(logging.Handler):
-  def __init__(self, formatter):
-    logging.Handler.__init__(self)
-    self.setFormatter(formatter)
-    self.pid = None
-
-    self.zctx = None
-    self.sock = None
-
-  def __del__(self):
-    self.close()
-
-  def close(self):
-    if self.sock is not None:
-      self.sock.close()
-    if self.zctx is not None:
-      self.zctx.term()
-
-  def connect(self):
-    self.zctx = zmq.Context()
-    self.sock = self.zctx.socket(zmq.PUSH)
-    self.sock.setsockopt(zmq.LINGER, 10)
-    self.sock.connect(Paths.swaglog_ipc())
-    self.pid = os.getpid()
-
-  def emit(self, record):
-    if os.getpid() != self.pid:
-      # TODO suppresses warning about forking proc with zmq socket, fix root cause
-      warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<zmq.*>")
-      self.connect()
-
-    msg = self.format(record).rstrip('\n')
-    # print("SEND".format(repr(msg)))
-    try:
-      s = chr(record.levelno)+msg
-      self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
-    except zmq.error.Again:
-      # drop :/
-      pass
-
-
-def add_file_handler(log):
-  """
-  Function to add the file log handler to swaglog.
-  This can be used to store logs when logmessaged is not running.
-  """
-  handler = get_file_handler()
-  handler.setFormatter(SwagLogFileFormatter(log))
-  log.addHandler(handler)
-
-
-cloudlog = log = SwagLogger()
-log.setLevel(logging.DEBUG)
-
-
-outhandler = logging.StreamHandler()
-
-print_level = os.environ.get('LOGPRINT', 'warning')
-if print_level == 'debug':
-  outhandler.setLevel(logging.DEBUG)
-elif print_level == 'info':
-  outhandler.setLevel(logging.INFO)
-elif print_level == 'warning':
-  outhandler.setLevel(logging.WARNING)
-
-ipchandler = UnixDomainSocketHandler(SwagFormatter(log))
-
-log.addHandler(outhandler)
-# logs are sent through IPC before writing to disk to prevent disk I/O blocking
-log.addHandler(ipchandler)

system/tests/test_logmessaged.py

@@ -7,7 +7,7 @@ import unittest
 import cereal.messaging as messaging
 from openpilot.selfdrive.manager.process_config import managed_processes
 from openpilot.system.hardware.hw import Paths
-from openpilot.system.swaglog import cloudlog, ipchandler
+from openpilot.common.swaglog import cloudlog, ipchandler
 
 
 class TestLogmessaged(unittest.TestCase):

system/timezoned.py

@@ -10,7 +10,7 @@ from timezonefinder import TimezoneFinder
 
 from openpilot.common.params import Params
 from openpilot.system.hardware import AGNOS
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 from openpilot.system.version import get_version
 
 REQUEST_HEADERS = {'User-Agent': "openpilot-" + get_version()}

system/version.py

@@ -5,7 +5,7 @@ from typing import List, Optional
 from functools import lru_cache
 
 from openpilot.common.basedir import BASEDIR
-from openpilot.system.swaglog import cloudlog
+from openpilot.common.swaglog import cloudlog
 
 RELEASE_BRANCHES = ['release3-staging', 'dashcam3-staging', 'release3', 'dashcam3', 'nightly']
 TESTED_BRANCHES = RELEASE_BRANCHES + ['devel', 'devel-staging']