mirror of
https://github.com/MoreTore/openpilot.git
synced 2026-09-30 11:23:55 +08:00
Move a bunch of stuff to system/ part 3 (#24829)
* move swaglog.py * timezoned * logmessaged * version.py * fix linter
This commit is contained in:
Adeeb Shihadeh
Executable
+42
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env python3
|
||||
import zmq
|
||||
from typing import NoReturn
|
||||
|
||||
import cereal.messaging as messaging
|
||||
from common.logging_extra import SwagLogFileFormatter
|
||||
from system.swaglog import get_file_handler
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
log_handler = get_file_handler()
|
||||
log_handler.setFormatter(SwagLogFileFormatter(None))
|
||||
log_level = 20 # logging.INFO
|
||||
|
||||
ctx = zmq.Context().instance()
|
||||
sock = ctx.socket(zmq.PULL)
|
||||
sock.bind("ipc:///tmp/logmessage")
|
||||
|
||||
# and we publish them
|
||||
log_message_sock = messaging.pub_sock('logMessage')
|
||||
error_log_message_sock = messaging.pub_sock('errorLogMessage')
|
||||
|
||||
while True:
|
||||
dat = b''.join(sock.recv_multipart())
|
||||
level = dat[0]
|
||||
record = dat[1:].decode("utf-8")
|
||||
if level >= log_level:
|
||||
log_handler.emit(record)
|
||||
|
||||
# then we publish them
|
||||
msg = messaging.new_message()
|
||||
msg.logMessage = record
|
||||
log_message_sock.send(msg.to_bytes())
|
||||
|
||||
if level >= 40: # logging.ERROR
|
||||
msg = messaging.new_message()
|
||||
msg.errorLogMessage = record
|
||||
error_log_message_sock.send(msg.to_bytes())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,122 @@
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
from logging.handlers import BaseRotatingHandler
|
||||
|
||||
import zmq
|
||||
|
||||
from common.logging_extra import SwagLogger, SwagFormatter, SwagLogFileFormatter
|
||||
from system.hardware import PC
|
||||
|
||||
if PC:
|
||||
SWAGLOG_DIR = os.path.join(str(Path.home()), ".comma", "log")
|
||||
else:
|
||||
SWAGLOG_DIR = "/data/log/"
|
||||
|
||||
def get_file_handler():
|
||||
Path(SWAGLOG_DIR).mkdir(parents=True, exist_ok=True)
|
||||
base_filename = os.path.join(SWAGLOG_DIR, "swaglog")
|
||||
handler = SwaglogRotatingFileHandler(base_filename)
|
||||
return handler
|
||||
|
||||
class SwaglogRotatingFileHandler(BaseRotatingHandler):
|
||||
def __init__(self, base_filename, interval=60, max_bytes=1024*256, backup_count=2500, encoding=None):
|
||||
super().__init__(base_filename, mode="a", encoding=encoding, delay=True)
|
||||
self.base_filename = base_filename
|
||||
self.interval = interval # seconds
|
||||
self.max_bytes = max_bytes
|
||||
self.backup_count = backup_count
|
||||
self.log_files = self.get_existing_logfiles()
|
||||
log_indexes = [f.split(".")[-1] for f in self.log_files]
|
||||
self.last_file_idx = max([int(i) for i in log_indexes if i.isdigit()] or [-1])
|
||||
self.last_rollover = None
|
||||
self.doRollover()
|
||||
|
||||
def _open(self):
|
||||
self.last_rollover = time.monotonic()
|
||||
self.last_file_idx += 1
|
||||
next_filename = f"{self.base_filename}.{self.last_file_idx:010}"
|
||||
stream = open(next_filename, self.mode, encoding=self.encoding)
|
||||
self.log_files.insert(0, next_filename)
|
||||
return stream
|
||||
|
||||
def get_existing_logfiles(self):
|
||||
log_files = list()
|
||||
base_dir = os.path.dirname(self.base_filename)
|
||||
for fn in os.listdir(base_dir):
|
||||
fp = os.path.join(base_dir, fn)
|
||||
if fp.startswith(self.base_filename) and os.path.isfile(fp):
|
||||
log_files.append(fp)
|
||||
return sorted(log_files)
|
||||
|
||||
def shouldRollover(self, record):
|
||||
size_exceeded = self.max_bytes > 0 and self.stream.tell() >= self.max_bytes
|
||||
time_exceeded = self.interval > 0 and self.last_rollover + self.interval <= time.monotonic()
|
||||
return size_exceeded or time_exceeded
|
||||
|
||||
def doRollover(self):
|
||||
if self.stream:
|
||||
self.stream.close()
|
||||
self.stream = self._open()
|
||||
|
||||
if self.backup_count > 0:
|
||||
while len(self.log_files) > self.backup_count:
|
||||
to_delete = self.log_files.pop()
|
||||
if os.path.exists(to_delete): # just being safe, should always exist
|
||||
os.remove(to_delete)
|
||||
|
||||
class UnixDomainSocketHandler(logging.Handler):
|
||||
def __init__(self, formatter):
|
||||
logging.Handler.__init__(self)
|
||||
self.setFormatter(formatter)
|
||||
self.pid = None
|
||||
|
||||
def connect(self):
|
||||
self.zctx = zmq.Context()
|
||||
self.sock = self.zctx.socket(zmq.PUSH)
|
||||
self.sock.setsockopt(zmq.LINGER, 10)
|
||||
self.sock.connect("ipc:///tmp/logmessage")
|
||||
self.pid = os.getpid()
|
||||
|
||||
def emit(self, record):
|
||||
if os.getpid() != self.pid:
|
||||
self.connect()
|
||||
|
||||
msg = self.format(record).rstrip('\n')
|
||||
# print("SEND".format(repr(msg)))
|
||||
try:
|
||||
s = chr(record.levelno)+msg
|
||||
self.sock.send(s.encode('utf8'), zmq.NOBLOCK)
|
||||
except zmq.error.Again:
|
||||
# drop :/
|
||||
pass
|
||||
|
||||
|
||||
def add_file_handler(log):
|
||||
"""
|
||||
Function to add the file log handler to swaglog.
|
||||
This can be used to store logs when logmessaged is not running.
|
||||
"""
|
||||
handler = get_file_handler()
|
||||
handler.setFormatter(SwagLogFileFormatter(log))
|
||||
log.addHandler(handler)
|
||||
|
||||
|
||||
cloudlog = log = SwagLogger()
|
||||
log.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
outhandler = logging.StreamHandler()
|
||||
|
||||
print_level = os.environ.get('LOGPRINT', 'warning')
|
||||
if print_level == 'debug':
|
||||
outhandler.setLevel(logging.DEBUG)
|
||||
elif print_level == 'info':
|
||||
outhandler.setLevel(logging.INFO)
|
||||
elif print_level == 'warning':
|
||||
outhandler.setLevel(logging.WARNING)
|
||||
|
||||
log.addHandler(outhandler)
|
||||
# logs are sent through IPC before writing to disk to prevent disk I/O blocking
|
||||
log.addHandler(UnixDomainSocketHandler(SwagFormatter(log)))
|
||||
Executable
+89
@@ -0,0 +1,89 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
import subprocess
|
||||
from typing import NoReturn
|
||||
|
||||
import requests
|
||||
from timezonefinder import TimezoneFinder
|
||||
|
||||
from common.params import Params
|
||||
from system.hardware import AGNOS
|
||||
from system.swaglog import cloudlog
|
||||
|
||||
|
||||
def set_timezone(valid_timezones, timezone):
|
||||
if timezone not in valid_timezones:
|
||||
cloudlog.error(f"Timezone not supported {timezone}")
|
||||
return
|
||||
|
||||
cloudlog.debug(f"Setting timezone to {timezone}")
|
||||
try:
|
||||
if AGNOS:
|
||||
tzpath = os.path.join("/usr/share/zoneinfo/", timezone)
|
||||
subprocess.check_call(f'sudo su -c "ln -snf {tzpath} /data/etc/tmptime && \
|
||||
mv /data/etc/tmptime /data/etc/localtime"', shell=True)
|
||||
subprocess.check_call(f'sudo su -c "echo \"{timezone}\" > /data/etc/timezone"', shell=True)
|
||||
else:
|
||||
subprocess.check_call(f'sudo timedatectl set-timezone {timezone}', shell=True)
|
||||
except subprocess.CalledProcessError:
|
||||
cloudlog.exception(f"Error setting timezone to {timezone}")
|
||||
|
||||
|
||||
def main() -> NoReturn:
|
||||
params = Params()
|
||||
tf = TimezoneFinder()
|
||||
|
||||
# Get allowed timezones
|
||||
valid_timezones = subprocess.check_output('timedatectl list-timezones', shell=True, encoding='utf8').strip().split('\n')
|
||||
|
||||
while True:
|
||||
time.sleep(60)
|
||||
|
||||
is_onroad = not params.get_bool("IsOffroad")
|
||||
if is_onroad:
|
||||
continue
|
||||
|
||||
# Set based on param
|
||||
timezone = params.get("Timezone", encoding='utf8')
|
||||
if timezone is not None:
|
||||
cloudlog.debug("Setting timezone based on param")
|
||||
set_timezone(valid_timezones, timezone)
|
||||
continue
|
||||
|
||||
location = params.get("LastGPSPosition", encoding='utf8')
|
||||
|
||||
# Find timezone based on IP geolocation if no gps location is available
|
||||
if location is None:
|
||||
cloudlog.debug("Setting timezone based on IP lookup")
|
||||
try:
|
||||
r = requests.get("https://ipapi.co/timezone", timeout=10)
|
||||
if r.status_code == 200:
|
||||
set_timezone(valid_timezones, r.text)
|
||||
else:
|
||||
cloudlog.error(f"Unexpected status code from api {r.status_code}")
|
||||
|
||||
time.sleep(3600) # Don't make too many API requests
|
||||
except requests.exceptions.RequestException:
|
||||
cloudlog.exception("Error getting timezone based on IP")
|
||||
continue
|
||||
|
||||
# Find timezone by reverse geocoding the last known gps location
|
||||
else:
|
||||
cloudlog.debug("Setting timezone based on GPS location")
|
||||
try:
|
||||
location = json.loads(location)
|
||||
except Exception:
|
||||
cloudlog.exception("Error parsing location")
|
||||
continue
|
||||
|
||||
timezone = tf.timezone_at(lng=location['longitude'], lat=location['latitude'])
|
||||
if timezone is None:
|
||||
cloudlog.error(f"No timezone found based on location, {location}")
|
||||
continue
|
||||
set_timezone(valid_timezones, timezone)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import subprocess
|
||||
from typing import List, Optional
|
||||
from functools import lru_cache
|
||||
|
||||
from common.basedir import BASEDIR
|
||||
from system.swaglog import cloudlog
|
||||
|
||||
TESTED_BRANCHES = ['devel', 'release3-staging', 'dashcam3-staging', 'release3', 'dashcam3']
|
||||
|
||||
training_version: bytes = b"0.2.0"
|
||||
terms_version: bytes = b"2"
|
||||
|
||||
|
||||
def cache(user_function, /):
|
||||
return lru_cache(maxsize=None)(user_function)
|
||||
|
||||
|
||||
def run_cmd(cmd: List[str]) -> str:
|
||||
return subprocess.check_output(cmd, encoding='utf8').strip()
|
||||
|
||||
|
||||
def run_cmd_default(cmd: List[str], default: Optional[str] = None) -> Optional[str]:
|
||||
try:
|
||||
return run_cmd(cmd)
|
||||
except subprocess.CalledProcessError:
|
||||
return default
|
||||
|
||||
|
||||
@cache
|
||||
def get_commit(branch: str = "HEAD", default: Optional[str] = None) -> Optional[str]:
|
||||
return run_cmd_default(["git", "rev-parse", branch], default=default)
|
||||
|
||||
|
||||
@cache
|
||||
def get_short_branch(default: Optional[str] = None) -> Optional[str]:
|
||||
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "HEAD"], default=default)
|
||||
|
||||
|
||||
@cache
|
||||
def get_branch(default: Optional[str] = None) -> Optional[str]:
|
||||
return run_cmd_default(["git", "rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], default=default)
|
||||
|
||||
|
||||
@cache
|
||||
def get_origin(default: Optional[str] = None) -> Optional[str]:
|
||||
try:
|
||||
local_branch = run_cmd(["git", "name-rev", "--name-only", "HEAD"])
|
||||
tracking_remote = run_cmd(["git", "config", "branch." + local_branch + ".remote"])
|
||||
return run_cmd(["git", "config", "remote." + tracking_remote + ".url"])
|
||||
except subprocess.CalledProcessError: # Not on a branch, fallback
|
||||
return run_cmd_default(["git", "config", "--get", "remote.origin.url"], default=default)
|
||||
|
||||
|
||||
@cache
|
||||
def get_normalized_origin(default: Optional[str] = None) -> Optional[str]:
|
||||
origin: Optional[str] = get_origin()
|
||||
|
||||
if origin is None:
|
||||
return default
|
||||
|
||||
return origin.replace("git@", "", 1) \
|
||||
.replace(".git", "", 1) \
|
||||
.replace("https://", "", 1) \
|
||||
.replace(":", "/", 1)
|
||||
|
||||
|
||||
@cache
|
||||
def get_version() -> str:
|
||||
with open(os.path.join(BASEDIR, "common", "version.h")) as _versionf:
|
||||
version = _versionf.read().split('"')[1]
|
||||
return version
|
||||
|
||||
@cache
|
||||
def get_short_version() -> str:
|
||||
return get_version().split('-')[0] # type: ignore
|
||||
|
||||
@cache
|
||||
def is_prebuilt() -> bool:
|
||||
return os.path.exists(os.path.join(BASEDIR, 'prebuilt'))
|
||||
|
||||
|
||||
@cache
|
||||
def is_comma_remote() -> bool:
|
||||
# note to fork maintainers, this is used for release metrics. please do not
|
||||
# touch this to get rid of the orange startup alert. there's better ways to do that
|
||||
origin: Optional[str] = get_origin()
|
||||
if origin is None:
|
||||
return False
|
||||
|
||||
return origin.startswith('git@github.com:commaai') or origin.startswith('https://github.com/commaai')
|
||||
|
||||
|
||||
@cache
|
||||
def is_tested_branch() -> bool:
|
||||
return get_short_branch() in TESTED_BRANCHES
|
||||
|
||||
|
||||
@cache
|
||||
def is_dirty() -> bool:
|
||||
origin = get_origin()
|
||||
branch = get_branch()
|
||||
if (origin is None) or (branch is None):
|
||||
return True
|
||||
|
||||
dirty = False
|
||||
try:
|
||||
# Actually check dirty files
|
||||
if not is_prebuilt():
|
||||
# This is needed otherwise touched files might show up as modified
|
||||
try:
|
||||
subprocess.check_call(["git", "update-index", "--refresh"])
|
||||
except subprocess.CalledProcessError:
|
||||
pass
|
||||
|
||||
dirty = (subprocess.call(["git", "diff-index", "--quiet", branch, "--"]) != 0)
|
||||
except subprocess.CalledProcessError:
|
||||
cloudlog.exception("git subprocess failed while checking dirty")
|
||||
dirty = True
|
||||
|
||||
return dirty
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from common.params import Params
|
||||
|
||||
params = Params()
|
||||
params.put("TermsVersion", terms_version)
|
||||
params.put("TrainingVersion", training_version)
|
||||
|
||||
print(f"Dirty: {is_dirty()}")
|
||||
print(f"Version: {get_version()}")
|
||||
print(f"Short version: {get_short_version()}")
|
||||
print(f"Origin: {get_origin()}")
|
||||
print(f"Normalized origin: {get_normalized_origin()}")
|
||||
print(f"Branch: {get_branch()}")
|
||||
print(f"Short branch: {get_short_branch()}")
|
||||
print(f"Prebuilt: {is_prebuilt()}")
|
||||
Reference in New Issue
Block a user