log to file and send through athena (#20250)

* log to file and send through athena

* rename logging level

* pass thru log formatter

* logMessage is TEXT

* send queue always strings

* switch to xattr and lower priority queue

* enable cloud logging for devices

* time or size based log rotation

* basename -> dirname

* remove HARDWARE.get_cloudlog_enabled

* fix errors

* fix another exception

* xattrs need to be bytes

* sending works

* cleanup files at start

* add id and adjust formatting

* do not send active log file

* better names

* separate log formatters

* fix formatter super init

* fix log file order

* ensure file always has file formatter

* i see why there was no formatter

* apply same formatting to cpp log msgs

* apply same formatting to cpp log msgs

* update queue names in tests

* strip deprecated keys in STATUS_PACKET

* strip DEPRECATED from dict recursively

* athena log queue test

* instanceof instead of type

* isinstance instead of type

* use super

* remove logentries

* last_scan param unused

* comment about special log msg attr names

* add dict_helpers.py to release files

* use monotonic time and counter for log rotation

* update for adjusted log file naming

* use monotonic clock for tracking last log file scan
old-commit-hash: 3d48bd934d6c84a5352c2a1d56e6f93ed59e28c4

common/dict_helpers.py

@@ -0,0 +1,9 @@
+# remove all keys that end in DEPRECATED
+def strip_deprecated_keys(d):
+  for k in list(d.keys()):
+    if isinstance(k, str):
+      if k.endswith('DEPRECATED'):
+        d.pop(k)
+      elif isinstance(d[k], dict):
+        strip_deprecated_keys(d[k])
+  return d

common/logging_extra.py

@@ -3,6 +3,7 @@ import os
 import sys
 import copy
 import json
+import uuid
 import socket
 import logging
 import traceback
@@ -62,8 +63,48 @@ class SwagFormatter(logging.Formatter):
     return record_dict
 
   def format(self, record):
+    if self.swaglogger is None:
+      raise Exception("must set swaglogger before calling format()")
     return json_robust_dumps(self.format_dict(record))
 
+class SwagLogFileFormatter(SwagFormatter):
+  def fix_kv(self, k, v):
+    # append type to names to preserve legacy naming in logs
+    # avoids overlapping key namespaces with different types
+    # e.g. log.info() creates 'msg' -> 'msg$s'
+    #      log.event() creates 'msg.health.logMonoTime' -> 'msg.health.logMonoTime$i'
+    #      because overlapping namespace 'msg' caused problems
+    if isinstance(v, (str, bytes)):
+      k += "$s"
+    elif isinstance(v, float):
+      k += "$f"
+    elif isinstance(v, bool):
+      k += "$b"
+    elif isinstance(v, int):
+      k += "$i"
+    elif isinstance(v, dict):
+      nv = {}
+      for ik, iv in v.items():
+        ik, iv = self.fix_kv(ik, iv)
+        nv[ik] = iv
+      v = nv
+    elif isinstance(v, list):
+      k += "$a"
+    return k, v
+
+  def format(self, record):
+    if isinstance(record, str):
+      v = json.loads(record)
+    else:
+      v = self.format_dict(record)
+
+    mk, mv = self.fix_kv('msg', v['msg'])
+    del v['msg']
+    v[mk] = mv
+    v['id'] = uuid.uuid4().hex
+
+    return json_robust_dumps(v)
+
 class SwagErrorFilter(logging.Filter):
   def filter(self, record):
     return record.levelno < logging.ERROR