Revert "athena upload: reduce memory usage and improve efficiency with streaming (#34528)"

This reverts commit 4c65f51a55.
This commit is contained in:
Adeeb Shihadeh
2025-02-06 11:50:04 -08:00
parent 61fa7e1ead
commit 9bc35c0919
5 changed files with 24 additions and 29 deletions
+10 -4
View File
@@ -1,4 +1,5 @@
#!/usr/bin/env python3
import io
import json
import os
import random
@@ -7,12 +8,12 @@ import threading
import time
import traceback
import datetime
import zstandard as zstd
from collections.abc import Iterator
from cereal import log
import cereal.messaging as messaging
from openpilot.common.api import Api
from openpilot.common.file_helpers import get_upload_stream
from openpilot.common.params import Params
from openpilot.common.realtime import set_core_affinity
from openpilot.system.hardware.hw import Paths
@@ -28,6 +29,7 @@ MAX_UPLOAD_SIZES = {
# bugs, including ones that can cause massive log sizes
"qcam": 5*1e6,
}
LOG_COMPRESSION_LEVEL = 10 # little benefit up to level 15. level ~17 is a small step change
allow_sleep = bool(os.getenv("UPLOADER_SLEEP", "1"))
force_wifi = os.getenv("FORCEWIFI") is not None
@@ -152,9 +154,13 @@ class Uploader:
if fake_upload:
return FakeResponse()
compress = key.endswith('.zst') and not fn.endswith('.zst')
with get_upload_stream(fn, compress) as stream:
return requests.put(url, data=stream, headers=headers, timeout=10)
with open(fn, "rb") as f:
content = f.read()
if key.endswith('.zst') and not fn.endswith('.zst'):
content = zstd.compress(content, LOG_COMPRESSION_LEVEL)
with io.BytesIO(content) as data:
return requests.put(url, data=data, headers=headers, timeout=10)
def upload(self, name: str, key: str, fn: str, network_type: int, metered: bool) -> bool:
try: