Files
onepilot/frogpilot/third_party/influxdb_client/client/_base.py
T
firestar5683 d0e1db6766 StarPilot
2026-03-22 03:15:05 -05:00

557 lines
24 KiB
Python

"""Commons function for Sync and Async client."""
from __future__ import absolute_import
import base64
import configparser
import logging
import os
from datetime import datetime, timedelta
from typing import List, Generator, Any, Union, Iterable, AsyncGenerator
from urllib3 import HTTPResponse
from influxdb_client import Configuration, Dialect, Query, OptionStatement, VariableAssignment, Identifier, \
Expression, BooleanLiteral, IntegerLiteral, FloatLiteral, DateTimeLiteral, UnaryExpression, DurationLiteral, \
Duration, StringLiteral, ArrayExpression, ImportDeclaration, MemberExpression, MemberAssignment, File, \
WriteService, QueryService, DeleteService, DeletePredicateRequest
from influxdb_client.client.flux_csv_parser import FluxResponseMetadataMode, FluxCsvParser, FluxSerializationMode
from influxdb_client.client.flux_table import FluxRecord, TableList, CSVIterator
from influxdb_client.client.util.date_utils import get_date_helper
from influxdb_client.client.util.helpers import get_org_query_param
from influxdb_client.client.warnings import MissingPivotFunction
from influxdb_client.client.write.dataframe_serializer import DataframeSerializer
from influxdb_client.rest import _UTF_8_encoding
try:
import dataclasses
_HAS_DATACLASS = True
except ModuleNotFoundError:
_HAS_DATACLASS = False
LOGGERS_NAMES = [
'influxdb_client.client.influxdb_client',
'influxdb_client.client.influxdb_client_async',
'influxdb_client.client.write_api',
'influxdb_client.client.write_api_async',
'influxdb_client.client.write.retry',
'influxdb_client.client.write.dataframe_serializer',
'influxdb_client.client.util.multiprocessing_helper',
'influxdb_client.client.http',
'influxdb_client.client.exceptions',
]
# noinspection PyMethodMayBeStatic
class _BaseClient(object):
def __init__(self, url, token, debug=None, timeout=10_000, enable_gzip=False, org: str = None,
default_tags: dict = None, http_client_logger: str = None, **kwargs) -> None:
self.url = url
self.token = token
self.org = org
self.default_tags = default_tags
self.conf = _Configuration()
if not isinstance(self.url, str):
raise ValueError('"url" attribute is not str instance')
if self.url.endswith("/"):
self.conf.host = self.url[:-1]
else:
self.conf.host = self.url
self.conf.enable_gzip = enable_gzip
self.conf.verify_ssl = kwargs.get('verify_ssl', True)
self.conf.ssl_ca_cert = kwargs.get('ssl_ca_cert', None)
self.conf.cert_file = kwargs.get('cert_file', None)
self.conf.cert_key_file = kwargs.get('cert_key_file', None)
self.conf.cert_key_password = kwargs.get('cert_key_password', None)
self.conf.ssl_context = kwargs.get('ssl_context', None)
self.conf.proxy = kwargs.get('proxy', None)
self.conf.proxy_headers = kwargs.get('proxy_headers', None)
self.conf.connection_pool_maxsize = kwargs.get('connection_pool_maxsize', self.conf.connection_pool_maxsize)
self.conf.timeout = timeout
# logging
self.conf.loggers["http_client_logger"] = logging.getLogger(http_client_logger)
for client_logger in LOGGERS_NAMES:
self.conf.loggers[client_logger] = logging.getLogger(client_logger)
self.conf.debug = debug
self.conf.username = kwargs.get('username', None)
self.conf.password = kwargs.get('password', None)
# defaults
self.auth_header_name = None
self.auth_header_value = None
# by token
if self.token:
self.auth_header_name = "Authorization"
self.auth_header_value = "Token " + self.token
# by HTTP basic
auth_basic = kwargs.get('auth_basic', False)
if auth_basic:
self.auth_header_name = "Authorization"
self.auth_header_value = "Basic " + base64.b64encode(token.encode()).decode()
# by username, password
if self.conf.username and self.conf.password:
self.auth_header_name = None
self.auth_header_value = None
self.retries = kwargs.get('retries', False)
self.profilers = kwargs.get('profilers', None)
pass
@classmethod
def _from_config_file(cls, config_file: str = "config.ini", debug=None, enable_gzip=False, **kwargs):
config = configparser.ConfigParser()
config_name = kwargs.get('config_name', 'influx2')
is_json = False
try:
config.read(config_file)
except configparser.ParsingError:
with open(config_file) as json_file:
import json
config = json.load(json_file)
is_json = True
def _config_value(key: str):
value = str(config[key]) if is_json else config[config_name][key]
return value.strip('"')
def _has_option(key: str):
return key in config if is_json else config.has_option(config_name, key)
def _has_section(key: str):
return key in config if is_json else config.has_section(key)
url = _config_value('url')
token = _config_value('token')
timeout = None
if _has_option('timeout'):
timeout = _config_value('timeout')
org = None
if _has_option('org'):
org = _config_value('org')
verify_ssl = True
if _has_option('verify_ssl'):
verify_ssl = _config_value('verify_ssl')
ssl_ca_cert = None
if _has_option('ssl_ca_cert'):
ssl_ca_cert = _config_value('ssl_ca_cert')
cert_file = None
if _has_option('cert_file'):
cert_file = _config_value('cert_file')
cert_key_file = None
if _has_option('cert_key_file'):
cert_key_file = _config_value('cert_key_file')
cert_key_password = None
if _has_option('cert_key_password'):
cert_key_password = _config_value('cert_key_password')
connection_pool_maxsize = None
if _has_option('connection_pool_maxsize'):
connection_pool_maxsize = _config_value('connection_pool_maxsize')
auth_basic = False
if _has_option('auth_basic'):
auth_basic = _config_value('auth_basic')
default_tags = None
if _has_section('tags'):
if is_json:
default_tags = config['tags']
else:
tags = {k: v.strip('"') for k, v in config.items('tags')}
default_tags = dict(tags)
profilers = None
if _has_option('profilers'):
profilers = [x.strip() for x in _config_value('profilers').split(',')]
proxy = None
if _has_option('proxy'):
proxy = _config_value('proxy')
return cls(url, token, debug=debug, timeout=_to_int(timeout), org=org, default_tags=default_tags,
enable_gzip=enable_gzip, verify_ssl=_to_bool(verify_ssl), ssl_ca_cert=ssl_ca_cert,
cert_file=cert_file, cert_key_file=cert_key_file, cert_key_password=cert_key_password,
connection_pool_maxsize=_to_int(connection_pool_maxsize), auth_basic=_to_bool(auth_basic),
profilers=profilers, proxy=proxy, **kwargs)
@classmethod
def _from_env_properties(cls, debug=None, enable_gzip=False, **kwargs):
url = os.getenv('INFLUXDB_V2_URL', "http://localhost:8086")
token = os.getenv('INFLUXDB_V2_TOKEN', "my-token")
timeout = os.getenv('INFLUXDB_V2_TIMEOUT', "10000")
org = os.getenv('INFLUXDB_V2_ORG', "my-org")
verify_ssl = os.getenv('INFLUXDB_V2_VERIFY_SSL', "True")
ssl_ca_cert = os.getenv('INFLUXDB_V2_SSL_CA_CERT', None)
cert_file = os.getenv('INFLUXDB_V2_CERT_FILE', None)
cert_key_file = os.getenv('INFLUXDB_V2_CERT_KEY_FILE', None)
cert_key_password = os.getenv('INFLUXDB_V2_CERT_KEY_PASSWORD', None)
connection_pool_maxsize = os.getenv('INFLUXDB_V2_CONNECTION_POOL_MAXSIZE', None)
auth_basic = os.getenv('INFLUXDB_V2_AUTH_BASIC', "False")
prof = os.getenv("INFLUXDB_V2_PROFILERS", None)
profilers = None
if prof is not None:
profilers = [x.strip() for x in prof.split(',')]
default_tags = dict()
for key, value in os.environ.items():
if key.startswith("INFLUXDB_V2_TAG_"):
default_tags[key[16:].lower()] = value
return cls(url, token, debug=debug, timeout=_to_int(timeout), org=org, default_tags=default_tags,
enable_gzip=enable_gzip, verify_ssl=_to_bool(verify_ssl), ssl_ca_cert=ssl_ca_cert,
cert_file=cert_file, cert_key_file=cert_key_file, cert_key_password=cert_key_password,
connection_pool_maxsize=_to_int(connection_pool_maxsize), auth_basic=_to_bool(auth_basic),
profilers=profilers, **kwargs)
# noinspection PyMethodMayBeStatic
class _BaseQueryApi(object):
default_dialect = Dialect(header=True, delimiter=",", comment_prefix="#",
annotations=["datatype", "group", "default"], date_time_format="RFC3339")
def __init__(self, influxdb_client, query_options=None):
from influxdb_client.client.query_api import QueryOptions
self._query_options = QueryOptions() if query_options is None else query_options
self._influxdb_client = influxdb_client
self._query_api = QueryService(influxdb_client.api_client)
"""Base implementation for Queryable API."""
def _to_tables(self, response, query_options=None, response_metadata_mode:
FluxResponseMetadataMode = FluxResponseMetadataMode.full) -> TableList:
"""
Parse HTTP response to TableList.
:param response: HTTP response from an HTTP client. Expected type: `urllib3.response.HTTPResponse`.
"""
_parser = self._to_tables_parser(response, query_options, response_metadata_mode)
list(_parser.generator())
return _parser.table_list()
async def _to_tables_async(self, response, query_options=None, response_metadata_mode:
FluxResponseMetadataMode = FluxResponseMetadataMode.full) -> TableList:
"""
Parse HTTP response to TableList.
:param response: HTTP response from an HTTP client. Expected type: `aiohttp.client_reqrep.ClientResponse`.
"""
async with self._to_tables_parser(response, query_options, response_metadata_mode) as parser:
async for _ in parser.generator_async():
pass
return parser.table_list()
def _to_csv(self, response: HTTPResponse) -> CSVIterator:
"""Parse HTTP response to CSV."""
return CSVIterator(response)
def _to_flux_record_stream(self, response, query_options=None,
response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full) -> \
Generator[FluxRecord, Any, None]:
"""
Parse HTTP response to FluxRecord stream.
:param response: HTTP response from an HTTP client. Expected type: `urllib3.response.HTTPResponse`.
"""
_parser = self._to_flux_record_stream_parser(query_options, response, response_metadata_mode)
return _parser.generator()
async def _to_flux_record_stream_async(self, response, query_options=None, response_metadata_mode:
FluxResponseMetadataMode = FluxResponseMetadataMode.full) -> \
AsyncGenerator['FluxRecord', None]:
"""
Parse HTTP response to FluxRecord stream.
:param response: HTTP response from an HTTP client. Expected type: `aiohttp.client_reqrep.ClientResponse`.
"""
_parser = self._to_flux_record_stream_parser(query_options, response, response_metadata_mode)
return (await _parser.__aenter__()).generator_async()
def _to_data_frame_stream(self, data_frame_index, response, query_options=None,
response_metadata_mode: FluxResponseMetadataMode = FluxResponseMetadataMode.full,
use_extension_dtypes=False):
"""
Parse HTTP response to DataFrame stream.
:param response: HTTP response from an HTTP client. Expected type: `urllib3.response.HTTPResponse`.
"""
_parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode,
use_extension_dtypes)
return _parser.generator()
async def _to_data_frame_stream_async(self, data_frame_index, response, query_options=None, response_metadata_mode:
FluxResponseMetadataMode = FluxResponseMetadataMode.full,
use_extension_dtypes=False):
"""
Parse HTTP response to DataFrame stream.
:param response: HTTP response from an HTTP client. Expected type: `aiohttp.client_reqrep.ClientResponse`.
"""
_parser = self._to_data_frame_stream_parser(data_frame_index, query_options, response, response_metadata_mode,
use_extension_dtypes)
return (await _parser.__aenter__()).generator_async()
def _to_tables_parser(self, response, query_options, response_metadata_mode):
return FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.tables,
query_options=query_options, response_metadata_mode=response_metadata_mode)
def _to_flux_record_stream_parser(self, query_options, response, response_metadata_mode):
return FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.stream,
query_options=query_options, response_metadata_mode=response_metadata_mode)
def _to_data_frame_stream_parser(self, data_frame_index, query_options, response, response_metadata_mode,
use_extension_dtypes):
return FluxCsvParser(response=response, serialization_mode=FluxSerializationMode.dataFrame,
data_frame_index=data_frame_index, query_options=query_options,
response_metadata_mode=response_metadata_mode,
use_extension_dtypes=use_extension_dtypes)
def _to_data_frames(self, _generator):
"""Parse stream of DataFrames into expected type."""
from ..extras import pd
if isinstance(_generator, list):
_dataFrames = _generator
else:
_dataFrames = list(_generator)
if len(_dataFrames) == 0:
return pd.DataFrame(columns=[], index=None)
elif len(_dataFrames) == 1:
return _dataFrames[0]
else:
return _dataFrames
def _org_param(self, org):
return get_org_query_param(org=org, client=self._influxdb_client)
def _get_query_options(self):
if self._query_options and self._query_options.profilers:
return self._query_options
elif self._influxdb_client.profilers:
from influxdb_client.client.query_api import QueryOptions
return QueryOptions(profilers=self._influxdb_client.profilers)
def _create_query(self, query, dialect=default_dialect, params: dict = None, **kwargs):
query_options = self._get_query_options()
profilers = query_options.profilers if query_options is not None else None
q = Query(query=query, dialect=dialect, extern=_BaseQueryApi._build_flux_ast(params, profilers))
if profilers:
print("\n===============")
print("Profiler: query")
print("===============")
print(query)
if kwargs.get('dataframe_query', False):
MissingPivotFunction.print_warning(query)
return q
@staticmethod
def _params_to_extern_ast(params: dict) -> List['OptionStatement']:
statements = []
for key, value in params.items():
expression = _BaseQueryApi._parm_to_extern_ast(value)
if expression is None:
continue
statements.append(OptionStatement("OptionStatement",
VariableAssignment("VariableAssignment", Identifier("Identifier", key),
expression)))
return statements
@staticmethod
def _parm_to_extern_ast(value) -> Union[Expression, None]:
if value is None:
return None
if isinstance(value, bool):
return BooleanLiteral("BooleanLiteral", value)
elif isinstance(value, int):
return IntegerLiteral("IntegerLiteral", str(value))
elif isinstance(value, float):
return FloatLiteral("FloatLiteral", value)
elif isinstance(value, datetime):
value = get_date_helper().to_utc(value)
nanoseconds = getattr(value, 'nanosecond', 0)
fraction = f'{(value.microsecond * 1000 + nanoseconds):09d}'
return DateTimeLiteral("DateTimeLiteral", value.strftime('%Y-%m-%dT%H:%M:%S.') + fraction + 'Z')
elif isinstance(value, timedelta):
_micro_delta = int(value / timedelta(microseconds=1))
if _micro_delta < 0:
return UnaryExpression("UnaryExpression", argument=DurationLiteral("DurationLiteral", [
Duration(magnitude=-_micro_delta, unit="us")]), operator="-")
else:
return DurationLiteral("DurationLiteral", [Duration(magnitude=_micro_delta, unit="us")])
elif isinstance(value, str):
return StringLiteral("StringLiteral", str(value))
elif isinstance(value, Iterable):
return ArrayExpression("ArrayExpression",
elements=list(map(lambda it: _BaseQueryApi._parm_to_extern_ast(it), value)))
else:
return value
@staticmethod
def _build_flux_ast(params: dict = None, profilers: List[str] = None):
imports = []
body = []
if profilers is not None and len(profilers) > 0:
imports.append(ImportDeclaration(
"ImportDeclaration",
path=StringLiteral("StringLiteral", "profiler")))
elements = []
for profiler in profilers:
elements.append(StringLiteral("StringLiteral", value=profiler))
member = MemberExpression(
"MemberExpression",
object=Identifier("Identifier", "profiler"),
_property=Identifier("Identifier", "enabledProfilers"))
prof = OptionStatement(
"OptionStatement",
assignment=MemberAssignment(
"MemberAssignment",
member=member,
init=ArrayExpression(
"ArrayExpression",
elements=elements)))
body.append(prof)
if params is not None:
body.extend(_BaseQueryApi._params_to_extern_ast(params))
return File(package=None, name=None, type=None, imports=imports, body=body)
class _BaseWriteApi(object):
def __init__(self, influxdb_client, point_settings=None):
self._influxdb_client = influxdb_client
self._point_settings = point_settings
self._write_service = WriteService(influxdb_client.api_client)
if influxdb_client.default_tags:
for key, value in influxdb_client.default_tags.items():
self._point_settings.add_default_tag(key, value)
def _append_default_tag(self, key, val, record):
from influxdb_client import Point
if isinstance(record, bytes) or isinstance(record, str):
pass
elif isinstance(record, Point):
record.tag(key, val)
elif isinstance(record, dict):
record.setdefault("tags", {})
record.get("tags")[key] = val
elif isinstance(record, Iterable):
for item in record:
self._append_default_tag(key, val, item)
def _append_default_tags(self, record):
if self._point_settings.defaultTags and record is not None:
for key, val in self._point_settings.defaultTags.items():
self._append_default_tag(key, val, record)
def _serialize(self, record, write_precision, payload, **kwargs):
from influxdb_client import Point
if isinstance(record, bytes):
payload[write_precision].append(record)
elif isinstance(record, str):
self._serialize(record.encode(_UTF_8_encoding), write_precision, payload, **kwargs)
elif isinstance(record, Point):
precision_from_point = kwargs.get('precision_from_point', True)
precision = record.write_precision if precision_from_point else write_precision
self._serialize(record.to_line_protocol(precision=precision), precision, payload, **kwargs)
elif isinstance(record, dict):
self._serialize(Point.from_dict(record, write_precision=write_precision, **kwargs),
write_precision, payload, **kwargs)
elif 'DataFrame' in type(record).__name__:
serializer = DataframeSerializer(record, self._point_settings, write_precision, **kwargs)
self._serialize(serializer.serialize(), write_precision, payload, **kwargs)
elif hasattr(record, "_asdict"):
# noinspection PyProtectedMember
self._serialize(record._asdict(), write_precision, payload, **kwargs)
elif _HAS_DATACLASS and dataclasses.is_dataclass(record):
self._serialize(dataclasses.asdict(record), write_precision, payload, **kwargs)
elif isinstance(record, Iterable):
for item in record:
self._serialize(item, write_precision, payload, **kwargs)
# noinspection PyMethodMayBeStatic
class _BaseDeleteApi(object):
def __init__(self, influxdb_client):
self._influxdb_client = influxdb_client
self._service = DeleteService(influxdb_client.api_client)
def _prepare_predicate_request(self, start, stop, predicate):
date_helper = get_date_helper()
if isinstance(start, datetime):
start = date_helper.to_utc(start)
if isinstance(stop, datetime):
stop = date_helper.to_utc(stop)
predicate_request = DeletePredicateRequest(start=start, stop=stop, predicate=predicate)
return predicate_request
class _Configuration(Configuration):
def __init__(self):
Configuration.__init__(self)
self.enable_gzip = False
self.username = None
self.password = None
def update_request_header_params(self, path: str, params: dict):
super().update_request_header_params(path, params)
if self.enable_gzip:
# GZIP Request
if path == '/api/v2/write':
params["Content-Encoding"] = "gzip"
params["Accept-Encoding"] = "identity"
pass
# GZIP Response
if path == '/api/v2/query':
# params["Content-Encoding"] = "gzip"
params["Accept-Encoding"] = "gzip"
pass
pass
pass
def update_request_body(self, path: str, body):
_body = super().update_request_body(path, body)
if self.enable_gzip:
# GZIP Request
if path == '/api/v2/write':
import gzip
if isinstance(_body, bytes):
return gzip.compress(data=_body)
else:
return gzip.compress(bytes(_body, _UTF_8_encoding))
return _body
def _to_bool(bool_value):
return str(bool_value).lower() in ("yes", "true")
def _to_int(int_value):
return int(int_value) if int_value is not None else None