from __future__ import annotations import json import logging import re import sys import traceback from typing import Any from datahub.timeutil import isoformat _SECRET_KEYS = ( "token", "password", "secret", "key", "authorization", "credential", "tushare_token", "datahub_token", "encryption_key", "cookie", ) _SECRET_JSON = re.compile( r'(?i)("(?:' + "|".join(re.escape(key) for key in _SECRET_KEYS) + r')"\s*:\s*")([^"\\]*(?:\\.[^"\\]*)*)(")' ) def redact_log_text(text: str) -> str: return _SECRET_JSON.sub(r"\1***\3", str(text)) def _redact(value: Any, key: str = "") -> Any: lowered = key.lower() if any(part in lowered for part in _SECRET_KEYS): return "***" if isinstance(value, dict): return {str(item_key): _redact(item_value, str(item_key)) for item_key, item_value in value.items()} if isinstance(value, list): return [_redact(item) for item in value] if isinstance(value, str): return redact_log_text(value) return value def _safe_exc_text(exc_info: tuple[Any, Any, Any]) -> str: exc = exc_info[1] if isinstance(exc, json.JSONDecodeError): return f"JSONDecodeError: invalid json at position {exc.pos}" cause = getattr(exc, "__cause__", None) if isinstance(cause, json.JSONDecodeError): return f"{type(exc).__name__}: invalid json in request" text = "".join(traceback.format_exception(*exc_info)) if isinstance(cause, json.JSONDecodeError) and cause.doc: text = text.replace(cause.doc, "") if isinstance(exc, json.JSONDecodeError) and exc.doc: text = text.replace(exc.doc, "") return redact_log_text(text) class JsonFormatter(logging.Formatter): def format(self, record: logging.LogRecord) -> str: payload: dict[str, Any] = { "ts": isoformat(), "level": record.levelname, "logger": record.name, "message": redact_log_text(record.getMessage()), } extra = getattr(record, "hub", None) if isinstance(extra, dict): payload.update(_redact(extra)) if record.exc_info: payload["exc"] = _safe_exc_text(record.exc_info) return json.dumps(payload, ensure_ascii=False, default=str) def configure_logging(level: str = "INFO") -> logging.Logger: logger = logging.getLogger("datahub") if logger.handlers: return logger handler = logging.StreamHandler(sys.stdout) handler.setFormatter(JsonFormatter()) logger.addHandler(handler) logger.setLevel(getattr(logging, level.upper(), logging.INFO)) logger.propagate = False return logger def get_logger() -> logging.Logger: return logging.getLogger("datahub")