diff --git a/src/common/logger.py b/src/common/logger.py index 89d0f1738..142cb3d92 100644 --- a/src/common/logger.py +++ b/src/common/logger.py @@ -1,5 +1,5 @@ import logging -import logging.handlers +# 不再需要logging.handlers，已切换到基于时间戳的处理器 from pathlib import Path from typing import Callable, Optional import json @@ -27,24 +27,19 @@ def get_file_handler(): if _file_handler is None: # 确保日志目录存在 LOG_DIR.mkdir(exist_ok=True) - - # 检查是否已有其他handler在使用同一个文件 - log_file_path = LOG_DIR / "app.log.jsonl" - root_logger = logging.getLogger() - + # 检查现有handler，避免重复创建 root_logger = logging.getLogger() for handler in root_logger.handlers: - if isinstance(handler, logging.handlers.RotatingFileHandler): - if hasattr(handler, "baseFilename") and Path(handler.baseFilename) == log_file_path: - _file_handler = handler - return _file_handler - - # 使用带压缩功能的handler，使用硬编码的默认值 - _file_handler = CompressedRotatingFileHandler( - log_file_path, - maxBytes=10 * 1024 * 1024, # 10MB - backupCount=5, + if isinstance(handler, TimestampedFileHandler): + _file_handler = handler + return _file_handler + + # 使用新的基于时间戳的handler，避免重命名操作 + _file_handler = TimestampedFileHandler( + log_dir=LOG_DIR, + max_bytes=10 * 1024 * 1024, # 10MB + backup_count=5, encoding="utf-8", compress=True, compress_level=6, @@ -112,79 +107,80 @@ class TimestampedFileHandler(logging.Handler): # 创建新文件 self._init_current_file() - def _safe_rename(self, source, dest): - """安全重命名文件，处理Windows文件占用问题""" - max_retries = 5 - retry_delay = 0.1 - - for attempt in range(max_retries): - try: - Path(source).rename(dest) - return True - except PermissionError as e: - if attempt < max_retries - 1: - print(f"[日志轮转] 重命名失败，重试 {attempt + 1}/{max_retries}: {source} -> {dest}") - time.sleep(retry_delay) - retry_delay *= 2 # 指数退避 - else: - print(f"[日志轮转] 重命名最终失败: {source} -> {dest}, 错误: {e}") - return False - except Exception as e: - print(f"[日志轮转] 重命名错误: {source} -> {dest}, 错误: {e}") - return False - return False - - def _safe_remove(self, filepath): - """安全删除文件，处理Windows文件占用问题""" - max_retries = 3 - retry_delay = 0.1 - - for attempt in range(max_retries): - try: - Path(filepath).unlink() - return True - except PermissionError as e: - if attempt < max_retries - 1: - print(f"[日志轮转] 删除失败，重试 {attempt + 1}/{max_retries}: {filepath}") - time.sleep(retry_delay) - retry_delay *= 2 - else: - print(f"[日志轮转] 删除最终失败: {filepath}, 错误: {e}") - return False - except Exception as e: - print(f"[日志轮转] 删除错误: {filepath}, 错误: {e}") - return False - return False - - def _compress_file(self, filepath): + def _compress_file(self, file_path): """在后台压缩文件""" - # 等待一段时间确保文件写入完成 - time.sleep(0.5) - try: - source_path = Path(filepath) - if not source_path.exists(): + time.sleep(0.5) # 等待文件写入完成 + + if not file_path.exists(): return - - compressed_path = Path(f"{filepath}.gz") - - # 记录原始大小 - original_size = source_path.stat().st_size - - with open(source_path, "rb") as f_in: - with gzip.open(compressed_path, "wb", compresslevel=self.compress_level) as f_out: + + compressed_path = file_path.with_suffix(file_path.suffix + '.gz') + original_size = file_path.stat().st_size + + with open(file_path, 'rb') as f_in: + with gzip.open(compressed_path, 'wb', compresslevel=self.compress_level) as f_out: shutil.copyfileobj(f_in, f_out) - - # 安全删除原文件 - if self._safe_remove(filepath): - compressed_size = compressed_path.stat().st_size - ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 - print(f"[日志压缩] {source_path.name} -> {compressed_path.name} (压缩率: {ratio:.1f}%)") - else: - print(f"[日志压缩] 压缩完成但原文件删除失败: {filepath}") - + + # 删除原文件 + file_path.unlink() + + compressed_size = compressed_path.stat().st_size + ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0 + print(f"[日志压缩] {file_path.name} -> {compressed_path.name} (压缩率: {ratio:.1f}%)") + except Exception as e: - print(f"[日志压缩] 压缩失败 {filepath}: {e}") + print(f"[日志压缩] 压缩失败 {file_path}: {e}") + + def _cleanup_old_files(self): + """清理旧的日志文件，保留指定数量""" + try: + # 获取所有日志文件（包括压缩的） + log_files = [] + for pattern in ["app_*.log.jsonl", "app_*.log.jsonl.gz"]: + log_files.extend(self.log_dir.glob(pattern)) + + # 按修改时间排序 + log_files.sort(key=lambda f: f.stat().st_mtime, reverse=True) + + # 删除超出数量限制的文件 + for old_file in log_files[self.backup_count:]: + try: + old_file.unlink() + print(f"[日志清理] 删除旧文件: {old_file.name}") + except Exception as e: + print(f"[日志清理] 删除失败 {old_file}: {e}") + + except Exception as e: + print(f"[日志清理] 清理过程出错: {e}") + + def emit(self, record): + """发出日志记录""" + try: + with self._lock: + # 检查是否需要轮转 + if self._should_rollover(): + self._do_rollover() + + # 写入日志 + if self.current_stream: + msg = self.format(record) + self.current_stream.write(msg + '\n') + self.current_stream.flush() + + except Exception: + self.handleError(record) + + def close(self): + """关闭处理器""" + with self._lock: + if self.current_stream: + self.current_stream.close() + self.current_stream = None + super().close() + + +# 旧的轮转文件处理器已移除，现在使用基于时间戳的处理器 def close_handlers(): @@ -201,17 +197,15 @@ def close_handlers(): def remove_duplicate_handlers(): - """移除重复的文件handler""" + """移除重复的handler，特别是文件handler""" root_logger = logging.getLogger() - log_file_path = str(LOG_DIR / "app.log.jsonl") - - # 收集所有文件handler + + # 收集所有时间戳文件handler file_handlers = [] for handler in root_logger.handlers[:]: - if isinstance(handler, logging.handlers.RotatingFileHandler): - if hasattr(handler, "baseFilename") and handler.baseFilename == log_file_path: - file_handlers.append(handler) - + if isinstance(handler, TimestampedFileHandler): + file_handlers.append(handler) + # 如果有多个文件handler，保留第一个，关闭其他的 if len(file_handlers) > 1: print(f"[日志系统] 检测到 {len(file_handlers)} 个重复的文件handler，正在清理...") @@ -219,7 +213,7 @@ def remove_duplicate_handlers(): print(f"[日志系统] 关闭重复的文件handler {i}") root_logger.removeHandler(handler) handler.close() - + # 更新全局引用 global _file_handler _file_handler = file_handlers[0] @@ -665,7 +659,7 @@ console_formatter = structlog.stdlib.ProcessorFormatter( # 获取根logger并配置格式化器 root_logger = logging.getLogger() for handler in root_logger.handlers: - if isinstance(handler, logging.handlers.RotatingFileHandler): + if isinstance(handler, TimestampedFileHandler): handler.setFormatter(file_formatter) else: handler.setFormatter(console_formatter) @@ -737,10 +731,10 @@ def configure_logging( # 更新文件handler配置 file_handler = get_file_handler() - if file_handler: - file_handler.maxBytes = max_bytes - file_handler.backupCount = backup_count - file_handler.baseFilename = str(log_path / "app.log.jsonl") + if file_handler and isinstance(file_handler, TimestampedFileHandler): + file_handler.max_bytes = max_bytes + file_handler.backup_count = backup_count + file_handler.log_dir = Path(log_dir) # 更新文件handler日志级别 if file_level: @@ -797,7 +791,7 @@ def reload_log_config(): # 重新配置console渲染器 root_logger = logging.getLogger() for handler in root_logger.handlers: - if isinstance(handler, logging.StreamHandler) and not isinstance(handler, logging.handlers.RotatingFileHandler): + if isinstance(handler, logging.StreamHandler): # 这是控制台处理器，更新其格式化器 handler.setFormatter( structlog.stdlib.ProcessorFormatter(