fix:用时间戳标记log
This commit is contained in:
@@ -1,5 +1,5 @@
|
|||||||
import logging
|
import logging
|
||||||
import logging.handlers
|
# 不再需要logging.handlers,已切换到基于时间戳的处理器
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from typing import Callable, Optional
|
from typing import Callable, Optional
|
||||||
import json
|
import json
|
||||||
@@ -27,24 +27,19 @@ def get_file_handler():
|
|||||||
if _file_handler is None:
|
if _file_handler is None:
|
||||||
# 确保日志目录存在
|
# 确保日志目录存在
|
||||||
LOG_DIR.mkdir(exist_ok=True)
|
LOG_DIR.mkdir(exist_ok=True)
|
||||||
|
|
||||||
# 检查是否已有其他handler在使用同一个文件
|
|
||||||
log_file_path = LOG_DIR / "app.log.jsonl"
|
|
||||||
root_logger = logging.getLogger()
|
|
||||||
|
|
||||||
# 检查现有handler,避免重复创建
|
# 检查现有handler,避免重复创建
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
for handler in root_logger.handlers:
|
for handler in root_logger.handlers:
|
||||||
if isinstance(handler, logging.handlers.RotatingFileHandler):
|
if isinstance(handler, TimestampedFileHandler):
|
||||||
if hasattr(handler, "baseFilename") and Path(handler.baseFilename) == log_file_path:
|
_file_handler = handler
|
||||||
_file_handler = handler
|
return _file_handler
|
||||||
return _file_handler
|
|
||||||
|
# 使用新的基于时间戳的handler,避免重命名操作
|
||||||
# 使用带压缩功能的handler,使用硬编码的默认值
|
_file_handler = TimestampedFileHandler(
|
||||||
_file_handler = CompressedRotatingFileHandler(
|
log_dir=LOG_DIR,
|
||||||
log_file_path,
|
max_bytes=10 * 1024 * 1024, # 10MB
|
||||||
maxBytes=10 * 1024 * 1024, # 10MB
|
backup_count=5,
|
||||||
backupCount=5,
|
|
||||||
encoding="utf-8",
|
encoding="utf-8",
|
||||||
compress=True,
|
compress=True,
|
||||||
compress_level=6,
|
compress_level=6,
|
||||||
@@ -112,79 +107,80 @@ class TimestampedFileHandler(logging.Handler):
|
|||||||
# 创建新文件
|
# 创建新文件
|
||||||
self._init_current_file()
|
self._init_current_file()
|
||||||
|
|
||||||
def _safe_rename(self, source, dest):
|
def _compress_file(self, file_path):
|
||||||
"""安全重命名文件,处理Windows文件占用问题"""
|
|
||||||
max_retries = 5
|
|
||||||
retry_delay = 0.1
|
|
||||||
|
|
||||||
for attempt in range(max_retries):
|
|
||||||
try:
|
|
||||||
Path(source).rename(dest)
|
|
||||||
return True
|
|
||||||
except PermissionError as e:
|
|
||||||
if attempt < max_retries - 1:
|
|
||||||
print(f"[日志轮转] 重命名失败,重试 {attempt + 1}/{max_retries}: {source} -> {dest}")
|
|
||||||
time.sleep(retry_delay)
|
|
||||||
retry_delay *= 2 # 指数退避
|
|
||||||
else:
|
|
||||||
print(f"[日志轮转] 重命名最终失败: {source} -> {dest}, 错误: {e}")
|
|
||||||
return False
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[日志轮转] 重命名错误: {source} -> {dest}, 错误: {e}")
|
|
||||||
return False
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _safe_remove(self, filepath):
|
|
||||||
"""安全删除文件,处理Windows文件占用问题"""
|
|
||||||
max_retries = 3
|
|
||||||
retry_delay = 0.1
|
|
||||||
|
|
||||||
for attempt in range(max_retries):
|
|
||||||
try:
|
|
||||||
Path(filepath).unlink()
|
|
||||||
return True
|
|
||||||
except PermissionError as e:
|
|
||||||
if attempt < max_retries - 1:
|
|
||||||
print(f"[日志轮转] 删除失败,重试 {attempt + 1}/{max_retries}: {filepath}")
|
|
||||||
time.sleep(retry_delay)
|
|
||||||
retry_delay *= 2
|
|
||||||
else:
|
|
||||||
print(f"[日志轮转] 删除最终失败: {filepath}, 错误: {e}")
|
|
||||||
return False
|
|
||||||
except Exception as e:
|
|
||||||
print(f"[日志轮转] 删除错误: {filepath}, 错误: {e}")
|
|
||||||
return False
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _compress_file(self, filepath):
|
|
||||||
"""在后台压缩文件"""
|
"""在后台压缩文件"""
|
||||||
# 等待一段时间确保文件写入完成
|
|
||||||
time.sleep(0.5)
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
source_path = Path(filepath)
|
time.sleep(0.5) # 等待文件写入完成
|
||||||
if not source_path.exists():
|
|
||||||
|
if not file_path.exists():
|
||||||
return
|
return
|
||||||
|
|
||||||
compressed_path = Path(f"{filepath}.gz")
|
compressed_path = file_path.with_suffix(file_path.suffix + '.gz')
|
||||||
|
original_size = file_path.stat().st_size
|
||||||
# 记录原始大小
|
|
||||||
original_size = source_path.stat().st_size
|
with open(file_path, 'rb') as f_in:
|
||||||
|
with gzip.open(compressed_path, 'wb', compresslevel=self.compress_level) as f_out:
|
||||||
with open(source_path, "rb") as f_in:
|
|
||||||
with gzip.open(compressed_path, "wb", compresslevel=self.compress_level) as f_out:
|
|
||||||
shutil.copyfileobj(f_in, f_out)
|
shutil.copyfileobj(f_in, f_out)
|
||||||
|
|
||||||
# 安全删除原文件
|
# 删除原文件
|
||||||
if self._safe_remove(filepath):
|
file_path.unlink()
|
||||||
compressed_size = compressed_path.stat().st_size
|
|
||||||
ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0
|
compressed_size = compressed_path.stat().st_size
|
||||||
print(f"[日志压缩] {source_path.name} -> {compressed_path.name} (压缩率: {ratio:.1f}%)")
|
ratio = (1 - compressed_size / original_size) * 100 if original_size > 0 else 0
|
||||||
else:
|
print(f"[日志压缩] {file_path.name} -> {compressed_path.name} (压缩率: {ratio:.1f}%)")
|
||||||
print(f"[日志压缩] 压缩完成但原文件删除失败: {filepath}")
|
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[日志压缩] 压缩失败 {filepath}: {e}")
|
print(f"[日志压缩] 压缩失败 {file_path}: {e}")
|
||||||
|
|
||||||
|
def _cleanup_old_files(self):
|
||||||
|
"""清理旧的日志文件,保留指定数量"""
|
||||||
|
try:
|
||||||
|
# 获取所有日志文件(包括压缩的)
|
||||||
|
log_files = []
|
||||||
|
for pattern in ["app_*.log.jsonl", "app_*.log.jsonl.gz"]:
|
||||||
|
log_files.extend(self.log_dir.glob(pattern))
|
||||||
|
|
||||||
|
# 按修改时间排序
|
||||||
|
log_files.sort(key=lambda f: f.stat().st_mtime, reverse=True)
|
||||||
|
|
||||||
|
# 删除超出数量限制的文件
|
||||||
|
for old_file in log_files[self.backup_count:]:
|
||||||
|
try:
|
||||||
|
old_file.unlink()
|
||||||
|
print(f"[日志清理] 删除旧文件: {old_file.name}")
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[日志清理] 删除失败 {old_file}: {e}")
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
print(f"[日志清理] 清理过程出错: {e}")
|
||||||
|
|
||||||
|
def emit(self, record):
|
||||||
|
"""发出日志记录"""
|
||||||
|
try:
|
||||||
|
with self._lock:
|
||||||
|
# 检查是否需要轮转
|
||||||
|
if self._should_rollover():
|
||||||
|
self._do_rollover()
|
||||||
|
|
||||||
|
# 写入日志
|
||||||
|
if self.current_stream:
|
||||||
|
msg = self.format(record)
|
||||||
|
self.current_stream.write(msg + '\n')
|
||||||
|
self.current_stream.flush()
|
||||||
|
|
||||||
|
except Exception:
|
||||||
|
self.handleError(record)
|
||||||
|
|
||||||
|
def close(self):
|
||||||
|
"""关闭处理器"""
|
||||||
|
with self._lock:
|
||||||
|
if self.current_stream:
|
||||||
|
self.current_stream.close()
|
||||||
|
self.current_stream = None
|
||||||
|
super().close()
|
||||||
|
|
||||||
|
|
||||||
|
# 旧的轮转文件处理器已移除,现在使用基于时间戳的处理器
|
||||||
|
|
||||||
|
|
||||||
def close_handlers():
|
def close_handlers():
|
||||||
@@ -201,17 +197,15 @@ def close_handlers():
|
|||||||
|
|
||||||
|
|
||||||
def remove_duplicate_handlers():
|
def remove_duplicate_handlers():
|
||||||
"""移除重复的文件handler"""
|
"""移除重复的handler,特别是文件handler"""
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
log_file_path = str(LOG_DIR / "app.log.jsonl")
|
|
||||||
|
# 收集所有时间戳文件handler
|
||||||
# 收集所有文件handler
|
|
||||||
file_handlers = []
|
file_handlers = []
|
||||||
for handler in root_logger.handlers[:]:
|
for handler in root_logger.handlers[:]:
|
||||||
if isinstance(handler, logging.handlers.RotatingFileHandler):
|
if isinstance(handler, TimestampedFileHandler):
|
||||||
if hasattr(handler, "baseFilename") and handler.baseFilename == log_file_path:
|
file_handlers.append(handler)
|
||||||
file_handlers.append(handler)
|
|
||||||
|
|
||||||
# 如果有多个文件handler,保留第一个,关闭其他的
|
# 如果有多个文件handler,保留第一个,关闭其他的
|
||||||
if len(file_handlers) > 1:
|
if len(file_handlers) > 1:
|
||||||
print(f"[日志系统] 检测到 {len(file_handlers)} 个重复的文件handler,正在清理...")
|
print(f"[日志系统] 检测到 {len(file_handlers)} 个重复的文件handler,正在清理...")
|
||||||
@@ -219,7 +213,7 @@ def remove_duplicate_handlers():
|
|||||||
print(f"[日志系统] 关闭重复的文件handler {i}")
|
print(f"[日志系统] 关闭重复的文件handler {i}")
|
||||||
root_logger.removeHandler(handler)
|
root_logger.removeHandler(handler)
|
||||||
handler.close()
|
handler.close()
|
||||||
|
|
||||||
# 更新全局引用
|
# 更新全局引用
|
||||||
global _file_handler
|
global _file_handler
|
||||||
_file_handler = file_handlers[0]
|
_file_handler = file_handlers[0]
|
||||||
@@ -665,7 +659,7 @@ console_formatter = structlog.stdlib.ProcessorFormatter(
|
|||||||
# 获取根logger并配置格式化器
|
# 获取根logger并配置格式化器
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
for handler in root_logger.handlers:
|
for handler in root_logger.handlers:
|
||||||
if isinstance(handler, logging.handlers.RotatingFileHandler):
|
if isinstance(handler, TimestampedFileHandler):
|
||||||
handler.setFormatter(file_formatter)
|
handler.setFormatter(file_formatter)
|
||||||
else:
|
else:
|
||||||
handler.setFormatter(console_formatter)
|
handler.setFormatter(console_formatter)
|
||||||
@@ -737,10 +731,10 @@ def configure_logging(
|
|||||||
|
|
||||||
# 更新文件handler配置
|
# 更新文件handler配置
|
||||||
file_handler = get_file_handler()
|
file_handler = get_file_handler()
|
||||||
if file_handler:
|
if file_handler and isinstance(file_handler, TimestampedFileHandler):
|
||||||
file_handler.maxBytes = max_bytes
|
file_handler.max_bytes = max_bytes
|
||||||
file_handler.backupCount = backup_count
|
file_handler.backup_count = backup_count
|
||||||
file_handler.baseFilename = str(log_path / "app.log.jsonl")
|
file_handler.log_dir = Path(log_dir)
|
||||||
|
|
||||||
# 更新文件handler日志级别
|
# 更新文件handler日志级别
|
||||||
if file_level:
|
if file_level:
|
||||||
@@ -797,7 +791,7 @@ def reload_log_config():
|
|||||||
# 重新配置console渲染器
|
# 重新配置console渲染器
|
||||||
root_logger = logging.getLogger()
|
root_logger = logging.getLogger()
|
||||||
for handler in root_logger.handlers:
|
for handler in root_logger.handlers:
|
||||||
if isinstance(handler, logging.StreamHandler) and not isinstance(handler, logging.handlers.RotatingFileHandler):
|
if isinstance(handler, logging.StreamHandler):
|
||||||
# 这是控制台处理器,更新其格式化器
|
# 这是控制台处理器,更新其格式化器
|
||||||
handler.setFormatter(
|
handler.setFormatter(
|
||||||
structlog.stdlib.ProcessorFormatter(
|
structlog.stdlib.ProcessorFormatter(
|
||||||
|
|||||||
Reference in New Issue
Block a user