gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(database): 完成API层、Utils层和兼容层重构 (Stage 4-6)

Browse Source 
Stage 4: API层重构
=================
新增文件:
- api/crud.py (430行): CRUDBase泛型类，提供12个CRUD方法
  * get, get_by, get_multi, create, update, delete
  * count, exists, get_or_create, bulk_create, bulk_update
  * 集成缓存: 自动缓存读操作，写操作清除缓存
  * 集成批处理: 可选use_batch参数透明使用AdaptiveBatchScheduler

- api/query.py (461行): 高级查询构建器
  * QueryBuilder: 链式调用，MongoDB风格操作符
    - 操作符: __gt, __lt, __gte, __lte, __ne, __in, __nin, __like, __isnull
    - 方法: filter, filter_or, order_by, limit, offset, no_cache
    - 执行: all, first, count, exists, paginate
  * AggregateQuery: 聚合查询
    - sum, avg, max, min, group_by_count

- api/specialized.py (461行): 业务特定API
  * ActionRecords: store_action_info, get_recent_actions
  * Messages: get_chat_history, get_message_count, save_message
  * PersonInfo: get_or_create_person, update_person_affinity
  * ChatStreams: get_or_create_chat_stream, get_active_streams
  * LLMUsage: record_llm_usage, get_usage_statistics
  * UserRelationships: get_user_relationship, update_relationship_affinity

- 更新api/__init__.py: 导出所有API接口

Stage 5: Utils层实现
===================
新增文件:
- utils/decorators.py (320行): 数据库操作装饰器
  * @retry: 自动重试失败操作，指数退避
  * @timeout: 超时控制
  * @cached: 自动缓存函数结果
  * @measure_time: 性能测量，慢查询日志
  * @transactional: 事务管理，自动提交/回滚
  * @db_operation: 组合装饰器

- utils/monitoring.py (330行): 性能监控系统
  * DatabaseMonitor: 单例监控器
  * OperationMetrics: 操作指标 (次数、时间、错误)
  * DatabaseMetrics: 全局指标
    - 连接池统计
    - 缓存命中率
    - 批处理统计
    - 预加载统计
  * 便捷函数: get_monitor, record_operation, print_stats

- 更新utils/__init__.py: 导出装饰器和监控函数

Stage 6: 兼容层实现
==================
新增目录: compatibility/
- adapter.py (370行): 向后兼容适配器
  * 完全兼容旧API签名: db_query, db_save, db_get, store_action_info
  * 支持MongoDB风格操作符 (\, \, \)
  * 内部使用新架构 (QueryBuilder + CRUDBase)
  * 保持返回dict格式不变
  * MODEL_MAPPING: 25个模型映射

- __init__.py: 导出兼容API

更新database/__init__.py:
- 导出核心层 (engine, session, models, migration)
- 导出优化层 (cache, preloader, batch_scheduler)
- 导出API层 (CRUD, Query, 业务API)
- 导出Utils层 (装饰器, 监控)
- 导出兼容层 (db_query, db_save等)

核心特性
========
 类型安全: Generic[T]提供完整类型推断
 缓存透明: 自动缓存，用户无需关心
 批处理透明: 可选批处理，自动优化高频写入
 链式查询: 流畅的API设计
 业务封装: 常用操作封装成便捷函数
 向后兼容: 兼容层保证现有代码无缝迁移
 性能监控: 完整的指标收集和报告

统计数据
========
- 新增文件: 7个
- 代码行数: ~2050行
- API函数: 14个业务API + 6个装饰器
- 兼容函数: 5个 (db_query, db_save, db_get等)

下一步
======
- 更新28个文件的import语句 (从sqlalchemy_database_api迁移)
- 移动旧文件到old/目录
- 编写Stage 4-6的测试
- 集成测试验证兼容性
This commit is contained in:
Windpicker-owo
2025-11-01 13:27:33 +08:00
parent aae84ec454
commit 61de975d73
10 changed files with 2563 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

322
src/common/database/utils/monitoring.py Normal file
View File

@@ -0,0 +1,322 @@
"""数据库性能监控
提供数据库操作的性能监控和统计功能
"""
import time
from collections import defaultdict
from dataclasses import dataclass, field
from typing import Any, Optional
from src.common.logger import get_logger
logger = get_logger("database.monitoring")
@dataclass
class OperationMetrics:
"""操作指标"""
count: int = 0
total_time: float = 0.0
min_time: float = float("inf")
max_time: float = 0.0
error_count: int = 0
last_execution_time: Optional[float] = None
@property
def avg_time(self) -> float:
"""平均执行时间"""
return self.total_time / self.count if self.count > 0 else 0.0
def record_success(self, execution_time: float):
"""记录成功执行"""
self.count += 1
self.total_time += execution_time
self.min_time = min(self.min_time, execution_time)
self.max_time = max(self.max_time, execution_time)
self.last_execution_time = time.time()
def record_error(self):
"""记录错误"""
self.error_count += 1
@dataclass
class DatabaseMetrics:
"""数据库指标"""
# 操作统计
operations: dict[str, OperationMetrics] = field(default_factory=dict)
# 连接池统计
connection_acquired: int = 0
connection_released: int = 0
connection_errors: int = 0
# 缓存统计
cache_hits: int = 0
cache_misses: int = 0
cache_sets: int = 0
cache_invalidations: int = 0
# 批处理统计
batch_operations: int = 0
batch_items_total: int = 0
batch_avg_size: float = 0.0
# 预加载统计
preload_operations: int = 0
preload_hits: int = 0
@property
def cache_hit_rate(self) -> float:
"""缓存命中率"""
total = self.cache_hits + self.cache_misses
return self.cache_hits / total if total > 0 else 0.0
@property
def error_rate(self) -> float:
"""错误率"""
total_ops = sum(m.count for m in self.operations.values())
total_errors = sum(m.error_count for m in self.operations.values())
return total_errors / total_ops if total_ops > 0 else 0.0
def get_operation_metrics(self, operation_name: str) -> OperationMetrics:
"""获取操作指标"""
if operation_name not in self.operations:
self.operations[operation_name] = OperationMetrics()
return self.operations[operation_name]
class DatabaseMonitor:
"""数据库监控器
单例模式，收集和报告数据库性能指标
"""
_instance: Optional["DatabaseMonitor"] = None
_metrics: DatabaseMetrics
def __new__(cls):
if cls._instance is None:
cls._instance = super().__new__(cls)
cls._instance._metrics = DatabaseMetrics()
return cls._instance
def record_operation(
self,
operation_name: str,
execution_time: float,
success: bool = True,
):
"""记录操作"""
metrics = self._metrics.get_operation_metrics(operation_name)
if success:
metrics.record_success(execution_time)
else:
metrics.record_error()
def record_connection_acquired(self):
"""记录连接获取"""
self._metrics.connection_acquired += 1
def record_connection_released(self):
"""记录连接释放"""
self._metrics.connection_released += 1
def record_connection_error(self):
"""记录连接错误"""
self._metrics.connection_errors += 1
def record_cache_hit(self):
"""记录缓存命中"""
self._metrics.cache_hits += 1
def record_cache_miss(self):
"""记录缓存未命中"""
self._metrics.cache_misses += 1
def record_cache_set(self):
"""记录缓存设置"""
self._metrics.cache_sets += 1
def record_cache_invalidation(self):
"""记录缓存失效"""
self._metrics.cache_invalidations += 1
def record_batch_operation(self, batch_size: int):
"""记录批处理操作"""
self._metrics.batch_operations += 1
self._metrics.batch_items_total += batch_size
self._metrics.batch_avg_size = (
self._metrics.batch_items_total / self._metrics.batch_operations
)
def record_preload_operation(self, hit: bool = False):
"""记录预加载操作"""
self._metrics.preload_operations += 1
if hit:
self._metrics.preload_hits += 1
def get_metrics(self) -> DatabaseMetrics:
"""获取指标"""
return self._metrics
def get_summary(self) -> dict[str, Any]:
"""获取统计摘要"""
metrics = self._metrics
operation_summary = {}
for op_name, op_metrics in metrics.operations.items():
operation_summary[op_name] = {
"count": op_metrics.count,
"avg_time": f"{op_metrics.avg_time:.3f}s",
"min_time": f"{op_metrics.min_time:.3f}s",
"max_time": f"{op_metrics.max_time:.3f}s",
"error_count": op_metrics.error_count,
}
return {
"operations": operation_summary,
"connections": {
"acquired": metrics.connection_acquired,
"released": metrics.connection_released,
"errors": metrics.connection_errors,
"active": metrics.connection_acquired - metrics.connection_released,
},
"cache": {
"hits": metrics.cache_hits,
"misses": metrics.cache_misses,
"sets": metrics.cache_sets,
"invalidations": metrics.cache_invalidations,
"hit_rate": f"{metrics.cache_hit_rate:.2%}",
},
"batch": {
"operations": metrics.batch_operations,
"total_items": metrics.batch_items_total,
"avg_size": f"{metrics.batch_avg_size:.1f}",
},
"preload": {
"operations": metrics.preload_operations,
"hits": metrics.preload_hits,
"hit_rate": (
f"{metrics.preload_hits / metrics.preload_operations:.2%}"
if metrics.preload_operations > 0
else "N/A"
),
},
"overall": {
"error_rate": f"{metrics.error_rate:.2%}",
},
}
def print_summary(self):
"""打印统计摘要"""
summary = self.get_summary()
logger.info("=" * 60)
logger.info("数据库性能统计")
logger.info("=" * 60)
# 操作统计
if summary["operations"]:
logger.info("\n操作统计:")
for op_name, stats in summary["operations"].items():
logger.info(
f" {op_name}: "
f"次数={stats['count']}, "
f"平均={stats['avg_time']}, "
f"最小={stats['min_time']}, "
f"最大={stats['max_time']}, "
f"错误={stats['error_count']}"
)
# 连接池统计
logger.info("\n连接池:")
conn = summary["connections"]
logger.info(
f" 获取={conn['acquired']}, "
f"释放={conn['released']}, "
f"活跃={conn['active']}, "
f"错误={conn['errors']}"
)
# 缓存统计
logger.info("\n缓存:")
cache = summary["cache"]
logger.info(
f" 命中={cache['hits']}, "
f"未命中={cache['misses']}, "
f"设置={cache['sets']}, "
f"失效={cache['invalidations']}, "
f"命中率={cache['hit_rate']}"
)
# 批处理统计
logger.info("\n批处理:")
batch = summary["batch"]
logger.info(
f" 操作={batch['operations']}, "
f"总项目={batch['total_items']}, "
f"平均大小={batch['avg_size']}"
)
# 预加载统计
logger.info("\n预加载:")
preload = summary["preload"]
logger.info(
f" 操作={preload['operations']}, "
f"命中={preload['hits']}, "
f"命中率={preload['hit_rate']}"
)
# 整体统计
logger.info("\n整体:")
overall = summary["overall"]
logger.info(f" 错误率={overall['error_rate']}")
logger.info("=" * 60)
def reset(self):
"""重置统计"""
self._metrics = DatabaseMetrics()
logger.info("数据库监控统计已重置")
# 全局监控器实例
_monitor: Optional[DatabaseMonitor] = None
def get_monitor() -> DatabaseMonitor:
"""获取监控器实例"""
global _monitor
if _monitor is None:
_monitor = DatabaseMonitor()
return _monitor
# 便捷函数
def record_operation(operation_name: str, execution_time: float, success: bool = True):
"""记录操作"""
get_monitor().record_operation(operation_name, execution_time, success)
def record_cache_hit():
"""记录缓存命中"""
get_monitor().record_cache_hit()
def record_cache_miss():
"""记录缓存未命中"""
get_monitor().record_cache_miss()
def print_stats():
"""打印统计信息"""
get_monitor().print_summary()
def reset_stats():
"""重置统计"""
get_monitor().reset()