refactor(chat): 重构消息处理流程引入缓冲队列机制
通过引入消息缓冲队列,解决了高频消息场景下的竞态条件和消息丢失问题。 新的处理机制将消息暂存于缓冲区,仅在流处理空闲时才释放到未读队列, 确保了消息处理的原子性和一致性。 核心变更: - 为每个聊天流创建独立的消息缓冲队列 - 实时跟踪流的处理状态,避免并发访问冲突 - 在处理开始前和完成后自动刷新缓冲区 - 仅在成功执行后清空未读消息,失败时保留消息 - 增加对取消任务和异常情况的容错处理 - 集成缓存统计和监控功能 此优化显著提升了消息处理的可靠性和性能表现。
This commit is contained in:
@@ -6,6 +6,7 @@
|
||||
import asyncio
|
||||
import random
|
||||
import time
|
||||
from collections import defaultdict, deque
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from src.chat.chatter_manager import ChatterManager
|
||||
@@ -46,6 +47,14 @@ class MessageManager:
|
||||
self.sleep_manager = SleepManager()
|
||||
self.wakeup_manager = WakeUpManager(self.sleep_manager)
|
||||
|
||||
# 消息缓存系统 - 直接集成到消息管理器
|
||||
self.message_caches: Dict[str, deque] = defaultdict(deque) # 每个流的消息缓存
|
||||
self.stream_processing_status: Dict[str, bool] = defaultdict(bool) # 流的处理状态
|
||||
self.cache_stats = {
|
||||
"total_cached_messages": 0,
|
||||
"total_flushed_messages": 0,
|
||||
}
|
||||
|
||||
# 不再需要全局上下文管理器,直接通过 ChatManager 访问各个 ChatStream 的 context_manager
|
||||
|
||||
async def start(self):
|
||||
@@ -72,6 +81,9 @@ class MessageManager:
|
||||
except Exception as e:
|
||||
logger.error(f"启动流缓存管理器失败: {e}")
|
||||
|
||||
# 启动消息缓存系统(内置)
|
||||
logger.info("📦 消息缓存系统已启动")
|
||||
|
||||
# 启动自适应流管理器
|
||||
try:
|
||||
from src.chat.message_manager.adaptive_stream_manager import init_adaptive_stream_manager
|
||||
@@ -115,6 +127,11 @@ class MessageManager:
|
||||
except Exception as e:
|
||||
logger.error(f"停止流缓存管理器失败: {e}")
|
||||
|
||||
# 停止消息缓存系统(内置)
|
||||
self.message_caches.clear()
|
||||
self.stream_processing_status.clear()
|
||||
logger.info("📦 消息缓存系统已停止")
|
||||
|
||||
# 停止自适应流管理器
|
||||
try:
|
||||
from src.chat.message_manager.adaptive_stream_manager import shutdown_adaptive_stream_manager
|
||||
@@ -429,6 +446,115 @@ class MessageManager:
|
||||
except Exception as e:
|
||||
logger.error(f"清除流 {stream_id} 的未读消息时发生错误: {e}")
|
||||
|
||||
# ===== 消息缓存系统方法 =====
|
||||
|
||||
def add_message_to_cache(self, stream_id: str, message: DatabaseMessages) -> bool:
|
||||
"""添加消息到缓存
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
message: 消息对象
|
||||
|
||||
Returns:
|
||||
bool: 是否成功添加到缓存
|
||||
"""
|
||||
try:
|
||||
if not self.is_running:
|
||||
return False
|
||||
|
||||
self.message_caches[stream_id].append(message)
|
||||
self.cache_stats["total_cached_messages"] += 1
|
||||
|
||||
logger.debug(f"消息已添加到缓存: stream={stream_id}, content={message.processed_plain_text[:50]}...")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"添加消息到缓存失败: stream={stream_id}, error={e}")
|
||||
return False
|
||||
|
||||
def flush_cached_messages(self, stream_id: str) -> list[DatabaseMessages]:
|
||||
"""刷新缓存消息到未读消息列表
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
|
||||
Returns:
|
||||
List[DatabaseMessages]: 缓存的消息列表
|
||||
"""
|
||||
try:
|
||||
if stream_id not in self.message_caches:
|
||||
return []
|
||||
|
||||
cached_messages = list(self.message_caches[stream_id])
|
||||
self.message_caches[stream_id].clear()
|
||||
|
||||
self.cache_stats["total_flushed_messages"] += len(cached_messages)
|
||||
|
||||
logger.debug(f"刷新缓存消息: stream={stream_id}, 数量={len(cached_messages)}")
|
||||
return cached_messages
|
||||
except Exception as e:
|
||||
logger.error(f"刷新缓存消息失败: stream={stream_id}, error={e}")
|
||||
return []
|
||||
|
||||
def set_stream_processing_status(self, stream_id: str, is_processing: bool):
|
||||
"""设置流的处理状态
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
is_processing: 是否正在处理
|
||||
"""
|
||||
try:
|
||||
self.stream_processing_status[stream_id] = is_processing
|
||||
logger.debug(f"设置流处理状态: stream={stream_id}, processing={is_processing}")
|
||||
except Exception as e:
|
||||
logger.error(f"设置流处理状态失败: stream={stream_id}, error={e}")
|
||||
|
||||
def get_stream_processing_status(self, stream_id: str) -> bool:
|
||||
"""获取流的处理状态
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
|
||||
Returns:
|
||||
bool: 是否正在处理
|
||||
"""
|
||||
return self.stream_processing_status.get(stream_id, False)
|
||||
|
||||
def has_cached_messages(self, stream_id: str) -> bool:
|
||||
"""检查流是否有缓存消息
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
|
||||
Returns:
|
||||
bool: 是否有缓存消息
|
||||
"""
|
||||
return stream_id in self.message_caches and len(self.message_caches[stream_id]) > 0
|
||||
|
||||
def get_cached_messages_count(self, stream_id: str) -> int:
|
||||
"""获取流的缓存消息数量
|
||||
|
||||
Args:
|
||||
stream_id: 流ID
|
||||
|
||||
Returns:
|
||||
int: 缓存消息数量
|
||||
"""
|
||||
return len(self.message_caches.get(stream_id, []))
|
||||
|
||||
def get_cache_stats(self) -> dict[str, Any]:
|
||||
"""获取缓存统计信息
|
||||
|
||||
Returns:
|
||||
Dict[str, Any]: 缓存统计信息
|
||||
"""
|
||||
return {
|
||||
"total_cached_messages": self.cache_stats["total_cached_messages"],
|
||||
"total_flushed_messages": self.cache_stats["total_flushed_messages"],
|
||||
"active_caches": len(self.message_caches),
|
||||
"cached_streams": len([s for s in self.message_caches.keys() if self.message_caches[s]]),
|
||||
"processing_streams": len([s for s in self.stream_processing_status.keys() if self.stream_processing_status[s]]),
|
||||
}
|
||||
|
||||
|
||||
# 创建全局消息管理器实例
|
||||
message_manager = MessageManager()
|
||||
|
||||
Reference in New Issue
Block a user