feat: 重构记忆系统配置,移除三层记忆相关配置,优化全局记忆管理逻辑,支持批量生成文本向量

This commit is contained in:
Windpicker-owo
2025-11-19 19:16:27 +08:00
parent 5231404852
commit cf48d02ed3
10 changed files with 164 additions and 139 deletions

View File

@@ -118,7 +118,7 @@ class SingleStreamContextManager:
# 三层记忆系统集成:将消息添加到感知记忆层
try:
if global_config.three_tier_memory and global_config.three_tier_memory.enable:
if global_config.memory and global_config.memory.enable:
unified_manager = _get_unified_memory_manager()
if unified_manager:
# 构建消息字典

View File

@@ -575,7 +575,7 @@ class DefaultReplyer:
str: 记忆信息字符串
"""
# 检查是否启用三层记忆系统
if not (global_config.three_tier_memory and global_config.three_tier_memory.enable):
if not (global_config.memory and global_config.memory.enable):
return ""
try:

View File

@@ -39,7 +39,6 @@ from src.config.official_configs import (
ReactionConfig,
ResponsePostProcessConfig,
ResponseSplitterConfig,
ThreeTierMemoryConfig,
ToolConfig,
VideoAnalysisConfig,
VoiceConfig,
@@ -382,7 +381,6 @@ class Config(ValidatedConfigBase):
emoji: EmojiConfig = Field(..., description="表情配置")
expression: ExpressionConfig = Field(..., description="表达配置")
memory: MemoryConfig | None = Field(default=None, description="记忆配置")
three_tier_memory: ThreeTierMemoryConfig | None = Field(default=None, description="三层记忆系统配置")
mood: MoodConfig = Field(..., description="情绪配置")
reaction: ReactionConfig = Field(default_factory=ReactionConfig, description="反应规则配置")
chinese_typo: ChineseTypoConfig = Field(..., description="中文错别字配置")

View File

@@ -434,8 +434,6 @@ class MemoryConfig(ValidatedConfigBase):
search_top_k: int = Field(default=10, description="默认检索返回数量")
search_min_importance: float = Field(default=0.3, description="最小重要性阈值")
search_similarity_threshold: float = Field(default=0.5, description="向量相似度阈值")
search_max_expand_depth: int = Field(default=2, description="检索时图扩展深度0-3")
search_expand_semantic_threshold: float = Field(default=0.3, description="图扩展时语义相似度阈值建议0.3-0.5,过低可能引入无关记忆,过高无法扩展)")
enable_query_optimization: bool = Field(default=True, description="启用查询优化")
# 路径扩展配置 (新算法)
@@ -453,30 +451,6 @@ class MemoryConfig(ValidatedConfigBase):
enable_memory_deduplication: bool = Field(default=True, description="启用检索结果去重(合并相似记忆)")
memory_deduplication_threshold: float = Field(default=0.85, description="记忆相似度阈值0.85表示85%相似即合并)")
# 检索权重配置 (记忆图系统)
search_vector_weight: float = Field(default=0.4, description="向量相似度权重")
search_graph_distance_weight: float = Field(default=0.2, description="图距离权重")
search_importance_weight: float = Field(default=0.2, description="重要性权重")
search_recency_weight: float = Field(default=0.2, description="时效性权重")
# 记忆整合配置
consolidation_enabled: bool = Field(default=False, description="是否启用记忆整合")
consolidation_interval_hours: float = Field(default=2.0, description="整合任务执行间隔(小时)")
consolidation_deduplication_threshold: float = Field(default=0.93, description="相似记忆去重阈值")
consolidation_time_window_hours: float = Field(default=2.0, description="整合时间窗口(小时)- 统一用于去重和关联")
consolidation_max_batch_size: int = Field(default=30, description="单次最多处理的记忆数量")
# 记忆关联配置(整合功能的子模块)
consolidation_linking_enabled: bool = Field(default=True, description="是否启用记忆关联建立")
consolidation_linking_max_candidates: int = Field(default=10, description="每个记忆最多关联的候选数")
consolidation_linking_max_memories: int = Field(default=20, description="单次最多处理的记忆总数")
consolidation_linking_min_importance: float = Field(default=0.5, description="最低重要性阈值")
consolidation_linking_pre_filter_threshold: float = Field(default=0.7, description="向量相似度预筛选阈值")
consolidation_linking_max_pairs_for_llm: int = Field(default=5, description="最多发送给LLM分析的候选对数")
consolidation_linking_min_confidence: float = Field(default=0.7, description="LLM分析最低置信度阈值")
consolidation_linking_llm_temperature: float = Field(default=0.2, description="LLM分析温度参数")
consolidation_linking_llm_max_tokens: int = Field(default=1500, description="LLM分析最大输出长度")
# 遗忘配置 (记忆图系统)
forgetting_enabled: bool = Field(default=True, description="是否启用自动遗忘")
forgetting_activation_threshold: float = Field(default=0.1, description="激活度阈值")
@@ -500,26 +474,13 @@ class MemoryConfig(ValidatedConfigBase):
node_merger_context_match_required: bool = Field(default=True, description="节点合并是否要求上下文匹配")
node_merger_merge_batch_size: int = Field(default=50, description="节点合并批量处理大小")
class MoodConfig(ValidatedConfigBase):
"""情绪配置类"""
enable_mood: bool = Field(default=False, description="启用情绪")
mood_update_threshold: float = Field(default=1.0, description="情绪更新阈值")
class ThreeTierMemoryConfig(ValidatedConfigBase):
"""三层记忆系统配置类"""
enable: bool = Field(default=False, description="启用三层记忆系统(实验性功能)")
data_dir: str = Field(default="data/memory_graph/three_tier", description="数据存储目录")
# ==================== 三层记忆系统配置 (Three-Tier Memory System) ====================
# 感知记忆层配置
perceptual_max_blocks: int = Field(default=50, description="记忆堆最大容量(全局)")
perceptual_block_size: int = Field(default=5, description="每个记忆块包含的消息数量")
perceptual_similarity_threshold: float = Field(default=0.55, description="相似度阈值0-1")
perceptual_topk: int = Field(default=3, description="TopK召回数量")
activation_threshold: int = Field(default=3, description="激活阈值(召回次数→短期)")
perceptual_activation_threshold: int = Field(default=3, description="激活阈值(召回次数→短期)")
# 短期记忆层配置
short_term_max_memories: int = Field(default=30, description="短期记忆最大数量")
@@ -532,10 +493,12 @@ class ThreeTierMemoryConfig(ValidatedConfigBase):
long_term_decay_factor: float = Field(default=0.95, description="衰减因子")
long_term_auto_transfer_interval: int = Field(default=60, description="自动转移间隔(秒)")
# Judge模型配置
judge_model_name: str = Field(default="utils_small", description="用于决策的LLM模型")
judge_temperature: float = Field(default=0.1, description="Judge模型的温度参数")
enable_judge_retrieval: bool = Field(default=True, description="启用智能检索判断")
class MoodConfig(ValidatedConfigBase):
"""情绪配置类"""
enable_mood: bool = Field(default=False, description="启用情绪")
mood_update_threshold: float = Field(default=1.0, description="情绪更新阈值")
class ReactionRuleConfig(ValidatedConfigBase):

View File

@@ -479,7 +479,7 @@ MoFox_Bot(第三方修改版)
# 初始化三层记忆系统(如果启用)
try:
if global_config.three_tier_memory and global_config.three_tier_memory.enable:
if global_config.memory and global_config.memory.enable:
from src.memory_graph.manager_singleton import initialize_unified_memory_manager
logger.info("三层记忆系统已启用,正在初始化...")
await initialize_unified_memory_manager()

View File

@@ -138,20 +138,24 @@ class MemoryManager:
)
# 检查配置值
expand_depth = self.config.search_max_expand_depth
expand_semantic_threshold = self.config.search_expand_semantic_threshold
search_top_k = self.config.search_top_k
# 兼容性处理:如果配置项不存在,使用默认值或映射到新配置项
expand_depth = getattr(self.config, "path_expansion_max_hops", 2)
expand_semantic_threshold = getattr(self.config, "search_similarity_threshold", 0.5)
search_top_k = getattr(self.config, "search_top_k", 10)
# 读取权重配置
search_vector_weight = self.config.search_vector_weight
search_importance_weight = self.config.search_importance_weight
search_recency_weight = self.config.search_recency_weight
search_vector_weight = getattr(self.config, "vector_weight", 0.65)
# context_weight 近似映射为 importance_weight
search_importance_weight = getattr(self.config, "context_weight", 0.25)
search_recency_weight = getattr(self.config, "recency_weight", 0.10)
# 读取阈值过滤配置
search_min_importance = self.config.search_min_importance
search_similarity_threshold = self.config.search_similarity_threshold
search_min_importance = getattr(self.config, "search_min_importance", 0.3)
search_similarity_threshold = getattr(self.config, "search_similarity_threshold", 0.5)
logger.info(
f"📊 配置检查: search_max_expand_depth={expand_depth}, "
f"search_expand_semantic_threshold={expand_semantic_threshold}, "
f"📊 配置检查: expand_depth={expand_depth}, "
f"expand_semantic_threshold={expand_semantic_threshold}, "
f"search_top_k={search_top_k}"
)
logger.info(
@@ -422,7 +426,7 @@ class MemoryManager:
"query": query,
"top_k": top_k,
"use_multi_query": use_multi_query,
"expand_depth": expand_depth or global_config.memory.search_max_expand_depth, # 传递图扩展深度
"expand_depth": expand_depth or getattr(global_config.memory, "path_expansion_max_hops", 2), # 传递图扩展深度
"context": context,
"prefer_node_types": prefer_node_types or [], # 🆕 传递偏好节点类型
}

View File

@@ -139,29 +139,29 @@ async def initialize_unified_memory_manager():
from src.memory_graph.unified_manager import UnifiedMemoryManager
# 检查是否启用三层记忆系统
if not hasattr(global_config, "three_tier_memory") or not getattr(
global_config.three_tier_memory, "enable", False
if not hasattr(global_config, "memory") or not getattr(
global_config.memory, "enable", False
):
logger.warning("三层记忆系统未启用,跳过初始化")
return None
config = global_config.three_tier_memory
config = global_config.memory
# 创建管理器实例
_unified_memory_manager = UnifiedMemoryManager(
data_dir=Path(getattr(config, "data_dir", "data/memory_graph/three_tier")),
data_dir=Path(getattr(config, "data_dir", "data/memory_graph")),
# 感知记忆配置
perceptual_max_blocks=getattr(config, "perceptual_max_blocks", 50),
perceptual_block_size=getattr(config, "perceptual_block_size", 5),
perceptual_activation_threshold=getattr(config, "perceptual_activation_threshold", 3),
perceptual_recall_top_k=getattr(config, "perceptual_recall_top_k", 5),
perceptual_recall_threshold=getattr(config, "perceptual_recall_threshold", 0.55),
perceptual_recall_top_k=getattr(config, "perceptual_topk", 5),
perceptual_recall_threshold=getattr(config, "perceptual_similarity_threshold", 0.55),
# 短期记忆配置
short_term_max_memories=getattr(config, "short_term_max_memories", 30),
short_term_transfer_threshold=getattr(config, "short_term_transfer_threshold", 0.6),
# 长期记忆配置
long_term_batch_size=getattr(config, "long_term_batch_size", 10),
long_term_search_top_k=getattr(config, "long_term_search_top_k", 5),
long_term_search_top_k=getattr(config, "search_top_k", 5),
long_term_decay_factor=getattr(config, "long_term_decay_factor", 0.95),
long_term_auto_transfer_interval=getattr(config, "long_term_auto_transfer_interval", 600),
# 智能检索配置

View File

@@ -279,6 +279,28 @@ class PerceptualMemoryManager:
logger.error(f"生成向量失败: {e}", exc_info=True)
return None
async def _generate_embeddings_batch(self, texts: list[str]) -> list[np.ndarray | None]:
"""
批量生成文本向量
Args:
texts: 文本列表
Returns:
向量列表,与输入一一对应
"""
try:
if not self.embedding_generator:
logger.error("嵌入生成器未初始化")
return [None] * len(texts)
embeddings = await self.embedding_generator.generate_batch(texts)
return embeddings
except Exception as e:
logger.error(f"批量生成向量失败: {e}", exc_info=True)
return [None] * len(texts)
async def recall_blocks(
self,
query_text: str,
@@ -528,11 +550,29 @@ class PerceptualMemoryManager:
logger.info("重新生成记忆块向量...")
for block in self.perceptual_memory.blocks:
if block.embedding is None and block.combined_text:
block.embedding = await self._generate_embedding(block.combined_text)
blocks_to_process = []
texts_to_process = []
logger.info(f"✅ 向量重新生成完成({len(self.perceptual_memory.blocks)} 个块)")
for block in self.perceptual_memory.blocks:
if block.embedding is None and block.combined_text and block.combined_text.strip():
blocks_to_process.append(block)
texts_to_process.append(block.combined_text)
if not blocks_to_process:
logger.info("没有需要重新生成向量的块")
return
logger.info(f"开始批量生成 {len(blocks_to_process)} 个块的向量...")
embeddings = await self._generate_embeddings_batch(texts_to_process)
success_count = 0
for block, embedding in zip(blocks_to_process, embeddings):
if embedding is not None:
block.embedding = embedding
success_count += 1
logger.info(f"✅ 向量重新生成完成(成功: {success_count}/{len(blocks_to_process)}")
async def shutdown(self) -> None:
"""关闭管理器"""

View File

@@ -492,6 +492,28 @@ class ShortTermMemoryManager:
logger.error(f"生成向量失败: {e}", exc_info=True)
return None
async def _generate_embeddings_batch(self, texts: list[str]) -> list[np.ndarray | None]:
"""
批量生成文本向量
Args:
texts: 文本列表
Returns:
向量列表,与输入一一对应
"""
try:
if not self.embedding_generator:
logger.error("嵌入生成器未初始化")
return [None] * len(texts)
embeddings = await self.embedding_generator.generate_batch(texts)
return embeddings
except Exception as e:
logger.error(f"批量生成向量失败: {e}", exc_info=True)
return [None] * len(texts)
def _parse_json_response(self, response: str) -> dict[str, Any] | None:
"""解析 LLM 的 JSON 响应"""
try:
@@ -684,11 +706,29 @@ class ShortTermMemoryManager:
"""重新生成记忆的向量"""
logger.info("重新生成短期记忆向量...")
for memory in self.memories:
if memory.embedding is None and memory.content:
memory.embedding = await self._generate_embedding(memory.content)
memories_to_process = []
texts_to_process = []
logger.info(f"✅ 向量重新生成完成({len(self.memories)} 条记忆)")
for memory in self.memories:
if memory.embedding is None and memory.content and memory.content.strip():
memories_to_process.append(memory)
texts_to_process.append(memory.content)
if not memories_to_process:
logger.info("没有需要重新生成向量的短期记忆")
return
logger.info(f"开始批量生成 {len(memories_to_process)} 条短期记忆的向量...")
embeddings = await self._generate_embeddings_batch(texts_to_process)
success_count = 0
for memory, embedding in zip(memories_to_process, embeddings):
if embedding is not None:
memory.embedding = embedding
success_count += 1
logger.info(f"✅ 向量重新生成完成(成功: {success_count}/{len(memories_to_process)}")
async def shutdown(self) -> None:
"""关闭管理器"""