feat: 重构记忆系统配置,移除三层记忆相关配置,优化全局记忆管理逻辑,支持批量生成文本向量
This commit is contained in:
@@ -118,7 +118,7 @@ class SingleStreamContextManager:
|
||||
|
||||
# 三层记忆系统集成:将消息添加到感知记忆层
|
||||
try:
|
||||
if global_config.three_tier_memory and global_config.three_tier_memory.enable:
|
||||
if global_config.memory and global_config.memory.enable:
|
||||
unified_manager = _get_unified_memory_manager()
|
||||
if unified_manager:
|
||||
# 构建消息字典
|
||||
|
||||
@@ -575,7 +575,7 @@ class DefaultReplyer:
|
||||
str: 记忆信息字符串
|
||||
"""
|
||||
# 检查是否启用三层记忆系统
|
||||
if not (global_config.three_tier_memory and global_config.three_tier_memory.enable):
|
||||
if not (global_config.memory and global_config.memory.enable):
|
||||
return ""
|
||||
|
||||
try:
|
||||
|
||||
@@ -39,7 +39,6 @@ from src.config.official_configs import (
|
||||
ReactionConfig,
|
||||
ResponsePostProcessConfig,
|
||||
ResponseSplitterConfig,
|
||||
ThreeTierMemoryConfig,
|
||||
ToolConfig,
|
||||
VideoAnalysisConfig,
|
||||
VoiceConfig,
|
||||
@@ -382,7 +381,6 @@ class Config(ValidatedConfigBase):
|
||||
emoji: EmojiConfig = Field(..., description="表情配置")
|
||||
expression: ExpressionConfig = Field(..., description="表达配置")
|
||||
memory: MemoryConfig | None = Field(default=None, description="记忆配置")
|
||||
three_tier_memory: ThreeTierMemoryConfig | None = Field(default=None, description="三层记忆系统配置")
|
||||
mood: MoodConfig = Field(..., description="情绪配置")
|
||||
reaction: ReactionConfig = Field(default_factory=ReactionConfig, description="反应规则配置")
|
||||
chinese_typo: ChineseTypoConfig = Field(..., description="中文错别字配置")
|
||||
|
||||
@@ -434,8 +434,6 @@ class MemoryConfig(ValidatedConfigBase):
|
||||
search_top_k: int = Field(default=10, description="默认检索返回数量")
|
||||
search_min_importance: float = Field(default=0.3, description="最小重要性阈值")
|
||||
search_similarity_threshold: float = Field(default=0.5, description="向量相似度阈值")
|
||||
search_max_expand_depth: int = Field(default=2, description="检索时图扩展深度(0-3)")
|
||||
search_expand_semantic_threshold: float = Field(default=0.3, description="图扩展时语义相似度阈值(建议0.3-0.5,过低可能引入无关记忆,过高无法扩展)")
|
||||
enable_query_optimization: bool = Field(default=True, description="启用查询优化")
|
||||
|
||||
# 路径扩展配置 (新算法)
|
||||
@@ -453,30 +451,6 @@ class MemoryConfig(ValidatedConfigBase):
|
||||
enable_memory_deduplication: bool = Field(default=True, description="启用检索结果去重(合并相似记忆)")
|
||||
memory_deduplication_threshold: float = Field(default=0.85, description="记忆相似度阈值(0.85表示85%相似即合并)")
|
||||
|
||||
# 检索权重配置 (记忆图系统)
|
||||
search_vector_weight: float = Field(default=0.4, description="向量相似度权重")
|
||||
search_graph_distance_weight: float = Field(default=0.2, description="图距离权重")
|
||||
search_importance_weight: float = Field(default=0.2, description="重要性权重")
|
||||
search_recency_weight: float = Field(default=0.2, description="时效性权重")
|
||||
|
||||
# 记忆整合配置
|
||||
consolidation_enabled: bool = Field(default=False, description="是否启用记忆整合")
|
||||
consolidation_interval_hours: float = Field(default=2.0, description="整合任务执行间隔(小时)")
|
||||
consolidation_deduplication_threshold: float = Field(default=0.93, description="相似记忆去重阈值")
|
||||
consolidation_time_window_hours: float = Field(default=2.0, description="整合时间窗口(小时)- 统一用于去重和关联")
|
||||
consolidation_max_batch_size: int = Field(default=30, description="单次最多处理的记忆数量")
|
||||
|
||||
# 记忆关联配置(整合功能的子模块)
|
||||
consolidation_linking_enabled: bool = Field(default=True, description="是否启用记忆关联建立")
|
||||
consolidation_linking_max_candidates: int = Field(default=10, description="每个记忆最多关联的候选数")
|
||||
consolidation_linking_max_memories: int = Field(default=20, description="单次最多处理的记忆总数")
|
||||
consolidation_linking_min_importance: float = Field(default=0.5, description="最低重要性阈值")
|
||||
consolidation_linking_pre_filter_threshold: float = Field(default=0.7, description="向量相似度预筛选阈值")
|
||||
consolidation_linking_max_pairs_for_llm: int = Field(default=5, description="最多发送给LLM分析的候选对数")
|
||||
consolidation_linking_min_confidence: float = Field(default=0.7, description="LLM分析最低置信度阈值")
|
||||
consolidation_linking_llm_temperature: float = Field(default=0.2, description="LLM分析温度参数")
|
||||
consolidation_linking_llm_max_tokens: int = Field(default=1500, description="LLM分析最大输出长度")
|
||||
|
||||
# 遗忘配置 (记忆图系统)
|
||||
forgetting_enabled: bool = Field(default=True, description="是否启用自动遗忘")
|
||||
forgetting_activation_threshold: float = Field(default=0.1, description="激活度阈值")
|
||||
@@ -500,26 +474,13 @@ class MemoryConfig(ValidatedConfigBase):
|
||||
node_merger_context_match_required: bool = Field(default=True, description="节点合并是否要求上下文匹配")
|
||||
node_merger_merge_batch_size: int = Field(default=50, description="节点合并批量处理大小")
|
||||
|
||||
|
||||
class MoodConfig(ValidatedConfigBase):
|
||||
"""情绪配置类"""
|
||||
|
||||
enable_mood: bool = Field(default=False, description="启用情绪")
|
||||
mood_update_threshold: float = Field(default=1.0, description="情绪更新阈值")
|
||||
|
||||
|
||||
class ThreeTierMemoryConfig(ValidatedConfigBase):
|
||||
"""三层记忆系统配置类"""
|
||||
|
||||
enable: bool = Field(default=False, description="启用三层记忆系统(实验性功能)")
|
||||
data_dir: str = Field(default="data/memory_graph/three_tier", description="数据存储目录")
|
||||
|
||||
# ==================== 三层记忆系统配置 (Three-Tier Memory System) ====================
|
||||
# 感知记忆层配置
|
||||
perceptual_max_blocks: int = Field(default=50, description="记忆堆最大容量(全局)")
|
||||
perceptual_block_size: int = Field(default=5, description="每个记忆块包含的消息数量")
|
||||
perceptual_similarity_threshold: float = Field(default=0.55, description="相似度阈值(0-1)")
|
||||
perceptual_topk: int = Field(default=3, description="TopK召回数量")
|
||||
activation_threshold: int = Field(default=3, description="激活阈值(召回次数→短期)")
|
||||
perceptual_activation_threshold: int = Field(default=3, description="激活阈值(召回次数→短期)")
|
||||
|
||||
# 短期记忆层配置
|
||||
short_term_max_memories: int = Field(default=30, description="短期记忆最大数量")
|
||||
@@ -532,10 +493,12 @@ class ThreeTierMemoryConfig(ValidatedConfigBase):
|
||||
long_term_decay_factor: float = Field(default=0.95, description="衰减因子")
|
||||
long_term_auto_transfer_interval: int = Field(default=60, description="自动转移间隔(秒)")
|
||||
|
||||
# Judge模型配置
|
||||
judge_model_name: str = Field(default="utils_small", description="用于决策的LLM模型")
|
||||
judge_temperature: float = Field(default=0.1, description="Judge模型的温度参数")
|
||||
enable_judge_retrieval: bool = Field(default=True, description="启用智能检索判断")
|
||||
|
||||
class MoodConfig(ValidatedConfigBase):
|
||||
"""情绪配置类"""
|
||||
|
||||
enable_mood: bool = Field(default=False, description="启用情绪")
|
||||
mood_update_threshold: float = Field(default=1.0, description="情绪更新阈值")
|
||||
|
||||
|
||||
class ReactionRuleConfig(ValidatedConfigBase):
|
||||
|
||||
@@ -479,7 +479,7 @@ MoFox_Bot(第三方修改版)
|
||||
|
||||
# 初始化三层记忆系统(如果启用)
|
||||
try:
|
||||
if global_config.three_tier_memory and global_config.three_tier_memory.enable:
|
||||
if global_config.memory and global_config.memory.enable:
|
||||
from src.memory_graph.manager_singleton import initialize_unified_memory_manager
|
||||
logger.info("三层记忆系统已启用,正在初始化...")
|
||||
await initialize_unified_memory_manager()
|
||||
|
||||
@@ -138,20 +138,24 @@ class MemoryManager:
|
||||
)
|
||||
|
||||
# 检查配置值
|
||||
expand_depth = self.config.search_max_expand_depth
|
||||
expand_semantic_threshold = self.config.search_expand_semantic_threshold
|
||||
search_top_k = self.config.search_top_k
|
||||
# 兼容性处理:如果配置项不存在,使用默认值或映射到新配置项
|
||||
expand_depth = getattr(self.config, "path_expansion_max_hops", 2)
|
||||
expand_semantic_threshold = getattr(self.config, "search_similarity_threshold", 0.5)
|
||||
search_top_k = getattr(self.config, "search_top_k", 10)
|
||||
|
||||
# 读取权重配置
|
||||
search_vector_weight = self.config.search_vector_weight
|
||||
search_importance_weight = self.config.search_importance_weight
|
||||
search_recency_weight = self.config.search_recency_weight
|
||||
search_vector_weight = getattr(self.config, "vector_weight", 0.65)
|
||||
# context_weight 近似映射为 importance_weight
|
||||
search_importance_weight = getattr(self.config, "context_weight", 0.25)
|
||||
search_recency_weight = getattr(self.config, "recency_weight", 0.10)
|
||||
|
||||
# 读取阈值过滤配置
|
||||
search_min_importance = self.config.search_min_importance
|
||||
search_similarity_threshold = self.config.search_similarity_threshold
|
||||
search_min_importance = getattr(self.config, "search_min_importance", 0.3)
|
||||
search_similarity_threshold = getattr(self.config, "search_similarity_threshold", 0.5)
|
||||
|
||||
logger.info(
|
||||
f"📊 配置检查: search_max_expand_depth={expand_depth}, "
|
||||
f"search_expand_semantic_threshold={expand_semantic_threshold}, "
|
||||
f"📊 配置检查: expand_depth={expand_depth}, "
|
||||
f"expand_semantic_threshold={expand_semantic_threshold}, "
|
||||
f"search_top_k={search_top_k}"
|
||||
)
|
||||
logger.info(
|
||||
@@ -422,7 +426,7 @@ class MemoryManager:
|
||||
"query": query,
|
||||
"top_k": top_k,
|
||||
"use_multi_query": use_multi_query,
|
||||
"expand_depth": expand_depth or global_config.memory.search_max_expand_depth, # 传递图扩展深度
|
||||
"expand_depth": expand_depth or getattr(global_config.memory, "path_expansion_max_hops", 2), # 传递图扩展深度
|
||||
"context": context,
|
||||
"prefer_node_types": prefer_node_types or [], # 🆕 传递偏好节点类型
|
||||
}
|
||||
|
||||
@@ -139,29 +139,29 @@ async def initialize_unified_memory_manager():
|
||||
from src.memory_graph.unified_manager import UnifiedMemoryManager
|
||||
|
||||
# 检查是否启用三层记忆系统
|
||||
if not hasattr(global_config, "three_tier_memory") or not getattr(
|
||||
global_config.three_tier_memory, "enable", False
|
||||
if not hasattr(global_config, "memory") or not getattr(
|
||||
global_config.memory, "enable", False
|
||||
):
|
||||
logger.warning("三层记忆系统未启用,跳过初始化")
|
||||
return None
|
||||
|
||||
config = global_config.three_tier_memory
|
||||
config = global_config.memory
|
||||
|
||||
# 创建管理器实例
|
||||
_unified_memory_manager = UnifiedMemoryManager(
|
||||
data_dir=Path(getattr(config, "data_dir", "data/memory_graph/three_tier")),
|
||||
data_dir=Path(getattr(config, "data_dir", "data/memory_graph")),
|
||||
# 感知记忆配置
|
||||
perceptual_max_blocks=getattr(config, "perceptual_max_blocks", 50),
|
||||
perceptual_block_size=getattr(config, "perceptual_block_size", 5),
|
||||
perceptual_activation_threshold=getattr(config, "perceptual_activation_threshold", 3),
|
||||
perceptual_recall_top_k=getattr(config, "perceptual_recall_top_k", 5),
|
||||
perceptual_recall_threshold=getattr(config, "perceptual_recall_threshold", 0.55),
|
||||
perceptual_recall_top_k=getattr(config, "perceptual_topk", 5),
|
||||
perceptual_recall_threshold=getattr(config, "perceptual_similarity_threshold", 0.55),
|
||||
# 短期记忆配置
|
||||
short_term_max_memories=getattr(config, "short_term_max_memories", 30),
|
||||
short_term_transfer_threshold=getattr(config, "short_term_transfer_threshold", 0.6),
|
||||
# 长期记忆配置
|
||||
long_term_batch_size=getattr(config, "long_term_batch_size", 10),
|
||||
long_term_search_top_k=getattr(config, "long_term_search_top_k", 5),
|
||||
long_term_search_top_k=getattr(config, "search_top_k", 5),
|
||||
long_term_decay_factor=getattr(config, "long_term_decay_factor", 0.95),
|
||||
long_term_auto_transfer_interval=getattr(config, "long_term_auto_transfer_interval", 600),
|
||||
# 智能检索配置
|
||||
|
||||
@@ -279,6 +279,28 @@ class PerceptualMemoryManager:
|
||||
logger.error(f"生成向量失败: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
async def _generate_embeddings_batch(self, texts: list[str]) -> list[np.ndarray | None]:
|
||||
"""
|
||||
批量生成文本向量
|
||||
|
||||
Args:
|
||||
texts: 文本列表
|
||||
|
||||
Returns:
|
||||
向量列表,与输入一一对应
|
||||
"""
|
||||
try:
|
||||
if not self.embedding_generator:
|
||||
logger.error("嵌入生成器未初始化")
|
||||
return [None] * len(texts)
|
||||
|
||||
embeddings = await self.embedding_generator.generate_batch(texts)
|
||||
return embeddings
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"批量生成向量失败: {e}", exc_info=True)
|
||||
return [None] * len(texts)
|
||||
|
||||
async def recall_blocks(
|
||||
self,
|
||||
query_text: str,
|
||||
@@ -528,11 +550,29 @@ class PerceptualMemoryManager:
|
||||
|
||||
logger.info("重新生成记忆块向量...")
|
||||
|
||||
for block in self.perceptual_memory.blocks:
|
||||
if block.embedding is None and block.combined_text:
|
||||
block.embedding = await self._generate_embedding(block.combined_text)
|
||||
blocks_to_process = []
|
||||
texts_to_process = []
|
||||
|
||||
logger.info(f"✅ 向量重新生成完成({len(self.perceptual_memory.blocks)} 个块)")
|
||||
for block in self.perceptual_memory.blocks:
|
||||
if block.embedding is None and block.combined_text and block.combined_text.strip():
|
||||
blocks_to_process.append(block)
|
||||
texts_to_process.append(block.combined_text)
|
||||
|
||||
if not blocks_to_process:
|
||||
logger.info("没有需要重新生成向量的块")
|
||||
return
|
||||
|
||||
logger.info(f"开始批量生成 {len(blocks_to_process)} 个块的向量...")
|
||||
|
||||
embeddings = await self._generate_embeddings_batch(texts_to_process)
|
||||
|
||||
success_count = 0
|
||||
for block, embedding in zip(blocks_to_process, embeddings):
|
||||
if embedding is not None:
|
||||
block.embedding = embedding
|
||||
success_count += 1
|
||||
|
||||
logger.info(f"✅ 向量重新生成完成(成功: {success_count}/{len(blocks_to_process)})")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""关闭管理器"""
|
||||
|
||||
@@ -492,6 +492,28 @@ class ShortTermMemoryManager:
|
||||
logger.error(f"生成向量失败: {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
async def _generate_embeddings_batch(self, texts: list[str]) -> list[np.ndarray | None]:
|
||||
"""
|
||||
批量生成文本向量
|
||||
|
||||
Args:
|
||||
texts: 文本列表
|
||||
|
||||
Returns:
|
||||
向量列表,与输入一一对应
|
||||
"""
|
||||
try:
|
||||
if not self.embedding_generator:
|
||||
logger.error("嵌入生成器未初始化")
|
||||
return [None] * len(texts)
|
||||
|
||||
embeddings = await self.embedding_generator.generate_batch(texts)
|
||||
return embeddings
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"批量生成向量失败: {e}", exc_info=True)
|
||||
return [None] * len(texts)
|
||||
|
||||
def _parse_json_response(self, response: str) -> dict[str, Any] | None:
|
||||
"""解析 LLM 的 JSON 响应"""
|
||||
try:
|
||||
@@ -684,11 +706,29 @@ class ShortTermMemoryManager:
|
||||
"""重新生成记忆的向量"""
|
||||
logger.info("重新生成短期记忆向量...")
|
||||
|
||||
for memory in self.memories:
|
||||
if memory.embedding is None and memory.content:
|
||||
memory.embedding = await self._generate_embedding(memory.content)
|
||||
memories_to_process = []
|
||||
texts_to_process = []
|
||||
|
||||
logger.info(f"✅ 向量重新生成完成({len(self.memories)} 条记忆)")
|
||||
for memory in self.memories:
|
||||
if memory.embedding is None and memory.content and memory.content.strip():
|
||||
memories_to_process.append(memory)
|
||||
texts_to_process.append(memory.content)
|
||||
|
||||
if not memories_to_process:
|
||||
logger.info("没有需要重新生成向量的短期记忆")
|
||||
return
|
||||
|
||||
logger.info(f"开始批量生成 {len(memories_to_process)} 条短期记忆的向量...")
|
||||
|
||||
embeddings = await self._generate_embeddings_batch(texts_to_process)
|
||||
|
||||
success_count = 0
|
||||
for memory, embedding in zip(memories_to_process, embeddings):
|
||||
if embedding is not None:
|
||||
memory.embedding = embedding
|
||||
success_count += 1
|
||||
|
||||
logger.info(f"✅ 向量重新生成完成(成功: {success_count}/{len(memories_to_process)})")
|
||||
|
||||
async def shutdown(self) -> None:
|
||||
"""关闭管理器"""
|
||||
|
||||
Reference in New Issue
Block a user