From 59081848e20c14f3b190eb1828d57180a5dc48b3 Mon Sep 17 00:00:00 2001 From: Windpicker-owo <3431391539@qq.com> Date: Thu, 6 Nov 2025 09:18:59 +0800 Subject: [PATCH] =?UTF-8?q?feat(memory):=20=E7=A7=BB=E9=99=A4=E4=BC=A0?= =?UTF-8?q?=E7=BB=9F=E5=86=85=E5=AD=98=E7=B3=BB=E7=BB=9F=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=86=85=E5=AD=98=E5=9B=BE=E8=B0=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除整个传统内存系统，包括内存系统模块及所有相关组件 - 删除弃用的内存组件：增强型内存激活器、海马体采样器、内存构建器、内存块、内存遗忘引擎、内存格式器、内存融合器、内存管理器、内存元数据索引、内存查询规划器、内存系统、消息集合处理器、消息集合存储、向量内存存储_v2 - 更新内存图配置，采用增强型检索设置 - 优化内存管理器查询功能，以分析完整对话上下文 - 更新机器人配置模板版本至7.6.1，新增内存图表检索参数 重大变更：旧版内存系统已被完全移除。所有内存功能现依赖于内存图系统。请更新配置以包含新的内存图检索参数。 --- src/chat/memory_system/__init__.py | 73 -- .../enhanced_memory_activator.py | 240 ---- src/chat/memory_system/hippocampus_sampler.py | 721 ----------- .../memory_system/memory_activator_new.py | 238 ---- src/chat/memory_system/memory_builder.py | 1135 ----------------- src/chat/memory_system/memory_chunk.py | 647 ---------- .../memory_system/memory_forgetting_engine.py | 355 ------ src/chat/memory_system/memory_formatter.py | 120 -- src/chat/memory_system/memory_fusion.py | 505 -------- src/chat/memory_system/memory_manager.py | 512 -------- .../memory_system/memory_metadata_index.py | 122 -- .../memory_system/memory_query_planner.py | 219 ---- .../message_collection_processor.py | 75 -- .../message_collection_storage.py | 193 --- .../memory_system/vector_memory_storage_v2.py | 1043 --------------- src/chat/replyer/default_generator.py | 41 +- src/chat/utils/prompt.py | 144 --- src/main.py | 22 +- src/memory_graph/config.py | 17 +- src/memory_graph/manager.py | 26 +- .../planner/plan_filter.py | 29 +- template/bot_config_template.toml | 28 +- 22 files changed, 77 insertions(+), 6428 deletions(-) delete mode 100644 src/chat/memory_system/__init__.py delete mode 100644 src/chat/memory_system/enhanced_memory_activator.py delete mode 100644 src/chat/memory_system/hippocampus_sampler.py delete mode 100644 src/chat/memory_system/memory_activator_new.py delete mode 100644 src/chat/memory_system/memory_builder.py delete mode 100644 src/chat/memory_system/memory_chunk.py delete mode 100644 src/chat/memory_system/memory_forgetting_engine.py delete mode 100644 src/chat/memory_system/memory_formatter.py delete mode 100644 src/chat/memory_system/memory_fusion.py delete mode 100644 src/chat/memory_system/memory_manager.py delete mode 100644 src/chat/memory_system/memory_metadata_index.py delete mode 100644 src/chat/memory_system/memory_query_planner.py delete mode 100644 src/chat/memory_system/message_collection_processor.py delete mode 100644 src/chat/memory_system/message_collection_storage.py delete mode 100644 src/chat/memory_system/vector_memory_storage_v2.py diff --git a/src/chat/memory_system/__init__.py b/src/chat/memory_system/__init__.py deleted file mode 100644 index 970cdef21..000000000 --- a/src/chat/memory_system/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -简化记忆系统模块 -移除即时记忆和长期记忆分类，实现统一记忆架构和智能遗忘机制 -""" - -# 核心数据结构 -# 激活器 -from .enhanced_memory_activator import MemoryActivator, enhanced_memory_activator, memory_activator -from .memory_chunk import ( - ConfidenceLevel, - ContentStructure, - ImportanceLevel, - MemoryChunk, - MemoryMetadata, - MemoryType, - create_memory_chunk, -) - -# 兼容性别名 -from .memory_chunk import MemoryChunk as Memory - -# 遗忘引擎 -from .memory_forgetting_engine import ForgettingConfig, MemoryForgettingEngine, get_memory_forgetting_engine -from .memory_formatter import format_memories_bracket_style - -# 记忆管理器 -from .memory_manager import MemoryManager, MemoryResult, memory_manager - -# 记忆核心系统 -from .memory_system import MemorySystem, MemorySystemConfig, get_memory_system, initialize_memory_system - -# Vector DB存储系统 -from .vector_memory_storage_v2 import VectorMemoryStorage, VectorStorageConfig, get_vector_memory_storage - -__all__ = [ - "ConfidenceLevel", - "ContentStructure", - "ForgettingConfig", - "ImportanceLevel", - "Memory", # 兼容性别名 - # 激活器 - "MemoryActivator", - # 核心数据结构 - "MemoryChunk", - # 遗忘引擎 - "MemoryForgettingEngine", - # 记忆管理器 - "MemoryManager", - "MemoryMetadata", - "MemoryResult", - # 记忆系统 - "MemorySystem", - "MemorySystemConfig", - "MemoryType", - # Vector DB存储 - "VectorMemoryStorage", - "VectorStorageConfig", - "create_memory_chunk", - "enhanced_memory_activator", # 兼容性别名 - # 格式化工具 - "format_memories_bracket_style", - "get_memory_forgetting_engine", - "get_memory_system", - "get_vector_memory_storage", - "initialize_memory_system", - "memory_activator", - "memory_manager", -] - -# 版本信息 -__version__ = "3.0.0" -__author__ = "MoFox Team" -__description__ = "简化记忆系统 - 统一记忆架构与智能遗忘机制" diff --git a/src/chat/memory_system/enhanced_memory_activator.py b/src/chat/memory_system/enhanced_memory_activator.py deleted file mode 100644 index 22b44c7a1..000000000 --- a/src/chat/memory_system/enhanced_memory_activator.py +++ /dev/null @@ -1,240 +0,0 @@ -""" -记忆激活器 -记忆系统的激活器组件 -""" - -import difflib -from datetime import datetime - -import orjson -from json_repair import repair_json - -from src.chat.memory_system.memory_manager import MemoryResult -from src.chat.utils.prompt import Prompt, global_prompt_manager -from src.common.logger import get_logger -from src.config.config import global_config, model_config -from src.llm_models.utils_model import LLMRequest - -logger = get_logger("memory_activator") - - -def get_keywords_from_json(json_str) -> list: - """ - 从JSON字符串中提取关键词列表 - - Args: - json_str: JSON格式的字符串 - - Returns: - List[str]: 关键词列表 - """ - try: - # 使用repair_json修复JSON格式 - fixed_json = repair_json(json_str) - - # 如果repair_json返回的是字符串，需要解析为Python对象 - result = orjson.loads(fixed_json) if isinstance(fixed_json, str) else fixed_json - return result.get("keywords", []) - except Exception as e: - logger.error(f"解析关键词JSON失败: {e}") - return [] - - -def init_prompt(): - # --- Memory Activator Prompt --- - memory_activator_prompt = """ - 你是一个记忆分析器，你需要根据以下信息来进行记忆检索 - - 以下是一段聊天记录，请根据这些信息，总结出几个关键词作为记忆检索的触发词 - - 聊天记录: - {obs_info_text} - - 用户想要回复的消息: - {target_message} - - 历史关键词（请避免重复提取这些关键词）： - {cached_keywords} - - 请输出一个json格式，包含以下字段： - {{ - "keywords": ["关键词1", "关键词2", "关键词3",......] - }} - - 不要输出其他多余内容，只输出json格式就好 - """ - - Prompt(memory_activator_prompt, "memory_activator_prompt") - - -class MemoryActivator: - """记忆激活器""" - - def __init__(self): - self.key_words_model = LLMRequest( - model_set=model_config.model_task_config.utils_small, - request_type="memory.activator", - ) - - self.running_memory = [] - self.cached_keywords = set() # 用于缓存历史关键词 - self.last_memory_query_time = 0 # 上次查询记忆的时间 - - async def activate_memory_with_chat_history(self, target_message, chat_history_prompt) -> list[dict]: - """ - 激活记忆 - """ - # 如果记忆系统被禁用，直接返回空列表 - if not global_config.memory.enable_memory: - return [] - - # 将缓存的关键词转换为字符串，用于prompt - cached_keywords_str = ", ".join(self.cached_keywords) if self.cached_keywords else "暂无历史关键词" - - prompt = await global_prompt_manager.format_prompt( - "memory_activator_prompt", - obs_info_text=chat_history_prompt, - target_message=target_message, - cached_keywords=cached_keywords_str, - ) - - # 生成关键词 - response, (reasoning_content, model_name, _) = await self.key_words_model.generate_response_async( - prompt, temperature=0.5 - ) - keywords = list(get_keywords_from_json(response)) - - # 更新关键词缓存 - if keywords: - # 限制缓存大小，最多保留10个关键词 - if len(self.cached_keywords) > 10: - # 转换为列表，移除最早的关键词 - cached_list = list(self.cached_keywords) - self.cached_keywords = set(cached_list[-8:]) - - # 添加新的关键词到缓存 - self.cached_keywords.update(keywords) - - logger.debug(f"记忆关键词: {self.cached_keywords}") - - # 使用记忆系统获取相关记忆 - memory_results = await self._query_unified_memory(keywords, target_message) - - # 处理和记忆结果 - if memory_results: - for result in memory_results: - # 检查是否已存在相似内容的记忆 - exists = any( - m["content"] == result.content - or difflib.SequenceMatcher(None, m["content"], result.content).ratio() >= 0.7 - for m in self.running_memory - ) - if not exists: - memory_entry = { - "topic": result.memory_type, - "content": result.content, - "timestamp": datetime.fromtimestamp(result.timestamp).isoformat(), - "duration": 1, - "confidence": result.confidence, - "importance": result.importance, - "source": result.source, - "relevance_score": result.relevance_score, # 添加相关度评分 - } - self.running_memory.append(memory_entry) - logger.debug(f"添加新记忆: {result.memory_type} - {result.content}") - - # 激活时，所有已有记忆的duration+1，达到3则移除 - for m in self.running_memory[:]: - m["duration"] = m.get("duration", 1) + 1 - self.running_memory = [m for m in self.running_memory if m["duration"] < 3] - - # 限制同时加载的记忆条数，最多保留最后5条 - if len(self.running_memory) > 5: - self.running_memory = self.running_memory[-5:] - - return self.running_memory - - async def _query_unified_memory(self, keywords: list[str], query_text: str) -> list[MemoryResult]: - """查询统一记忆系统""" - try: - # 使用记忆系统 - from src.chat.memory_system.memory_system import get_memory_system - - memory_system = get_memory_system() - if not memory_system or memory_system.status.value != "ready": - logger.warning("记忆系统未就绪") - return [] - - # 构建查询上下文 - context = {"keywords": keywords, "query_intent": "conversation_response"} - - # 查询记忆 - memories = await memory_system.retrieve_relevant_memories( - query_text=query_text, - user_id="global", # 使用全局作用域 - context=context, - limit=5, - ) - - # 转换为 MemoryResult 格式 - memory_results = [] - for memory in memories: - result = MemoryResult( - content=memory.display, - memory_type=memory.memory_type.value, - confidence=memory.metadata.confidence.value, - importance=memory.metadata.importance.value, - timestamp=memory.metadata.created_at, - source="unified_memory", - relevance_score=memory.metadata.relevance_score, - ) - memory_results.append(result) - - logger.debug(f"统一记忆查询返回 {len(memory_results)} 条结果") - return memory_results - - except Exception as e: - logger.error(f"查询统一记忆失败: {e}") - return [] - - async def get_instant_memory(self, target_message: str, chat_id: str) -> str | None: - """ - 获取即时记忆 - 兼容原有接口（使用统一存储） - """ - try: - # 使用统一存储系统获取相关记忆 - from src.chat.memory_system.memory_system import get_memory_system - - memory_system = get_memory_system() - if not memory_system or memory_system.status.value != "ready": - return None - - context = {"query_intent": "instant_response", "chat_id": chat_id} - - memories = await memory_system.retrieve_relevant_memories( - query_text=target_message, user_id="global", context=context, limit=1 - ) - - if memories: - return memories[0].display - - return None - - except Exception as e: - logger.error(f"获取即时记忆失败: {e}") - return None - - def clear_cache(self): - """清除缓存""" - self.cached_keywords.clear() - self.running_memory.clear() - logger.debug("记忆激活器缓存已清除") - - -# 创建全局实例 -memory_activator = MemoryActivator() - -# 兼容性别名 -enhanced_memory_activator = memory_activator - -init_prompt() diff --git a/src/chat/memory_system/hippocampus_sampler.py b/src/chat/memory_system/hippocampus_sampler.py deleted file mode 100644 index c3e0f856f..000000000 --- a/src/chat/memory_system/hippocampus_sampler.py +++ /dev/null @@ -1,721 +0,0 @@ -""" -海马体双峰分布采样器 -基于旧版海马体的采样策略，适配新版记忆系统 -实现低消耗、高效率的记忆采样模式 -""" - -import asyncio -import random -import time -from dataclasses import dataclass -from datetime import datetime, timedelta -from typing import Any - -import numpy as np - -from src.chat.utils.chat_message_builder import ( - build_readable_messages, - get_raw_msg_by_timestamp, - get_raw_msg_by_timestamp_with_chat, -) -from src.chat.utils.utils import translate_timestamp_to_human_readable -from src.common.logger import get_logger -from src.config.config import global_config -from src.llm_models.utils_model import LLMRequest - -logger = get_logger(__name__) - -# 全局背景任务集合 -_background_tasks = set() - - -@dataclass -class HippocampusSampleConfig: - """海马体采样配置""" - - # 双峰分布参数 - recent_mean_hours: float = 12.0 # 近期分布均值（小时） - recent_std_hours: float = 8.0 # 近期分布标准差（小时） - recent_weight: float = 0.7 # 近期分布权重 - - distant_mean_hours: float = 48.0 # 远期分布均值（小时） - distant_std_hours: float = 24.0 # 远期分布标准差（小时） - distant_weight: float = 0.3 # 远期分布权重 - - # 采样参数 - total_samples: int = 50 # 总采样数 - sample_interval: int = 1800 # 采样间隔（秒） - max_sample_length: int = 30 # 每次采样的最大消息数量 - batch_size: int = 5 # 批处理大小 - - @classmethod - def from_global_config(cls) -> "HippocampusSampleConfig": - """从全局配置创建海马体采样配置""" - config = global_config.memory.hippocampus_distribution_config - return cls( - recent_mean_hours=config[0], - recent_std_hours=config[1], - recent_weight=config[2], - distant_mean_hours=config[3], - distant_std_hours=config[4], - distant_weight=config[5], - total_samples=global_config.memory.hippocampus_sample_size, - sample_interval=global_config.memory.hippocampus_sample_interval, - max_sample_length=global_config.memory.hippocampus_batch_size, - batch_size=global_config.memory.hippocampus_batch_size, - ) - - -class HippocampusSampler: - """海马体双峰分布采样器""" - - def __init__(self, memory_system=None): - self.memory_system = memory_system - self.config = HippocampusSampleConfig.from_global_config() - self.last_sample_time = 0 - self.is_running = False - - # 记忆构建模型 - self.memory_builder_model: LLMRequest | None = None - - # 统计信息 - self.sample_count = 0 - self.success_count = 0 - self.last_sample_results: list[dict[str, Any]] = [] - - async def initialize(self): - """初始化采样器""" - try: - # 初始化LLM模型 - from src.config.config import model_config - - task_config = getattr(model_config.model_task_config, "utils", None) - if task_config: - self.memory_builder_model = LLMRequest(model_set=task_config, request_type="memory.hippocampus_build") - task = asyncio.create_task(self.start_background_sampling()) - _background_tasks.add(task) - task.add_done_callback(_background_tasks.discard) - logger.info("✅ 海马体采样器初始化成功") - else: - raise RuntimeError("未找到记忆构建模型配置") - - except Exception as e: - logger.error(f"❌ 海马体采样器初始化失败: {e}") - raise - - def generate_time_samples(self) -> list[datetime]: - """生成双峰分布的时间采样点""" - # 计算每个分布的样本数 - recent_samples = max(1, int(self.config.total_samples * self.config.recent_weight)) - distant_samples = max(1, self.config.total_samples - recent_samples) - - # 生成两个正态分布的小时偏移 - recent_offsets = np.random.normal( - loc=self.config.recent_mean_hours, scale=self.config.recent_std_hours, size=recent_samples - ) - distant_offsets = np.random.normal( - loc=self.config.distant_mean_hours, scale=self.config.distant_std_hours, size=distant_samples - ) - - # 合并两个分布的偏移 - all_offsets = np.concatenate([recent_offsets, distant_offsets]) - - # 转换为时间戳（使用绝对值确保时间点在过去） - base_time = datetime.now() - timestamps = [base_time - timedelta(hours=abs(offset)) for offset in all_offsets] - - # 按时间排序（从最早到最近） - return sorted(timestamps) - - async def collect_message_samples(self, target_timestamp: float) -> list[dict[str, Any]] | None: - """收集指定时间戳附近的消息样本""" - try: - # 随机时间窗口：5-30分钟 - time_window_seconds = random.randint(300, 1800) - - # 尝试3次获取消息 - for attempt in range(3): - timestamp_start = target_timestamp - timestamp_end = target_timestamp + time_window_seconds - - # 获取单条消息作为锚点 - anchor_messages = await get_raw_msg_by_timestamp( - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - limit=1, - limit_mode="earliest", - ) - - if not anchor_messages: - target_timestamp -= 120 # 向前调整2分钟 - continue - - anchor_message = anchor_messages[0] - chat_id = anchor_message.get("chat_id") - - if not chat_id: - continue - - # 获取同聊天的多条消息 - messages = await get_raw_msg_by_timestamp_with_chat( - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - limit=self.config.max_sample_length, - limit_mode="earliest", - chat_id=chat_id, - ) - - if messages and len(messages) >= 2: # 至少需要2条消息 - # 过滤掉已经记忆过的消息 - filtered_messages = [ - msg - for msg in messages - if msg.get("memorized_times", 0) < 2 # 最多记忆2次 - ] - - if filtered_messages: - logger.debug(f"成功收集 {len(filtered_messages)} 条消息样本") - return filtered_messages - - target_timestamp -= 120 # 向前调整再试 - - logger.debug(f"时间戳 {target_timestamp} 附近未找到有效消息样本") - return None - - except Exception as e: - logger.error(f"收集消息样本失败: {e}") - return None - - async def build_memory_from_samples(self, messages: list[dict[str, Any]], target_timestamp: float) -> str | None: - """从消息样本构建记忆""" - if not messages or not self.memory_system or not self.memory_builder_model: - return None - - try: - # 构建可读消息文本 - readable_text = await build_readable_messages( - messages, - merge_messages=True, - timestamp_mode="normal_no_YMD", - replace_bot_name=False, - ) - - if not readable_text: - logger.warning("无法从消息样本生成可读文本") - return None - - # 直接使用对话文本，不添加系统标识符 - input_text = readable_text - - logger.debug(f"开始构建记忆，文本长度: {len(input_text)}") - - # 构建上下文 - context = { - "user_id": "hippocampus_sampler", - "timestamp": time.time(), - "source": "hippocampus_sampling", - "message_count": len(messages), - "sample_mode": "bimodal_distribution", - "is_hippocampus_sample": True, # 标识为海马体样本 - "bypass_value_threshold": True, # 绕过价值阈值检查 - "hippocampus_sample_time": target_timestamp, # 记录样本时间 - } - - # 使用记忆系统构建记忆（绕过构建间隔检查） - memories = await self.memory_system.build_memory_from_conversation( - conversation_text=input_text, - context=context, - timestamp=time.time(), - bypass_interval=True, # 海马体采样器绕过构建间隔限制 - ) - - if memories: - memory_count = len(memories) - self.success_count += 1 - - # 记录采样结果 - result = { - "timestamp": time.time(), - "memory_count": memory_count, - "message_count": len(messages), - "text_preview": readable_text[:100] + "..." if len(readable_text) > 100 else readable_text, - "memory_types": [m.memory_type.value for m in memories], - } - self.last_sample_results.append(result) - - # 限制结果历史长度 - if len(self.last_sample_results) > 10: - self.last_sample_results.pop(0) - - logger.info(f"✅ 海马体采样成功构建 {memory_count} 条记忆") - return f"构建{memory_count}条记忆" - else: - logger.debug("海马体采样未生成有效记忆") - return None - - except Exception as e: - logger.error(f"海马体采样构建记忆失败: {e}") - return None - - async def perform_sampling_cycle(self) -> dict[str, Any]: - """执行一次完整的采样周期（优化版：批量融合构建）""" - if not self.should_sample(): - return {"status": "skipped", "reason": "interval_not_met"} - - start_time = time.time() - self.sample_count += 1 - - try: - # 生成时间采样点 - time_samples = self.generate_time_samples() - logger.debug(f"生成 {len(time_samples)} 个时间采样点") - - # 记录时间采样点（调试用） - readable_timestamps = [ - translate_timestamp_to_human_readable(int(ts.timestamp()), mode="normal") - for ts in time_samples[:5] # 只显示前5个 - ] - logger.debug(f"时间采样点示例: {readable_timestamps}") - - # 第一步：批量收集所有消息样本 - logger.debug("开始批量收集消息样本...") - collected_messages = await self._collect_all_message_samples(time_samples) - - if not collected_messages: - logger.info("未收集到有效消息样本，跳过本次采样") - self.last_sample_time = time.time() - return { - "status": "success", - "sample_count": self.sample_count, - "success_count": self.success_count, - "processed_samples": len(time_samples), - "successful_builds": 0, - "duration": time.time() - start_time, - "samples_generated": len(time_samples), - "message": "未收集到有效消息样本", - } - - logger.info(f"收集到 {len(collected_messages)} 组消息样本") - - # 第二步：融合和去重消息 - logger.debug("开始融合和去重消息...") - fused_messages = await self._fuse_and_deduplicate_messages(collected_messages) - - if not fused_messages: - logger.info("消息融合后为空，跳过记忆构建") - self.last_sample_time = time.time() - return { - "status": "success", - "sample_count": self.sample_count, - "success_count": self.success_count, - "processed_samples": len(time_samples), - "successful_builds": 0, - "duration": time.time() - start_time, - "samples_generated": len(time_samples), - "message": "消息融合后为空", - } - - logger.info(f"融合后得到 {len(fused_messages)} 组有效消息") - - # 第三步：一次性构建记忆 - logger.debug("开始批量构建记忆...") - build_result = await self._build_batch_memory(fused_messages, time_samples) - - # 更新最后采样时间 - self.last_sample_time = time.time() - - duration = time.time() - start_time - result = { - "status": "success", - "sample_count": self.sample_count, - "success_count": self.success_count, - "processed_samples": len(time_samples), - "successful_builds": build_result.get("memory_count", 0), - "duration": duration, - "samples_generated": len(time_samples), - "messages_collected": len(collected_messages), - "messages_fused": len(fused_messages), - "optimization_mode": "batch_fusion", - } - - logger.info( - f"✅ 海马体采样周期完成（批量融合模式） | " - f"采样点: {len(time_samples)} | " - f"收集消息: {len(collected_messages)} | " - f"融合消息: {len(fused_messages)} | " - f"构建记忆: {build_result.get('memory_count', 0)} | " - f"耗时: {duration:.2f}s" - ) - - return result - - except Exception as e: - logger.error(f"❌ 海马体采样周期失败: {e}") - return { - "status": "error", - "error": str(e), - "sample_count": self.sample_count, - "duration": time.time() - start_time, - } - - async def _collect_all_message_samples(self, time_samples: list[datetime]) -> list[list[dict[str, Any]]]: - """批量收集所有时间点的消息样本""" - collected_messages = [] - max_concurrent = min(5, len(time_samples)) # 提高并发数到5 - - for i in range(0, len(time_samples), max_concurrent): - batch = time_samples[i : i + max_concurrent] - tasks = [] - - # 创建并发收集任务 - for timestamp in batch: - target_ts = timestamp.timestamp() - task = self.collect_message_samples(target_ts) - tasks.append(task) - - # 执行并发收集 - results = await asyncio.gather(*tasks, return_exceptions=True) - - # 处理收集结果 - for result in results: - if isinstance(result, list) and result: - collected_messages.append(result) - elif isinstance(result, Exception): - logger.debug(f"消息收集异常: {result}") - - # 批次间短暂延迟 - if i + max_concurrent < len(time_samples): - await asyncio.sleep(0.5) - - return collected_messages - - async def _fuse_and_deduplicate_messages( - self, collected_messages: list[list[dict[str, Any]]] - ) -> list[list[dict[str, Any]]]: - """融合和去重消息样本""" - if not collected_messages: - return [] - - try: - # 展平所有消息 - all_messages = [] - for message_group in collected_messages: - all_messages.extend(message_group) - - logger.debug(f"展开后总消息数: {len(all_messages)}") - - # 去重逻辑：基于消息内容和时间戳 - unique_messages = [] - seen_hashes = set() - - for message in all_messages: - # 创建消息哈希用于去重 - content = message.get("processed_plain_text", "") or message.get("display_message", "") - timestamp = message.get("time", 0) - chat_id = message.get("chat_id", "") - - # 简单哈希：内容前50字符 + 时间戳(精确到分钟) + 聊天ID - hash_key = f"{content[:50]}_{int(timestamp // 60)}_{chat_id}" - - if hash_key not in seen_hashes and len(content.strip()) > 10: - seen_hashes.add(hash_key) - unique_messages.append(message) - - logger.debug(f"去重后消息数: {len(unique_messages)}") - - # 按时间排序 - unique_messages.sort(key=lambda x: x.get("time", 0)) - - # 按聊天ID分组重新组织 - chat_groups = {} - for message in unique_messages: - chat_id = message.get("chat_id", "unknown") - if chat_id not in chat_groups: - chat_groups[chat_id] = [] - chat_groups[chat_id].append(message) - - # 合并相邻时间范围内的消息 - fused_groups = [] - for chat_id, messages in chat_groups.items(): - fused_groups.extend(self._merge_adjacent_messages(messages)) - - logger.debug(f"融合后消息组数: {len(fused_groups)}") - return fused_groups - - except Exception as e: - logger.error(f"消息融合失败: {e}") - # 返回原始消息组作为备选 - return collected_messages[:5] # 限制返回数量 - - def _merge_adjacent_messages( - self, messages: list[dict[str, Any]], time_gap: int = 1800 - ) -> list[list[dict[str, Any]]]: - """合并时间间隔内的消息""" - if not messages: - return [] - - merged_groups = [] - current_group = [messages[0]] - - for i in range(1, len(messages)): - current_time = messages[i].get("time", 0) - prev_time = current_group[-1].get("time", 0) - - # 如果时间间隔小于阈值，合并到当前组 - if current_time - prev_time <= time_gap: - current_group.append(messages[i]) - else: - # 否则开始新组 - merged_groups.append(current_group) - current_group = [messages[i]] - - # 添加最后一组 - merged_groups.append(current_group) - - # 过滤掉只有一条消息的组（除非内容较长） - result_groups = [ - group for group in merged_groups - if len(group) > 1 or any(len(msg.get("processed_plain_text", "")) > 100 for msg in group) - ] - - return result_groups - - async def _build_batch_memory( - self, fused_messages: list[list[dict[str, Any]]], time_samples: list[datetime] - ) -> dict[str, Any]: - """批量构建记忆""" - if not fused_messages: - return {"memory_count": 0, "memories": []} - - try: - total_memories = [] - total_memory_count = 0 - - # 构建融合后的文本 - batch_input_text = await self._build_fused_conversation_text(fused_messages) - - if not batch_input_text: - logger.warning("无法构建融合文本，尝试单独构建") - # 备选方案：分别构建 - return await self._fallback_individual_build(fused_messages) - - # 创建批量上下文 - batch_context = { - "user_id": "hippocampus_batch_sampler", - "timestamp": time.time(), - "source": "hippocampus_batch_sampling", - "message_groups_count": len(fused_messages), - "total_messages": sum(len(group) for group in fused_messages), - "sample_count": len(time_samples), - "is_hippocampus_sample": True, - "bypass_value_threshold": True, - "optimization_mode": "batch_fusion", - } - - logger.debug(f"批量构建记忆，文本长度: {len(batch_input_text)}") - - # 一次性构建记忆 - memories = await self.memory_system.build_memory_from_conversation( - conversation_text=batch_input_text, context=batch_context, timestamp=time.time(), bypass_interval=True - ) - - if memories: - memory_count = len(memories) - self.success_count += 1 - total_memory_count += memory_count - total_memories.extend(memories) - - logger.info(f"✅ 批量海马体采样成功构建 {memory_count} 条记忆") - else: - logger.debug("批量海马体采样未生成有效记忆") - - # 记录采样结果 - result = { - "timestamp": time.time(), - "memory_count": total_memory_count, - "message_groups_count": len(fused_messages), - "total_messages": sum(len(group) for group in fused_messages), - "text_preview": batch_input_text[:200] + "..." if len(batch_input_text) > 200 else batch_input_text, - "memory_types": [m.memory_type.value for m in total_memories], - } - - self.last_sample_results.append(result) - - # 限制结果历史长度 - if len(self.last_sample_results) > 10: - self.last_sample_results.pop(0) - - return {"memory_count": total_memory_count, "memories": total_memories, "result": result} - - except Exception as e: - logger.error(f"批量构建记忆失败: {e}") - return {"memory_count": 0, "error": str(e)} - - async def _build_fused_conversation_text(self, fused_messages: list[list[dict[str, Any]]]) -> str: - """构建融合后的对话文本""" - try: - conversation_parts = [] - - for group_idx, message_group in enumerate(fused_messages): - if not message_group: - continue - - # 为每个消息组添加分隔符 - group_header = f"\n=== 对话片段 {group_idx + 1} ===" - conversation_parts.append(group_header) - - # 构建可读消息 - group_text = await build_readable_messages( - message_group, - merge_messages=True, - timestamp_mode="normal_no_YMD", - replace_bot_name=False, - ) - - if group_text and len(group_text.strip()) > 10: - conversation_parts.append(group_text.strip()) - - return "\n".join(conversation_parts) - - except Exception as e: - logger.error(f"构建融合文本失败: {e}") - return "" - - async def _fallback_individual_build(self, fused_messages: list[list[dict[str, Any]]]) -> dict[str, Any]: - """备选方案：单独构建每个消息组""" - total_memories = [] - total_count = 0 - - for group in fused_messages[:5]: # 限制最多5组 - try: - memories = await self.build_memory_from_samples(group, time.time()) - if memories: - total_memories.extend(memories) - total_count += len(memories) - except Exception as e: - logger.debug(f"单独构建失败: {e}") - - return {"memory_count": total_count, "memories": total_memories, "fallback_mode": True} - - async def process_sample_timestamp(self, target_timestamp: float) -> str | None: - """处理单个时间戳采样（保留作为备选方法）""" - try: - # 收集消息样本 - messages = await self.collect_message_samples(target_timestamp) - if not messages: - return None - - # 构建记忆 - result = await self.build_memory_from_samples(messages, target_timestamp) - return result - - except Exception as e: - logger.debug(f"处理时间戳采样失败 {target_timestamp}: {e}") - return None - - def should_sample(self) -> bool: - """检查是否应该进行采样""" - current_time = time.time() - - # 检查时间间隔 - if current_time - self.last_sample_time < self.config.sample_interval: - return False - - # 检查是否已初始化 - if not self.memory_builder_model: - logger.warning("海马体采样器未初始化") - return False - - return True - - async def start_background_sampling(self): - """启动后台采样""" - if self.is_running: - logger.warning("海马体后台采样已在运行") - return - - self.is_running = True - logger.info("🚀 启动海马体后台采样任务") - - try: - while self.is_running: - try: - # 执行采样周期 - result = await self.perform_sampling_cycle() - - # 如果是跳过状态，短暂睡眠 - if result.get("status") == "skipped": - await asyncio.sleep(60) # 1分钟后重试 - else: - # 正常等待下一个采样间隔 - await asyncio.sleep(self.config.sample_interval) - - except Exception as e: - logger.error(f"海马体后台采样异常: {e}") - await asyncio.sleep(300) # 异常时等待5分钟 - - except asyncio.CancelledError: - logger.info("海马体后台采样任务被取消") - finally: - self.is_running = False - - def stop_background_sampling(self): - """停止后台采样""" - self.is_running = False - logger.info("🛑 停止海马体后台采样任务") - - def get_sampling_stats(self) -> dict[str, Any]: - """获取采样统计信息""" - success_rate = (self.success_count / self.sample_count * 100) if self.sample_count > 0 else 0 - - # 计算最近的平均数据 - recent_avg_messages = 0 - recent_avg_memory_count = 0 - if self.last_sample_results: - recent_results = self.last_sample_results[-5:] # 最近5次 - recent_avg_messages = sum(r.get("total_messages", 0) for r in recent_results) / len(recent_results) - recent_avg_memory_count = sum(r.get("memory_count", 0) for r in recent_results) / len(recent_results) - - return { - "is_running": self.is_running, - "sample_count": self.sample_count, - "success_count": self.success_count, - "success_rate": f"{success_rate:.1f}%", - "last_sample_time": self.last_sample_time, - "optimization_mode": "batch_fusion", # 显示优化模式 - "performance_metrics": { - "avg_messages_per_sample": f"{recent_avg_messages:.1f}", - "avg_memories_per_sample": f"{recent_avg_memory_count:.1f}", - "fusion_efficiency": f"{(recent_avg_messages / max(recent_avg_memory_count, 1)):.1f}x" - if recent_avg_messages > 0 - else "N/A", - }, - "config": { - "sample_interval": self.config.sample_interval, - "total_samples": self.config.total_samples, - "recent_weight": f"{self.config.recent_weight:.1%}", - "distant_weight": f"{self.config.distant_weight:.1%}", - "max_concurrent": 5, # 批量模式并发数 - "fusion_time_gap": "30分钟", # 消息融合时间间隔 - }, - "recent_results": self.last_sample_results[-5:], # 最近5次结果 - } - - -# 全局海马体采样器实例 -_hippocampus_sampler: HippocampusSampler | None = None - - -def get_hippocampus_sampler(memory_system=None) -> HippocampusSampler: - """获取全局海马体采样器实例""" - global _hippocampus_sampler - if _hippocampus_sampler is None: - _hippocampus_sampler = HippocampusSampler(memory_system) - return _hippocampus_sampler - - -async def initialize_hippocampus_sampler(memory_system=None) -> HippocampusSampler: - """初始化全局海马体采样器""" - sampler = get_hippocampus_sampler(memory_system) - await sampler.initialize() - return sampler diff --git a/src/chat/memory_system/memory_activator_new.py b/src/chat/memory_system/memory_activator_new.py deleted file mode 100644 index 0b4e9a938..000000000 --- a/src/chat/memory_system/memory_activator_new.py +++ /dev/null @@ -1,238 +0,0 @@ -""" -记忆激活器 -记忆系统的激活器组件 -""" - -import difflib -from datetime import datetime - -import orjson -from json_repair import repair_json - -from src.chat.memory_system.memory_manager import MemoryResult -from src.chat.utils.prompt import Prompt, global_prompt_manager -from src.common.logger import get_logger -from src.config.config import global_config, model_config -from src.llm_models.utils_model import LLMRequest - -logger = get_logger("memory_activator") - - -def get_keywords_from_json(json_str) -> list: - """ - 从JSON字符串中提取关键词列表 - - Args: - json_str: JSON格式的字符串 - - Returns: - List[str]: 关键词列表 - """ - try: - # 使用repair_json修复JSON格式 - fixed_json = repair_json(json_str) - - # 如果repair_json返回的是字符串，需要解析为Python对象 - result = orjson.loads(fixed_json) if isinstance(fixed_json, str) else fixed_json - return result.get("keywords", []) - except Exception as e: - logger.error(f"解析关键词JSON失败: {e}") - return [] - - -def init_prompt(): - # --- Memory Activator Prompt --- - memory_activator_prompt = """ - 你是一个记忆分析器，你需要根据以下信息来进行记忆检索 - - 以下是一段聊天记录，请根据这些信息，总结出几个关键词作为记忆检索的触发词 - - 聊天记录: - {obs_info_text} - - 用户想要回复的消息: - {target_message} - - 历史关键词（请避免重复提取这些关键词）： - {cached_keywords} - - 请输出一个json格式，包含以下字段： - {{ - "keywords": ["关键词1", "关键词2", "关键词3",......] - }} - - 不要输出其他多余内容，只输出json格式就好 - """ - - Prompt(memory_activator_prompt, "memory_activator_prompt") - - -class MemoryActivator: - """记忆激活器""" - - def __init__(self): - self.key_words_model = LLMRequest( - model_set=model_config.model_task_config.utils_small, - request_type="memory.activator", - ) - - self.running_memory = [] - self.cached_keywords = set() # 用于缓存历史关键词 - self.last_memory_query_time = 0 # 上次查询记忆的时间 - - async def activate_memory_with_chat_history(self, target_message, chat_history_prompt) -> list[dict]: - """ - 激活记忆 - """ - # 如果记忆系统被禁用，直接返回空列表 - if not global_config.memory.enable_memory: - return [] - - # 将缓存的关键词转换为字符串，用于prompt - cached_keywords_str = ", ".join(self.cached_keywords) if self.cached_keywords else "暂无历史关键词" - - prompt = await global_prompt_manager.format_prompt( - "memory_activator_prompt", - obs_info_text=chat_history_prompt, - target_message=target_message, - cached_keywords=cached_keywords_str, - ) - - # 生成关键词 - response, (reasoning_content, model_name, _) = await self.key_words_model.generate_response_async( - prompt, temperature=0.5 - ) - keywords = list(get_keywords_from_json(response)) - - # 更新关键词缓存 - if keywords: - # 限制缓存大小，最多保留10个关键词 - if len(self.cached_keywords) > 10: - # 转换为列表，移除最早的关键词 - cached_list = list(self.cached_keywords) - self.cached_keywords = set(cached_list[-8:]) - - # 添加新的关键词到缓存 - self.cached_keywords.update(keywords) - - logger.debug(f"记忆关键词: {self.cached_keywords}") - - # 使用记忆系统获取相关记忆 - memory_results = await self._query_unified_memory(keywords, target_message) - - # 处理和记忆结果 - if memory_results: - for result in memory_results: - # 检查是否已存在相似内容的记忆 - exists = any( - m["content"] == result.content - or difflib.SequenceMatcher(None, m["content"], result.content).ratio() >= 0.7 - for m in self.running_memory - ) - if not exists: - memory_entry = { - "topic": result.memory_type, - "content": result.content, - "timestamp": datetime.fromtimestamp(result.timestamp).isoformat(), - "duration": 1, - "confidence": result.confidence, - "importance": result.importance, - "source": result.source, - "relevance_score": result.relevance_score, # 添加相关度评分 - } - self.running_memory.append(memory_entry) - logger.debug(f"添加新记忆: {result.memory_type} - {result.content}") - - # 激活时，所有已有记忆的duration+1，达到3则移除 - for m in self.running_memory[:]: - m["duration"] = m.get("duration", 1) + 1 - self.running_memory = [m for m in self.running_memory if m["duration"] < 3] - - # 限制同时加载的记忆条数，最多保留最后5条 - if len(self.running_memory) > 5: - self.running_memory = self.running_memory[-5:] - - return self.running_memory - - async def _query_unified_memory(self, keywords: list[str], query_text: str) -> list[MemoryResult]: - """查询统一记忆系统""" - try: - # 使用记忆系统 - from src.chat.memory_system.memory_system import get_memory_system - - memory_system = get_memory_system() - if not memory_system or memory_system.status.value != "ready": - logger.warning("记忆系统未就绪") - return [] - - # 构建查询上下文 - context = {"keywords": keywords, "query_intent": "conversation_response"} - - # 查询记忆 - memories = await memory_system.retrieve_relevant_memories( - query_text=query_text, - user_id="global", # 使用全局作用域 - context=context, - limit=5, - ) - - # 转换为 MemoryResult 格式 - memory_results = [] - for memory in memories: - result = MemoryResult( - content=memory.display, - memory_type=memory.memory_type.value, - confidence=memory.metadata.confidence.value, - importance=memory.metadata.importance.value, - timestamp=memory.metadata.created_at, - source="unified_memory", - relevance_score=memory.metadata.relevance_score, - ) - memory_results.append(result) - - logger.debug(f"统一记忆查询返回 {len(memory_results)} 条结果") - return memory_results - - except Exception as e: - logger.error(f"查询统一记忆失败: {e}") - return [] - - async def get_instant_memory(self, target_message: str, chat_id: str) -> str | None: - """ - 获取即时记忆 - 兼容原有接口（使用统一存储） - """ - try: - # 使用统一存储系统获取相关记忆 - from src.chat.memory_system.memory_system import get_memory_system - - memory_system = get_memory_system() - if not memory_system or memory_system.status.value != "ready": - return None - - context = {"query_intent": "instant_response", "chat_id": chat_id} - - memories = await memory_system.retrieve_relevant_memories( - query_text=target_message, user_id="global", context=context, limit=1 - ) - - if memories: - return memories[0].display - - return None - - except Exception as e: - logger.error(f"获取即时记忆失败: {e}") - return None - - def clear_cache(self): - """清除缓存""" - self.cached_keywords.clear() - self.running_memory.clear() - logger.debug("记忆激活器缓存已清除") - - -# 创建全局实例 -memory_activator = MemoryActivator() - - -init_prompt() diff --git a/src/chat/memory_system/memory_builder.py b/src/chat/memory_system/memory_builder.py deleted file mode 100644 index fba6dfe39..000000000 --- a/src/chat/memory_system/memory_builder.py +++ /dev/null @@ -1,1135 +0,0 @@ -""" -记忆构建模块 -从对话流中提取高质量、结构化记忆单元 -输出格式要求: -{ - "memories": [ - { - "type": "记忆类型", - "display": "一句优雅自然的中文描述，用于直接展示及提示词拼接", - "subject": ["主体1", "主体2"], - "predicate": "谓语(动作/状态)", - "object": "宾语(对象/属性或结构体)", - "keywords": ["关键词1", "关键词2"], - "importance": "重要性等级(1-4)", - "confidence": "置信度(1-4)", - "reasoning": "提取理由" - } - ] -} - -注意： -1. `subject` 可包含多个主体，请用数组表示；若主体不明确，请根据上下文给出最合理的称呼 -2. `display` 字段必填，必须是完整顺畅的自然语言，禁止依赖字符串拼接 -3. 主谓宾用于索引和检索结构化信息，提示词构建仅使用 `display` -4. 只提取确实值得记忆的信息，不要提取琐碎内容 -5. 确保信息准确、具体、有价值 -6. 重要性: 1=低, 2=一般, 3=高, 4=关键；置信度: 1=低, 2=中等, 3=高, 4=已验证 -""" - -import re -import time -from dataclasses import dataclass -from datetime import datetime -from enum import Enum -from typing import Any, TypeVar - -E = TypeVar("E", bound=Enum) - - -import orjson -from json_repair import repair_json - -from src.chat.memory_system.memory_chunk import ( - ConfidenceLevel, - ImportanceLevel, - MemoryChunk, - MemoryType, - create_memory_chunk, -) -from src.common.logger import get_logger -from src.llm_models.utils_model import LLMRequest - -logger = get_logger(__name__) - - -CHINESE_TO_MEMORY_TYPE: dict[str, MemoryType] = { - "个人事实": MemoryType.PERSONAL_FACT, - "事件": MemoryType.EVENT, - "偏好": MemoryType.PREFERENCE, - "观点": MemoryType.OPINION, - "关系": MemoryType.RELATIONSHIP, - "情感": MemoryType.EMOTION, - "知识": MemoryType.KNOWLEDGE, - "技能": MemoryType.SKILL, - "目标": MemoryType.GOAL, - "经验": MemoryType.EXPERIENCE, - "上下文": MemoryType.CONTEXTUAL, -} - - -class ExtractionStrategy(Enum): - """提取策略""" - - LLM_BASED = "llm_based" # 基于LLM的智能提取 - RULE_BASED = "rule_based" # 基于规则的提取 - HYBRID = "hybrid" # 混合策略 - - -@dataclass -class ExtractionResult: - """提取结果""" - - memories: list[MemoryChunk] - confidence_scores: list[float] - extraction_time: float - strategy_used: ExtractionStrategy - - -class MemoryExtractionError(Exception): - """记忆提取过程中发生的不可恢复错误""" - - -class MemoryBuilder: - """记忆构建器""" - - def __init__(self, llm_model: LLMRequest): - self.llm_model = llm_model - self.extraction_stats = { - "total_extractions": 0, - "successful_extractions": 0, - "failed_extractions": 0, - "average_confidence": 0.0, - } - - async def build_memories( - self, conversation_text: str, context: dict[str, Any], user_id: str, timestamp: float - ) -> list[MemoryChunk]: - """从对话中构建记忆""" - start_time = time.time() - - try: - logger.debug(f"开始从对话构建记忆，文本长度: {len(conversation_text)}") - - # 使用LLM提取记忆 - memories = await self._extract_with_llm(conversation_text, context, user_id, timestamp) - - # 后处理和验证 - validated_memories = self._validate_and_enhance_memories(memories, context) - - # 更新统计 - extraction_time = time.time() - start_time - self._update_extraction_stats(len(validated_memories), extraction_time) - - logger.info(f"✅ 成功构建 {len(validated_memories)} 条记忆，耗时 {extraction_time:.2f}秒") - return validated_memories - - except MemoryExtractionError as e: - logger.error(f"❌ 记忆构建失败（响应解析错误）: {e}") - self.extraction_stats["failed_extractions"] += 1 - raise - except Exception as e: - logger.error(f"❌ 记忆构建失败: {e}", exc_info=True) - self.extraction_stats["failed_extractions"] += 1 - raise - - async def _extract_with_llm( - self, text: str, context: dict[str, Any], user_id: str, timestamp: float - ) -> list[MemoryChunk]: - """使用LLM提取记忆""" - try: - prompt = self._build_llm_extraction_prompt(text, context) - - response, _ = await self.llm_model.generate_response_async(prompt, temperature=0.3) - - # 记录原始响应用于调试 - if response: - logger.debug(f"LLM记忆提取原始响应长度: {len(response)}, 前300字符: {response[:300]}") - else: - logger.warning("LLM记忆提取返回空响应") - - # 解析LLM响应 - memories = self._parse_llm_response(response, user_id, timestamp, context) - - return memories - - except MemoryExtractionError: - raise - except Exception as e: - logger.error(f"LLM提取失败: {e}") - raise MemoryExtractionError(str(e)) from e - - def _build_llm_extraction_prompt(self, text: str, context: dict[str, Any]) -> str: - """构建LLM提取提示""" - current_date = datetime.now().strftime("%Y-%m-%d %H:%M:%S") - message_type = context.get("message_type", "normal") - - bot_name = context.get("bot_name") - bot_identity = context.get("bot_identity") - bot_personality = context.get("bot_personality") - bot_personality_side = context.get("bot_personality_side") - bot_aliases = context.get("bot_aliases") or [] - if isinstance(bot_aliases, str): - bot_aliases = [bot_aliases] - - bot_name_display = bot_name or "机器人" - alias_display = "、".join(a for a in bot_aliases if a) or "无" - persona_details = [] - if bot_identity: - persona_details.append(f"身份: {bot_identity}") - if bot_personality: - persona_details.append(f"核心人设: {bot_personality}") - if bot_personality_side: - persona_details.append(f"侧写: {bot_personality_side}") - persona_display = "；".join(persona_details) if persona_details else "无" - - prompt = f""" -你是一个专业的记忆提取专家。请从以下对话中主动识别并提取所有可能重要的信息，特别是包含个人事实、事件、偏好、观点等要素的内容。 - -当前时间: {current_date} -消息类型: {message_type} - -## 🤖 机器人身份（仅供参考，禁止写入记忆） -- 机器人名称: {bot_name_display} -- 别名: {alias_display} -- 机器人人设概述: {persona_display} - -这些信息是机器人的固定设定，可用于帮助你理解对话。你可以在需要时记录机器人自身的状态、行为或设定，但要与用户信息清晰区分，避免误将系统ID写入记忆。 - -请务必遵守以下命名规范： -- 当说话者是机器人时，请使用“{bot_name_display}”或其他明确称呼作为主语； -- 记录关键事实时，请准确标记主体是机器人还是用户，避免混淆。 - -对话内容: -{text} - -## 🎯 重点记忆类型识别指南 - -### 1. **个人事实** (personal_fact) - 高优先级记忆 -**包括但不限于：** -- 基本信息：姓名、年龄、职业、学校、专业、工作地点 -- 生活状况：住址、电话、邮箱、社交账号 -- 身份特征：生日、星座、血型、国籍、语言能力 -- 健康信息：身体状况、疾病史、药物过敏、运动习惯 -- 家庭情况：家庭成员、婚姻状况、子女信息、宠物信息 - -**判断标准：** 涉及个人身份和生活的重要信息，都应该记忆 - -### 2. **事件** (event) - 高优先级记忆 -**包括但不限于：** -- 重要时刻：生日聚会、毕业典礼、婚礼、旅行 -- 日常活动：上班、上学、约会、看电影、吃饭 -- 特殊经历：考试、面试、会议、搬家、购物 -- 计划安排：约会、会议、旅行、活动 - - -**判断标准：** 涉及时间地点的具体活动和经历，都应该记忆 - -### 3. **偏好** (preference) - 高优先级记忆 -**包括但不限于：** -- 饮食偏好：喜欢的食物、餐厅、口味、禁忌 -- 娱乐喜好：喜欢的电影、音乐、游戏、书籍 -- 生活习惯：作息时间、运动方式、购物习惯 -- 消费偏好：品牌喜好、价格敏感度、购物场所 -- 风格偏好：服装风格、装修风格、颜色喜好 - -**判断标准：** 任何表达"喜欢"、"不喜欢"、"习惯"、"经常"等偏好的内容，都应该记忆 - -### 4. **观点** (opinion) - 高优先级记忆 -**包括但不限于：** -- 评价看法：对事物的评价、意见、建议 -- 价值判断：认为什么重要、什么不重要 -- 态度立场：支持、反对、中立的态度 -- 感受反馈：对经历的感受、反馈 - -**判断标准：** 任何表达主观看法和态度的内容，都应该记忆 - -### 5. **关系** (relationship) - 中等优先级记忆 -**包括但不限于：** -- 人际关系：朋友、同事、家人、恋人的关系状态 -- 社交互动：与他人的互动、交流、合作 -- 群体归属：所属团队、组织、社群 - -### 6. **情感** (emotion) - 中等优先级记忆 -**包括但不限于：** -- 情绪状态：开心、难过、生气、焦虑、兴奋 -- 情感变化：情绪的转变、原因和结果 - -### 7. **目标** (goal) - 中等优先级记忆 -**包括但不限于：** -- 计划安排：短期计划、长期目标 -- 愿望期待：想要实现的事情、期望的结果 - -## 📝 记忆提取原则 - -### ✅ 积极提取原则： -1. **宁可错记，不可遗漏** - 对于可能的个人信息优先记忆 -2. **持续追踪** - 相同信息的多次提及要强化记忆 -3. **上下文关联** - 结合对话背景理解信息重要性 -4. **细节丰富** - 记录具体的细节和描述 - -### 🚫 禁止使用模糊代称原则： -1. **绝对禁止使用"用户"作为代称** - 必须使用明确的名字或称呼 -2. **优先使用真实姓名** - 如果知道对方的名字，必须使用真实姓名 -3. **使用昵称或特定称呼** - 如果没有真实姓名，使用对话中出现的昵称 -4. **无法获取具体名字时拒绝构建** - 如果不知道对方的具体名字，宁可不构建这条记忆，也不要使用"用户"、"该对话者"等模糊代称 - -### 🕒 时间处理原则（重要）： -1. **绝对时间要求** - 涉及时间的记忆必须使用绝对时间（年月日） -2. **相对时间转换** - 将"明天"、"后天"、"下周"等相对时间转换为具体日期 -3. **时间格式规范** - 使用"YYYY-MM-DD"格式记录日期 -4. **当前时间参考** - 当前时间：{current_date}，基于此计算相对时间 - -**相对时间转换示例：** -- "明天" → "2024-09-30" -- "后天" → "2024-10-01" -- "下周" → "2024-10-07" -- "下个月" → "2024-10-01" -- "明年" → "2025-01-01" - -### 🎯 重要性等级标准： -- **4分 (关键)**：个人核心信息（姓名、联系方式、重要日期） -- **3分 (高)**：重要偏好、观点、经历事件 -- **2分 (一般)**：一般性信息、日常活动、感受表达 -- **1分 (低)**：琐碎细节、重复信息、临时状态 - -### 🔍 置信度标准： -- **4分 (已验证)**：用户明确确认的信息 -- **3分 (高)**：用户直接表达的清晰信息 -- **2分 (中等)**：需要推理或上下文判断的信息 -- **1分 (低)**：模糊或不完整的信息 - -输出格式要求: -{{ - "memories": [ - {{ - "type": "记忆类型", - "display": "一句自然流畅的中文描述，用于直接展示和提示词构建", - "subject": "主语(通常是用户)", - "predicate": "谓语(动作/状态)", - "object": "宾语(对象/属性)", - "keywords": ["关键词1", "关键词2"], - "importance": "重要性等级(1-4)", - "confidence": "置信度(1-4)", - "reasoning": "提取理由" - }} - ] -}} - -注意： -1. `display` 字段必填，必须是完整顺畅的自然语言，禁止依赖字符串拼接 -2. **display 字段格式要求**: 使用自然流畅的中文描述，**绝对禁止使用"用户"作为代称**，格式示例： - - 杰瑞喵养了一只名叫Whiskers的猫。 - - why ocean QAQ特别喜欢拿铁咖啡。 - - 在2024年5月15日，velida QAQ提到对新项目感到很有压力。 - - 杰瑞喵认为这个电影很有趣。 -3. **必须使用明确的名字**：如果知道对话者的名字（如杰瑞喵、why ocean QAQ等），必须直接使用其名字 -4. **不知道名字时不要构建**：如果无法从对话中确定对方的具体名字，宁可不构建这条记忆 -5. 主谓宾用于索引和检索，提示词构建仅使用 `display` 的自然语言描述 -6. 只提取确实值得记忆的信息，不要提取琐碎内容 -7. 确保提取的信息准确、具体、有价值 -8. 重要性等级: 1=低, 2=一般, 3=高, 4=关键；置信度: 1=低, 2=中等, 3=高, 4=已验证 - -## 🚨 时间处理要求（强制）： -- **绝对时间优先**：任何涉及时间的记忆都必须使用绝对日期格式 -- **相对时间转换**：遇到"明天"、"后天"、"下周"等相对时间必须转换为具体日期 -- **时间格式**：统一使用 "YYYY-MM-DD" 格式 -- **计算依据**：基于当前时间 {current_date} 进行转换计算 -""" - - return prompt - - def _extract_json_payload(self, response: str) -> str | None: - """从模型响应中提取JSON部分，兼容Markdown代码块等格式 - - 增强的JSON提取策略，支持多种格式： - 1. Markdown代码块: ```json ... ``` - 2. 普通代码块: ``` ... ``` - 3. 大括号包围的JSON对象 - 4. 直接的JSON字符串 - """ - if not response: - return None - - stripped = response.strip() - - # 策略1: 优先处理Markdown代码块格式 ```json ... ``` 或 ``` ... ``` - code_block_patterns = [ - r"```json\s*(.*?)```", # 明确标记json的代码块 - r"```\s*(.*?)```", # 普通代码块 - ] - - for pattern in code_block_patterns: - code_block_match = re.search(pattern, stripped, re.IGNORECASE | re.DOTALL) - if code_block_match: - candidate = code_block_match.group(1).strip() - if candidate and (candidate.startswith("{") or candidate.startswith("[")): - logger.debug(f"从代码块中提取JSON，长度: {len(candidate)}") - return candidate - - # 策略2: 查找第一个完整的JSON对象（大括号匹配） - start = stripped.find("{") - if start != -1: - # 使用栈来找到匹配的结束大括号 - brace_count = 0 - in_string = False - escape_next = False - - for i in range(start, len(stripped)): - char = stripped[i] - - if escape_next: - escape_next = False - continue - - if char == "\\": - escape_next = True - continue - - if char == '"' and not escape_next: - in_string = not in_string - continue - - if not in_string: - if char == "{": - brace_count += 1 - elif char == "}": - brace_count -= 1 - if brace_count == 0: - # 找到完整的JSON对象 - candidate = stripped[start : i + 1].strip() - logger.debug(f"通过大括号匹配提取JSON，长度: {len(candidate)}") - return candidate - - # 策略3: 简单的整体检查（作为最后的fallback） - if stripped.startswith("{") and stripped.endswith("}"): - logger.debug(f"整体作为JSON，长度: {len(stripped)}") - return stripped - - # 所有策略都失败 - logger.warning(f"无法从响应中提取JSON，响应预览: {stripped[:200]}") - return None - - def _parse_llm_response( - self, response: str, user_id: str, timestamp: float, context: dict[str, Any] - ) -> list[MemoryChunk]: - """解析LLM响应""" - if not response: - raise MemoryExtractionError("LLM未返回任何响应") - - json_payload = self._extract_json_payload(response) - if not json_payload: - preview = response[:200] if response else "空响应" - raise MemoryExtractionError(f"未在LLM响应中找到有效的JSON负载，响应片段: {preview}") - - try: - data = orjson.loads(json_payload) - logger.debug(f"JSON直接解析成功，数据keys: {list(data.keys()) if isinstance(data, dict) else type(data)}") - except Exception as e: - # 尝试使用 json_repair 修复 JSON - logger.warning(f"JSON直接解析失败: {type(e).__name__}: {e}，尝试使用json_repair修复") - logger.debug(f"失败的JSON片段(前500字符): {json_payload[:500]}") - try: - repaired_json = repair_json(json_payload) - # repair_json 可能返回字符串或已解析的对象 - if isinstance(repaired_json, str): - data = orjson.loads(repaired_json) - logger.info(f"✅ JSON修复成功(字符串模式)，数据keys: {list(data.keys()) if isinstance(data, dict) else type(data)}") - else: - data = repaired_json - logger.info(f"✅ JSON修复成功(对象模式)，数据keys: {list(data.keys()) if isinstance(data, dict) else type(data)}") - except Exception as repair_error: - preview = json_payload[:300] - logger.error(f"❌ JSON修复也失败: {type(repair_error).__name__}: {repair_error}") - logger.error(f"完整JSON payload(前800字符):\n{json_payload[:800]}") - raise MemoryExtractionError( - f"LLM响应JSON解析失败\n" - f"原始错误: {type(e).__name__}: {e}\n" - f"修复错误: {type(repair_error).__name__}: {repair_error}\n" - f"JSON片段(前300字符): {preview}" - ) from e - - # 提取 memories 列表，兼容多种格式 - memory_list = data.get("memories", []) - - # 如果没有 memories 字段，尝试其他可能的字段名 - if not memory_list: - for possible_key in ["memory", "results", "items", "data"]: - if possible_key in data: - memory_list = data[possible_key] - logger.debug(f"使用备选字段 '{possible_key}' 作为记忆列表") - break - - # 如果整个data就是一个列表，直接使用 - if not memory_list and isinstance(data, list): - memory_list = data - logger.debug("整个JSON就是记忆列表") - - if not isinstance(memory_list, list): - logger.warning(f"记忆列表格式错误，期望list但得到 {type(memory_list)}, 尝试包装为列表") - memory_list = [memory_list] if memory_list else [] - - logger.debug(f"提取到 {len(memory_list)} 个记忆候选项") - - bot_identifiers = self._collect_bot_identifiers(context) - system_identifiers = self._collect_system_identifiers(context) - default_subjects = self._resolve_conversation_participants(context, user_id) - - bot_display = None - if context: - primary_bot_name = context.get("bot_name") - if isinstance(primary_bot_name, str) and primary_bot_name.strip(): - bot_display = primary_bot_name.strip() - if bot_display is None: - aliases = context.get("bot_aliases") - if isinstance(aliases, list | tuple | set): - for alias in aliases: - if isinstance(alias, str) and alias.strip(): - bot_display = alias.strip() - break - elif isinstance(aliases, str) and aliases.strip(): - bot_display = aliases.strip() - if bot_display is None: - identity = context.get("bot_identity") - if isinstance(identity, str) and identity.strip(): - bot_display = identity.strip() - - if not bot_display: - bot_display = "机器人" - - bot_display = self._clean_subject_text(bot_display) - - memories: list[MemoryChunk] = [] - - for mem_data in memory_list: - try: - # 检查是否包含模糊代称 - display_text = mem_data.get("display", "") - if any( - ambiguous_term in display_text for ambiguous_term in ["用户", "user", "the user", "对方", "对手"] - ): - logger.debug(f"拒绝构建包含模糊代称的记忆，display字段: {display_text}") - continue - - subject_value = mem_data.get("subject") - normalized_subject = self._normalize_subjects( - subject_value, bot_identifiers, system_identifiers, default_subjects, bot_display - ) - - if not normalized_subject: - logger.debug("跳过疑似机器人自身信息的记忆: %s", mem_data) - continue - - # 创建记忆块 - importance_level = self._parse_enum_value( - ImportanceLevel, mem_data.get("importance"), ImportanceLevel.NORMAL, "importance" - ) - - confidence_level = self._parse_enum_value( - ConfidenceLevel, mem_data.get("confidence"), ConfidenceLevel.MEDIUM, "confidence" - ) - - predicate_value = mem_data.get("predicate", "") - object_value = mem_data.get("object", "") - - display_text = self._sanitize_display_text(mem_data.get("display")) - used_fallback_display = False - if not display_text: - display_text = self._compose_display_text(normalized_subject, predicate_value, object_value) - used_fallback_display = True - - memory = create_memory_chunk( - user_id=user_id, - subject=normalized_subject, - predicate=predicate_value, - obj=object_value, - memory_type=self._resolve_memory_type(mem_data.get("type")), - chat_id=context.get("chat_id"), - source_context=mem_data.get("reasoning", ""), - importance=importance_level, - confidence=confidence_level, - display=display_text, - ) - - if used_fallback_display: - logger.warning( - "LLM 记忆缺少自然语言 display 字段，已基于主谓宾临时生成描述", - fallback_generated=True, - memory_type=memory.memory_type.value, - subjects=memory.content.to_subject_list(), - predicate=predicate_value, - object_payload=object_value, - ) - - # 添加关键词 - keywords = mem_data.get("keywords", []) - for keyword in keywords: - memory.add_keyword(keyword) - - memories.append(memory) - - except Exception as e: - logger.warning(f"解析单个记忆失败: {e}, 数据: {mem_data}") - continue - - return memories - - def _resolve_memory_type(self, type_str: Any) -> MemoryType: - """健壮地解析记忆类型，兼容中文和英文""" - if not isinstance(type_str, str) or not type_str.strip(): - return MemoryType.CONTEXTUAL - - cleaned_type = type_str.strip() - - # 尝试中文映射 - if cleaned_type in CHINESE_TO_MEMORY_TYPE: - return CHINESE_TO_MEMORY_TYPE[cleaned_type] - - # 尝试直接作为枚举值解析 - try: - return MemoryType(cleaned_type.lower().replace(" ", "_")) - except ValueError: - pass - - # 尝试作为枚举名解析 - try: - return MemoryType[cleaned_type.upper()] - except KeyError: - pass - - logger.warning(f"无法解析未知的记忆类型 '{type_str}'，回退到上下文类型") - return MemoryType.CONTEXTUAL - - def _parse_enum_value(self, enum_cls: type[E], raw_value: Any, default: E, field_name: str) -> E: - """解析枚举值，兼容数字/字符串表示""" - if isinstance(raw_value, enum_cls): - return raw_value - - if raw_value is None: - return default - - # 直接尝试整数转换 - if isinstance(raw_value, int | float): - int_value = int(raw_value) - try: - return enum_cls(int_value) - except ValueError: - logger.debug("%s=%s 无法解析为 %s", field_name, raw_value, enum_cls.__name__) - return default - - if isinstance(raw_value, str): - value_str = raw_value.strip() - if not value_str: - return default - - if value_str.isdigit(): - try: - return enum_cls(int(value_str)) - except ValueError: - logger.debug("%s='%s' 无法解析为 %s", field_name, value_str, enum_cls.__name__) - else: - normalized = value_str.replace("-", "_").replace(" ", "_").upper() - for member in enum_cls: - if member.name == normalized: - return member - for member in enum_cls: - if str(member.value).lower() == value_str.lower(): - return member - - try: - return enum_cls(value_str) - except ValueError: - logger.debug("%s='%s' 无法解析为 %s", field_name, value_str, enum_cls.__name__) - - try: - return enum_cls(raw_value) - except Exception: - logger.debug( - "%s=%s 类型 %s 无法解析为 %s，使用默认值 %s", - field_name, - raw_value, - type(raw_value).__name__, - enum_cls.__name__, - default.name, - ) - return default - - def _collect_bot_identifiers(self, context: dict[str, Any] | None) -> set[str]: - identifiers: set[str] = {"bot", "机器人", "ai助手"} - if not context: - return identifiers - - for key in [ - "bot_name", - "bot_identity", - "bot_personality", - "bot_personality_side", - "bot_account", - ]: - value = context.get(key) - if isinstance(value, str) and value.strip(): - identifiers.add(value.strip().lower()) - - aliases = context.get("bot_aliases") - if isinstance(aliases, list | tuple | set): - for alias in aliases: - if isinstance(alias, str) and alias.strip(): - identifiers.add(alias.strip().lower()) - elif isinstance(aliases, str) and aliases.strip(): - identifiers.add(aliases.strip().lower()) - - return identifiers - - def _collect_system_identifiers(self, context: dict[str, Any] | None) -> set[str]: - identifiers: set[str] = set() - if not context: - return identifiers - - keys = [ - "chat_id", - "stream_id", - "stram_id", - "session_id", - "conversation_id", - "message_id", - "topic_id", - "thread_id", - ] - - for key in keys: - value = context.get(key) - if isinstance(value, str) and value.strip(): - identifiers.add(value.strip().lower()) - - user_id_value = context.get("user_id") - if isinstance(user_id_value, str) and user_id_value.strip(): - if self._looks_like_system_identifier(user_id_value): - identifiers.add(user_id_value.strip().lower()) - - return identifiers - - def _resolve_conversation_participants(self, context: dict[str, Any] | None, user_id: str) -> list[str]: - participants: list[str] = [] - - if context: - candidate_keys = [ - "participants", - "participant_names", - "speaker_names", - "members", - "member_names", - "mention_users", - "audiences", - ] - - for key in candidate_keys: - value = context.get(key) - if isinstance(value, list | tuple | set): - for item in value: - if isinstance(item, str): - cleaned = self._clean_subject_text(item) - if cleaned: - participants.append(cleaned) - elif isinstance(value, str): - participants.extend(part for part in self._split_subject_string(value) if part) - - fallback = self._resolve_user_display(context, user_id) - if fallback: - participants.append(fallback) - - if context: - bot_name = context.get("bot_name") or context.get("bot_identity") - if isinstance(bot_name, str): - cleaned = self._clean_subject_text(bot_name) - if cleaned: - participants.append(cleaned) - - if not participants: - participants = ["对话参与者"] - - deduplicated: list[str] = [] - seen = set() - for name in participants: - key = name.lower() - if key in seen: - continue - seen.add(key) - deduplicated.append(name) - - return deduplicated - - def _resolve_user_display(self, context: dict[str, Any] | None, user_id: str) -> str: - candidate_keys = [ - "user_display_name", - "user_name", - "nickname", - "sender_name", - "member_name", - "display_name", - "from_user_name", - "author_name", - "speaker_name", - ] - - if context: - for key in candidate_keys: - value = context.get(key) - if isinstance(value, str): - candidate = value.strip() - if candidate: - return self._clean_subject_text(candidate) - - if user_id and not self._looks_like_system_identifier(user_id): - return self._clean_subject_text(user_id) - - return "该用户" - - def _clean_subject_text(self, text: str) -> str: - if not text: - return "" - cleaned = re.sub(r"[\s\u3000]+", " ", text).strip() - cleaned = re.sub(r"[、，,；;]+$", "", cleaned) - return cleaned - - def _sanitize_display_text(self, value: Any) -> str: - if value is None: - return "" - - if isinstance(value, list | dict): - try: - value = orjson.dumps(value).decode("utf-8") - except Exception: - value = str(value) - - text = str(value).strip() - if not text or text.lower() in {"null", "none", "undefined"}: - return "" - - text = re.sub(r"[\s\u3000]+", " ", text) - return text.strip("\n ") - - def _looks_like_system_identifier(self, value: str) -> bool: - if not value: - return False - - condensed = value.replace("-", "").replace("_", "").strip() - if len(condensed) >= 16 and re.fullmatch(r"[0-9a-fA-F]+", condensed): - return True - - if len(value) >= 12 and re.fullmatch(r"[0-9A-Z_:-]+", value) and any(ch.isdigit() for ch in value): - return True - - return False - - def _split_subject_string(self, value: str) -> list[str]: - if not value: - return [] - - replaced = re.sub(r"\band\b", "、", value, flags=re.IGNORECASE) - replaced = replaced.replace("和", "、").replace("与", "、").replace("及", "、") - replaced = replaced.replace("&", "、").replace("/", "、").replace("+", "、") - - tokens = [self._clean_subject_text(token) for token in re.split(r"[、,，;；]+", replaced)] - return [token for token in tokens if token] - - def _normalize_subjects( - self, - subject: Any, - bot_identifiers: set[str], - system_identifiers: set[str], - default_subjects: list[str], - bot_display: str | None = None, - ) -> list[str]: - defaults = default_subjects or ["对话参与者"] - - raw_candidates: list[str] = [] - if isinstance(subject, list): - for item in subject: - if isinstance(item, str): - raw_candidates.extend(self._split_subject_string(item)) - elif item is not None: - raw_candidates.extend(self._split_subject_string(str(item))) - elif isinstance(subject, str): - raw_candidates.extend(self._split_subject_string(subject)) - elif subject is not None: - raw_candidates.extend(self._split_subject_string(str(subject))) - - normalized: list[str] = [] - bot_primary = self._clean_subject_text(bot_display or "") - - for candidate in raw_candidates: - if not candidate: - continue - - lowered = candidate.lower() - if lowered in bot_identifiers: - normalized.append(bot_primary or candidate) - continue - - if lowered in {"用户", "user", "the user", "对方", "对手"}: - # 直接拒绝构建包含模糊代称的记忆 - logger.debug(f"拒绝构建包含模糊代称的记忆: {candidate}") - return [] # 返回空列表表示拒绝构建 - - if lowered in system_identifiers or self._looks_like_system_identifier(candidate): - continue - - normalized.append(candidate) - - if not normalized: - normalized = list(defaults) - - deduplicated: list[str] = [] - seen = set() - for name in normalized: - key = name.lower() - if key in seen: - continue - seen.add(key) - deduplicated.append(name) - - return deduplicated - - def _extract_value_from_object(self, obj: str | dict[str, Any] | list[Any], keys: list[str]) -> str | None: - if isinstance(obj, dict): - for key in keys: - value = obj.get(key) - if value is None: - continue - if isinstance(value, list): - compact = "、".join(str(item) for item in value[:3]) - if compact: - return compact - else: - value_str = str(value).strip() - if value_str: - return value_str - elif isinstance(obj, list): - compact = "、".join(str(item) for item in obj[:3]) - return compact or None - elif isinstance(obj, str): - return obj.strip() or None - return None - - def _compose_display_text(self, subjects: list[str], predicate: str, obj: str | dict[str, Any] | list[Any]) -> str: - subject_phrase = "、".join(subjects) if subjects else "对话参与者" - predicate = (predicate or "").strip() - - if predicate == "is_named": - name = self._extract_value_from_object(obj, ["name", "nickname"]) or "" - name = self._clean_subject_text(name) - if name: - quoted = name if (name.startswith("「") and name.endswith("」")) else f"「{name}」" - return f"{subject_phrase}的昵称是{quoted}" - elif predicate == "is_age": - age = self._extract_value_from_object(obj, ["age"]) or "" - age = self._clean_subject_text(age) - if age: - return f"{subject_phrase}今年{age}岁" - elif predicate == "is_profession": - profession = self._extract_value_from_object(obj, ["profession", "job"]) or "" - profession = self._clean_subject_text(profession) - if profession: - return f"{subject_phrase}的职业是{profession}" - elif predicate == "lives_in": - location = self._extract_value_from_object(obj, ["location", "city", "place"]) or "" - location = self._clean_subject_text(location) - if location: - return f"{subject_phrase}居住在{location}" - elif predicate == "has_phone": - phone = self._extract_value_from_object(obj, ["phone", "number"]) or "" - phone = self._clean_subject_text(phone) - if phone: - return f"{subject_phrase}的电话号码是{phone}" - elif predicate == "has_email": - email = self._extract_value_from_object(obj, ["email"]) or "" - email = self._clean_subject_text(email) - if email: - return f"{subject_phrase}的邮箱是{email}" - elif predicate in {"likes", "likes_food", "favorite_is"}: - liked = self._extract_value_from_object(obj, ["item", "value", "name"]) or "" - liked = self._clean_subject_text(liked) - if liked: - verb = "喜欢" if predicate != "likes_food" else "爱吃" - if predicate == "favorite_is": - verb = "最喜欢" - return f"{subject_phrase}{verb}{liked}" - elif predicate in {"dislikes", "hates"}: - disliked = self._extract_value_from_object(obj, ["item", "value", "name"]) or "" - disliked = self._clean_subject_text(disliked) - if disliked: - verb = "不喜欢" if predicate == "dislikes" else "讨厌" - return f"{subject_phrase}{verb}{disliked}" - elif predicate == "mentioned_event": - description = self._extract_value_from_object(obj, ["event_text", "description"]) or "" - description = self._clean_subject_text(description) - if description: - return f"{subject_phrase}提到了：{description}" - - obj_text = self._extract_value_from_object(obj, ["value", "detail", "content"]) or "" - obj_text = self._clean_subject_text(obj_text) - - if predicate and obj_text: - return f"{subject_phrase}{predicate}{obj_text}".strip() - if obj_text: - return f"{subject_phrase}{obj_text}".strip() - if predicate: - return f"{subject_phrase}{predicate}".strip() - return subject_phrase - - def _validate_and_enhance_memories(self, memories: list[MemoryChunk], context: dict[str, Any]) -> list[MemoryChunk]: - """验证和增强记忆""" - validated_memories = [] - - for memory in memories: - # 基本验证 - if not self._validate_memory(memory): - continue - - # 增强记忆 - enhanced_memory = self._enhance_memory(memory, context) - validated_memories.append(enhanced_memory) - - return validated_memories - - def _validate_memory(self, memory: MemoryChunk) -> bool: - """验证记忆块""" - # 检查基本字段 - if not memory.content.subject or not memory.content.predicate: - logger.debug(f"记忆块缺少主语或谓语: {memory.memory_id}") - return False - - # 检查内容长度 - content_length = len(memory.text_content) - if content_length < 5 or content_length > 500: - logger.debug(f"记忆块内容长度异常: {content_length}") - return False - - # 检查置信度 - if memory.metadata.confidence == ConfidenceLevel.LOW: - logger.debug(f"记忆块置信度过低: {memory.memory_id}") - return False - - return True - - def _enhance_memory(self, memory: MemoryChunk, context: dict[str, Any]) -> MemoryChunk: - """增强记忆块""" - # 时间规范化处理 - self._normalize_time_in_memory(memory) - - # 添加时间上下文 - if not memory.temporal_context: - memory.temporal_context = { - "timestamp": memory.metadata.created_at, - "timezone": context.get("timezone", "UTC"), - "day_of_week": datetime.fromtimestamp(memory.metadata.created_at).strftime("%A"), - } - - # 添加情感上下文（如果有） - if context.get("sentiment"): - memory.metadata.emotional_context = context["sentiment"] - - # 自动添加标签 - self._auto_tag_memory(memory) - - return memory - - def _normalize_time_in_memory(self, memory: MemoryChunk): - """规范化记忆中的时间表达""" - import re - from datetime import datetime, timedelta - - # 获取当前时间作为参考 - current_time = datetime.fromtimestamp(memory.metadata.created_at) - - # 定义相对时间映射 - relative_time_patterns = { - r"今天|今日": current_time.strftime("%Y-%m-%d"), - r"昨天|昨日": (current_time - timedelta(days=1)).strftime("%Y-%m-%d"), - r"明天|明日": (current_time + timedelta(days=1)).strftime("%Y-%m-%d"), - r"后天": (current_time + timedelta(days=2)).strftime("%Y-%m-%d"), - r"大后天": (current_time + timedelta(days=3)).strftime("%Y-%m-%d"), - r"前天": (current_time - timedelta(days=2)).strftime("%Y-%m-%d"), - r"大前天": (current_time - timedelta(days=3)).strftime("%Y-%m-%d"), - r"本周|这周|这星期": current_time.strftime("%Y-%m-%d"), - r"上周|上星期": (current_time - timedelta(weeks=1)).strftime("%Y-%m-%d"), - r"下周|下星期": (current_time + timedelta(weeks=1)).strftime("%Y-%m-%d"), - r"本月|这个月": current_time.strftime("%Y-%m-01"), - r"上月|上个月": (current_time.replace(day=1) - timedelta(days=1)).strftime("%Y-%m-01"), - r"下月|下个月": (current_time.replace(day=1) + timedelta(days=32)).replace(day=1).strftime("%Y-%m-01"), - r"今年|今年": current_time.strftime("%Y"), - r"去年|上一年": str(current_time.year - 1), - r"明年|下一年": str(current_time.year + 1), - } - - def _normalize_value(value): - if isinstance(value, str): - normalized = value - for pattern, replacement in relative_time_patterns.items(): - normalized = re.sub(pattern, replacement, normalized) - return normalized - if isinstance(value, dict): - return {k: _normalize_value(v) for k, v in value.items()} - if isinstance(value, list): - return [_normalize_value(item) for item in value] - return value - - # 规范化主语和谓语（通常是字符串） - memory.content.subject = _normalize_value(memory.content.subject) - memory.content.predicate = _normalize_value(memory.content.predicate) - - # 规范化宾语（可能是字符串、列表或字典） - memory.content.object = _normalize_value(memory.content.object) - - # 记录时间规范化操作 - logger.debug(f"记忆 {memory.memory_id} 已进行时间规范化") - - def _auto_tag_memory(self, memory: MemoryChunk): - """自动为记忆添加标签""" - # 基于记忆类型的自动标签 - type_tags = { - MemoryType.PERSONAL_FACT: ["个人信息", "基本资料"], - MemoryType.EVENT: ["事件", "日程"], - MemoryType.PREFERENCE: ["偏好", "喜好"], - MemoryType.OPINION: ["观点", "态度"], - MemoryType.RELATIONSHIP: ["关系", "社交"], - MemoryType.EMOTION: ["情感", "情绪"], - MemoryType.KNOWLEDGE: ["知识", "信息"], - MemoryType.SKILL: ["技能", "能力"], - MemoryType.GOAL: ["目标", "计划"], - MemoryType.EXPERIENCE: ["经验", "经历"], - } - - tags = type_tags.get(memory.memory_type, []) - for tag in tags: - memory.add_tag(tag) - - def _update_extraction_stats(self, success_count: int, extraction_time: float): - """更新提取统计""" - self.extraction_stats["total_extractions"] += 1 - self.extraction_stats["successful_extractions"] += success_count - self.extraction_stats["failed_extractions"] += max(0, 1 - success_count) - - # 更新平均置信度 - if self.extraction_stats["successful_extractions"] > 0: - total_confidence = self.extraction_stats["average_confidence"] * ( - self.extraction_stats["successful_extractions"] - success_count - ) - # 假设新记忆的平均置信度为0.8 - total_confidence += 0.8 * success_count - self.extraction_stats["average_confidence"] = ( - total_confidence / self.extraction_stats["successful_extractions"] - ) - - def get_extraction_stats(self) -> dict[str, Any]: - """获取提取统计信息""" - return self.extraction_stats.copy() - - def reset_stats(self): - """重置统计信息""" - self.extraction_stats = { - "total_extractions": 0, - "successful_extractions": 0, - "failed_extractions": 0, - "average_confidence": 0.0, - } diff --git a/src/chat/memory_system/memory_chunk.py b/src/chat/memory_system/memory_chunk.py deleted file mode 100644 index c3c5fe0ee..000000000 --- a/src/chat/memory_system/memory_chunk.py +++ /dev/null @@ -1,647 +0,0 @@ -""" -结构化记忆单元设计 -实现高质量、结构化的记忆单元，符合文档设计规范 -""" - -import hashlib -import time -import uuid -from collections.abc import Iterable -from dataclasses import dataclass, field -from enum import Enum -from typing import Any - -import numpy as np -import orjson - -from src.common.logger import get_logger - -logger = get_logger(__name__) - - -class MemoryType(Enum): - """记忆类型分类""" - - PERSONAL_FACT = "personal_fact" # 个人事实（姓名、职业、住址等） - EVENT = "event" # 事件（重要经历、约会等） - PREFERENCE = "preference" # 偏好（喜好、习惯等） - OPINION = "opinion" # 观点（对事物的看法） - RELATIONSHIP = "relationship" # 关系（与他人的关系） - EMOTION = "emotion" # 情感状态 - KNOWLEDGE = "knowledge" # 知识信息 - SKILL = "skill" # 技能能力 - GOAL = "goal" # 目标计划 - EXPERIENCE = "experience" # 经验教训 - CONTEXTUAL = "contextual" # 上下文信息 - - -class ConfidenceLevel(Enum): - """置信度等级""" - - LOW = 1 # 低置信度，可能不准确 - MEDIUM = 2 # 中等置信度，有一定依据 - HIGH = 3 # 高置信度，有明确来源 - VERIFIED = 4 # 已验证，非常可靠 - - -class ImportanceLevel(Enum): - """重要性等级""" - - LOW = 1 # 低重要性，普通信息 - NORMAL = 2 # 一般重要性，日常信息 - HIGH = 3 # 高重要性，重要信息 - CRITICAL = 4 # 关键重要性，核心信息 - - -@dataclass -class ContentStructure: - """主谓宾结构，包含自然语言描述""" - - subject: str | list[str] - predicate: str - object: str | dict - display: str = "" - - def to_dict(self) -> dict[str, Any]: - """转换为字典格式""" - return {"subject": self.subject, "predicate": self.predicate, "object": self.object, "display": self.display} - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "ContentStructure": - """从字典创建实例""" - return cls( - subject=data.get("subject", ""), - predicate=data.get("predicate", ""), - object=data.get("object", ""), - display=data.get("display", ""), - ) - - def to_subject_list(self) -> list[str]: - """将主语转换为列表形式""" - if isinstance(self.subject, list): - return [s for s in self.subject if isinstance(s, str) and s.strip()] - if isinstance(self.subject, str) and self.subject.strip(): - return [self.subject.strip()] - return [] - - def __str__(self) -> str: - """字符串表示""" - if self.display: - return self.display - subjects = "、".join(self.to_subject_list()) or str(self.subject) - object_str = self.object if isinstance(self.object, str) else str(self.object) - return f"{subjects} {self.predicate} {object_str}".strip() - - -@dataclass -class MemoryMetadata: - """记忆元数据 - 简化版本""" - - # 基础信息 - memory_id: str # 唯一标识符 - user_id: str # 用户ID - chat_id: str | None = None # 聊天ID（群聊或私聊） - - # 时间信息 - created_at: float = 0.0 # 创建时间戳 - last_accessed: float = 0.0 # 最后访问时间 - last_modified: float = 0.0 # 最后修改时间 - - # 激活频率管理 - last_activation_time: float = 0.0 # 最后激活时间 - activation_frequency: int = 0 # 激活频率（单位时间内的激活次数） - total_activations: int = 0 # 总激活次数 - - # 统计信息 - access_count: int = 0 # 访问次数 - relevance_score: float = 0.0 # 相关度评分 - - # 信心和重要性（核心字段） - confidence: ConfidenceLevel = ConfidenceLevel.MEDIUM - importance: ImportanceLevel = ImportanceLevel.NORMAL - - # 遗忘机制相关 - forgetting_threshold: float = 0.0 # 遗忘阈值（动态计算） - last_forgetting_check: float = 0.0 # 上次遗忘检查时间 - - # 来源信息 - source_context: str | None = None # 来源上下文片段 - # 兼容旧字段: 一些代码或旧版本可能直接访问 metadata.source - source: str | None = None - - def __post_init__(self): - """后初始化处理""" - if not self.memory_id: - self.memory_id = str(uuid.uuid4()) - - current_time = time.time() - - if self.created_at == 0: - self.created_at = current_time - - if self.last_accessed == 0: - self.last_accessed = current_time - - if self.last_modified == 0: - self.last_modified = current_time - - if self.last_activation_time == 0: - self.last_activation_time = current_time - - if self.last_forgetting_check == 0: - self.last_forgetting_check = current_time - - # 兼容性：如果旧字段 source 被使用，保证 source 与 source_context 同步 - if not getattr(self, "source", None) and getattr(self, "source_context", None): - try: - self.source = str(self.source_context) - except Exception: - self.source = None - # 如果有 source 字段但 source_context 为空，也同步回去 - if not getattr(self, "source_context", None) and getattr(self, "source", None): - try: - self.source_context = str(self.source) - except Exception: - self.source_context = None - - def update_access(self): - """更新访问信息""" - current_time = time.time() - self.last_accessed = current_time - self.access_count += 1 - self.total_activations += 1 - - # 更新激活频率 - self._update_activation_frequency(current_time) - - def _update_activation_frequency(self, current_time: float): - """更新激活频率（24小时内的激活次数）""" - - # 如果超过24小时，重置激活频率 - if current_time - self.last_activation_time > 86400: # 24小时 = 86400秒 - self.activation_frequency = 1 - else: - self.activation_frequency += 1 - - self.last_activation_time = current_time - - def update_relevance(self, new_score: float): - """更新相关度评分""" - self.relevance_score = max(0.0, min(1.0, new_score)) - self.last_modified = time.time() - - def calculate_forgetting_threshold(self) -> float: - """计算遗忘阈值（天数）""" - # 基础天数 - base_days = 30.0 - - # 重要性权重 (1-4 -> 0-3) - importance_weight = (self.importance.value - 1) * 15 # 0, 15, 30, 45 - - # 置信度权重 (1-4 -> 0-3) - confidence_weight = (self.confidence.value - 1) * 10 # 0, 10, 20, 30 - - # 激活频率权重（每5次激活增加1天） - frequency_weight = min(self.activation_frequency, 20) * 0.5 # 最多10天 - - # 计算最终阈值 - threshold = base_days + importance_weight + confidence_weight + frequency_weight - - # 设置最小和最大阈值 - return max(7.0, min(threshold, 365.0)) # 7天到1年之间 - - def should_forget(self, current_time: float | None = None) -> bool: - """判断是否应该遗忘""" - if current_time is None: - current_time = time.time() - - # 计算遗忘阈值 - self.forgetting_threshold = self.calculate_forgetting_threshold() - - # 计算距离最后激活的时间 - days_since_activation = (current_time - self.last_activation_time) / 86400 - - return days_since_activation > self.forgetting_threshold - - def is_dormant(self, current_time: float | None = None, inactive_days: int = 90) -> bool: - """判断是否处于休眠状态（长期未激活）""" - if current_time is None: - current_time = time.time() - - days_since_last_access = (current_time - self.last_accessed) / 86400 - return days_since_last_access > inactive_days - - def to_dict(self) -> dict[str, Any]: - """转换为字典格式""" - return { - "memory_id": self.memory_id, - "user_id": self.user_id, - "chat_id": self.chat_id, - "created_at": self.created_at, - "last_accessed": self.last_accessed, - "last_modified": self.last_modified, - "last_activation_time": self.last_activation_time, - "activation_frequency": self.activation_frequency, - "total_activations": self.total_activations, - "access_count": self.access_count, - "relevance_score": self.relevance_score, - "confidence": self.confidence.value, - "importance": self.importance.value, - "forgetting_threshold": self.forgetting_threshold, - "last_forgetting_check": self.last_forgetting_check, - "source_context": self.source_context, - } - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "MemoryMetadata": - """从字典创建实例""" - return cls( - memory_id=data.get("memory_id", ""), - user_id=data.get("user_id", ""), - chat_id=data.get("chat_id"), - created_at=data.get("created_at", 0), - last_accessed=data.get("last_accessed", 0), - last_modified=data.get("last_modified", 0), - last_activation_time=data.get("last_activation_time", 0), - activation_frequency=data.get("activation_frequency", 0), - total_activations=data.get("total_activations", 0), - access_count=data.get("access_count", 0), - relevance_score=data.get("relevance_score", 0.0), - confidence=ConfidenceLevel(data.get("confidence", ConfidenceLevel.MEDIUM.value)), - importance=ImportanceLevel(data.get("importance", ImportanceLevel.NORMAL.value)), - forgetting_threshold=data.get("forgetting_threshold", 0.0), - last_forgetting_check=data.get("last_forgetting_check", 0), - source_context=data.get("source_context"), - ) - - -@dataclass -class MemoryChunk: - """结构化记忆单元 - 核心数据结构""" - - # 元数据 - metadata: MemoryMetadata - - # 内容结构 - content: ContentStructure # 主谓宾结构 - memory_type: MemoryType # 记忆类型 - - # 扩展信息 - keywords: list[str] = field(default_factory=list) # 关键词列表 - tags: list[str] = field(default_factory=list) # 标签列表 - categories: list[str] = field(default_factory=list) # 分类列表 - - # 语义信息 - embedding: list[float] | None = None # 语义向量 - semantic_hash: str | None = None # 语义哈希值 - - # 关联信息 - related_memories: list[str] = field(default_factory=list) # 关联记忆ID列表 - temporal_context: dict[str, Any] | None = None # 时间上下文 - - def __post_init__(self): - """后初始化处理""" - if self.embedding and len(self.embedding) > 0: - self._generate_semantic_hash() - - def _generate_semantic_hash(self): - """生成语义哈希值""" - if not self.embedding: - return - - try: - # 使用向量和内容生成稳定的哈希 - content_str = f"{self.content.subject}:{self.content.predicate}:{self.content.object!s}" - embedding_str = ",".join(map(str, [round(x, 6) for x in self.embedding])) - - hash_input = f"{content_str}|{embedding_str}" - hash_object = hashlib.sha256(hash_input.encode("utf-8")) - self.semantic_hash = hash_object.hexdigest()[:16] - - except Exception as e: - logger.warning(f"生成语义哈希失败: {e}") - self.semantic_hash = str(uuid.uuid4())[:16] - - @property - def memory_id(self) -> str: - """获取记忆ID""" - return self.metadata.memory_id - - @property - def user_id(self) -> str: - """获取用户ID""" - return self.metadata.user_id - - @property - def text_content(self) -> str: - """获取文本内容（优先使用display）""" - return str(self.content) - - @property - def display(self) -> str: - """获取展示文本""" - return self.content.display or str(self.content) - - @property - def subjects(self) -> list[str]: - """获取主语列表""" - return self.content.to_subject_list() - - def update_access(self): - """更新访问信息""" - self.metadata.update_access() - - def update_relevance(self, new_score: float): - """更新相关度评分""" - self.metadata.update_relevance(new_score) - - def should_forget(self, current_time: float | None = None) -> bool: - """判断是否应该遗忘""" - return self.metadata.should_forget(current_time) - - def is_dormant(self, current_time: float | None = None, inactive_days: int = 90) -> bool: - """判断是否处于休眠状态（长期未激活）""" - return self.metadata.is_dormant(current_time, inactive_days) - - def calculate_forgetting_threshold(self) -> float: - """计算遗忘阈值（天数）""" - return self.metadata.calculate_forgetting_threshold() - - def add_keyword(self, keyword: str): - """添加关键词""" - if keyword and keyword not in self.keywords: - self.keywords.append(keyword.strip()) - - def add_tag(self, tag: str): - """添加标签""" - if tag and tag not in self.tags: - self.tags.append(tag.strip()) - - def add_category(self, category: str): - """添加分类""" - if category and category not in self.categories: - self.categories.append(category.strip()) - - def add_related_memory(self, memory_id: str): - """添加关联记忆""" - if memory_id and memory_id not in self.related_memories: - self.related_memories.append(memory_id) - - def set_embedding(self, embedding: list[float]): - """设置语义向量""" - self.embedding = embedding - self._generate_semantic_hash() - - def calculate_similarity(self, other: "MemoryChunk") -> float: - """计算与另一个记忆块的相似度""" - if not self.embedding or not other.embedding: - return 0.0 - - try: - # 计算余弦相似度 - v1 = np.array(self.embedding) - v2 = np.array(other.embedding) - - dot_product = np.dot(v1, v2) - norm1 = np.linalg.norm(v1) - norm2 = np.linalg.norm(v2) - - if norm1 == 0 or norm2 == 0: - return 0.0 - - similarity = dot_product / (norm1 * norm2) - return max(0.0, min(1.0, similarity)) - - except Exception as e: - logger.warning(f"计算记忆相似度失败: {e}") - return 0.0 - - def to_dict(self) -> dict[str, Any]: - """转换为完整的字典格式""" - return { - "metadata": self.metadata.to_dict(), - "content": self.content.to_dict(), - "memory_type": self.memory_type.value, - "keywords": self.keywords, - "tags": self.tags, - "categories": self.categories, - "embedding": self.embedding, - "semantic_hash": self.semantic_hash, - "related_memories": self.related_memories, - "temporal_context": self.temporal_context, - } - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "MemoryChunk": - """从字典创建实例""" - metadata = MemoryMetadata.from_dict(data.get("metadata", {})) - content = ContentStructure.from_dict(data.get("content", {})) - - chunk = cls( - metadata=metadata, - content=content, - memory_type=MemoryType(data.get("memory_type", MemoryType.CONTEXTUAL.value)), - keywords=data.get("keywords", []), - tags=data.get("tags", []), - categories=data.get("categories", []), - embedding=data.get("embedding"), - semantic_hash=data.get("semantic_hash"), - related_memories=data.get("related_memories", []), - temporal_context=data.get("temporal_context"), - ) - - return chunk - - def to_json(self) -> str: - """转换为JSON字符串""" - return orjson.dumps(self.to_dict()).decode("utf-8") - - @classmethod - def from_json(cls, json_str: str) -> "MemoryChunk": - """从JSON字符串创建实例""" - try: - data = orjson.loads(json_str) - return cls.from_dict(data) - except Exception as e: - logger.error(f"从JSON创建记忆块失败: {e}") - raise - - def is_similar_to(self, other: "MemoryChunk", threshold: float = 0.8) -> bool: - """判断是否与另一个记忆块相似""" - if self.semantic_hash and other.semantic_hash: - return self.semantic_hash == other.semantic_hash - - return self.calculate_similarity(other) >= threshold - - def merge_with(self, other: "MemoryChunk") -> bool: - """与另一个记忆块合并（如果相似）""" - if not self.is_similar_to(other): - return False - - try: - # 合并关键词 - for keyword in other.keywords: - self.add_keyword(keyword) - - # 合并标签 - for tag in other.tags: - self.add_tag(tag) - - # 合并分类 - for category in other.categories: - self.add_category(category) - - # 合并关联记忆 - for memory_id in other.related_memories: - self.add_related_memory(memory_id) - - # 更新元数据 - self.metadata.last_modified = time.time() - self.metadata.access_count += other.metadata.access_count - self.metadata.relevance_score = max(self.metadata.relevance_score, other.metadata.relevance_score) - - # 更新置信度 - if other.metadata.confidence.value > self.metadata.confidence.value: - self.metadata.confidence = other.metadata.confidence - - # 更新重要性 - if other.metadata.importance.value > self.metadata.importance.value: - self.metadata.importance = other.metadata.importance - - logger.debug(f"记忆块 {self.memory_id} 合并了记忆块 {other.memory_id}") - return True - - except Exception as e: - logger.error(f"合并记忆块失败: {e}") - return False - - def __str__(self) -> str: - """字符串表示""" - type_emoji = { - MemoryType.PERSONAL_FACT: "👤", - MemoryType.EVENT: "📅", - MemoryType.PREFERENCE: "❤️", - MemoryType.OPINION: "💭", - MemoryType.RELATIONSHIP: "👥", - MemoryType.EMOTION: "😊", - MemoryType.KNOWLEDGE: "📚", - MemoryType.SKILL: "🛠️", - MemoryType.GOAL: "🎯", - MemoryType.EXPERIENCE: "💡", - MemoryType.CONTEXTUAL: "📝", - } - - emoji = type_emoji.get(self.memory_type, "📝") - confidence_icon = "●" * self.metadata.confidence.value - importance_icon = "★" * self.metadata.importance.value - - return f"{emoji} [{self.memory_type.value}] {self.display} {confidence_icon} {importance_icon}" - - def __repr__(self) -> str: - """调试表示""" - return f"MemoryChunk(id={self.memory_id[:8]}..., type={self.memory_type.value}, user={self.user_id})" - - -def _build_display_text(subjects: Iterable[str], predicate: str, obj: str | dict) -> str: - """根据主谓宾生成自然语言描述""" - subjects_clean = [s.strip() for s in subjects if s and isinstance(s, str)] - subject_part = "、".join(subjects_clean) if subjects_clean else "对话参与者" - - if isinstance(obj, dict): - object_candidates = [] - for key, value in obj.items(): - if isinstance(value, str | int | float): - object_candidates.append(f"{key}:{value}") - elif isinstance(value, list): - compact = "、".join(str(item) for item in value[:3]) - object_candidates.append(f"{key}:{compact}") - object_part = "，".join(object_candidates) if object_candidates else str(obj) - else: - object_part = str(obj).strip() - - predicate_clean = predicate.strip() - if not predicate_clean: - return f"{subject_part} {object_part}".strip() - - if object_part: - return f"{subject_part}{predicate_clean}{object_part}".strip() - return f"{subject_part}{predicate_clean}".strip() - - -def create_memory_chunk( - user_id: str, - subject: str | list[str], - predicate: str, - obj: str | dict, - memory_type: MemoryType, - chat_id: str | None = None, - source_context: str | None = None, - importance: ImportanceLevel = ImportanceLevel.NORMAL, - confidence: ConfidenceLevel = ConfidenceLevel.MEDIUM, - display: str | None = None, - **kwargs, -) -> MemoryChunk: - """便捷的内存块创建函数""" - metadata = MemoryMetadata( - memory_id="", - user_id=user_id, - chat_id=chat_id, - created_at=time.time(), - last_accessed=0, - last_modified=0, - confidence=confidence, - importance=importance, - source_context=source_context, - ) - - subjects: list[str] - if isinstance(subject, list): - subjects = [s for s in subject if isinstance(s, str) and s.strip()] - subject_payload: str | list[str] = subjects - else: - cleaned = subject.strip() if isinstance(subject, str) else "" - subjects = [cleaned] if cleaned else [] - subject_payload = cleaned - - display_text = display or _build_display_text(subjects, predicate, obj) - - content = ContentStructure(subject=subject_payload, predicate=predicate, object=obj, display=display_text) - - chunk = MemoryChunk(metadata=metadata, content=content, memory_type=memory_type, **kwargs) - - return chunk - - -@dataclass -class MessageCollection: - """消息集合数据结构""" - - collection_id: str = field(default_factory=lambda: str(uuid.uuid4())) - chat_id: str | None = None # 聊天ID（群聊或私聊） - messages: list[str] = field(default_factory=list) - combined_text: str = "" - created_at: float = field(default_factory=time.time) - embedding: list[float] | None = None - - def to_dict(self) -> dict[str, Any]: - """转换为字典格式""" - return { - "collection_id": self.collection_id, - "chat_id": self.chat_id, - "messages": self.messages, - "combined_text": self.combined_text, - "created_at": self.created_at, - "embedding": self.embedding, - } - - @classmethod - def from_dict(cls, data: dict[str, Any]) -> "MessageCollection": - """从字典创建实例""" - return cls( - collection_id=data.get("collection_id", str(uuid.uuid4())), - chat_id=data.get("chat_id"), - messages=data.get("messages", []), - combined_text=data.get("combined_text", ""), - created_at=data.get("created_at", time.time()), - embedding=data.get("embedding"), - ) diff --git a/src/chat/memory_system/memory_forgetting_engine.py b/src/chat/memory_system/memory_forgetting_engine.py deleted file mode 100644 index e41d1149c..000000000 --- a/src/chat/memory_system/memory_forgetting_engine.py +++ /dev/null @@ -1,355 +0,0 @@ -""" -智能记忆遗忘引擎 -基于重要程度、置信度和激活频率的智能遗忘机制 -""" - -import asyncio -import time -from dataclasses import dataclass -from datetime import datetime - -from src.chat.memory_system.memory_chunk import ConfidenceLevel, ImportanceLevel, MemoryChunk -from src.common.logger import get_logger - -logger = get_logger(__name__) - - -@dataclass -class ForgettingStats: - """遗忘统计信息""" - - total_checked: int = 0 - marked_for_forgetting: int = 0 - actually_forgotten: int = 0 - dormant_memories: int = 0 - last_check_time: float = 0.0 - check_duration: float = 0.0 - - -@dataclass -class ForgettingConfig: - """遗忘引擎配置""" - - # 检查频率配置 - check_interval_hours: int = 24 # 定期检查间隔（小时） - batch_size: int = 100 # 批处理大小 - - # 遗忘阈值配置 - base_forgetting_days: float = 30.0 # 基础遗忘天数 - min_forgetting_days: float = 7.0 # 最小遗忘天数 - max_forgetting_days: float = 365.0 # 最大遗忘天数 - - # 重要程度权重 - critical_importance_bonus: float = 45.0 # 关键重要性额外天数 - high_importance_bonus: float = 30.0 # 高重要性额外天数 - normal_importance_bonus: float = 15.0 # 一般重要性额外天数 - low_importance_bonus: float = 0.0 # 低重要性额外天数 - - # 置信度权重 - verified_confidence_bonus: float = 30.0 # 已验证置信度额外天数 - high_confidence_bonus: float = 20.0 # 高置信度额外天数 - medium_confidence_bonus: float = 10.0 # 中等置信度额外天数 - low_confidence_bonus: float = 0.0 # 低置信度额外天数 - - # 激活频率权重 - activation_frequency_weight: float = 0.5 # 每次激活增加的天数权重 - max_frequency_bonus: float = 10.0 # 最大激活频率奖励天数 - - # 休眠配置 - dormant_threshold_days: int = 90 # 休眠状态判定天数 - force_forget_dormant_days: int = 180 # 强制遗忘休眠记忆的天数 - - -class MemoryForgettingEngine: - """智能记忆遗忘引擎""" - - def __init__(self, config: ForgettingConfig | None = None): - self.config = config or ForgettingConfig() - self.stats = ForgettingStats() - self._last_forgetting_check = 0.0 - self._forgetting_lock = asyncio.Lock() - - logger.info("MemoryForgettingEngine 初始化完成") - - def calculate_forgetting_threshold(self, memory: MemoryChunk) -> float: - """ - 计算记忆的遗忘阈值（天数） - - Args: - memory: 记忆块 - - Returns: - 遗忘阈值（天数） - """ - # 基础天数 - threshold = self.config.base_forgetting_days - - # 重要性权重 - importance = memory.metadata.importance - if importance == ImportanceLevel.CRITICAL: - threshold += self.config.critical_importance_bonus - elif importance == ImportanceLevel.HIGH: - threshold += self.config.high_importance_bonus - elif importance == ImportanceLevel.NORMAL: - threshold += self.config.normal_importance_bonus - # LOW 级别不增加额外天数 - - # 置信度权重 - confidence = memory.metadata.confidence - if confidence == ConfidenceLevel.VERIFIED: - threshold += self.config.verified_confidence_bonus - elif confidence == ConfidenceLevel.HIGH: - threshold += self.config.high_confidence_bonus - elif confidence == ConfidenceLevel.MEDIUM: - threshold += self.config.medium_confidence_bonus - # LOW 级别不增加额外天数 - - # 激活频率权重 - frequency_bonus = min( - memory.metadata.activation_frequency * self.config.activation_frequency_weight, - self.config.max_frequency_bonus, - ) - threshold += frequency_bonus - - # 确保在合理范围内 - return max(self.config.min_forgetting_days, min(threshold, self.config.max_forgetting_days)) - - def should_forget_memory(self, memory: MemoryChunk, current_time: float | None = None) -> bool: - """ - 判断记忆是否应该被遗忘 - - Args: - memory: 记忆块 - current_time: 当前时间戳 - - Returns: - 是否应该遗忘 - """ - if current_time is None: - current_time = time.time() - - # 关键重要性的记忆永不遗忘 - if memory.metadata.importance == ImportanceLevel.CRITICAL: - return False - - # 计算遗忘阈值 - forgetting_threshold = self.calculate_forgetting_threshold(memory) - - # 计算距离最后激活的时间 - days_since_activation = (current_time - memory.metadata.last_activation_time) / 86400 - - # 判断是否超过阈值 - should_forget = days_since_activation > forgetting_threshold - - if should_forget: - logger.debug( - f"记忆 {memory.memory_id[:8]} 触发遗忘条件: " - f"重要性={memory.metadata.importance.name}, " - f"置信度={memory.metadata.confidence.name}, " - f"激活频率={memory.metadata.activation_frequency}, " - f"阈值={forgetting_threshold:.1f}天, " - f"未激活天数={days_since_activation:.1f}天" - ) - - return should_forget - - def is_dormant_memory(self, memory: MemoryChunk, current_time: float | None = None) -> bool: - """ - 判断记忆是否处于休眠状态 - - Args: - memory: 记忆块 - current_time: 当前时间戳 - - Returns: - 是否处于休眠状态 - """ - return memory.is_dormant(current_time, self.config.dormant_threshold_days) - - def should_force_forget_dormant(self, memory: MemoryChunk, current_time: float | None = None) -> bool: - """ - 判断是否应该强制遗忘休眠记忆 - - Args: - memory: 记忆块 - current_time: 当前时间戳 - - Returns: - 是否应该强制遗忘 - """ - if current_time is None: - current_time = time.time() - - # 只有非关键重要性的记忆才会被强制遗忘 - if memory.metadata.importance == ImportanceLevel.CRITICAL: - return False - - days_since_last_access = (current_time - memory.metadata.last_accessed) / 86400 - return days_since_last_access > self.config.force_forget_dormant_days - - async def check_memories_for_forgetting(self, memories: list[MemoryChunk]) -> tuple[list[str], list[str]]: - """ - 检查记忆列表，识别需要遗忘的记忆 - - Args: - memories: 记忆块列表 - - Returns: - (普通遗忘列表, 强制遗忘列表) - """ - start_time = time.time() - current_time = start_time - - normal_forgetting_ids = [] - force_forgetting_ids = [] - - self.stats.total_checked = len(memories) - self.stats.last_check_time = current_time - - for memory in memories: - try: - # 检查休眠状态 - if self.is_dormant_memory(memory, current_time): - self.stats.dormant_memories += 1 - - # 检查是否应该强制遗忘休眠记忆 - if self.should_force_forget_dormant(memory, current_time): - force_forgetting_ids.append(memory.memory_id) - logger.debug(f"休眠记忆 {memory.memory_id[:8]} 被标记为强制遗忘") - continue - - # 检查普通遗忘条件 - if self.should_forget_memory(memory, current_time): - normal_forgetting_ids.append(memory.memory_id) - self.stats.marked_for_forgetting += 1 - - except Exception as e: - logger.warning(f"检查记忆 {memory.memory_id[:8]} 遗忘状态失败: {e}") - continue - - self.stats.check_duration = time.time() - start_time - - logger.info( - f"遗忘检查完成 | 总数={self.stats.total_checked}, " - f"标记遗忘={len(normal_forgetting_ids)}, " - f"强制遗忘={len(force_forgetting_ids)}, " - f"休眠={self.stats.dormant_memories}, " - f"耗时={self.stats.check_duration:.3f}s" - ) - - return normal_forgetting_ids, force_forgetting_ids - - async def perform_forgetting_check(self, memories: list[MemoryChunk]) -> dict[str, any]: - """ - 执行完整的遗忘检查流程 - - Args: - memories: 记忆块列表 - - Returns: - 检查结果统计 - """ - async with self._forgetting_lock: - normal_forgetting, force_forgetting = await self.check_memories_for_forgetting(memories) - - # 更新统计 - self.stats.actually_forgotten = len(normal_forgetting) + len(force_forgetting) - - return { - "normal_forgetting": normal_forgetting, - "force_forgetting": force_forgetting, - "stats": { - "total_checked": self.stats.total_checked, - "marked_for_forgetting": self.stats.marked_for_forgetting, - "actually_forgotten": self.stats.actually_forgotten, - "dormant_memories": self.stats.dormant_memories, - "check_duration": self.stats.check_duration, - "last_check_time": self.stats.last_check_time, - }, - } - - def is_forgetting_check_needed(self) -> bool: - """检查是否需要进行遗忘检查""" - current_time = time.time() - hours_since_last_check = (current_time - self._last_forgetting_check) / 3600 - - return hours_since_last_check >= self.config.check_interval_hours - - async def schedule_periodic_check(self, memories_provider, enable_auto_cleanup: bool = True): - """ - 定期执行遗忘检查（可以在后台任务中调用） - - Args: - memories_provider: 提供记忆列表的函数 - enable_auto_cleanup: 是否启用自动清理 - """ - if not self.is_forgetting_check_needed(): - return - - try: - logger.info("开始执行定期遗忘检查...") - - # 获取记忆列表 - memories = await memories_provider() - - if not memories: - logger.debug("无记忆数据需要检查") - return - - # 执行遗忘检查 - result = await self.perform_forgetting_check(memories) - - # 如果启用自动清理，执行实际的遗忘操作 - if enable_auto_cleanup and (result["normal_forgetting"] or result["force_forgetting"]): - logger.info( - f"检测到 {len(result['normal_forgetting'])} 条普通遗忘和 {len(result['force_forgetting'])} 条强制遗忘记忆" - ) - # 这里可以调用实际的删除逻辑 - # await self.cleanup_forgotten_memories(result["normal_forgetting"] + result["force_forgetting"]) - - self._last_forgetting_check = time.time() - - except Exception as e: - logger.error(f"定期遗忘检查失败: {e}", exc_info=True) - - def get_forgetting_stats(self) -> dict[str, any]: - """获取遗忘统计信息""" - return { - "total_checked": self.stats.total_checked, - "marked_for_forgetting": self.stats.marked_for_forgetting, - "actually_forgotten": self.stats.actually_forgotten, - "dormant_memories": self.stats.dormant_memories, - "last_check_time": datetime.fromtimestamp(self.stats.last_check_time).isoformat() - if self.stats.last_check_time - else None, - "last_check_duration": self.stats.check_duration, - "config": { - "check_interval_hours": self.config.check_interval_hours, - "base_forgetting_days": self.config.base_forgetting_days, - "min_forgetting_days": self.config.min_forgetting_days, - "max_forgetting_days": self.config.max_forgetting_days, - }, - } - - def reset_stats(self): - """重置统计信息""" - self.stats = ForgettingStats() - logger.debug("遗忘统计信息已重置") - - def update_config(self, **kwargs): - """更新配置""" - for key, value in kwargs.items(): - if hasattr(self.config, key): - setattr(self.config, key, value) - logger.debug(f"遗忘配置更新: {key} = {value}") - else: - logger.warning(f"未知的配置项: {key}") - - -# 创建全局遗忘引擎实例 -memory_forgetting_engine = MemoryForgettingEngine() - - -def get_memory_forgetting_engine() -> MemoryForgettingEngine: - """获取全局遗忘引擎实例""" - return memory_forgetting_engine diff --git a/src/chat/memory_system/memory_formatter.py b/src/chat/memory_system/memory_formatter.py deleted file mode 100644 index 5d69b32c3..000000000 --- a/src/chat/memory_system/memory_formatter.py +++ /dev/null @@ -1,120 +0,0 @@ -"""记忆格式化工具 - -提供统一的记忆块格式化函数，供构建 Prompt 时使用。 - -当前使用的函数: format_memories_bracket_style -输入: list[dict] 其中每个元素包含： - - display: str 记忆可读内容 - - memory_type: str 记忆类型 (personal_fact/opinion/preference/event 等) - - metadata: dict 可选，包括 - - confidence: 置信度 (str|float) - - importance: 重要度 (str|float) - - timestamp: 时间戳 (float|str) - - source: 来源 (str) - - relevance_score: 相关度 (float) - -返回: 适合直接嵌入提示词的大段文本；若无有效记忆返回空串。 -""" - -from __future__ import annotations - -import time -from collections.abc import Iterable -from typing import Any - - -def _format_timestamp(ts: Any) -> str: - try: - if ts in (None, ""): - return "" - if isinstance(ts, int | float) and ts > 0: - return time.strftime("%Y-%m-%d %H:%M", time.localtime(float(ts))) - return str(ts) - except Exception: - return "" - - -def _coerce_str(v: Any) -> str: - if v is None: - return "" - return str(v) - - -def format_memories_bracket_style( - memories: Iterable[dict[str, Any]] | None, - query_context: str | None = None, - max_items: int = 15, -) -> str: - """以方括号 + 标注字段的方式格式化记忆列表。 - - 例子输出: - ## 相关记忆回顾 - - [类型:personal_fact|重要:高|置信:0.83|相关:0.72] 他喜欢黑咖啡 (来源: chat, 2025-10-05 09:30) - - Args: - memories: 记忆字典迭代器 - query_context: 当前查询/用户的消息，用于在首行提示（可选） - max_items: 最多输出的记忆条数 - Returns: - str: 格式化文本；若无内容返回空串 - """ - if not memories: - return "" - - lines: list[str] = ["## 相关记忆回顾"] - if query_context: - lines.append(f"（与当前消息相关：{query_context[:60]}{'...' if len(query_context) > 60 else ''}）") - lines.append("") - - count = 0 - for mem in memories: - if count >= max_items: - break - if not isinstance(mem, dict): - continue - display = _coerce_str(mem.get("display", "")).strip() - if not display: - continue - - mtype = _coerce_str(mem.get("memory_type", "fact")) or "fact" - meta = mem.get("metadata", {}) if isinstance(mem.get("metadata"), dict) else {} - confidence = _coerce_str(meta.get("confidence", "")) - importance = _coerce_str(meta.get("importance", "")) - source = _coerce_str(meta.get("source", "")) - rel = meta.get("relevance_score") - try: - rel_str = f"{float(rel):.2f}" if rel is not None else "" - except Exception: - rel_str = "" - ts = _format_timestamp(meta.get("timestamp")) - - # 构建标签段 - tags: list[str] = [f"类型:{mtype}"] - if importance: - tags.append(f"重要:{importance}") - if confidence: - tags.append(f"置信:{confidence}") - if rel_str: - tags.append(f"相关:{rel_str}") - - tag_block = "|".join(tags) - suffix_parts = [] - if source: - suffix_parts.append(source) - if ts: - suffix_parts.append(ts) - suffix = (" (" + ", ".join(suffix_parts) + ")") if suffix_parts else "" - - lines.append(f"- [{tag_block}] {display}{suffix}") - count += 1 - - if count == 0: - return "" - - if count >= max_items: - lines.append(f"\n(已截断，仅显示前 {max_items} 条相关记忆)") - - return "\n".join(lines) - - -__all__ = ["format_memories_bracket_style"] diff --git a/src/chat/memory_system/memory_fusion.py b/src/chat/memory_system/memory_fusion.py deleted file mode 100644 index 6e384ca8c..000000000 --- a/src/chat/memory_system/memory_fusion.py +++ /dev/null @@ -1,505 +0,0 @@ -""" -记忆融合与去重机制 -避免记忆碎片化，确保长期记忆库的高质量 -""" - -import time -from dataclasses import dataclass -from typing import Any - -from src.chat.memory_system.memory_chunk import ConfidenceLevel, ImportanceLevel, MemoryChunk -from src.common.logger import get_logger - -logger = get_logger(__name__) - - -@dataclass -class FusionResult: - """融合结果""" - - original_count: int - fused_count: int - removed_duplicates: int - merged_memories: list[MemoryChunk] - fusion_time: float - details: list[str] - - -@dataclass -class DuplicateGroup: - """重复记忆组""" - - group_id: str - memories: list[MemoryChunk] - similarity_matrix: list[list[float]] - representative_memory: MemoryChunk | None = None - - -class MemoryFusionEngine: - """记忆融合引擎""" - - def __init__(self, similarity_threshold: float = 0.85): - self.similarity_threshold = similarity_threshold - self.fusion_stats = { - "total_fusions": 0, - "memories_fused": 0, - "duplicates_removed": 0, - "average_similarity": 0.0, - } - - # 融合策略配置 - self.fusion_strategies = { - "semantic_similarity": True, # 语义相似性融合 - "temporal_proximity": True, # 时间接近性融合 - "logical_consistency": True, # 逻辑一致性融合 - "confidence_boosting": True, # 置信度提升 - "importance_preservation": True, # 重要性保持 - } - - async def fuse_memories( - self, new_memories: list[MemoryChunk], existing_memories: list[MemoryChunk] | None = None - ) -> list[MemoryChunk]: - """融合记忆列表""" - start_time = time.time() - - try: - if not new_memories: - return [] - - logger.info(f"开始记忆融合，新记忆: {len(new_memories)}，现有记忆: {len(existing_memories or [])}") - - # 1. 检测重复记忆组 - duplicate_groups = await self._detect_duplicate_groups(new_memories, existing_memories or []) - - if not duplicate_groups: - fusion_time = time.time() - start_time - self._update_fusion_stats(len(new_memories), 0, fusion_time) - logger.info("✅ 记忆融合完成: %d 条记忆，移除 0 条重复", len(new_memories)) - return new_memories - - # 2. 对每个重复组进行融合 - fused_memories = [] - removed_count = 0 - - for group in duplicate_groups: - if len(group.memories) == 1: - # 单个记忆，直接添加 - fused_memories.append(group.memories[0]) - else: - # 多个记忆，进行融合 - fused_memory = await self._fuse_memory_group(group) - if fused_memory: - fused_memories.append(fused_memory) - removed_count += len(group.memories) - 1 - - # 3. 更新统计 - fusion_time = time.time() - start_time - self._update_fusion_stats(len(new_memories), removed_count, fusion_time) - - logger.info(f"✅ 记忆融合完成: {len(fused_memories)} 条记忆，移除 {removed_count} 条重复") - return fused_memories - - except Exception as e: - logger.error(f"❌ 记忆融合失败: {e}", exc_info=True) - return new_memories # 失败时返回原始记忆 - - async def _detect_duplicate_groups( - self, new_memories: list[MemoryChunk], existing_memories: list[MemoryChunk] - ) -> list[DuplicateGroup]: - """检测重复记忆组""" - all_memories = new_memories + existing_memories - new_memory_ids = {memory.memory_id for memory in new_memories} - groups = [] - processed_ids = set() - - for i, memory1 in enumerate(all_memories): - if memory1.memory_id in processed_ids: - continue - - # 创建新的重复组 - group = DuplicateGroup(group_id=f"group_{len(groups)}", memories=[memory1], similarity_matrix=[[1.0]]) - - processed_ids.add(memory1.memory_id) - - # 寻找相似记忆 - for j, memory2 in enumerate(all_memories[i + 1 :], i + 1): - if memory2.memory_id in processed_ids: - continue - - similarity = self._calculate_comprehensive_similarity(memory1, memory2) - - if similarity >= self.similarity_threshold: - group.memories.append(memory2) - processed_ids.add(memory2.memory_id) - - # 更新相似度矩阵 - self._update_similarity_matrix(group, memory2, similarity) - - if len(group.memories) > 1: - # 选择代表性记忆 - group.representative_memory = self._select_representative_memory(group) - groups.append(group) - else: - # 仅包含单条记忆，只有当其来自新记忆列表时保留 - if memory1.memory_id in new_memory_ids: - groups.append(group) - - logger.debug(f"检测到 {len(groups)} 个重复记忆组") - return groups - - def _calculate_comprehensive_similarity(self, mem1: MemoryChunk, mem2: MemoryChunk) -> float: - """计算综合相似度""" - similarity_scores = [] - - # 1. 语义向量相似度 - if self.fusion_strategies["semantic_similarity"]: - semantic_sim = mem1.calculate_similarity(mem2) - similarity_scores.append(("semantic", semantic_sim)) - - # 2. 文本相似度 - text_sim = self._calculate_text_similarity(mem1.text_content, mem2.text_content) - similarity_scores.append(("text", text_sim)) - - # 3. 关键词重叠度 - keyword_sim = self._calculate_keyword_similarity(mem1.keywords, mem2.keywords) - similarity_scores.append(("keyword", keyword_sim)) - - # 4. 类型一致性 - type_consistency = 1.0 if mem1.memory_type == mem2.memory_type else 0.0 - similarity_scores.append(("type", type_consistency)) - - # 5. 时间接近性 - if self.fusion_strategies["temporal_proximity"]: - temporal_sim = self._calculate_temporal_similarity(mem1.metadata.created_at, mem2.metadata.created_at) - similarity_scores.append(("temporal", temporal_sim)) - - # 6. 逻辑一致性 - if self.fusion_strategies["logical_consistency"]: - logical_sim = self._calculate_logical_similarity(mem1, mem2) - similarity_scores.append(("logical", logical_sim)) - - # 计算加权平均相似度 - weights = {"semantic": 0.35, "text": 0.25, "keyword": 0.15, "type": 0.10, "temporal": 0.10, "logical": 0.05} - - weighted_sum = 0.0 - total_weight = 0.0 - - for score_type, score in similarity_scores: - weight = weights.get(score_type, 0.1) - weighted_sum += weight * score - total_weight += weight - - final_similarity = weighted_sum / total_weight if total_weight > 0 else 0.0 - - logger.debug(f"综合相似度计算: {final_similarity:.3f} - {[(t, f'{s:.3f}') for t, s in similarity_scores]}") - - return final_similarity - - def _calculate_text_similarity(self, text1: str, text2: str) -> float: - """计算文本相似度""" - # 简单的词汇重叠度计算 - words1 = set(text1.lower().split()) - words2 = set(text2.lower().split()) - - if not words1 or not words2: - return 0.0 - - intersection = words1 & words2 - union = words1 | words2 - - jaccard_similarity = len(intersection) / len(union) - return jaccard_similarity - - def _calculate_keyword_similarity(self, keywords1: list[str], keywords2: list[str]) -> float: - """计算关键词相似度""" - if not keywords1 or not keywords2: - return 0.0 - - set1 = set(k.lower() for k in keywords1) # noqa: C401 - set2 = set(k.lower() for k in keywords2) # noqa: C401 - - intersection = set1 & set2 - union = set1 | set2 - - return len(intersection) / len(union) if union else 0.0 - - def _calculate_temporal_similarity(self, time1: float, time2: float) -> float: - """计算时间相似度""" - time_diff = abs(time1 - time2) - hours_diff = time_diff / 3600 - - # 24小时内相似度较高 - if hours_diff <= 24: - return 1.0 - (hours_diff / 24) - elif hours_diff <= 168: # 一周内 - return 0.7 - ((hours_diff - 24) / 168) * 0.5 - else: - return 0.2 - - def _calculate_logical_similarity(self, mem1: MemoryChunk, mem2: MemoryChunk) -> float: - """计算逻辑一致性""" - # 检查主谓宾结构的逻辑一致性 - consistency_score = 0.0 - - # 主语一致性 - subjects1 = set(mem1.subjects) - subjects2 = set(mem2.subjects) - if subjects1 or subjects2: - overlap = len(subjects1 & subjects2) - union_count = max(len(subjects1 | subjects2), 1) - consistency_score += (overlap / union_count) * 0.4 - - # 谓语相似性 - predicate_sim = self._calculate_text_similarity(mem1.content.predicate, mem2.content.predicate) - consistency_score += predicate_sim * 0.3 - - # 宾语相似性 - if isinstance(mem1.content.object, str) and isinstance(mem2.content.object, str): - object_sim = self._calculate_text_similarity(str(mem1.content.object), str(mem2.content.object)) - consistency_score += object_sim * 0.3 - - return consistency_score - - def _update_similarity_matrix(self, group: DuplicateGroup, new_memory: MemoryChunk, similarity: float): - """更新组的相似度矩阵""" - # 为新记忆添加行和列 - for i in range(len(group.similarity_matrix)): - group.similarity_matrix[i].append(similarity) - - # 添加新行 - new_row = [similarity] + [1.0] * len(group.similarity_matrix) - group.similarity_matrix.append(new_row) - - def _select_representative_memory(self, group: DuplicateGroup) -> MemoryChunk: - """选择代表性记忆""" - if not group.memories: - return None - - # 评分标准 - best_memory = None - best_score = -1.0 - - for memory in group.memories: - score = 0.0 - - # 置信度权重 - score += memory.metadata.confidence.value * 0.3 - - # 重要性权重 - score += memory.metadata.importance.value * 0.3 - - # 访问次数权重 - score += min(memory.metadata.access_count * 0.1, 0.2) - - # 相关度权重 - score += memory.metadata.relevance_score * 0.2 - - if score > best_score: - best_score = score - best_memory = memory - - return best_memory - - async def _fuse_memory_group(self, group: DuplicateGroup) -> MemoryChunk | None: - """融合记忆组""" - if not group.memories: - return None - - if len(group.memories) == 1: - return group.memories[0] - - try: - # 选择基础记忆（通常是代表性记忆） - base_memory = group.representative_memory or group.memories[0] - - # 融合其他记忆的属性 - fused_memory = await self._merge_memory_attributes(base_memory, group.memories) - - # 更新元数据 - self._update_fused_metadata(fused_memory, group) - - logger.debug(f"成功融合记忆组，包含 {len(group.memories)} 条原始记忆") - return fused_memory - - except Exception as e: - logger.error(f"融合记忆组失败: {e}") - # 返回置信度最高的记忆 - return max(group.memories, key=lambda m: m.metadata.confidence.value) - - async def _merge_memory_attributes(self, base_memory: MemoryChunk, memories: list[MemoryChunk]) -> MemoryChunk: - """合并记忆属性""" - # 创建基础记忆的深拷贝 - fused_memory = MemoryChunk.from_dict(base_memory.to_dict()) - - # 合并关键词 - all_keywords = set() - for memory in memories: - all_keywords.update(memory.keywords) - fused_memory.keywords = sorted(all_keywords) - - # 合并标签 - all_tags = set() - for memory in memories: - all_tags.update(memory.tags) - fused_memory.tags = sorted(all_tags) - - # 合并分类 - all_categories = set() - for memory in memories: - all_categories.update(memory.categories) - fused_memory.categories = sorted(all_categories) - - # 合并关联记忆 - all_related = set() - for memory in memories: - all_related.update(memory.related_memories) - # 移除对自身和组内记忆的引用 - all_related = {rid for rid in all_related if rid not in [m.memory_id for m in memories]} - fused_memory.related_memories = sorted(all_related) - - # 合并时间上下文 - if self.fusion_strategies["temporal_proximity"]: - fused_memory.temporal_context = self._merge_temporal_context(memories) - - return fused_memory - - def _update_fused_metadata(self, fused_memory: MemoryChunk, group: DuplicateGroup): - """更新融合记忆的元数据""" - # 更新修改时间 - fused_memory.metadata.last_modified = time.time() - - # 计算平均访问次数 - total_access = sum(m.metadata.access_count for m in group.memories) - fused_memory.metadata.access_count = total_access - - # 提升置信度（如果有多个来源支持） - if self.fusion_strategies["confidence_boosting"] and len(group.memories) > 1: - max_confidence = max(m.metadata.confidence.value for m in group.memories) - if max_confidence < ConfidenceLevel.VERIFIED.value: - fused_memory.metadata.confidence = ConfidenceLevel( - min(max_confidence + 1, ConfidenceLevel.VERIFIED.value) - ) - - # 保持最高重要性 - if self.fusion_strategies["importance_preservation"]: - max_importance = max(m.metadata.importance.value for m in group.memories) - fused_memory.metadata.importance = ImportanceLevel(max_importance) - - # 计算平均相关度 - avg_relevance = sum(m.metadata.relevance_score for m in group.memories) / len(group.memories) - fused_memory.metadata.relevance_score = min(avg_relevance * 1.1, 1.0) # 稍微提升相关度 - - # 设置来源信息 - source_ids = [m.memory_id[:8] for m in group.memories] - fused_memory.metadata.source_context = f"Fused from {len(group.memories)} memories: {', '.join(source_ids)}" - - def _merge_temporal_context(self, memories: list[MemoryChunk]) -> dict[str, Any]: - """合并时间上下文""" - contexts = [m.temporal_context for m in memories if m.temporal_context] - - if not contexts: - return {} - - # 计算时间范围 - timestamps = [m.metadata.created_at for m in memories] - earliest_time = min(timestamps) - latest_time = max(timestamps) - - merged_context = { - "earliest_timestamp": earliest_time, - "latest_timestamp": latest_time, - "time_span_hours": (latest_time - earliest_time) / 3600, - "source_memories": len(memories), - } - - # 合并其他上下文信息 - for context in contexts: - for key, value in context.items(): - if key not in ["timestamp", "earliest_timestamp", "latest_timestamp"]: - if key not in merged_context: - merged_context[key] = value - elif merged_context[key] != value: - merged_context[key] = f"multiple: {value}" - - return merged_context - - async def incremental_fusion( - self, new_memory: MemoryChunk, existing_memories: list[MemoryChunk] - ) -> tuple[MemoryChunk, list[MemoryChunk]]: - """增量融合（单个新记忆与现有记忆融合）""" - # 寻找相似记忆 - similar_memories = [] - - for existing in existing_memories: - similarity = self._calculate_comprehensive_similarity(new_memory, existing) - if similarity >= self.similarity_threshold: - similar_memories.append((existing, similarity)) - - if not similar_memories: - # 没有相似记忆，直接返回 - return new_memory, existing_memories - - # 按相似度排序 - similar_memories.sort(key=lambda x: x[1], reverse=True) - - # 与最相似的记忆融合 - best_match, similarity = similar_memories[0] - - # 创建融合组 - group = DuplicateGroup( - group_id=f"incremental_{int(time.time())}", - memories=[new_memory, best_match], - similarity_matrix=[[1.0, similarity], [similarity, 1.0]], - ) - - # 执行融合 - fused_memory = await self._fuse_memory_group(group) - - # 从现有记忆中移除被融合的记忆 - updated_existing = [m for m in existing_memories if m.memory_id != best_match.memory_id] - updated_existing.append(fused_memory) - - logger.debug(f"增量融合完成，相似度: {similarity:.3f}") - - return fused_memory, updated_existing - - def _update_fusion_stats(self, original_count: int, removed_count: int, fusion_time: float): - """更新融合统计""" - self.fusion_stats["total_fusions"] += 1 - self.fusion_stats["memories_fused"] += original_count - self.fusion_stats["duplicates_removed"] += removed_count - - # 更新平均相似度（估算） - if removed_count > 0: - avg_similarity = 0.9 # 假设平均相似度较高 - total_similarity = self.fusion_stats["average_similarity"] * (self.fusion_stats["total_fusions"] - 1) - total_similarity += avg_similarity - self.fusion_stats["average_similarity"] = total_similarity / self.fusion_stats["total_fusions"] - - async def maintenance(self): - """维护操作""" - try: - logger.info("开始记忆融合引擎维护...") - - # 可以在这里添加定期维护任务，如： - # - 重新评估低置信度记忆 - # - 清理孤立记忆引用 - # - 优化融合策略参数 - - logger.info("✅ 记忆融合引擎维护完成") - - except Exception as e: - logger.error(f"❌ 记忆融合引擎维护失败: {e}", exc_info=True) - - def get_fusion_stats(self) -> dict[str, Any]: - """获取融合统计信息""" - return self.fusion_stats.copy() - - def reset_stats(self): - """重置统计信息""" - self.fusion_stats = { - "total_fusions": 0, - "memories_fused": 0, - "duplicates_removed": 0, - "average_similarity": 0.0, - } diff --git a/src/chat/memory_system/memory_manager.py b/src/chat/memory_system/memory_manager.py deleted file mode 100644 index 3b6ef6a46..000000000 --- a/src/chat/memory_system/memory_manager.py +++ /dev/null @@ -1,512 +0,0 @@ -""" -记忆系统管理器 -替代原有的 Hippocampus 和 instant_memory 系统 -""" - -import re -from dataclasses import dataclass -from typing import Any - -from src.chat.memory_system.memory_chunk import MemoryChunk, MemoryType -from src.chat.memory_system.memory_system import MemorySystem -from src.chat.memory_system.message_collection_processor import MessageCollectionProcessor -from src.chat.memory_system.message_collection_storage import MessageCollectionStorage -from src.common.logger import get_logger - -logger = get_logger(__name__) - - -@dataclass -class MemoryResult: - """记忆查询结果""" - - content: str - memory_type: str - confidence: float - importance: float - timestamp: float - source: str = "memory" - relevance_score: float = 0.0 - structure: dict[str, Any] | None = None - - -class MemoryManager: - """记忆系统管理器 - 替代原有的 HippocampusManager""" - - def __init__(self): - self.memory_system: MemorySystem | None = None - self.message_collection_storage: MessageCollectionStorage | None = None - self.message_collection_processor: MessageCollectionProcessor | None = None - self.is_initialized = False - self.user_cache = {} # 用户记忆缓存 - - def _clean_text(self, text: Any) -> str: - if text is None: - return "" - - cleaned = re.sub(r"[\s\u3000]+", " ", str(text)).strip() - cleaned = re.sub(r"[、，,；;]+$", "", cleaned) - return cleaned - - async def initialize(self): - """初始化记忆系统""" - if self.is_initialized: - return - - try: - from src.config.config import global_config - - # 检查是否启用记忆系统 - if not global_config.memory.enable_memory: - logger.info("记忆系统已禁用，跳过初始化") - self.is_initialized = True - return - - logger.info("正在初始化记忆系统...") - - # 初始化记忆系统 - from src.chat.memory_system.memory_system import get_memory_system - self.memory_system = get_memory_system() - - # 初始化消息集合系统 - self.message_collection_storage = MessageCollectionStorage() - self.message_collection_processor = MessageCollectionProcessor(self.message_collection_storage) - - self.is_initialized = True - logger.info(" 记忆系统初始化完成") - - except Exception as e: - logger.error(f"记忆系统初始化失败: {e}") - # 如果系统初始化失败，创建一个空的管理器避免系统崩溃 - self.memory_system = None - self.message_collection_storage = None - self.message_collection_processor = None - self.is_initialized = True # 标记为已初始化但系统不可用 - - def get_hippocampus(self): - """兼容原有接口 - 返回空""" - logger.debug("get_hippocampus 调用 - 记忆系统不使用此方法") - return {} - - async def build_memory(self): - """兼容原有接口 - 构建记忆""" - if not self.is_initialized or not self.memory_system: - return - - try: - # 记忆系统使用实时构建，不需要定时构建 - logger.debug("build_memory 调用 - 记忆系统使用实时构建") - except Exception as e: - logger.error(f"build_memory 失败: {e}") - - async def forget_memory(self, percentage: float = 0.005): - """兼容原有接口 - 遗忘机制""" - if not self.is_initialized or not self.memory_system: - return - - try: - # 增强记忆系统有内置的遗忘机制 - logger.debug(f"forget_memory 调用 - 参数: {percentage}") - # 可以在这里调用增强系统的维护功能 - await self.memory_system.maintenance() - except Exception as e: - logger.error(f"forget_memory 失败: {e}") - - async def get_memory_from_text( - self, - text: str, - chat_id: str, - user_id: str, - max_memory_num: int = 3, - max_memory_length: int = 2, - time_weight: float = 1.0, - keyword_weight: float = 1.0, - ) -> list[tuple[str, str]]: - """从文本获取相关记忆 - 兼容原有接口""" - if not self.is_initialized or not self.memory_system: - return [] - - try: - # 使用增强记忆系统检索 - context = { - "chat_id": chat_id, - "expected_memory_types": [MemoryType.PERSONAL_FACT, MemoryType.EVENT, MemoryType.PREFERENCE], - } - - relevant_memories = await self.memory_system.retrieve_relevant_memories( - query=text, user_id=user_id, context=context, limit=max_memory_num - ) - - # 转换为原有格式 (topic, content) - results = [] - for memory in relevant_memories: - topic = memory.memory_type.value - content = memory.text_content - results.append((topic, content)) - - logger.debug(f"从文本检索到 {len(results)} 条相关记忆") - - # 如果检索到有效记忆，打印详细信息 - if results: - logger.info(f"📚 从文本 '{text[:50]}...' 检索到 {len(results)} 条有效记忆:") - for i, (topic, content) in enumerate(results, 1): - # 处理长内容，如果超过150字符则截断 - display_content = content - if len(content) > 150: - display_content = content[:150] + "..." - logger.info(f" 记忆#{i} [{topic}]: {display_content}") - - return results - - except Exception as e: - logger.error(f"get_memory_from_text 失败: {e}") - return [] - - async def get_memory_from_topic( - self, valid_keywords: list[str], max_memory_num: int = 3, max_memory_length: int = 2, max_depth: int = 3 - ) -> list[tuple[str, str]]: - """从关键词获取记忆 - 兼容原有接口""" - if not self.is_initialized or not self.memory_system: - return [] - - try: - # 将关键词转换为查询文本 - query_text = " ".join(valid_keywords) - - # 使用增强记忆系统检索 - context = { - "keywords": valid_keywords, - "expected_memory_types": [ - MemoryType.PERSONAL_FACT, - MemoryType.EVENT, - MemoryType.PREFERENCE, - MemoryType.OPINION, - ], - } - - relevant_memories = await self.memory_system.retrieve_relevant_memories( - query_text=query_text, - user_id="default_user", # 可以根据实际需要传递 - context=context, - limit=max_memory_num, - ) - - # 转换为原有格式 (topic, content) - results = [] - for memory in relevant_memories: - topic = memory.memory_type.value - content = memory.text_content - results.append((topic, content)) - - logger.debug(f"从关键词 {valid_keywords} 检索到 {len(results)} 条相关记忆") - - # 如果检索到有效记忆，打印详细信息 - if results: - keywords_str = ", ".join(valid_keywords[:5]) # 最多显示5个关键词 - if len(valid_keywords) > 5: - keywords_str += f" ... (共{len(valid_keywords)}个关键词)" - logger.info(f"🔍 从关键词 [{keywords_str}] 检索到 {len(results)} 条有效记忆:") - for i, (topic, content) in enumerate(results, 1): - # 处理长内容，如果超过150字符则截断 - display_content = content - if len(content) > 150: - display_content = content[:150] + "..." - logger.info(f" 记忆#{i} [{topic}]: {display_content}") - - return results - - except Exception as e: - logger.error(f"get_memory_from_topic 失败: {e}") - return [] - - def get_memory_from_keyword(self, keyword: str, max_depth: int = 2) -> list: - """从单个关键词获取记忆 - 兼容原有接口""" - if not self.is_initialized or not self.memory_system: - return [] - - try: - # 同步方法，返回空列表 - logger.debug(f"get_memory_from_keyword 调用 - 关键词: {keyword}") - return [] - except Exception as e: - logger.error(f"get_memory_from_keyword 失败: {e}") - return [] - - async def process_conversation( - self, conversation_text: str, context: dict[str, Any], user_id: str, timestamp: float | None = None - ) -> list[MemoryChunk]: - """处理对话并构建记忆 - 新增功能""" - if not self.is_initialized or not self.memory_system: - return [] - - try: - # 将消息添加到消息集合处理器 - chat_id = context.get("chat_id") - if self.message_collection_processor and chat_id: - await self.message_collection_processor.add_message(conversation_text, chat_id) - - payload_context = dict(context or {}) - payload_context.setdefault("conversation_text", conversation_text) - if timestamp is not None: - payload_context.setdefault("timestamp", timestamp) - - result = await self.memory_system.process_conversation_memory(payload_context) - - # 从结果中提取记忆块 - memory_chunks = [] - if result.get("success"): - memory_chunks = result.get("created_memories", []) - - logger.info(f"从对话构建了 {len(memory_chunks)} 条记忆") - return memory_chunks - - except Exception as e: - logger.error(f"process_conversation 失败: {e}") - return [] - - async def get_enhanced_memory_context( - self, query_text: str, user_id: str, context: dict[str, Any] | None = None, limit: int = 5 - ) -> list[MemoryResult]: - """获取增强记忆上下文 - 新增功能""" - if not self.is_initialized or not self.memory_system: - return [] - - try: - relevant_memories = await self.memory_system.retrieve_relevant_memories( - query=query_text, user_id=None, context=context or {}, limit=limit - ) - - results = [] - for memory in relevant_memories: - formatted_content, structure = self._format_memory_chunk(memory) - result = MemoryResult( - content=formatted_content, - memory_type=memory.memory_type.value, - confidence=memory.metadata.confidence.value, - importance=memory.metadata.importance.value, - timestamp=memory.metadata.created_at, - source="enhanced_memory", - relevance_score=memory.metadata.relevance_score, - structure=structure, - ) - results.append(result) - - return results - - except Exception as e: - logger.error(f"get_enhanced_memory_context 失败: {e}") - return [] - - def _format_memory_chunk(self, memory: MemoryChunk) -> tuple[str, dict[str, Any]]: - """将记忆块转换为更易读的文本描述""" - structure = memory.content.to_dict() - if memory.display: - return self._clean_text(memory.display), structure - - subject = structure.get("subject") - predicate = structure.get("predicate") or "" - obj = structure.get("object") - - subject_display = self._format_subject(subject, memory) - formatted = self._apply_predicate_format(subject_display, predicate, obj) - - if not formatted: - predicate_display = self._format_predicate(predicate) - object_display = self._format_object(obj) - formatted = f"{subject_display}{predicate_display}{object_display}".strip() - - formatted = self._clean_text(formatted) - - return formatted, structure - - def _format_subject(self, subject: str | None, memory: MemoryChunk) -> str: - if not subject: - return "该用户" - - if subject == memory.metadata.user_id: - return "该用户" - if memory.metadata.chat_id and subject == memory.metadata.chat_id: - return "该聊天" - return self._clean_text(subject) - - def _apply_predicate_format(self, subject: str, predicate: str, obj: Any) -> str | None: - predicate = (predicate or "").strip() - obj_value = obj - - if predicate == "is_named": - name = self._extract_from_object(obj_value, ["name", "nickname"]) or self._format_object(obj_value) - name = self._clean_text(name) - if not name: - return None - name_display = name if (name.startswith("「") and name.endswith("」")) else f"「{name}」" - return f"{subject}的昵称是{name_display}" - if predicate == "is_age": - age = self._extract_from_object(obj_value, ["age"]) or self._format_object(obj_value) - age = self._clean_text(age) - if not age: - return None - return f"{subject}今年{age}岁" - if predicate == "is_profession": - profession = self._extract_from_object(obj_value, ["profession", "job"]) or self._format_object(obj_value) - profession = self._clean_text(profession) - if not profession: - return None - return f"{subject}的职业是{profession}" - if predicate == "lives_in": - location = self._extract_from_object(obj_value, ["location", "city", "place"]) or self._format_object( - obj_value - ) - location = self._clean_text(location) - if not location: - return None - return f"{subject}居住在{location}" - if predicate == "has_phone": - phone = self._extract_from_object(obj_value, ["phone", "number"]) or self._format_object(obj_value) - phone = self._clean_text(phone) - if not phone: - return None - return f"{subject}的电话号码是{phone}" - if predicate == "has_email": - email = self._extract_from_object(obj_value, ["email"]) or self._format_object(obj_value) - email = self._clean_text(email) - if not email: - return None - return f"{subject}的邮箱是{email}" - if predicate == "likes": - liked = self._format_object(obj_value) - if not liked: - return None - return f"{subject}喜欢{liked}" - if predicate == "likes_food": - food = self._format_object(obj_value) - if not food: - return None - return f"{subject}爱吃{food}" - if predicate == "dislikes": - disliked = self._format_object(obj_value) - if not disliked: - return None - return f"{subject}不喜欢{disliked}" - if predicate == "hates": - hated = self._format_object(obj_value) - if not hated: - return None - return f"{subject}讨厌{hated}" - if predicate == "favorite_is": - favorite = self._format_object(obj_value) - if not favorite: - return None - return f"{subject}最喜欢{favorite}" - if predicate == "mentioned_event": - event_text = self._extract_from_object(obj_value, ["event_text", "description"]) or self._format_object( - obj_value - ) - event_text = self._clean_text(self._truncate(event_text)) - if not event_text: - return None - return f"{subject}提到了计划或事件：{event_text}" - if predicate in {"正在", "在", "正在进行"}: - action = self._format_object(obj_value) - if not action: - return None - return f"{subject}{predicate}{action}" - if predicate in {"感到", "觉得", "表示", "提到", "说道", "说"}: - feeling = self._format_object(obj_value) - if not feeling: - return None - return f"{subject}{predicate}{feeling}" - if predicate in {"与", "和", "跟"}: - counterpart = self._format_object(obj_value) - if counterpart: - return f"{subject}{predicate}{counterpart}" - return f"{subject}{predicate}" - - return None - - def _format_predicate(self, predicate: str) -> str: - if not predicate: - return "" - predicate_map = { - "is_named": "的昵称是", - "is_profession": "的职业是", - "lives_in": "居住在", - "has_phone": "的电话是", - "has_email": "的邮箱是", - "likes": "喜欢", - "dislikes": "不喜欢", - "likes_food": "爱吃", - "hates": "讨厌", - "favorite_is": "最喜欢", - "mentioned_event": "提到的事件", - } - if predicate in predicate_map: - connector = predicate_map[predicate] - if connector.startswith("的"): - return connector - return f" {connector} " - cleaned = predicate.replace("_", " ").strip() - if re.search(r"[\u4e00-\u9fff]", cleaned): - return cleaned - return f" {cleaned} " - - def _format_object(self, obj: Any) -> str: - if obj is None: - return "" - if isinstance(obj, dict): - parts = [] - for key, value in obj.items(): - formatted_value = self._format_object(value) - if not formatted_value: - continue - pretty_key = { - "name": "名字", - "profession": "职业", - "location": "位置", - "event_text": "内容", - "timestamp": "时间", - }.get(key, key) - parts.append(f"{pretty_key}: {formatted_value}") - return self._clean_text("；".join(parts)) - if isinstance(obj, list): - formatted_items = [self._format_object(item) for item in obj] - filtered = [item for item in formatted_items if item] - return self._clean_text("、".join(filtered)) if filtered else "" - if isinstance(obj, int | float): - return str(obj) - text = self._truncate(str(obj).strip()) - return self._clean_text(text) - - def _extract_from_object(self, obj: Any, keys: list[str]) -> str | None: - if isinstance(obj, dict): - for key in keys: - if obj.get(key): - value = obj[key] - if isinstance(value, dict | list): - return self._clean_text(self._format_object(value)) - return self._clean_text(value) - if isinstance(obj, list) and obj: - return self._clean_text(self._format_object(obj[0])) - if isinstance(obj, str | int | float): - return self._clean_text(obj) - return None - - def _truncate(self, text: str, max_length: int = 80) -> str: - if len(text) <= max_length: - return text - return text[: max_length - 1] + "…" - - async def shutdown(self): - """关闭增强记忆系统""" - if not self.is_initialized: - return - - try: - if self.memory_system: - await self.memory_system.shutdown() - logger.info(" 记忆系统已关闭") - except Exception as e: - logger.error(f"关闭记忆系统失败: {e}") - - -# 全局记忆管理器实例 -memory_manager = MemoryManager() diff --git a/src/chat/memory_system/memory_metadata_index.py b/src/chat/memory_system/memory_metadata_index.py deleted file mode 100644 index 4b92c410a..000000000 --- a/src/chat/memory_system/memory_metadata_index.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -记忆元数据索引。 -""" - -from dataclasses import asdict, dataclass -from typing import Any - -from src.common.logger import get_logger - -logger = get_logger(__name__) - -from inkfox.memory import PyMetadataIndex as _RustIndex # type: ignore - - -@dataclass -class MemoryMetadataIndexEntry: - memory_id: str - user_id: str - memory_type: str - subjects: list[str] - objects: list[str] - keywords: list[str] - tags: list[str] - importance: int - confidence: int - created_at: float - access_count: int - chat_id: str | None = None - content_preview: str | None = None - - -class MemoryMetadataIndex: - """Rust 加速版本唯一实现。""" - - def __init__(self, index_file: str = "data/memory_metadata_index.json"): - self._rust = _RustIndex(index_file) - # 仅为向量层和调试提供最小缓存（长度判断、get_entry 返回） - self.index: dict[str, MemoryMetadataIndexEntry] = {} - logger.info("✅ MemoryMetadataIndex (Rust) 初始化完成，仅支持加速实现") - - # 向后代码仍调用的接口：batch_add_or_update / add_or_update - def batch_add_or_update(self, entries: list[MemoryMetadataIndexEntry]): - if not entries: - return - payload = [] - for e in entries: - if not e.memory_id: - continue - self.index[e.memory_id] = e - payload.append(asdict(e)) - if payload: - try: - self._rust.batch_add(payload) - except Exception as ex: - logger.error(f"Rust 元数据批量添加失败: {ex}") - - def add_or_update(self, entry: MemoryMetadataIndexEntry): - self.batch_add_or_update([entry]) - - def search( - self, - memory_types: list[str] | None = None, - subjects: list[str] | None = None, - keywords: list[str] | None = None, - tags: list[str] | None = None, - importance_min: int | None = None, - importance_max: int | None = None, - created_after: float | None = None, - created_before: float | None = None, - user_id: str | None = None, - limit: int | None = None, - flexible_mode: bool = True, - ) -> list[str]: - params: dict[str, Any] = { - "user_id": user_id, - "memory_types": memory_types, - "subjects": subjects, - "keywords": keywords, - "tags": tags, - "importance_min": importance_min, - "importance_max": importance_max, - "created_after": created_after, - "created_before": created_before, - "limit": limit, - } - params = {k: v for k, v in params.items() if v is not None} - try: - if flexible_mode: - return list(self._rust.search_flexible(params)) - return list(self._rust.search_strict(params)) - except Exception as ex: - logger.error(f"Rust 搜索失败返回空: {ex}") - return [] - - def get_entry(self, memory_id: str) -> MemoryMetadataIndexEntry | None: - return self.index.get(memory_id) - - def get_stats(self) -> dict[str, Any]: - try: - raw = self._rust.stats() - return { - "total_memories": raw.get("total", 0), - "types": raw.get("types_dist", {}), - "subjects_count": raw.get("subjects_indexed", 0), - "keywords_count": raw.get("keywords_indexed", 0), - "tags_count": raw.get("tags_indexed", 0), - } - except Exception as ex: - logger.warning(f"读取 Rust stats 失败: {ex}") - return {"total_memories": 0} - - def save(self): # 仅调用 rust save - try: - self._rust.save() - except Exception as ex: - logger.warning(f"Rust save 失败: {ex}") - - -__all__ = [ - "MemoryMetadataIndex", - "MemoryMetadataIndexEntry", -] diff --git a/src/chat/memory_system/memory_query_planner.py b/src/chat/memory_system/memory_query_planner.py deleted file mode 100644 index e3bef8ce2..000000000 --- a/src/chat/memory_system/memory_query_planner.py +++ /dev/null @@ -1,219 +0,0 @@ -"""记忆检索查询规划器""" - -from __future__ import annotations - -import re -from dataclasses import dataclass, field -from typing import Any - -import orjson - -from src.chat.memory_system.memory_chunk import MemoryType -from src.common.logger import get_logger -from src.llm_models.utils_model import LLMRequest -from src.utils.json_parser import extract_and_parse_json - -logger = get_logger(__name__) - - -@dataclass -class MemoryQueryPlan: - """查询规划结果""" - - semantic_query: str - memory_types: list[MemoryType] = field(default_factory=list) - subject_includes: list[str] = field(default_factory=list) - object_includes: list[str] = field(default_factory=list) - required_keywords: list[str] = field(default_factory=list) - optional_keywords: list[str] = field(default_factory=list) - owner_filters: list[str] = field(default_factory=list) - recency_preference: str = "any" - limit: int = 10 - emphasis: str | None = None - raw_plan: dict[str, Any] = field(default_factory=dict) - - def ensure_defaults(self, fallback_query: str, default_limit: int) -> None: - if not self.semantic_query: - self.semantic_query = fallback_query - if self.limit <= 0: - self.limit = default_limit - self.recency_preference = (self.recency_preference or "any").lower() - if self.recency_preference not in {"any", "recent", "historical"}: - self.recency_preference = "any" - self.emphasis = (self.emphasis or "balanced").lower() - - -class MemoryQueryPlanner: - """基于小模型的记忆检索查询规划器""" - - def __init__(self, planner_model: LLMRequest | None, default_limit: int = 10): - self.model = planner_model - self.default_limit = default_limit - - async def plan_query(self, query_text: str, context: dict[str, Any]) -> MemoryQueryPlan: - if not self.model: - logger.debug("未提供查询规划模型，使用默认规划") - return self._default_plan(query_text) - - prompt = self._build_prompt(query_text, context) - - try: - response, _ = await self.model.generate_response_async(prompt, temperature=0.2) - # 使用统一的 JSON 解析工具 - data = extract_and_parse_json(response, strict=False) - if not data or not isinstance(data, dict): - logger.debug("查询规划模型未返回有效的结构化结果，使用默认规划") - return self._default_plan(query_text) - - plan = self._parse_plan_dict(data, query_text) - plan.ensure_defaults(query_text, self.default_limit) - return plan - - except Exception as exc: - logger.error("查询规划模型调用失败: %s", exc, exc_info=True) - return self._default_plan(query_text) - - def _default_plan(self, query_text: str) -> MemoryQueryPlan: - return MemoryQueryPlan(semantic_query=query_text, limit=self.default_limit) - - def _parse_plan_dict(self, data: dict[str, Any], fallback_query: str) -> MemoryQueryPlan: - semantic_query = self._safe_str(data.get("semantic_query")) or fallback_query - - def _collect_list(key: str) -> list[str]: - value = data.get(key) - if isinstance(value, str): - return [value] - if isinstance(value, list): - return [self._safe_str(item) for item in value if self._safe_str(item)] - return [] - - memory_type_values = _collect_list("memory_types") - memory_types: list[MemoryType] = [] - for item in memory_type_values: - if not item: - continue - try: - memory_types.append(MemoryType(item)) - except ValueError: - # 尝试匹配value值 - normalized = item.lower() - for mt in MemoryType: - if mt.value == normalized: - memory_types.append(mt) - break - - plan = MemoryQueryPlan( - semantic_query=semantic_query, - memory_types=memory_types, - subject_includes=_collect_list("subject_includes"), - object_includes=_collect_list("object_includes"), - required_keywords=_collect_list("required_keywords"), - optional_keywords=_collect_list("optional_keywords"), - owner_filters=_collect_list("owner_filters"), - recency_preference=self._safe_str(data.get("recency")) or "any", - limit=self._safe_int(data.get("limit"), self.default_limit), - emphasis=self._safe_str(data.get("emphasis")) or "balanced", - raw_plan=data, - ) - return plan - - def _build_prompt(self, query_text: str, context: dict[str, Any]) -> str: - participants = context.get("participants") or context.get("speaker_names") or [] - if isinstance(participants, str): - participants = [participants] - participants = [p for p in participants if isinstance(p, str) and p.strip()] - participant_preview = "、".join(participants[:5]) or "未知" - - persona = context.get("bot_personality") or context.get("bot_identity") or "未知" - - # 构建未读消息上下文信息 - context_section = "" - if context.get("has_unread_context") and context.get("unread_messages_context"): - unread_context = context["unread_messages_context"] - unread_messages = unread_context.get("messages", []) - unread_keywords = unread_context.get("keywords", []) - unread_participants = unread_context.get("participants", []) - context_summary = unread_context.get("context_summary", "") - - if unread_messages: - # 构建未读消息摘要 - message_previews = [] - for msg in unread_messages[:5]: # 最多显示5条 - sender = msg.get("sender", "未知") - content = msg.get("content", "")[:100] # 限制每条消息长度 - message_previews.append(f"{sender}: {content}") - - context_section = f""" - -## 📋 未读消息上下文 (共{unread_context.get("total_count", 0)}条未读消息) -### 最近消息预览: -{chr(10).join(message_previews)} - -### 上下文关键词: -{", ".join(unread_keywords[:15]) if unread_keywords else "无"} - -### 对话参与者: -{", ".join(unread_participants) if unread_participants else "无"} - -### 上下文摘要: -{context_summary[:300] if context_summary else "无"} -""" - else: - context_section = """ - -## 📋 未读消息上下文: -无未读消息或上下文信息不可用 -""" - - return f""" -你是一名记忆检索规划助手，请基于输入生成一个简洁的 JSON 检索计划。 -你的任务是分析当前查询并结合未读消息的上下文，生成更精准的记忆检索策略。 - -仅需提供以下字段： -- semantic_query: 用于向量召回的自然语言描述，要求具体且贴合当前查询和上下文； -- memory_types: 建议检索的记忆类型列表，取值范围来自 MemoryType 枚举 (personal_fact,event,preference,opinion,relationship,emotion,knowledge,skill,goal,experience,contextual)； -- subject_includes: 建议出现在记忆主语中的人物或角色； -- object_includes: 建议关注的对象、主题或关键信息； -- required_keywords: 建议必须包含的关键词（从上下文中提取）； -- recency: 推荐的时间偏好，可选 recent/any/historical； -- limit: 推荐的最大返回数量 (1-15)； -- emphasis: 检索重点，可选 balanced/contextual/recent/comprehensive。 - -请不要生成谓语字段，也不要额外补充其它参数。 - -## 当前查询: -"{query_text}" - -## 已知对话参与者: -{participant_preview} - -## 机器人设定: -{persona}{context_section} - -## 🎯 指导原则: -1. **上下文关联**: 优先分析与当前查询相关的未读消息内容和关键词 -2. **语义理解**: 结合上下文理解查询的真实意图，而非字面意思 -3. **参与者感知**: 考虑未读消息中的参与者，检索与他们相关的记忆 -4. **主题延续**: 关注未读消息中讨论的主题，检索相关的历史记忆 -5. **时间相关性**: 如果未读消息讨论最近的事件，偏向检索相关时期的记忆 - -请直接输出符合要求的 JSON 对象，禁止添加额外文本或 Markdown 代码块。 -""" - - @staticmethod - def _safe_str(value: Any) -> str: - if isinstance(value, str): - return value.strip() - if value is None: - return "" - return str(value).strip() - - @staticmethod - def _safe_int(value: Any, default: int) -> int: - try: - number = int(value) - if number <= 0: - return default - return number - except (TypeError, ValueError): - return default diff --git a/src/chat/memory_system/message_collection_processor.py b/src/chat/memory_system/message_collection_processor.py deleted file mode 100644 index b930aa3c9..000000000 --- a/src/chat/memory_system/message_collection_processor.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -消息集合处理器 -负责收集消息、创建集合并将其存入向量存储。 -""" - -import asyncio -from collections import deque -from typing import Any - -from src.chat.memory_system.memory_chunk import MessageCollection -from src.chat.memory_system.message_collection_storage import MessageCollectionStorage -from src.common.logger import get_logger - -logger = get_logger(__name__) - - -class MessageCollectionProcessor: - """处理消息集合的创建和存储""" - - def __init__(self, storage: MessageCollectionStorage, buffer_size: int = 5): - self.storage = storage - self.buffer_size = buffer_size - self.message_buffers: dict[str, deque[str]] = {} - self._lock = asyncio.Lock() - - async def add_message(self, message_text: str, chat_id: str): - """添加一条新消息到指定聊天的缓冲区，并在满时触发处理""" - async with self._lock: - if not isinstance(message_text, str) or not message_text.strip(): - return - - if chat_id not in self.message_buffers: - self.message_buffers[chat_id] = deque(maxlen=self.buffer_size) - - buffer = self.message_buffers[chat_id] - buffer.append(message_text) - logger.debug(f"消息已添加到聊天 '{chat_id}' 的缓冲区，当前数量: {len(buffer)}/{self.buffer_size}") - - if len(buffer) == self.buffer_size: - await self._process_buffer(chat_id) - - async def _process_buffer(self, chat_id: str): - """处理指定聊天缓冲区中的消息，创建并存储一个集合""" - buffer = self.message_buffers.get(chat_id) - if not buffer or len(buffer) < self.buffer_size: - return - - messages_to_process = list(buffer) - buffer.clear() - - logger.info(f"聊天 '{chat_id}' 的消息缓冲区已满，开始创建消息集合...") - - try: - combined_text = "\n".join(messages_to_process) - - collection = MessageCollection( - chat_id=chat_id, - messages=messages_to_process, - combined_text=combined_text, - ) - - await self.storage.add_collection(collection) - logger.info(f"成功为聊天 '{chat_id}' 创建并存储了新的消息集合: {collection.collection_id}") - - except Exception as e: - logger.error(f"处理聊天 '{chat_id}' 的消息缓冲区失败: {e}", exc_info=True) - - def get_stats(self) -> dict[str, Any]: - """获取处理器统计信息""" - total_buffered_messages = sum(len(buf) for buf in self.message_buffers.values()) - return { - "active_buffers": len(self.message_buffers), - "total_buffered_messages": total_buffered_messages, - "buffer_capacity_per_chat": self.buffer_size, - } diff --git a/src/chat/memory_system/message_collection_storage.py b/src/chat/memory_system/message_collection_storage.py deleted file mode 100644 index 8392ffa86..000000000 --- a/src/chat/memory_system/message_collection_storage.py +++ /dev/null @@ -1,193 +0,0 @@ -""" -消息集合向量存储系统 -专用于存储和检索消息集合，以提供即时上下文。 -""" - -import time -from typing import Any - -from src.chat.memory_system.memory_chunk import MessageCollection -from src.chat.utils.utils import get_embedding -from src.common.logger import get_logger -from src.common.vector_db import vector_db_service -from src.config.config import global_config - -logger = get_logger(__name__) - -class MessageCollectionStorage: - """消息集合向量存储""" - - def __init__(self): - self.config = global_config.memory - self.vector_db_service = vector_db_service - self.collection_name = "message_collections" - self._initialize_storage() - - def _initialize_storage(self): - """初始化存储""" - try: - self.vector_db_service.get_or_create_collection( - name=self.collection_name, - metadata={"description": "短期消息集合记忆", "hnsw:space": "cosine"}, - ) - logger.info(f"消息集合存储初始化完成，集合: '{self.collection_name}'") - except Exception as e: - logger.error(f"消息集合存储初始化失败: {e}", exc_info=True) - raise - - async def add_collection(self, collection: MessageCollection): - """添加一个新的消息集合，并处理容量和时间限制""" - try: - # 清理过期和超额的集合 - await self._cleanup_collections() - - # 向量化并存储 - embedding = await get_embedding(collection.combined_text) - if not embedding: - logger.warning(f"无法为消息集合 {collection.collection_id} 生成向量，跳过存储。") - return - - collection.embedding = embedding - - self.vector_db_service.add( - collection_name=self.collection_name, - embeddings=[embedding], - ids=[collection.collection_id], - documents=[collection.combined_text], - metadatas=[collection.to_dict()], - ) - logger.debug(f"成功存储消息集合: {collection.collection_id}") - - except Exception as e: - logger.error(f"存储消息集合失败: {e}", exc_info=True) - - async def _cleanup_collections(self): - """清理超额和过期的消息集合""" - try: - # 基于时间清理 - if self.config.instant_memory_retention_hours > 0: - expiration_time = time.time() - self.config.instant_memory_retention_hours * 3600 - expired_docs = self.vector_db_service.get( - collection_name=self.collection_name, - where={"created_at": {"$lt": expiration_time}}, - include=[], # 只获取ID - ) - if expired_docs and expired_docs.get("ids"): - self.vector_db_service.delete(collection_name=self.collection_name, ids=expired_docs["ids"]) - logger.info(f"删除了 {len(expired_docs['ids'])} 个过期的瞬时记忆") - - # 基于数量清理 - current_count = self.vector_db_service.count(self.collection_name) - if current_count > self.config.instant_memory_max_collections: - num_to_delete = current_count - self.config.instant_memory_max_collections - - # 获取所有文档的元数据以进行排序 - all_docs = self.vector_db_service.get( - collection_name=self.collection_name, - include=["metadatas"] - ) - - if all_docs and all_docs.get("ids"): - # 在内存中排序找到最旧的文档 - sorted_docs = sorted( - zip(all_docs["ids"], all_docs["metadatas"]), - key=lambda item: item[1].get("created_at", 0), - ) - - ids_to_delete = [doc[0] for doc in sorted_docs[:num_to_delete]] - - if ids_to_delete: - self.vector_db_service.delete(collection_name=self.collection_name, ids=ids_to_delete) - logger.info(f"消息集合已满，删除最旧的 {len(ids_to_delete)} 个集合") - - except Exception as e: - logger.error(f"清理消息集合失败: {e}", exc_info=True) - - - async def get_relevant_collection(self, query_text: str, n_results: int = 1) -> list[MessageCollection]: - """根据查询文本检索最相关的消息集合""" - if not query_text.strip(): - return [] - - try: - query_embedding = await get_embedding(query_text) - if not query_embedding: - return [] - - results = self.vector_db_service.query( - collection_name=self.collection_name, - query_embeddings=[query_embedding], - n_results=n_results, - ) - - collections = [] - if results and results.get("ids") and results["ids"][0]: - collections.extend(MessageCollection.from_dict(metadata) for metadata in results["metadatas"][0]) - - return collections - except Exception as e: - logger.error(f"检索相关消息集合失败: {e}", exc_info=True) - return [] - - async def get_message_collection_context(self, query_text: str, chat_id: str) -> str: - """获取消息集合上下文，用于添加到 prompt 中。优先展示当前聊天的上下文。""" - try: - collections = await self.get_relevant_collection(query_text, n_results=5) - if not collections: - return "" - - # 根据传入的 chat_id 对集合进行排序 - collections.sort(key=lambda c: c.chat_id == chat_id, reverse=True) - - context_parts = [] - for collection in collections: - if not collection.combined_text: - continue - - header = "## 📝 相关对话上下文\n" - if collection.chat_id == chat_id: - # 匹配的ID，使用更明显的标识 - context_parts.append( - f"{header} [🔥 来自当前聊天的上下文]\n```\n{collection.combined_text}\n```" - ) - else: - # 不匹配的ID - context_parts.append( - f"{header} [💡 来自其他聊天的相关上下文 (ID: {collection.chat_id})]\n```\n{collection.combined_text}\n```" - ) - - if not context_parts: - return "" - - # 格式化消息集合为 prompt 上下文 - final_context = "\n\n---\n\n".join(context_parts) + "\n\n---" - - logger.info(f"🔗 为查询 '{query_text[:50]}...' 在聊天 '{chat_id}' 中找到 {len(collections)} 个相关消息集合上下文") - return f"\n{final_context}\n" - - except Exception as e: - logger.error(f"get_message_collection_context 失败: {e}") - return "" - - def clear_all(self): - """清空所有消息集合""" - try: - # In ChromaDB, the easiest way to clear a collection is to delete and recreate it. - self.vector_db_service.delete_collection(name=self.collection_name) - self._initialize_storage() - logger.info(f"已清空所有消息集合: '{self.collection_name}'") - except Exception as e: - logger.error(f"清空消息集合失败: {e}", exc_info=True) - - def get_stats(self) -> dict[str, Any]: - """获取存储统计信息""" - try: - count = self.vector_db_service.count(self.collection_name) - return { - "collection_name": self.collection_name, - "total_collections": count, - "storage_limit": self.config.instant_memory_max_collections, - } - except Exception as e: - logger.error(f"获取消息集合存储统计失败: {e}") - return {} diff --git a/src/chat/memory_system/vector_memory_storage_v2.py b/src/chat/memory_system/vector_memory_storage_v2.py deleted file mode 100644 index e799fc31b..000000000 --- a/src/chat/memory_system/vector_memory_storage_v2.py +++ /dev/null @@ -1,1043 +0,0 @@ -""" -基于Vector DB的统一记忆存储系统 V2 -使用ChromaDB作为底层存储，替代JSON存储方式 - -主要特性: -- 统一的向量存储接口 -- 高效的语义检索 -- 元数据过滤支持 -- 批量操作优化 -- 自动清理过期记忆 -""" - -import asyncio -import threading -import time -from dataclasses import dataclass -from datetime import datetime -from typing import Any - -import orjson - -from src.chat.memory_system.memory_chunk import ConfidenceLevel, ImportanceLevel, MemoryChunk -from src.chat.memory_system.memory_forgetting_engine import MemoryForgettingEngine -from src.chat.memory_system.memory_metadata_index import MemoryMetadataIndex, MemoryMetadataIndexEntry -from src.chat.utils.utils import get_embedding -from src.common.logger import get_logger -from src.common.vector_db import vector_db_service - -logger = get_logger(__name__) - -# 全局枚举映射表缓存 -_ENUM_MAPPINGS_CACHE = {} - - -def _build_enum_mapping(enum_class: type) -> dict[str, Any]: - """构建枚举类的完整映射表 - - Args: - enum_class: 枚举类 - - Returns: - Dict[str, Any]: 包含各种映射格式的字典 - """ - cache_key = f"{enum_class.__module__}.{enum_class.__name__}" - - # 如果已经缓存过，直接返回 - if cache_key in _ENUM_MAPPINGS_CACHE: - return _ENUM_MAPPINGS_CACHE[cache_key] - - mapping = { - "name_to_enum": {}, # 枚举名称 -> 枚举实例 (HIGH -> ImportanceLevel.HIGH) - "value_to_enum": {}, # 整数值 -> 枚举实例 (3 -> ImportanceLevel.HIGH) - "value_str_to_enum": {}, # 字符串value -> 枚举实例 ("3" -> ImportanceLevel.HIGH) - "enum_value_to_name": {}, # 枚举实例 -> 名称映射 (反向) - "all_possible_strings": set(), # 所有可能的字符串表示 - } - - for member in enum_class: - # 名称映射 (支持大小写) - mapping["name_to_enum"][member.name] = member - mapping["name_to_enum"][member.name.lower()] = member - mapping["name_to_enum"][member.name.upper()] = member - - # 值映射 - mapping["value_to_enum"][member.value] = member - mapping["value_str_to_enum"][str(member.value)] = member - - # 反向映射 - mapping["enum_value_to_name"][member] = member.name - - # 收集所有可能的字符串表示 - mapping["all_possible_strings"].add(member.name) - mapping["all_possible_strings"].add(member.name.lower()) - mapping["all_possible_strings"].add(member.name.upper()) - mapping["all_possible_strings"].add(str(member.value)) - - # 缓存结果 - _ENUM_MAPPINGS_CACHE[cache_key] = mapping - logger.debug( - f"构建枚举映射表: {enum_class.__name__} -> {len(mapping['name_to_enum'])} 个名称映射, {len(mapping['value_to_enum'])} 个值映射" - ) - - return mapping - - -@dataclass -class VectorStorageConfig: - """Vector存储配置""" - - # 集合配置 - memory_collection: str = "unified_memory_v2" - metadata_collection: str = "memory_metadata_v2" - - # 检索配置 - similarity_threshold: float = 0.5 # 降低阈值以提高召回率（0.5-0.6 是合理范围） - search_limit: int = 20 - batch_size: int = 100 - - # 性能配置 - enable_caching: bool = True - cache_size_limit: int = 1000 - auto_cleanup_interval: int = 3600 # 1小时 - - # 遗忘配置 - enable_forgetting: bool = True - retention_hours: int = 24 * 30 # 30天 - - @classmethod - def from_global_config(cls): - """从全局配置创建实例""" - from src.config.config import global_config - - memory_cfg = global_config.memory - - return cls( - similarity_threshold=getattr(memory_cfg, "vector_db_similarity_threshold", 0.5), - search_limit=getattr(memory_cfg, "vector_db_search_limit", 20), - batch_size=getattr(memory_cfg, "vector_db_batch_size", 100), - enable_caching=getattr(memory_cfg, "vector_db_enable_caching", True), - cache_size_limit=getattr(memory_cfg, "vector_db_cache_size_limit", 1000), - auto_cleanup_interval=getattr(memory_cfg, "vector_db_auto_cleanup_interval", 3600), - enable_forgetting=getattr(memory_cfg, "enable_memory_forgetting", True), - retention_hours=getattr(memory_cfg, "vector_db_retention_hours", 720), - ) - - -class VectorMemoryStorage: - @property - def keyword_index(self) -> dict: - """ - 动态构建关键词倒排索引（仅兼容旧接口，基于当前缓存） - 返回: {keyword: [memory_id, ...]} - """ - index = {} - for memory in self.memory_cache.values(): - for kw in getattr(memory, "keywords", []): - if not kw: - continue - kw_norm = kw.strip().lower() - if kw_norm: - index.setdefault(kw_norm, []).append(getattr(memory.metadata, "memory_id", None)) - return index - - """基于Vector DB的记忆存储系统""" - - def __init__(self, config: VectorStorageConfig | None = None): - # 默认从全局配置读取，如果没有传入config - if config is None: - try: - self.config = VectorStorageConfig.from_global_config() - logger.info("✅ Vector存储配置已从全局配置加载") - except Exception as e: - logger.warning(f"从全局配置加载失败，使用默认配置: {e}") - self.config = VectorStorageConfig() - else: - self.config = config - - # 从配置中获取批处理大小和集合名称 - self.batch_size = self.config.batch_size - self.collection_name = self.config.memory_collection - self.vector_db_service = vector_db_service - - # 内存缓存 - self.memory_cache: dict[str, MemoryChunk] = {} - self.cache_timestamps: dict[str, float] = {} - self._cache = self.memory_cache # 别名，兼容旧代码 - - # 元数据索引管理器（JSON文件索引） - self.metadata_index = MemoryMetadataIndex() - - # 遗忘引擎 - self.forgetting_engine: MemoryForgettingEngine | None = None - if self.config.enable_forgetting: - self.forgetting_engine = MemoryForgettingEngine() - - # 统计信息 - self.stats = { - "total_memories": 0, - "cache_hits": 0, - "cache_misses": 0, - "total_searches": 0, - "total_stores": 0, - "last_cleanup_time": 0.0, - "forgetting_stats": {}, - } - - # 线程锁 - self._lock = threading.RLock() - - # 定时清理任务 - self._cleanup_task = None - self._stop_cleanup = False - - # 初始化系统 - self._initialize_storage() - self._start_cleanup_task() - - def _initialize_storage(self): - """初始化Vector DB存储""" - try: - # 创建记忆集合 - vector_db_service.get_or_create_collection( - name=self.config.memory_collection, - metadata={"description": "统一记忆存储V2", "hnsw:space": "cosine", "version": "2.0"}, - ) - - # 创建元数据集合（用于复杂查询） - vector_db_service.get_or_create_collection( - name=self.config.metadata_collection, - metadata={"description": "记忆元数据索引", "hnsw:space": "cosine", "version": "2.0"}, - ) - - # 获取当前记忆总数 - self.stats["total_memories"] = vector_db_service.count(self.config.memory_collection) - - logger.info(f"Vector记忆存储初始化完成，当前记忆数: {self.stats['total_memories']}") - - except Exception as e: - logger.error(f"Vector存储系统初始化失败: {e}", exc_info=True) - raise - - def _start_cleanup_task(self): - """启动定时清理任务""" - if self.config.auto_cleanup_interval > 0: - - def cleanup_worker(): - # 在新线程中创建事件循环 - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - try: - while not self._stop_cleanup: - try: - time.sleep(self.config.auto_cleanup_interval) - if not self._stop_cleanup: - # 在线程的事件循环中运行异步清理任务 - loop.run_until_complete(self._perform_auto_cleanup()) - except Exception as e: - logger.error(f"定时清理任务出错: {e}") - finally: - loop.close() - logger.debug("清理任务事件循环已关闭") - - self._cleanup_task = threading.Thread(target=cleanup_worker, daemon=True) - self._cleanup_task.start() - logger.info(f"定时清理任务已启动，间隔: {self.config.auto_cleanup_interval}秒") - - async def _perform_auto_cleanup(self): - """执行自动清理""" - try: - current_time = time.time() - - # 清理过期缓存 - if self.config.enable_caching: - expired_keys = [ - memory_id - for memory_id, timestamp in self.cache_timestamps.items() - if current_time - timestamp > 3600 # 1小时过期 - ] - - for key in expired_keys: - self.memory_cache.pop(key, None) - self.cache_timestamps.pop(key, None) - - if expired_keys: - logger.debug(f"清理了 {len(expired_keys)} 个过期缓存项") - - # 执行遗忘检查 - if self.forgetting_engine: - await self.perform_forgetting_check() - - self.stats["last_cleanup_time"] = current_time - - except Exception as e: - logger.error(f"自动清理失败: {e}") - - def _memory_to_vector_format(self, memory: MemoryChunk) -> dict[str, Any]: - """将MemoryChunk转换为向量存储格式""" - try: - # 获取memory_id - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", None) - - # 生成向量表示的文本 - display_text = ( - getattr(memory, "display", None) or getattr(memory, "text_content", None) or str(memory.content) - ) - if not display_text.strip(): - logger.warning(f"记忆 {memory_id} 缺少有效的显示文本") - display_text = f"{memory.memory_type.value}: {', '.join(memory.subjects)}" - - # 构建元数据 - 修复枚举值和列表序列化 - metadata = { - "memory_id": memory_id, - "user_id": memory.metadata.user_id or "unknown", - "memory_type": memory.memory_type.value, - "importance": memory.metadata.importance.name, # 使用 .name 而不是枚举对象 - "confidence": memory.metadata.confidence.name, # 使用 .name 而不是枚举对象 - "created_at": memory.metadata.created_at, - "last_accessed": memory.metadata.last_accessed or memory.metadata.created_at, - "access_count": memory.metadata.access_count, - "subjects": orjson.dumps(memory.subjects).decode("utf-8"), # 列表转JSON字符串 - "keywords": orjson.dumps(memory.keywords).decode("utf-8"), # 列表转JSON字符串 - "tags": orjson.dumps(memory.tags).decode("utf-8"), # 列表转JSON字符串 - "categories": orjson.dumps(memory.categories).decode("utf-8"), # 列表转JSON字符串 - "relevance_score": memory.metadata.relevance_score, - } - - # 添加可选字段 - if memory.metadata.source_context: - metadata["source_context"] = str(memory.metadata.source_context) - - if memory.content.predicate: - metadata["predicate"] = memory.content.predicate - - if memory.content.object: - if isinstance(memory.content.object, dict | list): - metadata["object"] = orjson.dumps(memory.content.object).decode() - else: - metadata["object"] = str(memory.content.object) - - return { - "id": memory_id, - "embedding": None, # 将由vector_db_service生成 - "metadata": metadata, - "document": display_text, - } - - except Exception as e: - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", "unknown") - logger.error(f"转换记忆 {memory_id} 到向量格式失败: {e}", exc_info=True) - raise - - def _vector_result_to_memory(self, document: str, metadata: dict[str, Any]) -> MemoryChunk | None: - """将Vector DB结果转换为MemoryChunk""" - try: - # 从元数据中恢复完整记忆 - if "memory_data" in metadata: - memory_dict = orjson.loads(metadata["memory_data"]) - return MemoryChunk.from_dict(memory_dict) - - # 兜底：从基础字段重建（使用新的结构化格式） - logger.warning(f"未找到memory_data，使用兜底逻辑重建记忆 (id={metadata.get('memory_id', 'unknown')})") - - # 构建符合MemoryChunk.from_dict期望的结构 - memory_dict = { - "metadata": { - "memory_id": metadata.get("memory_id", f"recovered_{int(time.time())}"), - "user_id": metadata.get("user_id", "unknown"), - "created_at": metadata.get("timestamp", time.time()), - "last_accessed": metadata.get("last_access_time", time.time()), - "last_modified": metadata.get("timestamp", time.time()), - "access_count": metadata.get("access_count", 0), - "relevance_score": 0.0, - "confidence": self._parse_enum_value( - metadata.get("confidence", 2), ConfidenceLevel, ConfidenceLevel.MEDIUM - ), - "importance": self._parse_enum_value( - metadata.get("importance", 2), ImportanceLevel, ImportanceLevel.NORMAL - ), - "source_context": None, - }, - "content": { - "subject": "", - "predicate": "", - "object": "", - "display": document, # 使用document作为显示文本 - }, - "memory_type": metadata.get("memory_type", "contextual"), - "keywords": orjson.loads(metadata.get("keywords", "[]")) - if isinstance(metadata.get("keywords"), str) - else metadata.get("keywords", []), - "tags": [], - "categories": [], - "embedding": None, - "semantic_hash": None, - "related_memories": [], - "temporal_context": None, - } - - return MemoryChunk.from_dict(memory_dict) - - except Exception as e: - logger.error(f"转换Vector结果到MemoryChunk失败: {e}", exc_info=True) - return None - - def _parse_enum_value(self, value: Any, enum_class: type, default: Any) -> Any: - """解析枚举值，支持字符串、整数和枚举实例 - - Args: - value: 要解析的值（可能是字符串、整数或枚举实例） - enum_class: 目标枚举类 - default: 默认值 - - Returns: - 解析后的枚举实例 - """ - if value is None: - return default - - # 如果已经是枚举实例，直接返回 - if isinstance(value, enum_class): - return value - - # 如果是整数，尝试按value值匹配 - if isinstance(value, int): - try: - for member in enum_class: - if member.value == value: - return member - # 如果没找到匹配的，返回默认值 - logger.warning(f"无法找到{enum_class.__name__}中value={value}的枚举项，使用默认值") - return default - except Exception as e: - logger.warning(f"解析{enum_class.__name__}整数值{value}时出错: {e}，使用默认值") - return default - - # 如果是字符串，尝试按名称或value值匹配 - if isinstance(value, str): - str_value = value.strip().upper() - - # 先尝试按枚举名称匹配 - try: - if hasattr(enum_class, str_value): - return getattr(enum_class, str_value) - except AttributeError: - pass - - # 再尝试按value值匹配（如果value是字符串形式的数字） - try: - int_value = int(str_value) - return self._parse_enum_value(int_value, enum_class, default) - except ValueError: - pass - - # 最后尝试按小写名称匹配 - try: - for member in enum_class: - if member.value.upper() == str_value: - return member - logger.warning(f"无法找到{enum_class.__name__}中名称或value为'{value}'的枚举项，使用默认值") - return default - except Exception as e: - logger.warning(f"解析{enum_class.__name__}字符串值'{value}'时出错: {e}，使用默认值") - return default - - # 其他类型，返回默认值 - logger.warning(f"不支持的{enum_class.__name__}值类型: {type(value)}，使用默认值") - return default - - def _get_from_cache(self, memory_id: str) -> MemoryChunk | None: - """从缓存获取记忆""" - if not self.config.enable_caching: - return None - - with self._lock: - if memory_id in self.memory_cache: - self.cache_timestamps[memory_id] = time.time() - self.stats["cache_hits"] += 1 - return self.memory_cache[memory_id] - - self.stats["cache_misses"] += 1 - return None - - def _add_to_cache(self, memory: MemoryChunk): - """添加记忆到缓存""" - if not self.config.enable_caching: - return - - with self._lock: - # 检查缓存大小限制 - if len(self.memory_cache) >= self.config.cache_size_limit: - # 移除最老的缓存项 - oldest_id = min(self.cache_timestamps.keys(), key=lambda k: self.cache_timestamps[k]) - self.memory_cache.pop(oldest_id, None) - self.cache_timestamps.pop(oldest_id, None) - - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", None) - if memory_id: - self.memory_cache[memory_id] = memory - self.cache_timestamps[memory_id] = time.time() - - async def store_memories(self, memories: list[MemoryChunk]) -> int: - """批量存储记忆""" - if not memories: - return 0 - - start_time = datetime.now() - success_count = 0 - - try: - # 转换为向量格式 - vector_data_list = [] - for memory in memories: - try: - vector_data = self._memory_to_vector_format(memory) - vector_data_list.append(vector_data) - except Exception as e: - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", "unknown") - logger.error(f"处理记忆 {memory_id} 失败: {e}") - continue - - if not vector_data_list: - logger.warning("没有有效的记忆数据可存储") - return 0 - - # 批量存储到向量数据库 - for i in range(0, len(vector_data_list), self.batch_size): - batch = vector_data_list[i : i + self.batch_size] - - try: - # 生成embeddings - embeddings = [] - for item in batch: - try: - embedding = await get_embedding(item["document"]) - embeddings.append(embedding) - except Exception as e: - logger.error(f"生成embedding失败: {e}") - # 使用零向量作为后备 - embeddings.append([0.0] * 768) # 默认维度 - - # vector_db_service.add 需要embeddings参数 - self.vector_db_service.add( - collection_name=self.collection_name, - embeddings=embeddings, - ids=[item["id"] for item in batch], - documents=[item["document"] for item in batch], - metadatas=[item["metadata"] for item in batch], - ) - success = True - - if success: - # 更新缓存和元数据索引 - metadata_entries = [] - for item in batch: - memory_id = item["id"] - # 从原始 memories 列表中找到对应的 MemoryChunk - memory = next( - ( - m - for m in memories - if (getattr(m.metadata, "memory_id", None) or getattr(m, "memory_id", None)) - == memory_id - ), - None, - ) - if memory: - # 更新缓存 - self._cache[memory_id] = memory - success_count += 1 - - # 创建元数据索引条目 - try: - index_entry = MemoryMetadataIndexEntry( - memory_id=memory_id, - user_id=memory.metadata.user_id or "unknown", - memory_type=memory.memory_type.value, - subjects=memory.subjects, - objects=[str(memory.content.object)] if memory.content.object else [], - keywords=memory.keywords, - tags=memory.tags, - importance=memory.metadata.importance.value, - confidence=memory.metadata.confidence.value, - created_at=memory.metadata.created_at, - access_count=memory.metadata.access_count, - chat_id=memory.metadata.chat_id, - content_preview=str(memory.content)[:100] if memory.content else None, - ) - metadata_entries.append(index_entry) - except Exception as e: - logger.warning(f"创建元数据索引条目失败 (memory_id={memory_id}): {e}") - - # 批量更新元数据索引 - if metadata_entries: - try: - self.metadata_index.batch_add_or_update(metadata_entries) - logger.debug(f"更新元数据索引: {len(metadata_entries)} 条") - except Exception as e: - logger.error(f"批量更新元数据索引失败: {e}") - else: - logger.warning(f"批次存储失败，跳过 {len(batch)} 条记忆") - - except Exception as e: - logger.error(f"批量存储失败: {e}", exc_info=True) - continue - - duration = (datetime.now() - start_time).total_seconds() - logger.info(f"成功存储 {success_count}/{len(memories)} 条记忆，耗时 {duration:.2f}秒") - - # 保存元数据索引到磁盘 - if success_count > 0: - try: - self.metadata_index.save() - logger.debug("元数据索引已保存到磁盘") - except Exception as e: - logger.error(f"保存元数据索引失败: {e}") - - return success_count - - except Exception as e: - logger.error(f"批量存储记忆失败: {e}", exc_info=True) - return success_count - - async def store_memory(self, memory: MemoryChunk) -> bool: - """存储单条记忆""" - result = await self.store_memories([memory]) - return result > 0 - - async def search_similar_memories( - self, - query_text: str, - limit: int = 10, - similarity_threshold: float | None = None, - filters: dict[str, Any] | None = None, - # 新增：元数据过滤参数（用于JSON索引粗筛） - metadata_filters: dict[str, Any] | None = None, - ) -> list[tuple[MemoryChunk, float]]: - """ - 搜索相似记忆（混合索引模式） - - Args: - query_text: 查询文本 - limit: 返回数量限制 - similarity_threshold: 相似度阈值 - filters: ChromaDB where条件（保留用于兼容） - metadata_filters: JSON元数据索引过滤条件，支持: - - memory_types: List[str] - - subjects: List[str] - - keywords: List[str] - - tags: List[str] - - importance_min: int - - importance_max: int - - created_after: float - - created_before: float - - user_id: str - """ - if not query_text.strip(): - return [] - - try: - # === 阶段一：JSON元数据粗筛（可选） === - candidate_ids: list[str] | None = None - if metadata_filters: - logger.debug(f"[JSON元数据粗筛] 开始，过滤条件: {metadata_filters}") - candidate_ids = self.metadata_index.search( - memory_types=metadata_filters.get("memory_types"), - subjects=metadata_filters.get("subjects"), - keywords=metadata_filters.get("keywords"), - tags=metadata_filters.get("tags"), - importance_min=metadata_filters.get("importance_min"), - importance_max=metadata_filters.get("importance_max"), - created_after=metadata_filters.get("created_after"), - created_before=metadata_filters.get("created_before"), - user_id=metadata_filters.get("user_id"), - limit=self.config.search_limit * 2, # 粗筛返回更多候选 - flexible_mode=True, # 使用灵活匹配模式 - ) - logger.debug(f"[JSON元数据粗筛] 完成，筛选出 {len(candidate_ids)} 个候选ID") - - # 如果粗筛后没有结果，回退到全部记忆搜索 - if not candidate_ids: - total_memories = len(self.metadata_index.index) - logger.warning( - f"JSON元数据粗筛后无候选，启用回退机制：在全部 {total_memories} 条记忆中进行向量搜索" - ) - logger.info("💡 提示：这可能是因为查询条件过于严格，或相关记忆的元数据与查询条件不完全匹配") - candidate_ids = None # 设为None表示不限制候选ID - else: - logger.debug("[JSON元数据粗筛] 成功筛选出候选，进入向量精筛阶段") - - # === 阶段二：向量精筛 === - # 生成查询向量 - query_embedding = await get_embedding(query_text) - if not query_embedding: - return [] - - threshold = similarity_threshold or self.config.similarity_threshold - - # 构建where条件 - where_conditions = filters or {} - - # 如果有候选ID列表，添加到where条件 - if candidate_ids: - # ChromaDB的where条件需要使用$in操作符 - where_conditions["memory_id"] = {"$in": candidate_ids} - logger.debug(f"[向量精筛] 限制在 {len(candidate_ids)} 个候选ID内搜索") - else: - logger.debug("[向量精筛] 在全部记忆中搜索（元数据筛选无结果回退）") - - # 查询Vector DB - logger.debug(f"[向量精筛] 开始，limit={min(limit, self.config.search_limit)}") - results = vector_db_service.query( - collection_name=self.config.memory_collection, - query_embeddings=[query_embedding], - n_results=min(limit, self.config.search_limit), - where=where_conditions if where_conditions else None, - ) - - # 处理结果 - similar_memories = [] - - if results.get("documents") and results["documents"][0]: - documents = results["documents"][0] - distances = results.get("distances", [[]])[0] - metadatas = results.get("metadatas", [[]])[0] - ids = results.get("ids", [[]])[0] - - logger.debug( - f"向量检索返回原始结果：documents={len(documents)}, ids={len(ids)}, metadatas={len(metadatas)}" - ) - for i, (doc, metadata, memory_id) in enumerate(zip(documents, metadatas, ids, strict=False)): - # 计算相似度 - distance = distances[i] if i < len(distances) else 1.0 - similarity = 1 - distance # ChromaDB返回距离，转换为相似度 - - if similarity < threshold: - continue - - # 首先尝试从缓存获取 - memory = self._get_from_cache(memory_id) - - if not memory: - # 从Vector结果重建 - memory = self._vector_result_to_memory(doc, metadata) - if memory: - self._add_to_cache(memory) - - if memory: - similar_memories.append((memory, similarity)) - # 记录单条结果的关键日志（id，相似度，简短文本） - try: - short_text = ( - (str(memory.content)[:120]) - if hasattr(memory, "content") - else (doc[:120] if isinstance(doc, str) else "") - ) - except Exception: - short_text = "" - logger.debug(f"检索结果 - id={memory_id}, similarity={similarity:.4f}, summary={short_text}") - - # 按相似度排序 - similar_memories.sort(key=lambda x: x[1], reverse=True) - - self.stats["total_searches"] += 1 - logger.debug( - f"搜索相似记忆: query='{query_text[:60]}...', limit={limit}, threshold={threshold}, filters={where_conditions}, 返回数={len(similar_memories)}" - ) - logger.debug(f"搜索相似记忆 详细结果数={len(similar_memories)}") - - return similar_memories - - except Exception as e: - logger.error(f"搜索相似记忆失败: {e}") - return [] - - async def get_memory_by_id(self, memory_id: str) -> MemoryChunk | None: - """根据ID获取记忆""" - # 首先尝试从缓存获取 - memory = self._get_from_cache(memory_id) - if memory: - return memory - - try: - # 从Vector DB获取 - results = vector_db_service.get(collection_name=self.config.memory_collection, ids=[memory_id]) - - if results.get("documents") and results["documents"]: - document = results["documents"][0] - metadata = results["metadatas"][0] if results.get("metadatas") else {} - - memory = self._vector_result_to_memory(document, metadata) - if memory: - self._add_to_cache(memory) - - return memory - - except Exception as e: - logger.error(f"获取记忆 {memory_id} 失败: {e}") - - return None - - async def get_memories_by_filters(self, filters: dict[str, Any], limit: int = 100) -> list[MemoryChunk]: - """根据过滤条件获取记忆""" - try: - results = vector_db_service.get(collection_name=self.config.memory_collection, where=filters, limit=limit) - - memories = [] - if results.get("documents"): - documents = results["documents"] - metadatas = results.get("metadatas", [{}] * len(documents)) - ids = results.get("ids", []) - - logger.debug(f"按过滤条件获取返回: docs={len(documents)}, ids={len(ids)}") - for i, (doc, metadata) in enumerate(zip(documents, metadatas, strict=False)): - memory_id = ids[i] if i < len(ids) else None - - # 首先尝试从缓存获取 - if memory_id: - memory = self._get_from_cache(memory_id) - if memory: - memories.append(memory) - logger.debug(f"过滤获取命中缓存: id={memory_id}") - continue - - # 从Vector结果重建 - memory = self._vector_result_to_memory(doc, metadata) - if memory: - memories.append(memory) - if memory_id: - self._add_to_cache(memory) - logger.debug(f"过滤获取结果: id={memory_id}, meta_keys={list(metadata.keys())}") - - return memories - - except Exception as e: - logger.error(f"根据过滤条件获取记忆失败: {e}") - return [] - - async def update_memory(self, memory: MemoryChunk) -> bool: - """更新记忆""" - try: - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", None) - if not memory_id: - logger.error("无法更新记忆：缺少memory_id") - return False - - # 先删除旧记忆 - await self.delete_memory(memory_id) - - # 重新存储更新后的记忆 - return await self.store_memory(memory) - - except Exception as e: - memory_id = getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", "unknown") - logger.error(f"更新记忆 {memory_id} 失败: {e}") - return False - - async def delete_memory(self, memory_id: str) -> bool: - """删除记忆""" - try: - # 从Vector DB删除 - vector_db_service.delete(collection_name=self.config.memory_collection, ids=[memory_id]) - - # 从缓存删除 - with self._lock: - self.memory_cache.pop(memory_id, None) - self.cache_timestamps.pop(memory_id, None) - - self.stats["total_memories"] = max(0, self.stats["total_memories"] - 1) - logger.debug(f"删除记忆: {memory_id}") - - return True - - except Exception as e: - logger.error(f"删除记忆 {memory_id} 失败: {e}") - return False - - async def delete_memories_by_filters(self, filters: dict[str, Any]) -> int: - """根据过滤条件批量删除记忆""" - try: - # 先获取要删除的记忆ID - results = vector_db_service.get( - collection_name=self.config.memory_collection, where=filters, include=["metadatas"] - ) - - if not results.get("ids"): - return 0 - - memory_ids = results["ids"] - - # 批量删除 - vector_db_service.delete(collection_name=self.config.memory_collection, where=filters) - - # 从缓存删除 - with self._lock: - for memory_id in memory_ids: - self.memory_cache.pop(memory_id, None) - self.cache_timestamps.pop(memory_id, None) - - deleted_count = len(memory_ids) - self.stats["total_memories"] = max(0, self.stats["total_memories"] - deleted_count) - logger.info(f"批量删除记忆: {deleted_count} 条") - - return deleted_count - - except Exception as e: - logger.error(f"批量删除记忆失败: {e}") - return 0 - - async def perform_forgetting_check(self) -> dict[str, Any]: - """执行遗忘检查""" - if not self.forgetting_engine: - return {"error": "遗忘引擎未启用"} - - try: - # 获取所有记忆进行遗忘检查 - # 注意：对于大型数据集，这里应该分批处理 - current_time = time.time() - cutoff_time = current_time - (self.config.retention_hours * 3600) - - # 先删除明显过期的记忆 - expired_filters = {"timestamp": {"$lt": cutoff_time}} - expired_count = await self.delete_memories_by_filters(expired_filters) - - # 对剩余记忆执行智能遗忘检查 - # 这里为了性能考虑，只检查一部分记忆 - sample_memories = await self.get_memories_by_filters({}, limit=500) - - if sample_memories: - result = await self.forgetting_engine.perform_forgetting_check(sample_memories) - - # 遗忘标记的记忆 - forgetting_ids = result.get("normal_forgetting", []) + result.get("force_forgetting", []) - forgotten_count = 0 - - for memory_id in forgetting_ids: - if await self.delete_memory(memory_id): - forgotten_count += 1 - - result["forgotten_count"] = forgotten_count - result["expired_count"] = expired_count - - # 更新统计 - self.stats["forgetting_stats"] = self.forgetting_engine.get_forgetting_stats() - - logger.info(f"遗忘检查完成: 过期删除 {expired_count}, 智能遗忘 {forgotten_count}") - return result - - return {"expired_count": expired_count, "forgotten_count": 0} - - except Exception as e: - logger.error(f"执行遗忘检查失败: {e}") - return {"error": str(e)} - - def get_storage_stats(self) -> dict[str, Any]: - """获取存储统计信息""" - try: - current_total = vector_db_service.count(self.config.memory_collection) - self.stats["total_memories"] = current_total - except Exception: - pass - - return { - **self.stats, - "cache_size": len(self.memory_cache), - "collection_name": self.config.memory_collection, - "storage_type": "vector_db_v2", - "uptime": time.time() - self.stats.get("start_time", time.time()), - } - - def stop(self): - """停止存储系统""" - self._stop_cleanup = True - - if self._cleanup_task and self._cleanup_task.is_alive(): - logger.info("正在停止定时清理任务...") - - # 清空缓存 - with self._lock: - self.memory_cache.clear() - self.cache_timestamps.clear() - - logger.info("Vector记忆存储系统已停止") - - def cleanup(self): - """清理资源，兼容旧接口""" - logger.info("正在清理VectorMemoryStorage资源...") - self.stop() - - -# 全局实例（可选） -_global_vector_storage = None - - -def get_vector_memory_storage(config: VectorStorageConfig | None = None) -> VectorMemoryStorage: - """获取全局Vector记忆存储实例""" - global _global_vector_storage - - if _global_vector_storage is None: - _global_vector_storage = VectorMemoryStorage(config) - - return _global_vector_storage - - -# 兼容性接口 -class VectorMemoryStorageAdapter: - """适配器类，提供与原UnifiedMemoryStorage兼容的接口""" - - def __init__(self, config: VectorStorageConfig | None = None): - self.storage = VectorMemoryStorage(config) - - async def store_memories(self, memories: list[MemoryChunk]) -> int: - return await self.storage.store_memories(memories) - - async def search_similar_memories( - self, query_text: str, limit: int = 10, scope_id: str | None = None, filters: dict[str, Any] | None = None - ) -> list[tuple[str, float]]: - results = await self.storage.search_similar_memories(query_text, limit, filters=filters) - # 转换为原格式：(memory_id, similarity) - return [ - (getattr(memory.metadata, "memory_id", None) or getattr(memory, "memory_id", "unknown"), similarity) - for memory, similarity in results - ] - - def get_stats(self) -> dict[str, Any]: - return self.storage.get_storage_stats() - - -if __name__ == "__main__": - # 简单测试 - async def test_vector_storage(): - storage = VectorMemoryStorage() - - # 创建测试记忆 - from src.chat.memory_system.memory_chunk import MemoryType - - test_memory = MemoryChunk( - memory_id="test_001", - user_id="test_user", - text_content="今天天气很好，适合出门散步", - memory_type=MemoryType.FACT, - keywords=["天气", "散步"], - importance=0.7, - ) - - # 存储记忆 - success = await storage.store_memory(test_memory) - print(f"存储结果: {success}") - - # 搜索记忆 - results = await storage.search_similar_memories("天气怎么样", limit=5) - print(f"搜索结果: {len(results)} 条") - - for memory, similarity in results: - print(f" - {memory.text_content[:50]}... (相似度: {similarity:.3f})") - - # 获取统计信息 - stats = storage.get_storage_stats() - print(f"存储统计: {stats}") - - storage.stop() - - asyncio.run(test_vector_storage()) diff --git a/src/chat/replyer/default_generator.py b/src/chat/replyer/default_generator.py index 3c729cad7..ecf82c9b5 100644 --- a/src/chat/replyer/default_generator.py +++ b/src/chat/replyer/default_generator.py @@ -256,8 +256,6 @@ class DefaultReplyer: self._chat_info_initialized = False self.heart_fc_sender = HeartFCSender() - # 使用新的增强记忆系统 - # from src.chat.memory_system.enhanced_memory_activator import EnhancedMemoryActivator self._chat_info_initialized = False async def _initialize_chat_info(self): @@ -401,19 +399,9 @@ class DefaultReplyer: f"插件{result.get_summary().get('stopped_handlers', '')}于请求后取消了内容生成" ) - # 回复生成成功后，异步存储聊天记忆（不阻塞返回） - try: - # 将记忆存储作为子任务创建，可以被取消 - memory_task = asyncio.create_task( - self._store_chat_memory_async(reply_to, reply_message), - name=f"store_memory_{self.chat_stream.stream_id}" - ) - # 不等待完成，让它在后台运行 - # 如果父任务被取消，这个子任务也会被垃圾回收 - logger.debug(f"创建记忆存储子任务: {memory_task.get_name()}") - except Exception as memory_e: - # 记忆存储失败不应该影响回复生成的成功返回 - logger.warning(f"记忆存储失败，但不影响回复生成: {memory_e}") + # 旧的自动记忆存储已移除，现在使用记忆图系统通过工具创建记忆 + # 记忆由LLM在对话过程中通过CreateMemoryTool主动创建，而非自动存储 + pass return True, llm_response, prompt @@ -545,19 +533,6 @@ class DefaultReplyer: Returns: str: 记忆信息字符串 """ - chat_talking_prompt_short = build_readable_messages( - chat_history, - replace_bot_name=True, - merge_messages=False, - timestamp_mode="relative", - read_mark=0.0, - show_actions=True, - ) - - - if not global_config.memory.enable_memory: - return "" - # 使用新的记忆图系统检索记忆（带智能查询优化） all_memories = [] try: @@ -1886,14 +1861,22 @@ class DefaultReplyer: return f"你与{sender}是普通朋友关系。" + # 已废弃：旧的自动记忆存储逻辑 + # 新的记忆图系统通过LLM工具(CreateMemoryTool)主动创建记忆，而非自动存储 async def _store_chat_memory_async(self, reply_to: str, reply_message: DatabaseMessages | dict[str, Any] | None = None): """ - 异步存储聊天记忆（从build_memory_block迁移而来） + [已废弃] 异步存储聊天记忆（从build_memory_block迁移而来） + + 此函数已被记忆图系统的工具调用方式替代。 + 记忆现在由LLM在对话过程中通过CreateMemoryTool主动创建。 Args: reply_to: 回复对象 reply_message: 回复的原始消息 """ + return # 已禁用，保留函数签名以防其他地方有引用 + + # 以下代码已废弃，不再执行 try: if not global_config.memory.enable_memory: return diff --git a/src/chat/utils/prompt.py b/src/chat/utils/prompt.py index 97e3ebdba..3502842f5 100644 --- a/src/chat/utils/prompt.py +++ b/src/chat/utils/prompt.py @@ -641,150 +641,6 @@ class Prompt: logger.error(f"构建表达习惯失败: {e}") return {"expression_habits_block": ""} - # _build_memory_block 和 _build_memory_block_fast 已移除 - # 记忆构建现在完全由 default_generator.py 的 build_memory_block 方法处理 - # 使用新的记忆图系统，通过 pre_built_params 传入 - - async def _REMOVED_build_memory_block(self) -> dict[str, Any]: - """已废弃：构建与当前对话相关的记忆上下文块（完整版）.""" - if not global_config.memory.enable_memory: - return {"memory_block": ""} - - try: - from src.chat.memory_system.enhanced_memory_activator import enhanced_memory_activator - - # 准备用于记忆激活的聊天历史 - chat_history = "" - if self.parameters.message_list_before_now_long: - recent_messages = self.parameters.message_list_before_now_long[-20:] - chat_history = await build_readable_messages( - recent_messages, replace_bot_name=True, timestamp_mode="normal", truncate=True - ) - - # 并行查询长期记忆和即时记忆以提高性能 - import asyncio - - memory_tasks = [ - enhanced_memory_activator.activate_memory_with_chat_history( - target_message=self.parameters.target, chat_history_prompt=chat_history - ), - enhanced_memory_activator.get_instant_memory( - target_message=self.parameters.target, chat_id=self.parameters.chat_id - ), - ] - - try: - # 使用 `return_exceptions=True` 来防止一个任务的失败导致所有任务失败 - running_memories, instant_memory = await asyncio.gather(*memory_tasks, return_exceptions=True) - - # 单独处理每个任务的结果，如果是异常则记录并使用默认值 - if isinstance(running_memories, BaseException): - logger.warning(f"长期记忆查询失败: {running_memories}") - running_memories = [] - if isinstance(instant_memory, BaseException): - logger.warning(f"即时记忆查询失败: {instant_memory}") - instant_memory = None - - except asyncio.TimeoutError: - logger.warning("记忆查询超时，使用部分结果") - running_memories = [] - instant_memory = None - - # 将检索到的记忆格式化为提示词 - if running_memories: - try: - from src.chat.memory_system.memory_formatter import format_memories_bracket_style - - # 将原始记忆数据转换为格式化器所需的标准格式 - formatted_memories = [] - for memory in running_memories: - content = memory.get("content", "") - display_text = content - # 清理内容，移除元数据括号 - if "（类型:" in content and "）" in content: - display_text = content.split("（类型:")[0].strip() - - # 映射记忆主题到标准类型 - topic = memory.get("topic", "personal_fact") - memory_type_mapping = { - "relationship": "personal_fact", - "opinion": "opinion", - "personal_fact": "personal_fact", - "preference": "preference", - "event": "event", - } - mapped_type = memory_type_mapping.get(topic, "personal_fact") - - formatted_memories.append( - { - "display": display_text, - "memory_type": mapped_type, - "metadata": { - "confidence": memory.get("confidence", "未知"), - "importance": memory.get("importance", "一般"), - "timestamp": memory.get("timestamp", ""), - "source": memory.get("source", "unknown"), - "relevance_score": memory.get("relevance_score", 0.0), - }, - } - ) - - # 使用指定的风格进行格式化 - memory_block = format_memories_bracket_style( - formatted_memories, query_context=self.parameters.target - ) - except Exception as e: - # 如果格式化失败，提供一个简化的、健壮的备用格式 - logger.warning(f"记忆格式化失败，使用简化格式: {e}") - memory_parts = ["## 相关记忆回顾", ""] - for memory in running_memories: - content = memory.get("content", "") - if "（类型:" in content and "）" in content: - clean_content = content.split("（类型:")[0].strip() - memory_parts.append(f"- {clean_content}") - else: - memory_parts.append(f"- {content}") - memory_block = "\n".join(memory_parts) - else: - memory_block = "" - - # 将即时记忆附加到记忆块的末尾 - if instant_memory: - if memory_block: - memory_block += f"\n- 最相关记忆：{instant_memory}" - else: - memory_block = f"- 最相关记忆：{instant_memory}" - - return {"memory_block": memory_block} - - except Exception as e: - logger.error(f"构建记忆块失败: {e}") - return {"memory_block": ""} - - async def _REMOVED_build_memory_block_fast(self) -> dict[str, Any]: - """已废弃：快速构建记忆块（简化版），作为未预构建时的后备方案.""" - if not global_config.memory.enable_memory: - return {"memory_block": ""} - - try: - from src.chat.memory_system.enhanced_memory_activator import enhanced_memory_activator - - # 这个快速版本只查询最高优先级的“即时记忆”，速度更快 - instant_memory = await enhanced_memory_activator.get_instant_memory( - target_message=self.parameters.target, chat_id=self.parameters.chat_id - ) - - if instant_memory: - memory_block = f"- 相关记忆：{instant_memory}" - else: - memory_block = "" - - return {"memory_block": memory_block} - - except Exception as e: - logger.warning(f"快速构建记忆块失败: {e}") - return {"memory_block": ""} - async def _build_relation_info(self) -> dict[str, Any]: """构建与对话目标相关的关系信息.""" try: diff --git a/src/main.py b/src/main.py index dae43ac2c..2b3b72a9f 100644 --- a/src/main.py +++ b/src/main.py @@ -13,7 +13,6 @@ from maim_message import MessageServer from rich.traceback import install from src.chat.emoji_system.emoji_manager import get_emoji_manager -from src.chat.memory_system.memory_manager import memory_manager from src.chat.message_receive.bot import chat_bot from src.chat.message_receive.chat_stream import get_chat_manager from src.chat.utils.statistic import OnlineTimeRecordTask, StatisticOutputTask @@ -76,8 +75,6 @@ class MainSystem: """主系统类，负责协调所有组件""" def __init__(self) -> None: - # 使用增强记忆系统 - self.memory_manager = memory_manager self.individuality: Individuality = get_individuality() # 使用消息API替代直接的FastAPI实例 @@ -250,12 +247,6 @@ class MainSystem: logger.error(f"准备停止消息重组器时出错: {e}") # 停止增强记忆系统 - try: - if global_config.memory and getattr(global_config.memory, 'enable', False): - cleanup_tasks.append(("增强记忆系统", self.memory_manager.shutdown())) - except Exception as e: - logger.error(f"准备停止增强记忆系统时出错: {e}") - # 停止统一调度器 try: from src.schedule.unified_scheduler import shutdown_scheduler @@ -468,13 +459,12 @@ MoFox_Bot(第三方修改版) _background_tasks.add(task) task.add_done_callback(_background_tasks.discard) - # 初始化增强记忆系统 - if global_config.memory and getattr(global_config.memory, 'enable', False): - from src.chat.memory_system.memory_system import initialize_memory_system - await self._safe_init("增强记忆系统", initialize_memory_system)() - await self._safe_init("记忆管理器", self.memory_manager.initialize)() - else: - logger.info("记忆系统已禁用，跳过初始化") + # 初始化记忆图系统 + try: + from src.memory_graph.manager_singleton import initialize_memory_manager + await self._safe_init("记忆图系统", initialize_memory_manager)() + except Exception as e: + logger.error(f"记忆图系统初始化失败: {e}") # 初始化消息兴趣值计算组件 await self._initialize_interest_calculator() diff --git a/src/memory_graph/config.py b/src/memory_graph/config.py index c7c531bd5..4aa8c94da 100644 --- a/src/memory_graph/config.py +++ b/src/memory_graph/config.py @@ -163,6 +163,14 @@ class MemoryGraphConfig: activation_propagation_depth=getattr(mg_config, 'activation_propagation_depth', 1), max_memory_nodes_per_memory=getattr(mg_config, 'max_memory_nodes_per_memory', 10), max_related_memories=getattr(mg_config, 'max_related_memories', 5), + # 检索配置 + retrieval=RetrievalConfig( + max_expand_depth=getattr(mg_config, 'search_max_expand_depth', 2), + vector_weight=getattr(mg_config, 'search_vector_weight', 0.4), + graph_distance_weight=getattr(mg_config, 'search_graph_distance_weight', 0.2), + importance_weight=getattr(mg_config, 'search_importance_weight', 0.2), + recency_weight=getattr(mg_config, 'search_recency_weight', 0.2), + ), ) return config @@ -206,7 +214,14 @@ class MemoryGraphConfig: max_related_memories=config_dict.get("max_related_memories", 5), # 旧配置字段（向后兼容） consolidation=ConsolidationConfig(**config_dict.get("consolidation", {})), - retrieval=RetrievalConfig(**config_dict.get("retrieval", {})), + retrieval=RetrievalConfig( + max_expand_depth=config_dict.get("search_max_expand_depth", 2), + vector_weight=config_dict.get("search_vector_weight", 0.4), + graph_distance_weight=config_dict.get("search_graph_distance_weight", 0.2), + importance_weight=config_dict.get("search_importance_weight", 0.2), + recency_weight=config_dict.get("search_recency_weight", 0.2), + **config_dict.get("retrieval", {}) + ), node_merger=NodeMergerConfig(**config_dict.get("node_merger", {})), storage=StorageConfig(**config_dict.get("storage", {})), decay_rates=config_dict.get("decay_rates", cls().decay_rates), diff --git a/src/memory_graph/manager.py b/src/memory_graph/manager.py index b1e0aabf1..6a8705a96 100644 --- a/src/memory_graph/manager.py +++ b/src/memory_graph/manager.py @@ -365,20 +365,26 @@ class MemoryManager: chat_history = context.get("chat_history", "") sender = context.get("sender", "") - prompt = f"""你是一个记忆检索查询优化助手。请将用户的查询转换为更适合语义搜索的表述。 + prompt = f"""你是一个记忆检索查询优化助手。你的任务是分析对话历史，生成一个综合性的搜索查询。 -要求： -1. 提取查询的核心意图和关键信息 -2. 使用更具体、描述性的语言 -3. 如果查询涉及人物，明确指出是谁 -4. 保持简洁，只输出优化后的查询文本 +**任务说明：** +不要只优化单个消息，而是要综合分析整个对话上下文，提取出最核心的检索意图。 -当前查询: {query} +**要求：** +1. 仔细阅读对话历史，理解对话的主题和脉络 +2. 识别关键人物、事件、关系和话题 +3. 提取最值得检索的核心信息点 +4. 生成一个简洁但信息丰富的搜索查询（15-30字） +5. 如果涉及特定人物，必须明确指出人名 +6. 只输出查询文本，不要解释 -{f"发言人: {sender}" if sender else ""} -{f"最近对话: {chat_history[-200:]}" if chat_history else ""} +**对话上下文：** +{chat_history[-500:] if chat_history else "（无历史对话）"} -优化后的查询:""" +**当前消息：** +{sender}: {query} + +**生成综合查询：**""" optimized_query, _ = await llm.generate_response_async( prompt, diff --git a/src/plugins/built_in/affinity_flow_chatter/planner/plan_filter.py b/src/plugins/built_in/affinity_flow_chatter/planner/plan_filter.py index f9514a8d8..e13335e75 100644 --- a/src/plugins/built_in/affinity_flow_chatter/planner/plan_filter.py +++ b/src/plugins/built_in/affinity_flow_chatter/planner/plan_filter.py @@ -648,18 +648,20 @@ class ChatterPlanFilter: else: keywords.append("晚上") - # 使用新的统一记忆系统检索记忆 + # 使用记忆图系统检索记忆 try: - from src.chat.memory_system import get_memory_system + from src.memory_graph.manager_singleton import get_memory_manager - memory_system = get_memory_system() + memory_manager = get_memory_manager() + if not memory_manager: + return "记忆系统未初始化。" + # 将关键词转换为查询字符串 query = " ".join(keywords) - enhanced_memories = await memory_system.retrieve_relevant_memories( - query_text=query, - user_id="system", # 系统查询 - scope_id="system", - limit=5, + enhanced_memories = await memory_manager.search_memories( + query=query, + top_k=5, + optimize_query=False, # 直接使用关键词查询 ) if not enhanced_memories: @@ -667,9 +669,14 @@ class ChatterPlanFilter: # 转换格式以兼容现有代码 retrieved_memories = [] - for memory_chunk in enhanced_memories: - content = memory_chunk.display or memory_chunk.text_content or "" - memory_type = memory_chunk.memory_type.value if memory_chunk.memory_type else "unknown" + for memory in enhanced_memories: + # 从记忆图的节点中提取内容 + content_parts = [] + for node in memory.nodes: + if node.content: + content_parts.append(node.content) + content = " ".join(content_parts) if content_parts else "无内容" + memory_type = memory.memory_type.value retrieved_memories.append((memory_type, content)) memory_statements = [ diff --git a/template/bot_config_template.toml b/template/bot_config_template.toml index 59b4b2590..7385fc3c0 100644 --- a/template/bot_config_template.toml +++ b/template/bot_config_template.toml @@ -1,5 +1,5 @@ [inner] -version = "7.6.0" +version = "7.6.1" #----以下是给开发人员阅读的，如果你只是部署了MoFox-Bot，不需要阅读---- #如果你想要修改配置文件，请递增version的值 @@ -254,7 +254,7 @@ search_min_importance = 0.3 # 最小重要性阈值 (0.0-1.0) search_similarity_threshold = 0.5 # 向量相似度阈值 # 智能查询优化 -enable_query_optimization = true # 启用查询优化（使用小模型优化搜索查询） +enable_query_optimization = true # 启用查询优化（使用小模型分析对话历史，生成综合性搜索查询） # === 记忆整合配置 === consolidation_enabled = true # 是否启用记忆整合 @@ -272,27 +272,17 @@ activation_decay_rate = 0.9 # 激活度衰减率（每天衰减10%） activation_propagation_strength = 0.5 # 激活传播强度（传播到相关记忆的激活度比例） activation_propagation_depth = 1 # 激活传播深度（最多传播几层） +# === 记忆检索配置 === +search_max_expand_depth = 2 # 检索时图扩展深度（0=仅直接匹配，1=扩展1跳，2=扩展2跳，推荐1-2） +search_vector_weight = 0.4 # 向量相似度权重 +search_graph_distance_weight = 0.2 # 图距离权重 +search_importance_weight = 0.2 # 重要性权重 +search_recency_weight = 0.2 # 时效性权重 + # === 性能配置 === max_memory_nodes_per_memory = 10 # 每条记忆最多包含的节点数 max_related_memories = 5 # 激活传播时最多影响的相关记忆数 -# ==================== 旧记忆系统配置 (已弃用) ==================== -# 注意：以下配置仅用于向后兼容，新系统不使用这些配置 -# 旧的 enhanced memory 系统已被 memory_graph 系统取代 - -[memory_legacy] -# 旧系统已禁用，所有配置保留仅供参考 -enable_legacy_memory = false # 旧记忆系统已禁用 - -# Vector DB配置 (ChromaDB) - 保留用于其他系统 -[vector_db] -type = "chromadb" # Vector DB类型 -path = "data/chroma_db" # Vector DB数据路径（用于其他系统，非memory_graph） - -[vector_db.settings] -anonymized_telemetry = false # 禁用匿名遥测 -allow_reset = true # 允许重置 - [voice] enable_asr = true # 是否启用语音识别，启用后MoFox-Bot可以识别语音消息，启用该功能需要配置语音识别模型[model.voice] # [语音识别提供商] 可选值: "api", "local". 默认使用 "api".