From 0c70749a6668d767c7d3f378077428cf93d7f0e0 Mon Sep 17 00:00:00 2001 From: Furina-1013-create <189647097+Furina-1013-create@users.noreply.github.com> Date: Thu, 18 Sep 2025 15:51:42 +0800 Subject: [PATCH 1/4] Clean up deleted web_search_tool files after merge --- .../built_in/web_search_tool/_manifest.json | 25 --- .../built_in/web_search_tool/plugin.py | 160 ------------------ 2 files changed, 185 deletions(-) delete mode 100644 src/plugins/built_in/web_search_tool/_manifest.json delete mode 100644 src/plugins/built_in/web_search_tool/plugin.py diff --git a/src/plugins/built_in/web_search_tool/_manifest.json b/src/plugins/built_in/web_search_tool/_manifest.json deleted file mode 100644 index 549781c2a..000000000 --- a/src/plugins/built_in/web_search_tool/_manifest.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "manifest_version": 1, - "name": "web_search_tool", - "version": "1.0.0", - "description": "一个用于在互联网上搜索信息的工具", - "author": { - "name": "MoFox-Studio", - "url": "https://github.com/MoFox-Studio" - }, - "license": "GPL-v3.0-or-later", - - "host_application": { - "min_version": "0.10.0" - }, - "keywords": ["web_search", "url_parser"], - "categories": ["web_search", "url_parser"], - - "default_locale": "zh-CN", - "locales_path": "_locales", - - "plugin_info": { - "is_built_in": false, - "plugin_type": "web_search" - } -} \ No newline at end of file diff --git a/src/plugins/built_in/web_search_tool/plugin.py b/src/plugins/built_in/web_search_tool/plugin.py deleted file mode 100644 index 1789062ae..000000000 --- a/src/plugins/built_in/web_search_tool/plugin.py +++ /dev/null @@ -1,160 +0,0 @@ -""" -Web Search Tool Plugin - -一个功能强大的网络搜索和URL解析插件，支持多种搜索引擎和解析策略。 -""" -from typing import List, Tuple, Type - -from src.plugin_system import ( - BasePlugin, - register_plugin, - ComponentInfo, - ConfigField, - PythonDependency -) -from src.plugin_system.apis import config_api -from src.common.logger import get_logger - -from .tools.web_search import WebSurfingTool -from .tools.url_parser import URLParserTool - -logger = get_logger("web_search_plugin") - - -@register_plugin -class WEBSEARCHPLUGIN(BasePlugin): - """ - 网络搜索工具插件 - - 提供网络搜索和URL解析功能，支持多种搜索引擎： - - Exa (需要API密钥) - - Tavily (需要API密钥) - - DuckDuckGo (免费) - - Bing (免费) - """ - - # 插件基本信息 - plugin_name: str = "web_search_tool" # 内部标识符 - enable_plugin: bool = True - dependencies: List[str] = [] # 插件依赖列表 - - def __init__(self, *args, **kwargs): - """初始化插件，立即加载所有搜索引擎""" - super().__init__(*args, **kwargs) - - # 立即初始化所有搜索引擎，触发API密钥管理器的日志输出 - logger.info("🚀 正在初始化所有搜索引擎...") - try: - from .engines.exa_engine import ExaSearchEngine - from .engines.tavily_engine import TavilySearchEngine - from .engines.ddg_engine import DDGSearchEngine - from .engines.bing_engine import BingSearchEngine - - # 实例化所有搜索引擎，这会触发API密钥管理器的初始化 - exa_engine = ExaSearchEngine() - tavily_engine = TavilySearchEngine() - ddg_engine = DDGSearchEngine() - bing_engine = BingSearchEngine() - - # 报告每个引擎的状态 - engines_status = { - "Exa": exa_engine.is_available(), - "Tavily": tavily_engine.is_available(), - "DuckDuckGo": ddg_engine.is_available(), - "Bing": bing_engine.is_available() - } - - available_engines = [name for name, available in engines_status.items() if available] - unavailable_engines = [name for name, available in engines_status.items() if not available] - - if available_engines: - logger.info(f"✅ 可用搜索引擎: {', '.join(available_engines)}") - if unavailable_engines: - logger.info(f"❌ 不可用搜索引擎: {', '.join(unavailable_engines)}") - - except Exception as e: - logger.error(f"❌ 搜索引擎初始化失败: {e}", exc_info=True) - - # Python包依赖列表 - python_dependencies: List[PythonDependency] = [ - PythonDependency( - package_name="asyncddgs", - description="异步DuckDuckGo搜索库", - optional=False - ), - PythonDependency( - package_name="exa_py", - description="Exa搜索API客户端库", - optional=True # 如果没有API密钥，这个是可选的 - ), - PythonDependency( - package_name="tavily", - install_name="tavily-python", # 安装时使用这个名称 - description="Tavily搜索API客户端库", - optional=True # 如果没有API密钥，这个是可选的 - ), - PythonDependency( - package_name="httpx", - version=">=0.20.0", - install_name="httpx[socks]", # 安装时使用这个名称（包含可选依赖） - description="支持SOCKS代理的HTTP客户端库", - optional=False - ) - ] - config_file_name: str = "config.toml" # 配置文件名 - - # 配置节描述 - config_section_descriptions = { - "plugin": "插件基本信息", - "proxy": "链接本地解析代理配置" - } - - # 配置Schema定义 - # 注意：EXA配置和组件设置已迁移到主配置文件(bot_config.toml)的[exa]和[web_search]部分 - config_schema: dict = { - "plugin": { - "name": ConfigField(type=str, default="WEB_SEARCH_PLUGIN", description="插件名称"), - "version": ConfigField(type=str, default="1.0.0", description="插件版本"), - "enabled": ConfigField(type=bool, default=False, description="是否启用插件"), - }, - "proxy": { - "http_proxy": ConfigField( - type=str, - default=None, - description="HTTP代理地址，格式如: http://proxy.example.com:8080" - ), - "https_proxy": ConfigField( - type=str, - default=None, - description="HTTPS代理地址，格式如: http://proxy.example.com:8080" - ), - "socks5_proxy": ConfigField( - type=str, - default=None, - description="SOCKS5代理地址，格式如: socks5://proxy.example.com:1080" - ), - "enable_proxy": ConfigField( - type=bool, - default=False, - description="是否启用代理" - ) - }, - } - - def get_plugin_components(self) -> List[Tuple[ComponentInfo, Type]]: - """ - 获取插件组件列表 - - Returns: - 组件信息和类型的元组列表 - """ - enable_tool = [] - - # 从主配置文件读取组件启用配置 - if config_api.get_global_config("web_search.enable_web_search_tool", True): - enable_tool.append((WebSurfingTool.get_tool_info(), WebSurfingTool)) - - if config_api.get_global_config("web_search.enable_url_tool", True): - enable_tool.append((URLParserTool.get_tool_info(), URLParserTool)) - - return enable_tool From 9912d7f643d347cbadcf1e3d618aa78bcbf89cc4 Mon Sep 17 00:00:00 2001 From: minecraft1024a Date: Tue, 23 Sep 2025 19:15:58 +0800 Subject: [PATCH 2/4] =?UTF-8?q?perf(memory):=20=E4=BC=98=E5=8C=96=E8=AE=B0?= =?UTF-8?q?=E5=BF=86=E7=B3=BB=E7=BB=9F=E6=95=B0=E6=8D=AE=E5=BA=93=E6=93=8D?= =?UTF-8?q?=E4=BD=9C=E5=B9=B6=E4=BF=AE=E5=A4=8D=E5=B9=B6=E5=8F=91=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 将消息记忆次数的更新方式从单次写入重构为批量更新，在记忆构建任务结束时统一执行，大幅减少数据库写入次数，显著提升性能。 此外，为 `HippocampusManager` 添加了异步锁，以防止记忆巩固和遗忘操作并发执行时产生竞争条件。同时，增加了节点去重逻辑，在插入数据库前检查重复的概念，确保数据一致性。 --- src/chat/memory_system/Hippocampus.py | 67 ++++++++++++------- .../src/recv_handler/notice_handler.py | 2 +- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/src/chat/memory_system/Hippocampus.py b/src/chat/memory_system/Hippocampus.py index fcc8e65d2..ca726c1a8 100644 --- a/src/chat/memory_system/Hippocampus.py +++ b/src/chat/memory_system/Hippocampus.py @@ -3,6 +3,7 @@ import datetime import math import random import time +import asyncio import re import orjson import jieba @@ -789,7 +790,7 @@ class EntorhinalCortex: self.hippocampus = hippocampus self.memory_graph = hippocampus.memory_graph - async def get_memory_sample(self): + async def get_memory_sample(self) -> tuple[list, list[str]]: """从数据库获取记忆样本""" # 硬编码：每条消息最大记忆次数 max_memorized_time_per_msg = 2 @@ -811,24 +812,27 @@ class EntorhinalCortex: for _, readable_timestamp in zip(timestamps, readable_timestamps, strict=False): logger.debug(f"回忆往事: {readable_timestamp}") chat_samples = [] + all_message_ids_to_update = [] for timestamp in timestamps: - if messages := await self.random_get_msg_snippet( + if result := await self.random_get_msg_snippet( timestamp, global_config.memory.memory_build_sample_length, max_memorized_time_per_msg, ): + messages, message_ids_to_update = result time_diff = (datetime.datetime.now().timestamp() - timestamp) / 3600 logger.info(f"成功抽取 {time_diff:.1f} 小时前的消息样本，共{len(messages)}条") chat_samples.append(messages) + all_message_ids_to_update.extend(message_ids_to_update) else: logger.debug(f"时间戳 {timestamp} 的消息无需记忆") - return chat_samples + return chat_samples, all_message_ids_to_update @staticmethod async def random_get_msg_snippet( target_timestamp: float, chat_size: int, max_memorized_time_per_msg: int - ) -> list | None: + ) -> tuple[list, list[str]] | None: # sourcery skip: invert-any-all, use-any, use-named-expression, use-next """从数据库中随机获取指定时间戳附近的消息片段 (使用 chat_message_builder)""" time_window_seconds = random.randint(300, 1800) # 随机时间窗口，5到30分钟 @@ -862,18 +866,9 @@ class EntorhinalCortex: # 如果所有消息都有效 if all_valid: - # 更新数据库中的记忆次数 - for message in messages: - # 确保在更新前获取最新的 memorized_times - current_memorized_times = message.get("memorized_times", 0) - async with get_db_session() as session: - await session.execute( - update(Messages) - .where(Messages.message_id == message["message_id"]) - .values(memorized_times=current_memorized_times + 1) - ) - await session.commit() - return messages # 直接返回原始的消息列表 + # 返回消息和需要更新的message_id + message_ids_to_update = [msg["message_id"] for msg in messages] + return messages, message_ids_to_update target_timestamp -= 120 # 如果第一次尝试失败，稍微向前调整时间戳再试 @@ -953,10 +948,20 @@ class EntorhinalCortex: # 批量处理节点 if nodes_to_create: - batch_size = 100 - for i in range(0, len(nodes_to_create), batch_size): - batch = nodes_to_create[i : i + batch_size] - await session.execute(insert(GraphNodes), batch) + # 在插入前进行去重检查 + unique_nodes_to_create = [] + seen_concepts = set(db_nodes.keys()) + for node_data in nodes_to_create: + concept = node_data["concept"] + if concept not in seen_concepts: + unique_nodes_to_create.append(node_data) + seen_concepts.add(concept) + + if unique_nodes_to_create: + batch_size = 100 + for i in range(0, len(unique_nodes_to_create), batch_size): + batch = unique_nodes_to_create[i : i + batch_size] + await session.execute(insert(GraphNodes), batch) if nodes_to_update: batch_size = 100 @@ -1346,7 +1351,7 @@ class ParahippocampalGyrus: # sourcery skip: merge-list-appends-into-extend logger.info("------------------------------------开始构建记忆--------------------------------------") start_time = time.time() - memory_samples = await self.hippocampus.entorhinal_cortex.get_memory_sample() + memory_samples, all_message_ids_to_update = await self.hippocampus.entorhinal_cortex.get_memory_sample() all_added_nodes = [] all_connected_nodes = [] all_added_edges = [] @@ -1409,8 +1414,21 @@ class ParahippocampalGyrus: if all_connected_nodes: logger.info(f"强化连接节点: {', '.join(all_connected_nodes)}") + # 先同步记忆图 await self.hippocampus.entorhinal_cortex.sync_memory_to_db() + # 最后批量更新消息的记忆次数 + if all_message_ids_to_update: + async with get_db_session() as session: + # 使用 in_ 操作符进行批量更新 + await session.execute( + update(Messages) + .where(Messages.message_id.in_(all_message_ids_to_update)) + .values(memorized_times=Messages.memorized_times + 1) + ) + await session.commit() + logger.info(f"批量更新了 {len(all_message_ids_to_update)} 条消息的记忆次数") + end_time = time.time() logger.info(f"---------------------记忆构建耗时: {end_time - start_time:.2f} 秒---------------------") @@ -1617,6 +1635,7 @@ class HippocampusManager: def __init__(self): self._hippocampus: Hippocampus = None # type: ignore self._initialized = False + self._db_lock = asyncio.Lock() def initialize(self): """初始化海马体实例""" @@ -1665,14 +1684,16 @@ class HippocampusManager: """遗忘记忆的公共接口""" if not self._initialized: raise RuntimeError("HippocampusManager 尚未初始化，请先调用 initialize 方法") - return await self._hippocampus.parahippocampal_gyrus.operation_forget_topic(percentage) + async with self._db_lock: + return await self._hippocampus.parahippocampal_gyrus.operation_forget_topic(percentage) async def consolidate_memory(self): """整合记忆的公共接口""" if not self._initialized: raise RuntimeError("HippocampusManager 尚未初始化，请先调用 initialize 方法") # 使用 operation_build_memory 方法来整合记忆 - return await self._hippocampus.parahippocampal_gyrus.operation_build_memory() + async with self._db_lock: + return await self._hippocampus.parahippocampal_gyrus.operation_build_memory() async def get_memory_from_text( self, diff --git a/src/plugins/built_in/napcat_adapter_plugin/src/recv_handler/notice_handler.py b/src/plugins/built_in/napcat_adapter_plugin/src/recv_handler/notice_handler.py index 4a32657a7..58b7f23b9 100644 --- a/src/plugins/built_in/napcat_adapter_plugin/src/recv_handler/notice_handler.py +++ b/src/plugins/built_in/napcat_adapter_plugin/src/recv_handler/notice_handler.py @@ -339,7 +339,7 @@ class NoticeHandler: message_id=raw_message.get("message_id",""), emoji_id=like_emoji_id ) - seg_data = Seg(type="text",data=f"{user_name}使用Emoji表情{QQ_FACE.get(like_emoji_id,"")}回复了你的消息[{target_message_text}]") + seg_data = Seg(type="text",data=f"{user_name}使用Emoji表情{QQ_FACE.get(like_emoji_id, '')}回复了你的消息[{target_message_text}]") return seg_data, user_info async def handle_ban_notify(self, raw_message: dict, group_id: int) -> Tuple[Seg, UserInfo] | Tuple[None, None]: From 46f210cc6bface94bcccbec41c54a8a2f7338390 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ikun=E4=B8=A4=E5=B9=B4=E5=8D=8A?= <334495606@qq.com> Date: Tue, 14 Oct 2025 22:22:05 +0800 Subject: [PATCH 3/4] =?UTF-8?q?=E5=B0=86=E4=BE=9D=E8=B5=96=E5=90=8D?= =?UTF-8?q?=E5=AD=97=E4=BF=AE=E6=AD=A3=E5=96=B5=EF=BC=81=EF=BC=81=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f5de2e25f..ad982f968 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "google>=3.0.0", "google-genai>=1.29.0", "httpx>=0.28.1", - "rjieba>=0.42.1", + "jieba>=0.42.1", "json-repair>=0.47.6", "json5>=0.12.1", "jsonlines>=4.0.0", @@ -75,7 +75,7 @@ dependencies = [ "aiomysql>=0.2.0", "aiosqlite>=0.21.0", "inkfox>=0.1.1", - "rrjieba>=0.1.13", + "rjieba>=0.1.13", "mcp>=0.9.0", "sse-starlette>=2.2.1", ] From 9d1b0bc30456214349b50411ebde4fd8d1c9d493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?ikun=E4=B8=A4=E5=B9=B4=E5=8D=8A?= <334495606@qq.com> Date: Tue, 14 Oct 2025 22:56:49 +0800 Subject: [PATCH 4/4] =?UTF-8?q?=E6=9B=B4=E6=AD=A3=E4=BA=86python=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=96=B5=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ad982f968..3b7682736 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,7 +2,7 @@ name = "MaiBot" version = "0.8.1" description = "MaiCore 是一个基于大语言模型的可交互智能体" -requires-python = ">=3.10" +requires-python = ">=3.11" dependencies = [ "aiohttp>=3.12.14", "aiohttp-cors>=0.8.1",