From 669f9e400a31d4ea7a624bbf198aa7d754b136ef Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Wed, 5 Mar 2025 09:49:19 +0800 Subject: [PATCH 01/53] =?UTF-8?q?feat:=20=E4=BF=AE=E6=94=B9emoji=E4=B8=BAe?= =?UTF-8?q?mbedding=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 260 +++++++++++------------------- 1 file changed, 90 insertions(+), 170 deletions(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 2311b2459..a0164d065 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -14,10 +14,12 @@ import asyncio import time from PIL import Image import io +from loguru import logger from nonebot import get_driver from ..chat.config import global_config from ..models.utils_model import LLM_request +from utils import get_embedding driver = get_driver() config = driver.config @@ -27,16 +29,6 @@ class EmojiManager: _instance = None EMOJI_DIR = "data/emoji" # 表情包存储目录 - EMOTION_KEYWORDS = { - 'happy': ['开心', '快乐', '高兴', '欢喜', '笑', '喜悦', '兴奋', '愉快', '乐', '好'], - 'angry': ['生气', '愤怒', '恼火', '不爽', '火大', '怒', '气愤', '恼怒', '发火', '不满'], - 'sad': ['伤心', '难过', '悲伤', '痛苦', '哭', '忧伤', '悲痛', '哀伤', '委屈', '失落'], - 'surprised': ['惊讶', '震惊', '吃惊', '意外', '惊', '诧异', '惊奇', '惊喜', '不敢相信', '目瞪口呆'], - 'disgusted': ['恶心', '讨厌', '厌恶', '反感', '嫌弃', '恶', '嫌恶', '憎恶', '不喜欢', '烦'], - 'fearful': ['害怕', '恐惧', '惊恐', '担心', '怕', '惊吓', '惊慌', '畏惧', '胆怯', '惧'], - 'neutral': ['普通', '一般', '还行', '正常', '平静', '平淡', '一般般', '凑合', '还好', '就这样'] - } - def __new__(cls): if cls._instance is None: cls._instance = super().__new__(cls) @@ -64,7 +56,7 @@ class EmojiManager: # 启动时执行一次完整性检查 self.check_emoji_file_integrity() except Exception as e: - print(f"\033[1;31m[错误]\033[0m 初始化表情管理器失败: {str(e)}") + logger.error(f"初始化表情管理器失败: {str(e)}") def _ensure_db(self): """确保数据库已初始化""" @@ -77,7 +69,7 @@ class EmojiManager: """确保emoji集合存在并创建索引""" if 'emoji' not in self.db.db.list_collection_names(): self.db.db.create_collection('emoji') - self.db.db.emoji.create_index([('tags', 1)]) + self.db.db.emoji.create_index([('embedding', '2dsphere')]) self.db.db.emoji.create_index([('filename', 1)], unique=True) def record_usage(self, emoji_id: str): @@ -89,79 +81,8 @@ class EmojiManager: {'$inc': {'usage_count': 1}} ) except Exception as e: - print(f"\033[1;31m[错误]\033[0m 记录表情使用失败: {str(e)}") + logger.error(f"记录表情使用失败: {str(e)}") - async def _get_emotion_from_text(self, text: str) -> List[str]: - """从文本中识别情感关键词 - Args: - text: 输入文本 - Returns: - List[str]: 匹配到的情感标签列表 - """ - try: - prompt = f'分析这段文本："{text}"，从"happy,angry,sad,surprised,disgusted,fearful,neutral"中选出最匹配的1个情感标签。只需要返回标签，不要输出其他任何内容。' - - content, _ = await self.llm.generate_response(prompt) - emotion = content.strip().lower() - - if emotion in self.EMOTION_KEYWORDS: - print(f"\033[1;32m[成功]\033[0m 识别到的情感: {emotion}") - return [emotion] - - return ['neutral'] - - except Exception as e: - print(f"\033[1;31m[错误]\033[0m 情感分析失败: {str(e)}") - return ['neutral'] - - async def get_emoji_for_emotion(self, emotion_tag: str) -> Optional[str]: - try: - self._ensure_db() - - # 构建查询条件：标签匹配任一情感 - query = {'tags': {'$in': emotion_tag}} - - # print(f"\033[1;34m[调试]\033[0m 表情查询条件: {query}") - - try: - # 随机获取一个匹配的表情 - emoji = self.db.db.emoji.aggregate([ - {'$match': query}, - {'$sample': {'size': 1}} - ]).next() - print(f"\033[1;32m[成功]\033[0m 找到匹配的表情") - if emoji and 'path' in emoji: - # 更新使用次数 - self.db.db.emoji.update_one( - {'_id': emoji['_id']}, - {'$inc': {'usage_count': 1}} - ) - return emoji['path'] - except StopIteration: - # 如果没有匹配的表情，从所有表情中随机选择一个 - print(f"\033[1;33m[提示]\033[0m 未找到匹配的表情，随机选择一个") - try: - emoji = self.db.db.emoji.aggregate([ - {'$sample': {'size': 1}} - ]).next() - if emoji and 'path' in emoji: - # 更新使用次数 - self.db.db.emoji.update_one( - {'_id': emoji['_id']}, - {'$inc': {'usage_count': 1}} - ) - return emoji['path'] - except StopIteration: - print(f"\033[1;31m[错误]\033[0m 数据库中没有任何表情") - return None - - return None - - except Exception as e: - print(f"\033[1;31m[错误]\033[0m 获取表情包失败: {str(e)}") - return None - - async def get_emoji_for_text(self, text: str) -> Optional[str]: """根据文本内容获取相关表情包 Args: @@ -171,77 +92,84 @@ class EmojiManager: """ try: self._ensure_db() - # 获取情感标签 - emotions = await self._get_emotion_from_text(text) - print("为 ‘"+ str(text) + "’ 获取到的情感标签为：" + str(emotions)) - if not emotions: + + # 获取文本的embedding + text_embedding = get_embedding(text) + if not text_embedding: + logger.error("无法获取文本的embedding") return None - # 构建查询条件：标签匹配任一情感 - query = {'tags': {'$in': emotions}} - - print(f"\033[1;34m[调试]\033[0m 表情查询条件: {query}") - print(f"\033[1;34m[调试]\033[0m 匹配到的情感: {emotions}") + # 使用embedding进行相似度搜索，获取最相似的3个表情包 + pipeline = [ + { + "$search": { + "index": "default", + "knnBeta": { + "vector": text_embedding, + "path": "embedding", + "k": 3 + } + } + } + ] try: - # 随机获取一个匹配的表情 - emoji = self.db.db.emoji.aggregate([ - {'$match': query}, - {'$sample': {'size': 1}} - ]).next() - print(f"\033[1;32m[成功]\033[0m 找到匹配的表情") - if emoji and 'path' in emoji: + # 获取搜索结果 + results = list(self.db.db.emoji.aggregate(pipeline)) + + if not results: + logger.warning("未找到匹配的表情包，尝试随机选择") + # 如果没有匹配的表情，随机选择一个 + try: + emoji = self.db.db.emoji.aggregate([ + {'$sample': {'size': 1}} + ]).next() + if emoji and 'path' in emoji: + # 更新使用次数 + self.db.db.emoji.update_one( + {'_id': emoji['_id']}, + {'$inc': {'usage_count': 1}} + ) + return emoji['path'] + except StopIteration: + logger.error("数据库中没有任何表情") + return None + + # 从最相似的3个表情包中随机选择一个 + selected_emoji = random.choice(results) + + if selected_emoji and 'path' in selected_emoji: # 更新使用次数 self.db.db.emoji.update_one( - {'_id': emoji['_id']}, + {'_id': selected_emoji['_id']}, {'$inc': {'usage_count': 1}} ) - return emoji['path'] - except StopIteration: - # 如果没有匹配的表情，从所有表情中随机选择一个 - print(f"\033[1;33m[提示]\033[0m 未找到匹配的表情，随机选择一个") - try: - emoji = self.db.db.emoji.aggregate([ - {'$sample': {'size': 1}} - ]).next() - if emoji and 'path' in emoji: - # 更新使用次数 - self.db.db.emoji.update_one( - {'_id': emoji['_id']}, - {'$inc': {'usage_count': 1}} - ) - return emoji['path'] - except StopIteration: - print(f"\033[1;31m[错误]\033[0m 数据库中没有任何表情") - return None + logger.success(f"找到匹配的表情包: {selected_emoji.get('discription', '无描述')}") + return selected_emoji['path'] + + except Exception as search_error: + logger.error(f"搜索表情包失败: {str(search_error)}") + return None return None except Exception as e: - print(f"\033[1;31m[错误]\033[0m 获取表情包失败: {str(e)}") + logger.error(f"获取表情包失败: {str(e)}") return None async def _get_emoji_tag(self, image_base64: str) -> str: """获取表情包的标签""" try: - prompt = '这是一个表情包，请从"happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"中选出1个情感标签。只输出标签，不要输出其他任何内容，只输出情感标签就好' + prompt = '这是一个表情包，请为其生成简洁的描述，同时生成表情包所蕴含的情绪的描述。' content, _ = await self.llm.generate_response_for_image(prompt, image_base64) - tag_result = content.strip().lower() - - valid_tags = ["happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"] - for tag_match in valid_tags: - if tag_match in tag_result or tag_match == tag_result: - return tag_match - print(f"\033[1;33m[警告]\033[0m 无效的标签: {tag_result}, 跳过") + logger.debug(f"输出描述: {content}") + return content except Exception as e: - print(f"\033[1;31m[错误]\033[0m 获取标签失败: {str(e)}") - return "skip" + logger.error(f"获取标签失败: {str(e)}") + return None - print(f"\033[1;32m[调试信息]\033[0m 使用默认标签: neutral") - return "skip" # 默认标签 - async def _compress_image(self, image_path: str, target_size: int = 0.8 * 1024 * 1024) -> Optional[str]: """压缩图片并返回base64编码 Args: @@ -303,12 +231,12 @@ class EmojiManager: # 获取压缩后的数据并转换为base64 compressed_data = output_buffer.getvalue() - print(f"\033[1;32m[成功]\033[0m 压缩图片: {os.path.basename(image_path)} ({original_width}x{original_height} -> {new_width}x{new_height})") + logger.success(f"压缩图片: {os.path.basename(image_path)} ({original_width}x{original_height} -> {new_width}x{new_height})") return base64.b64encode(compressed_data).decode('utf-8') except Exception as e: - print(f"\033[1;31m[错误]\033[0m 压缩图片失败: {os.path.basename(image_path)}, 错误: {str(e)}") + logger.error(f"压缩图片失败: {os.path.basename(image_path)}, 错误: {str(e)}") return None async def scan_new_emojis(self): @@ -334,35 +262,29 @@ class EmojiManager: os.remove(image_path) continue - # 获取表情包的情感标签 - tag = await self._get_emoji_tag(image_base64) - if not tag == "skip": + # 获取表情包的描述 + discription = await self._get_emoji_tag(image_base64) + embedding = get_embedding(discription) + if discription is not None: # 准备数据库记录 emoji_record = { 'filename': filename, 'path': image_path, - 'tags': [tag], + 'embedding':embedding, + 'discription': discription, 'timestamp': int(time.time()) } # 保存到数据库 self.db.db['emoji'].insert_one(emoji_record) - print(f"\033[1;32m[成功]\033[0m 注册新表情包: {filename}") - print(f"标签: {tag}") + logger.success(f"注册新表情包: {filename}") + logger.info(f"描述: {discription}") else: - print(f"\033[1;33m[警告]\033[0m 跳过表情包: {filename}") + logger.warning(f"跳过表情包: {filename}") except Exception as e: - print(f"\033[1;31m[错误]\033[0m 扫描表情包失败: {str(e)}") - import traceback - print(traceback.format_exc()) - - async def _periodic_scan(self, interval_MINS: int = 10): - """定期扫描新表情包""" - while True: - print(f"\033[1;36m[表情包]\033[0m 开始扫描新表情包...") - await self.scan_new_emojis() - await asyncio.sleep(interval_MINS * 60) # 每600秒扫描一次 + logger.error(f"扫描表情包失败: {str(e)}") + logger.error(traceback.format_exc()) def check_emoji_file_integrity(self): """检查表情包文件完整性 @@ -378,44 +300,42 @@ class EmojiManager: for emoji in all_emojis: try: if 'path' not in emoji: - print(f"\033[1;33m[提示]\033[0m 发现无效记录（缺少path字段），ID: {emoji.get('_id', 'unknown')}") + logger.warning(f"发现无效记录（缺少path字段），ID: {emoji.get('_id', 'unknown')}") + self.db.db.emoji.delete_one({'_id': emoji['_id']}) + removed_count += 1 + continue + + if 'embedding' not in emoji: + logger.warning(f"发现过时记录（缺少embedding字段），ID: {emoji.get('_id', 'unknown')}") self.db.db.emoji.delete_one({'_id': emoji['_id']}) removed_count += 1 continue # 检查文件是否存在 if not os.path.exists(emoji['path']): - print(f"\033[1;33m[提示]\033[0m 表情包文件已被删除: {emoji['path']}") + logger.warning(f"表情包文件已被删除: {emoji['path']}") # 从数据库中删除记录 result = self.db.db.emoji.delete_one({'_id': emoji['_id']}) if result.deleted_count > 0: - print(f"\033[1;32m[成功]\033[0m 成功删除数据库记录: {emoji['_id']}") + logger.success(f"成功删除数据库记录: {emoji['_id']}") removed_count += 1 else: - print(f"\033[1;31m[错误]\033[0m 删除数据库记录失败: {emoji['_id']}") + logger.error(f"删除数据库记录失败: {emoji['_id']}") except Exception as item_error: - print(f"\033[1;31m[错误]\033[0m 处理表情包记录时出错: {str(item_error)}") + logger.error(f"处理表情包记录时出错: {str(item_error)}") continue # 验证清理结果 remaining_count = self.db.db.emoji.count_documents({}) if removed_count > 0: - print(f"\033[1;32m[成功]\033[0m 已清理 {removed_count} 个失效的表情包记录") - print(f"\033[1;34m[统计]\033[0m 清理前总数: {total_count} | 清理后总数: {remaining_count}") - # print(f"\033[1;34m[统计]\033[0m 应删除数量: {removed_count} | 实际删除数量: {total_count - remaining_count}") - # 执行数据库压缩 - try: - self.db.db.command({"compact": "emoji"}) - print(f"\033[1;32m[成功]\033[0m 数据库集合压缩完成") - except Exception as compact_error: - print(f"\033[1;31m[错误]\033[0m 数据库压缩失败: {str(compact_error)}") + logger.success(f"已清理 {removed_count} 个失效的表情包记录") + logger.info(f"清理前总数: {total_count} | 清理后总数: {remaining_count}") else: - print(f"\033[1;36m[表情包]\033[0m 已检查 {total_count} 个表情包记录") + logger.info(f"已检查 {total_count} 个表情包记录") except Exception as e: - print(f"\033[1;31m[错误]\033[0m 检查表情包完整性失败: {str(e)}") - import traceback - print(f"\033[1;31m[错误追踪]\033[0m\n{traceback.format_exc()}") + logger.error(f"检查表情包完整性失败: {str(e)}") + logger.error(traceback.format_exc()) async def start_periodic_check(self, interval_MINS: int = 120): while True: From 97bddb83e71cfcd2a965ddaaef29c3970a4d21df Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Wed, 5 Mar 2025 10:43:08 +0800 Subject: [PATCH 02/53] =?UTF-8?q?feat:=20=E8=A1=A8=E6=83=85=E5=8C=85?= =?UTF-8?q?=E5=8C=B9=E9=85=8D=E4=BB=8E=E6=83=85=E7=BB=AA=E5=8C=B9=E9=85=8D?= =?UTF-8?q?=E6=94=B9=E6=88=90=E5=B5=8C=E5=85=A5=E7=9B=B8=E4=BC=BC=E5=BA=A6?= =?UTF-8?q?=E5=8C=B9=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/bot.py | 2 +- src/plugins/chat/emoji_manager.py | 84 +++++++++++++++++-------------- 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 6b0e76db5..f9488b96f 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -174,7 +174,7 @@ class ChatBot: bot_response_time = tinking_time_point if random() < global_config.emoji_chance: - emoji_path = await emoji_manager.get_emoji_for_emotion(emotion) + emoji_path = await emoji_manager.get_emoji_for_text(response) if emoji_path: emoji_cq = CQCode.create_emoji_cq(emoji_path) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index a0164d065..aa0bc1fb5 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -15,11 +15,12 @@ import time from PIL import Image import io from loguru import logger +import traceback from nonebot import get_driver from ..chat.config import global_config from ..models.utils_model import LLM_request -from utils import get_embedding +from ..chat.utils import get_embedding driver = get_driver() config = driver.config @@ -39,7 +40,7 @@ class EmojiManager: def __init__(self): self.db = Database.get_instance() self._scan_task = None - self.llm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=50) + self.llm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=1000) def _ensure_emoji_dir(self): """确保表情存储目录存在""" @@ -98,45 +99,44 @@ class EmojiManager: if not text_embedding: logger.error("无法获取文本的embedding") return None - - # 使用embedding进行相似度搜索，获取最相似的3个表情包 - pipeline = [ - { - "$search": { - "index": "default", - "knnBeta": { - "vector": text_embedding, - "path": "embedding", - "k": 3 - } - } - } - ] try: - # 获取搜索结果 - results = list(self.db.db.emoji.aggregate(pipeline)) + # 获取所有表情包 + all_emojis = list(self.db.db.emoji.find({}, {'_id': 1, 'path': 1, 'embedding': 1, 'discription': 1})) - if not results: - logger.warning("未找到匹配的表情包，尝试随机选择") - # 如果没有匹配的表情，随机选择一个 - try: - emoji = self.db.db.emoji.aggregate([ - {'$sample': {'size': 1}} - ]).next() - if emoji and 'path' in emoji: - # 更新使用次数 - self.db.db.emoji.update_one( - {'_id': emoji['_id']}, - {'$inc': {'usage_count': 1}} - ) - return emoji['path'] - except StopIteration: - logger.error("数据库中没有任何表情") - return None + if not all_emojis: + logger.warning("数据库中没有任何表情包") + return None - # 从最相似的3个表情包中随机选择一个 - selected_emoji = random.choice(results) + # 计算余弦相似度并排序 + def cosine_similarity(v1, v2): + if not v1 or not v2: + return 0 + dot_product = sum(a * b for a, b in zip(v1, v2)) + norm_v1 = sum(a * a for a in v1) ** 0.5 + norm_v2 = sum(b * b for b in v2) ** 0.5 + if norm_v1 == 0 or norm_v2 == 0: + return 0 + return dot_product / (norm_v1 * norm_v2) + + # 计算所有表情包与输入文本的相似度 + emoji_similarities = [ + (emoji, cosine_similarity(text_embedding, emoji.get('embedding', []))) + for emoji in all_emojis + ] + + # 按相似度降序排序 + emoji_similarities.sort(key=lambda x: x[1], reverse=True) + + # 获取前3个最相似的表情包 + top_3_emojis = emoji_similarities[:3] + + if not top_3_emojis: + logger.warning("未找到匹配的表情包") + return None + + # 从前3个中随机选择一个 + selected_emoji, similarity = random.choice(top_3_emojis) if selected_emoji and 'path' in selected_emoji: # 更新使用次数 @@ -144,7 +144,7 @@ class EmojiManager: {'_id': selected_emoji['_id']}, {'$inc': {'usage_count': 1}} ) - logger.success(f"找到匹配的表情包: {selected_emoji.get('discription', '无描述')}") + logger.success(f"找到匹配的表情包: {selected_emoji.get('discription', '无描述')} (相似度: {similarity:.4f})") return selected_emoji['path'] except Exception as search_error: @@ -285,6 +285,14 @@ class EmojiManager: except Exception as e: logger.error(f"扫描表情包失败: {str(e)}") logger.error(traceback.format_exc()) + + async def _periodic_scan(self, interval_MINS: int = 10): + """定期扫描新表情包""" + while True: + print(f"\033[1;36m[表情包]\033[0m 开始扫描新表情包...") + await self.scan_new_emojis() + await asyncio.sleep(interval_MINS * 60) # 每600秒扫描一次 + def check_emoji_file_integrity(self): """检查表情包文件完整性 From a445c222505cd7ff724ffdea93a0ba2ac6eb2d5b Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Thu, 6 Mar 2025 01:08:26 +0800 Subject: [PATCH 03/53] =?UTF-8?q?=E4=B9=8B=E6=88=91=E7=9A=84LLM?= =?UTF-8?q?=E4=B8=BA=E4=BB=80=E4=B9=88=E5=8F=AA=E6=9C=89=E4=B8=80=E5=8D=8A?= =?UTF-8?q?TAG?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 3ba873d74..c7dbc6ffd 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -36,6 +36,7 @@ class LLM_request: data = { "model": self.model_name, "messages": [{"role": "user", "content": prompt}], + "max_tokens": 8000, **self.params } @@ -65,10 +66,10 @@ class LLM_request: think_match = None reasoning_content = message.get("reasoning_content", "") if not reasoning_content: - think_match = re.search(r'(.*?)', content, re.DOTALL) + think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) if think_match: reasoning_content = think_match.group(1).strip() - content = re.sub(r'.*?', '', content, flags=re.DOTALL).strip() + content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() return content, reasoning_content return "没有返回结果", "" @@ -112,9 +113,10 @@ class LLM_request: ] } ], + "max_tokens": 8000, **self.params } - + # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" @@ -122,9 +124,9 @@ class LLM_request: max_retries = 3 base_wait_time = 15 - + current_image_base64 = image_base64 - + for retry in range(max_retries): try: @@ -141,7 +143,7 @@ class LLM_request: logger.warning("图片太大(413)，尝试压缩...") current_image_base64 = compress_base64_image_by_scale(current_image_base64) continue - + response.raise_for_status() # 检查其他响应状态 result = await response.json() @@ -151,10 +153,10 @@ class LLM_request: think_match = None reasoning_content = message.get("reasoning_content", "") if not reasoning_content: - think_match = re.search(r'(.*?)', content, re.DOTALL) + think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) if think_match: reasoning_content = think_match.group(1).strip() - content = re.sub(r'.*?', '', content, flags=re.DOTALL).strip() + content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() return content, reasoning_content return "没有返回结果", "" @@ -197,6 +199,7 @@ class LLM_request: ] } ], + "max_tokens": 8000, **self.params } @@ -226,10 +229,10 @@ class LLM_request: think_match = None reasoning_content = message.get("reasoning_content", "") if not reasoning_content: - think_match = re.search(r'(.*?)', content, re.DOTALL) + think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) if think_match: reasoning_content = think_match.group(1).strip() - content = re.sub(r'.*?', '', content, flags=re.DOTALL).strip() + content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() return content, reasoning_content return "没有返回结果", "" From a896cf5ec4d31d358bf3cef3e5721f01da817d07 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 02:13:54 +0800 Subject: [PATCH 04/53] =?UTF-8?q?fix:=20=E5=85=BC=E5=AE=B9tag?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 38 +++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index aa0bc1fb5..2a74e8b02 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -29,6 +29,16 @@ config = driver.config class EmojiManager: _instance = None EMOJI_DIR = "data/emoji" # 表情包存储目录 + + EMOTION_KEYWORDS = { + 'happy': ['开心', '快乐', '高兴', '欢喜', '笑', '喜悦', '兴奋', '愉快', '乐', '好'], + 'angry': ['生气', '愤怒', '恼火', '不爽', '火大', '怒', '气愤', '恼怒', '发火', '不满'], + 'sad': ['伤心', '难过', '悲伤', '痛苦', '哭', '忧伤', '悲痛', '哀伤', '委屈', '失落'], + 'surprised': ['惊讶', '震惊', '吃惊', '意外', '惊', '诧异', '惊奇', '惊喜', '不敢相信', '目瞪口呆'], + 'disgusted': ['恶心', '讨厌', '厌恶', '反感', '嫌弃', '恶', '嫌恶', '憎恶', '不喜欢', '烦'], + 'fearful': ['害怕', '恐惧', '惊恐', '担心', '怕', '惊吓', '惊慌', '畏惧', '胆怯', '惧'], + 'neutral': ['普通', '一般', '还行', '正常', '平静', '平淡', '一般般', '凑合', '还好', '就这样'] + } def __new__(cls): if cls._instance is None: @@ -71,6 +81,7 @@ class EmojiManager: if 'emoji' not in self.db.db.list_collection_names(): self.db.db.create_collection('emoji') self.db.db.emoji.create_index([('embedding', '2dsphere')]) + self.db.db.emoji.create_index([('tags', 1)]) self.db.db.emoji.create_index([('filename', 1)], unique=True) def record_usage(self, emoji_id: str): @@ -160,7 +171,28 @@ class EmojiManager: async def _get_emoji_tag(self, image_base64: str) -> str: """获取表情包的标签""" try: - prompt = '这是一个表情包，请为其生成简洁的描述，同时生成表情包所蕴含的情绪的描述。' + prompt = '这是一个表情包，请从"happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"中选出1个情感标签。只输出标签，不要输出其他任何内容，只输出情感标签就好' + + content, _ = await self.llm.generate_response_for_image(prompt, image_base64) + tag_result = content.strip().lower() + + valid_tags = ["happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"] + for tag_match in valid_tags: + if tag_match in tag_result or tag_match == tag_result: + return tag_match + print(f"\033[1;33m[警告]\033[0m 无效的标签: {tag_result}, 跳过") + + except Exception as e: + print(f"\033[1;31m[错误]\033[0m 获取标签失败: {str(e)}") + return "neutral" + + print(f"\033[1;32m[调试信息]\033[0m 使用默认标签: neutral") + return "neutral" # 默认标签 + + async def _get_emoji_discription(self, image_base64: str) -> str: + """获取表情包的标签""" + try: + prompt = '这是一个表情包，简洁的描述一下表情包的内容和表情包所表达的情感' content, _ = await self.llm.generate_response_for_image(prompt, image_base64) logger.debug(f"输出描述: {content}") @@ -263,7 +295,8 @@ class EmojiManager: continue # 获取表情包的描述 - discription = await self._get_emoji_tag(image_base64) + discription = await self._get_emoji_discription(image_base64) + tag = await self._get_emoji_tag(image_base64) embedding = get_embedding(discription) if discription is not None: # 准备数据库记录 @@ -272,6 +305,7 @@ class EmojiManager: 'path': image_path, 'embedding':embedding, 'discription': discription, + 'tag':tag, 'timestamp': int(time.time()) } From dbfe9c04917930cd10b64643bdce75a411cc491f Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 02:23:58 +0800 Subject: [PATCH 05/53] =?UTF-8?q?fix:=20=E4=B8=BAlog=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0modelname?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 11d7e2b72..3e1a825a4 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -41,7 +41,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -122,7 +122,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -264,7 +264,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -329,7 +329,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -385,7 +385,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 From fea3285d2012d572a856a0bdf1aaf692eeb15fc7 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 06:30:27 +0800 Subject: [PATCH 06/53] =?UTF-8?q?feat:=20emoji=E9=80=89=E6=8B=A9=E9=80=BB?= =?UTF-8?q?=E8=BE=91=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 2a74e8b02..f2bee4fb5 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -51,6 +51,7 @@ class EmojiManager: self.db = Database.get_instance() self._scan_task = None self.llm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=1000) + self.lm = LLM_request(model=global_config.llm_reasoning_minor, max_tokens=1000) def _ensure_emoji_dir(self): """确保表情存储目录存在""" @@ -106,7 +107,8 @@ class EmojiManager: self._ensure_db() # 获取文本的embedding - text_embedding = get_embedding(text) + text_for_search= await self._get_kimoji_for_text(text) + text_embedding = get_embedding(text_for_search) if not text_embedding: logger.error("无法获取文本的embedding") return None @@ -202,6 +204,18 @@ class EmojiManager: logger.error(f"获取标签失败: {str(e)}") return None + async def _get_kimoji_for_text(self, text:str): + try: + prompt = f'这是{global_config.BOT_NICKNAME}将要发送的消息内容:\n{text}\n若要为其配上表情包，请你输出这个表情包应该表达怎样的情感，应该给人什么样的感觉，不要太简洁也不要太长，注意不要输出任何对内容的分析内容，只输出\"一种什么样的感觉\"中间的形容词部分。' + + content, _ = await self.llm.generate_response_async(prompt) + logger.info(f"输出描述: {content}") + return content + + except Exception as e: + logger.error(f"获取标签失败: {str(e)}") + return None + async def _compress_image(self, image_path: str, target_size: int = 0.8 * 1024 * 1024) -> Optional[str]: """压缩图片并返回base64编码 Args: From 3897c9787a59ba241138df7528ce0e03575006dc Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 06:45:20 +0800 Subject: [PATCH 07/53] =?UTF-8?q?fix:=20=E5=90=8E=E7=BD=AEemotion=E7=94=9F?= =?UTF-8?q?=E6=88=90=EF=BC=8C=E5=A4=A7=E5=B9=85=E6=8F=90=E9=AB=98=E5=93=8D?= =?UTF-8?q?=E5=BA=94=E9=80=9F=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/bot.py | 9 ++++++++- src/plugins/chat/llm_generator.py | 15 +++++---------- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index e3525b3bb..398cb37e3 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -119,7 +119,7 @@ class ChatBot: willing_manager.change_reply_willing_sent(thinking_message.group_id) - response, emotion = await self.gpt.generate_response(message) + response,raw_content = await self.gpt.generate_response(message) # if response is None: # thinking_message.interupt=True @@ -171,6 +171,13 @@ class ChatBot: message_manager.add_message(message_set) bot_response_time = tinking_time_point + emotion = await self.gpt._get_emotion_tags(raw_content) + print(f"为 '{response}' 获取到的情感标签为：{emotion}") + valuedict={ + 'happy':0.5,'angry':-1,'sad':-0.5,'surprised':0.5,'disgusted':-1.5,'fearful':-0.25,'neutral':0.25 + } + await relationship_manager.update_relationship_value(message.user_id, relationship_value=valuedict[emotion[0]]) + if random() < global_config.emoji_chance: emoji_path = await emoji_manager.get_emoji_for_emotion(emotion) if emoji_path: diff --git a/src/plugins/chat/llm_generator.py b/src/plugins/chat/llm_generator.py index 04f2e73ad..ab0f4e12c 100644 --- a/src/plugins/chat/llm_generator.py +++ b/src/plugins/chat/llm_generator.py @@ -44,19 +44,15 @@ class ResponseGenerator: print(f"+++++++++++++++++{global_config.BOT_NICKNAME}{self.current_model_type}思考中+++++++++++++++++") model_response = await self._generate_response_with_model(message, current_model) + raw_content=model_response if model_response: print(f'{global_config.BOT_NICKNAME}的回复是：{model_response}') - model_response, emotion = await self._process_response(model_response) + model_response = await self._process_response(model_response) if model_response: - print(f"为 '{model_response}' 获取到的情感标签为：{emotion}") - valuedict={ - 'happy':0.5,'angry':-1,'sad':-0.5,'surprised':0.5,'disgusted':-1.5,'fearful':-0.25,'neutral':0.25 - } - await relationship_manager.update_relationship_value(message.user_id, relationship_value=valuedict[emotion[0]]) - return model_response, emotion - return None, [] + return model_response ,raw_content + return None,raw_content async def _generate_response_with_model(self, message: Message, model: LLM_request) -> Optional[str]: """使用指定的模型生成回复""" @@ -158,10 +154,9 @@ class ResponseGenerator: if not content: return None, [] - emotion_tags = await self._get_emotion_tags(content) processed_response = process_llm_response(content) - return processed_response, emotion_tags + return processed_response class InitiativeMessageGenerate: From a612519d56c57ddcfc22a07e4235fa4b2924af85 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 07:22:36 +0800 Subject: [PATCH 08/53] =?UTF-8?q?feat:=20=E6=96=B0=E5=A2=9E=E8=A1=A8?= =?UTF-8?q?=E6=83=85=E5=8C=85=E8=BF=87=E6=BB=A4=EF=BC=8C=E5=A5=B6=E9=BE=99?= =?UTF-8?q?=E5=86=8D=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 23 +++++++++++++++++++++-- src/plugins/chat/llm_generator.py | 3 ++- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index f2bee4fb5..e7ff85803 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -194,7 +194,19 @@ class EmojiManager: async def _get_emoji_discription(self, image_base64: str) -> str: """获取表情包的标签""" try: - prompt = '这是一个表情包，简洁的描述一下表情包的内容和表情包所表达的情感' + prompt = '这是一个表情包，使用中文简洁的描述一下表情包的内容和表情包所表达的情感' + + content, _ = await self.llm.generate_response_for_image(prompt, image_base64) + logger.debug(f"输出描述: {content}") + return content + + except Exception as e: + logger.error(f"获取标签失败: {str(e)}") + return None + + async def _check_emoji(self, image_base64: str) -> str: + try: + prompt = '这是一个表情包，请回答这个表情包是否满足\"动漫风格，画风可爱\"的要求，是则回答是，否则回答否，不要出现任何其他内容' content, _ = await self.llm.generate_response_for_image(prompt, image_base64) logger.debug(f"输出描述: {content}") @@ -208,7 +220,7 @@ class EmojiManager: try: prompt = f'这是{global_config.BOT_NICKNAME}将要发送的消息内容:\n{text}\n若要为其配上表情包，请你输出这个表情包应该表达怎样的情感，应该给人什么样的感觉，不要太简洁也不要太长，注意不要输出任何对内容的分析内容，只输出\"一种什么样的感觉\"中间的形容词部分。' - content, _ = await self.llm.generate_response_async(prompt) + content, _ = await self.lm.generate_response_async(prompt) logger.info(f"输出描述: {content}") return content @@ -310,6 +322,13 @@ class EmojiManager: # 获取表情包的描述 discription = await self._get_emoji_discription(image_base64) + check = await self._check_emoji(image_base64) + if '是' not in check: + os.remove(image_path) + logger.info(f"描述: {discription}") + logger.info(f"其不满足过滤规则，被剔除 {check}") + continue + logger.info(f"check通过 {check}") tag = await self._get_emoji_tag(image_base64) embedding = get_embedding(discription) if discription is not None: diff --git a/src/plugins/chat/llm_generator.py b/src/plugins/chat/llm_generator.py index ab0f4e12c..a2f981c9e 100644 --- a/src/plugins/chat/llm_generator.py +++ b/src/plugins/chat/llm_generator.py @@ -24,6 +24,7 @@ class ResponseGenerator: self.model_r1 = LLM_request(model=global_config.llm_reasoning, temperature=0.7,max_tokens=1000) self.model_v3 = LLM_request(model=global_config.llm_normal, temperature=0.7,max_tokens=1000) self.model_r1_distill = LLM_request(model=global_config.llm_reasoning_minor, temperature=0.7,max_tokens=1000) + self.model_v25 = LLM_request(model=global_config.llm_normal_minor, temperature=0.7,max_tokens=1000) self.db = Database.get_instance() self.current_model_type = 'r1' # 默认使用 R1 @@ -138,7 +139,7 @@ class ResponseGenerator: 内容：{content} 输出： ''' - content, _ = await self.model_v3.generate_response(prompt) + content, _ = await self.model_v25.generate_response(prompt) content=content.strip() if content in ['happy','angry','sad','surprised','disgusted','fearful','neutral']: return [content] From 7c3fb28f10bb41499c910b3087aa698512cc8782 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 08:18:33 +0800 Subject: [PATCH 09/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=89=93?= =?UTF-8?q?=E5=8D=B0=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 3e1a825a4..88fd831b8 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -41,7 +41,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -122,7 +122,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -264,7 +264,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -329,7 +329,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -385,7 +385,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}+{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 From eaa711ada78d5a113caf4ecded75d19edeeb362e Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Thu, 6 Mar 2025 14:27:22 +0800 Subject: [PATCH 10/53] v0.5.7 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 记忆系统接入关键词，重新启动自主发言功能 --- src/plugins/chat/bot.py | 32 ++--- src/plugins/chat/config.py | 4 + src/plugins/chat/llm_generator.py | 5 +- src/plugins/chat/message_sender.py | 10 +- src/plugins/chat/prompt_builder.py | 115 ++++++---------- src/plugins/chat/topic_identifier.py | 4 +- src/plugins/chat/utils.py | 63 ++++++++- src/plugins/chat/willing_manager.py | 6 +- src/plugins/memory_system/memory.py | 197 ++++++++++++++++++++++++++- src/plugins/models/utils_model.py | 196 ++++++++++++++++++++++++++ 10 files changed, 520 insertions(+), 112 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 398cb37e3..dc82cf236 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -15,7 +15,7 @@ from .message import Message_Thinking # 导入 Message_Thinking 类 from .relationship_manager import relationship_manager from .willing_manager import willing_manager # 导入意愿管理器 from .utils import is_mentioned_bot_in_txt, calculate_typing_time -from ..memory_system.memory import memory_graph +from ..memory_system.memory import memory_graph,hippocampus from loguru import logger class ChatBot: @@ -70,24 +70,12 @@ class ChatBot: - topic=await topic_identifier.identify_topic_llm(message.processed_plain_text) - - - # topic1 = topic_identifier.identify_topic_jieba(message.processed_plain_text) - # topic2 = await topic_identifier.identify_topic_llm(message.processed_plain_text) - # topic3 = topic_identifier.identify_topic_snownlp(message.processed_plain_text) - logger.info(f"\033[1;32m[主题识别]\033[0m 使用{global_config.topic_extract}主题: {topic}") - - all_num = 0 - interested_num = 0 - if topic: - for current_topic in topic: - all_num += 1 - first_layer_items, second_layer_items = memory_graph.get_related_item(current_topic, depth=2) - if first_layer_items: - interested_num += 1 - print(f"\033[1;32m[前额叶]\033[0m 对|{current_topic}|有印象") - interested_rate = interested_num / all_num if all_num > 0 else 0 + # topic=await topic_identifier.identify_topic_llm(message.processed_plain_text) + topic = '' + interested_rate = 0 + interested_rate = await hippocampus.memory_activate_value(message.processed_plain_text)/100 + print(f"\033[1;32m[记忆激活]\033[0m 对{message.processed_plain_text}的激活度:---------------------------------------{interested_rate}\n") + # logger.info(f"\033[1;32m[主题识别]\033[0m 使用{global_config.topic_extract}主题: {topic}") await self.storage.store_message(message, topic[0] if topic else None) @@ -134,7 +122,7 @@ class ChatBot: if isinstance(msg, Message_Thinking) and msg.message_id == think_id: thinking_message = msg container.messages.remove(msg) - print(f"\033[1;32m[思考消息删除]\033[0m 已找到思考消息对象，开始删除") + # print(f"\033[1;32m[思考消息删除]\033[0m 已找到思考消息对象，开始删除") break #记录开始思考的时间，避免从思考到回复的时间太久 @@ -167,7 +155,7 @@ class ChatBot: message_set.add_message(bot_message) #message_set 可以直接加入 message_manager - print(f"\033[1;32m[回复]\033[0m 将回复载入发送容器") + # print(f"\033[1;32m[回复]\033[0m 将回复载入发送容器") message_manager.add_message(message_set) bot_response_time = tinking_time_point @@ -205,7 +193,7 @@ class ChatBot: ) message_manager.add_message(bot_message) - willing_manager.change_reply_willing_after_sent(event.group_id) + # willing_manager.change_reply_willing_after_sent(event.group_id) # 创建全局ChatBot实例 chat_bot = ChatBot() \ No newline at end of file diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index be599f48a..d5ee364ce 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -40,6 +40,7 @@ class BotConfig: llm_normal_minor: Dict[str, str] = field(default_factory=lambda: {}) embedding: Dict[str, str] = field(default_factory=lambda: {}) vlm: Dict[str, str] = field(default_factory=lambda: {}) + rerank: Dict[str, str] = field(default_factory=lambda: {}) # 主题提取配置 topic_extract: str = 'snownlp' # 只支持jieba,snownlp,llm @@ -136,6 +137,9 @@ class BotConfig: if "embedding" in model_config: config.embedding = model_config["embedding"] + if "rerank" in model_config: + config.rerank = model_config["rerank"] + if 'topic' in toml_dict: topic_config=toml_dict['topic'] if 'topic_extract' in topic_config: diff --git a/src/plugins/chat/llm_generator.py b/src/plugins/chat/llm_generator.py index ab0f4e12c..004cd0450 100644 --- a/src/plugins/chat/llm_generator.py +++ b/src/plugins/chat/llm_generator.py @@ -63,10 +63,11 @@ class ResponseGenerator: # 获取关系值 relationship_value = relationship_manager.get_relationship(message.user_id).relationship_value if relationship_manager.get_relationship(message.user_id) else 0.0 if relationship_value != 0.0: - print(f"\033[1;32m[关系管理]\033[0m 回复中_当前关系值: {relationship_value}") + # print(f"\033[1;32m[关系管理]\033[0m 回复中_当前关系值: {relationship_value}") + pass # 构建prompt - prompt, prompt_check = prompt_builder._build_prompt( + prompt, prompt_check = await prompt_builder._build_prompt( message_txt=message.processed_plain_text, sender_name=sender_name, relationship_value=relationship_value, diff --git a/src/plugins/chat/message_sender.py b/src/plugins/chat/message_sender.py index 970fd3682..c81dec1bb 100644 --- a/src/plugins/chat/message_sender.py +++ b/src/plugins/chat/message_sender.py @@ -103,7 +103,7 @@ class MessageContainer: def add_message(self, message: Union[Message_Thinking, Message_Sending]) -> None: """添加消息到队列""" - print(f"\033[1;32m[添加消息]\033[0m 添加消息到对应群") + # print(f"\033[1;32m[添加消息]\033[0m 添加消息到对应群") if isinstance(message, MessageSet): for single_message in message.messages: self.messages.append(single_message) @@ -156,17 +156,13 @@ class MessageManager: #最早的对象，可能是思考消息，也可能是发送消息 message_earliest = container.get_earliest_message() #一个message_thinking or message_sending - #一个月后删了 - if not message_earliest: - print(f"\033[1;34m[BUG，如果出现这个，说明有BUG，3月4日留]\033[0m ") - return - #如果是思考消息 if isinstance(message_earliest, Message_Thinking): #优先等待这条消息 message_earliest.update_thinking_time() thinking_time = message_earliest.thinking_time - print(f"\033[1;34m[调试]\033[0m 消息正在思考中，已思考{int(thinking_time)}秒") + if thinking_time % 10 == 0: + print(f"\033[1;34m[调试]\033[0m 消息正在思考中，已思考{int(thinking_time)}秒") else:# 如果不是message_thinking就只能是message_sending print(f"\033[1;34m[调试]\033[0m 消息'{message_earliest.processed_plain_text}'正在发送中") #直接发，等什么呢 diff --git a/src/plugins/chat/prompt_builder.py b/src/plugins/chat/prompt_builder.py index ba22a403d..1c510e251 100644 --- a/src/plugins/chat/prompt_builder.py +++ b/src/plugins/chat/prompt_builder.py @@ -2,13 +2,15 @@ import time import random from ..schedule.schedule_generator import bot_schedule import os -from .utils import get_embedding, combine_messages, get_recent_group_detailed_plain_text +from .utils import get_embedding, combine_messages, get_recent_group_detailed_plain_text,find_similar_topics from ...common.database import Database from .config import global_config from .topic_identifier import topic_identifier -from ..memory_system.memory import memory_graph +from ..memory_system.memory import memory_graph,hippocampus from random import choice - +import numpy as np +import jieba +from collections import Counter class PromptBuilder: def __init__(self): @@ -16,7 +18,9 @@ class PromptBuilder: self.activate_messages = '' self.db = Database.get_instance() - def _build_prompt(self, + + + async def _build_prompt(self, message_txt: str, sender_name: str = "某人", relationship_value: float = 0.0, @@ -31,60 +35,7 @@ class PromptBuilder: Returns: str: 构建好的prompt - """ - - - memory_prompt = '' - start_time = time.time() # 记录开始时间 - # topic = await topic_identifier.identify_topic_llm(message_txt) - topic = topic_identifier.identify_topic_snownlp(message_txt) - - # print(f"\033[1;32m[pb主题识别]\033[0m 主题: {topic}") - - all_first_layer_items = [] # 存储所有第一层记忆 - all_second_layer_items = {} # 用字典存储每个topic的第二层记忆 - overlapping_second_layer = set() # 存储重叠的第二层记忆 - - if topic: - # 遍历所有topic - for current_topic in topic: - first_layer_items, second_layer_items = memory_graph.get_related_item(current_topic, depth=2) - # if first_layer_items: - # print(f"\033[1;32m[前额叶]\033[0m 主题 '{current_topic}' 的第一层记忆: {first_layer_items}") - - # 记录第一层数据 - all_first_layer_items.extend(first_layer_items) - - # 记录第二层数据 - all_second_layer_items[current_topic] = second_layer_items - - # 检查是否有重叠的第二层数据 - for other_topic, other_second_layer in all_second_layer_items.items(): - if other_topic != current_topic: - # 找到重叠的记忆 - overlap = set(second_layer_items) & set(other_second_layer) - if overlap: - # print(f"\033[1;32m[前额叶]\033[0m 发现主题 '{current_topic}' 和 '{other_topic}' 有共同的第二层记忆: {overlap}") - overlapping_second_layer.update(overlap) - - selected_first_layer = random.sample(all_first_layer_items, min(2, len(all_first_layer_items))) if all_first_layer_items else [] - selected_second_layer = random.sample(list(overlapping_second_layer), min(2, len(overlapping_second_layer))) if overlapping_second_layer else [] - - # 合并并去重 - all_memories = list(set(selected_first_layer + selected_second_layer)) - if all_memories: - print(f"\033[1;32m[前额叶]\033[0m 合并所有需要的记忆: {all_memories}") - random_item = " ".join(all_memories) - memory_prompt = f"看到这些聊天，你想起来{random_item}\n" - else: - memory_prompt = "" # 如果没有记忆，则返回空字符串 - - end_time = time.time() # 记录结束时间 - print(f"\033[1;32m[回忆耗时]\033[0m 耗时: {(end_time - start_time):.3f}秒") # 输出耗时 - - - - + """ #先禁用关系 if 0 > 30: relation_prompt = "关系特别特别好，你很喜欢喜欢他" @@ -112,22 +63,48 @@ class PromptBuilder: prompt_info = self.get_prompt_info(message_txt,threshold=0.5) if prompt_info: prompt_info = f'''\n----------------------------------------------------\n你有以下这些[知识]：\n{prompt_info}\n请你记住上面的[知识]，之后可能会用到\n----------------------------------------------------\n''' - # promt_info_prompt = '你有一些[知识]，在上面可以参考。' end_time = time.time() print(f"\033[1;32m[知识检索]\033[0m 耗时: {(end_time - start_time):.3f}秒") - # print(f"\033[1;34m[调试]\033[0m 获取知识库内容结果: {prompt_info}") - - # print(f"\033[1;34m[调试信息]\033[0m 正在构建聊天上下文") - + # 获取聊天上下文 chat_talking_prompt = '' if group_id: chat_talking_prompt = get_recent_group_detailed_plain_text(self.db, group_id, limit=global_config.MAX_CONTEXT_SIZE,combine = True) chat_talking_prompt = f"以下是群里正在聊天的内容：\n{chat_talking_prompt}" - # print(f"\033[1;34m[调试]\033[0m 已从数据库获取群 {group_id} 的消息记录:{chat_talking_prompt}") + + + # 使用新的记忆获取方法 + memory_prompt = '' + start_time = time.time() + + # 调用 hippocampus 的 get_relevant_memories 方法 + relevant_memories = await hippocampus.get_relevant_memories( + text=message_txt, + max_topics=5, + similarity_threshold=0.4 + ) + + if relevant_memories: + # 格式化记忆内容 + memory_items = [] + for memory in relevant_memories: + memory_items.append(f"关于「{memory['topic']}」的记忆：{memory['content']}") + + memory_prompt = f"看到这些聊天，你想起来：\n" + "\n".join(memory_items) + "\n" + + # 打印调试信息 + print("\n\033[1;32m[记忆检索]\033[0m 找到以下相关记忆：") + for memory in relevant_memories: + print(f"- 主题「{memory['topic']}」[相似度: {memory['similarity']:.2f}]: {memory['content']}") + + end_time = time.time() + print(f"\033[1;32m[回忆耗时]\033[0m 耗时: {(end_time - start_time):.3f}秒") + + + #激活prompt构建 activate_prompt = '' activate_prompt = f"以上是群里正在进行的聊天，{memory_prompt} 现在昵称为 '{sender_name}' 的用户说的:{message_txt}。引起了你的注意,你和他{relation_prompt}，你想要{relation_prompt_2}。" @@ -162,29 +139,19 @@ class PromptBuilder: if random.random() < 0.01: prompt_ger += '你喜欢用文言文' - #额外信息要求 extra_info = '''但是记得回复平淡一些，简短一些，尤其注意在没明确提到时不要过多提及自身的背景, 记住不要输出多余内容(包括前后缀，冒号和引号，括号，表情等)，只需要输出回复内容就好，不要输出其他任何内容''' - - #合并prompt prompt = "" prompt += f"{prompt_info}\n" prompt += f"{prompt_date}\n" prompt += f"{chat_talking_prompt}\n" - - # prompt += f"{memory_prompt}\n" - - # prompt += f"{activate_prompt}\n" prompt += f"{prompt_personality}\n" prompt += f"{prompt_ger}\n" prompt += f"{extra_info}\n" - - '''读空气prompt处理''' - activate_prompt_check=f"以上是群里正在进行的聊天，昵称为 '{sender_name}' 的用户说的:{message_txt}。引起了你的注意,你和他{relation_prompt}，你想要{relation_prompt_2}，但是这不一定是合适的时机，请你决定是否要回应这条消息。" prompt_personality_check = '' extra_check_info=f"请注意把握群里的聊天内容的基础上，综合群内的氛围，例如，和{global_config.BOT_NICKNAME}相关的话题要积极回复,如果是at自己的消息一定要回复，如果自己正在和别人聊天一定要回复，其他话题如果合适搭话也可以回复，如果认为应该回复请输出yes，否则输出no，请注意是决定是否需要回复，而不是编写回复内容，除了yes和no不要输出任何回复内容。" diff --git a/src/plugins/chat/topic_identifier.py b/src/plugins/chat/topic_identifier.py index 812d4e321..60c5b0051 100644 --- a/src/plugins/chat/topic_identifier.py +++ b/src/plugins/chat/topic_identifier.py @@ -42,7 +42,7 @@ class TopicIdentifier: print(f"\033[1;32m[主题识别]\033[0m 主题: {topic_list}") return topic_list if topic_list else None - def identify_topic_snownlp(self, text: str) -> Optional[List[str]]: + def identify_topic_snownlp(self, text: str,num:int=5) -> Optional[List[str]]: """使用 SnowNLP 进行主题识别 Args: @@ -57,7 +57,7 @@ class TopicIdentifier: try: s = SnowNLP(text) # 提取前3个关键词作为主题 - keywords = s.keywords(5) + keywords = s.keywords(num) return keywords if keywords else None except Exception as e: print(f"\033[1;31m[错误]\033[0m SnowNLP 处理失败: {str(e)}") diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index aa16268ef..ddc698bc7 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -11,6 +11,8 @@ from collections import Counter import math from nonebot import get_driver from ..models.utils_model import LLM_request +import aiohttp +import jieba driver = get_driver() config = driver.config @@ -117,7 +119,7 @@ def get_cloest_chat_from_db(db, length: int, timestamp: str): chat_text += record["detailed_plain_text"] return chat_text - print(f"消息已读取3次，跳过") + # print(f"消息已读取3次，跳过") return '' def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: @@ -421,3 +423,62 @@ def calculate_typing_time(input_string: str, chinese_time: float = 0.2, english_ return total_time +def find_similar_topics(message_txt: str, all_memory_topic: list, top_k: int = 5) -> list: + """使用重排序API找出与输入文本最相似的话题 + + Args: + message_txt: 输入文本 + all_memory_topic: 所有记忆主题列表 + top_k: 返回最相似的话题数量 + + Returns: + list: 最相似话题列表及其相似度分数 + """ + + if not all_memory_topic: + return [] + + try: + llm = LLM_request(model=global_config.rerank) + return llm.rerank_sync(message_txt, all_memory_topic, top_k) + except Exception as e: + print(f"重排序API调用出错: {str(e)}") + return [] + +def cosine_similarity(v1, v2): + """计算余弦相似度""" + dot_product = np.dot(v1, v2) + norm1 = np.linalg.norm(v1) + norm2 = np.linalg.norm(v2) + if norm1 == 0 or norm2 == 0: + return 0 + return dot_product / (norm1 * norm2) + +def text_to_vector(text): + """将文本转换为词频向量""" + # 分词 + words = jieba.lcut(text) + # 统计词频 + word_freq = Counter(words) + return word_freq + +def find_similar_topics_simple(text: str, topics: list, top_k: int = 5) -> list: + """使用简单的余弦相似度计算文本相似度""" + # 将输入文本转换为词频向量 + text_vector = text_to_vector(text) + + # 计算每个主题的相似度 + similarities = [] + for topic in topics: + topic_vector = text_to_vector(topic) + # 获取所有唯一词 + all_words = set(text_vector.keys()) | set(topic_vector.keys()) + # 构建向量 + v1 = [text_vector.get(word, 0) for word in all_words] + v2 = [topic_vector.get(word, 0) for word in all_words] + # 计算相似度 + similarity = cosine_similarity(v1, v2) + similarities.append((topic, similarity)) + + # 按相似度降序排序并返回前k个 + return sorted(similarities, key=lambda x: x[1], reverse=True)[:top_k] \ No newline at end of file diff --git a/src/plugins/chat/willing_manager.py b/src/plugins/chat/willing_manager.py index 7559406f9..acc8543da 100644 --- a/src/plugins/chat/willing_manager.py +++ b/src/plugins/chat/willing_manager.py @@ -37,13 +37,13 @@ class WillingManager: current_willing *= 0.15 print(f"表情包, 当前意愿: {current_willing}") - if interested_rate > 0.65: + if interested_rate > 0.4: print(f"兴趣度: {interested_rate}, 当前意愿: {current_willing}") - current_willing += interested_rate-0.6 + current_willing += interested_rate-0.1 self.group_reply_willing[group_id] = min(current_willing, 3.0) - reply_probability = max((current_willing - 0.55) * 1.9, 0) + reply_probability = max((current_willing - 0.45) * 2, 0) if group_id not in config.talk_allowed_groups: current_willing = 0 reply_probability = 0 diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index 4d20d05a9..cdb6e6e1b 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -11,7 +11,7 @@ from ..chat.config import global_config from ...common.database import Database # 使用正确的导入语法 from ..models.utils_model import LLM_request import math -from ..chat.utils import calculate_information_content, get_cloest_chat_from_db +from ..chat.utils import calculate_information_content, get_cloest_chat_from_db ,find_similar_topics,text_to_vector,cosine_similarity @@ -135,6 +135,14 @@ class Hippocampus: self.llm_model_get_topic = LLM_request(model = global_config.llm_normal_minor,temperature=0.5) self.llm_model_summary = LLM_request(model = global_config.llm_normal,temperature=0.5) + def get_all_node_names(self) -> list: + """获取记忆图中所有节点的名字列表 + + Returns: + list: 包含所有节点名字的列表 + """ + return list(self.memory_graph.G.nodes()) + def calculate_node_hash(self, concept, memory_items): """计算节点的特征值""" if not isinstance(memory_items, list): @@ -483,6 +491,193 @@ class Hippocampus: prompt = f'这是一段文字：{text}。我想让你基于这段文字来概括"{topic}"这个概念，帮我总结成一句自然的话，可以包含时间和人物，以及具体的观点。只输出这句话就好' return prompt + async def _identify_topics(self, text: str) -> list: + """从文本中识别可能的主题 + + Args: + text: 输入文本 + + Returns: + list: 识别出的主题列表 + """ + topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(text, 5)) + print(f"话题: {topics_response[0]}") + topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] + print(f"话题: {topics}") + + return topics + + def _find_similar_topics(self, topics: list, similarity_threshold: float = 0.4, debug_info: str = "") -> list: + """查找与给定主题相似的记忆主题 + + Args: + topics: 主题列表 + similarity_threshold: 相似度阈值 + debug_info: 调试信息前缀 + + Returns: + list: (主题, 相似度) 元组列表 + """ + all_memory_topics = self.get_all_node_names() + all_similar_topics = [] + + # 计算每个识别出的主题与记忆主题的相似度 + for topic in topics: + if debug_info: + print(f"\033[1;32m[{debug_info}]\033[0m 正在思考有没有见过: {topic}") + + topic_vector = text_to_vector(topic) + has_similar_topic = False + + for memory_topic in all_memory_topics: + memory_vector = text_to_vector(memory_topic) + # 获取所有唯一词 + all_words = set(topic_vector.keys()) | set(memory_vector.keys()) + # 构建向量 + v1 = [topic_vector.get(word, 0) for word in all_words] + v2 = [memory_vector.get(word, 0) for word in all_words] + # 计算相似度 + similarity = cosine_similarity(v1, v2) + + if similarity >= similarity_threshold: + has_similar_topic = True + if debug_info: + print(f"\033[1;32m[{debug_info}]\033[0m 找到相似主题: {topic} -> {memory_topic} (相似度: {similarity:.2f})") + all_similar_topics.append((memory_topic, similarity)) + + if not has_similar_topic and debug_info: + print(f"\033[1;31m[{debug_info}]\033[0m 没有见过: {topic} ，呃呃") + + return all_similar_topics + + def _get_top_topics(self, similar_topics: list, max_topics: int = 5) -> list: + """获取相似度最高的主题 + + Args: + similar_topics: (主题, 相似度) 元组列表 + max_topics: 最大主题数量 + + Returns: + list: (主题, 相似度) 元组列表 + """ + seen_topics = set() + top_topics = [] + + for topic, score in sorted(similar_topics, key=lambda x: x[1], reverse=True): + if topic not in seen_topics and len(top_topics) < max_topics: + seen_topics.add(topic) + top_topics.append((topic, score)) + + return top_topics + + async def memory_activate_value(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.3) -> int: + """计算输入文本对记忆的激活程度""" + print(f"\033[1;32m[记忆激活]\033[0m 开始计算文本的记忆激活度: {text}") + + # 识别主题 + identified_topics = await self._identify_topics(text) + print(f"\033[1;32m[记忆激活]\033[0m 识别出的主题: {identified_topics}") + + if not identified_topics: + print(f"\033[1;32m[记忆激活]\033[0m 未识别出主题,返回0") + return 0 + + # 查找相似主题 + all_similar_topics = self._find_similar_topics( + identified_topics, + similarity_threshold=similarity_threshold, + debug_info="记忆激活" + ) + + if not all_similar_topics: + print(f"\033[1;32m[记忆激活]\033[0m 未找到相似主题,返回0") + return 0 + + # 获取最相关的主题 + top_topics = self._get_top_topics(all_similar_topics, max_topics) + + # 如果只找到一个主题，进行惩罚 + if len(top_topics) == 1: + topic, score = top_topics[0] + activation = int(score * 50) # 单主题情况下，直接用相似度*50作为激活值 + print(f"\033[1;32m[记忆激活]\033[0m 只找到一个主题，进行惩罚:") + print(f"\033[1;32m[记忆激活]\033[0m - 主题: {topic}") + print(f"\033[1;32m[记忆激活]\033[0m - 相似度: {score:.3f}") + print(f"\033[1;32m[记忆激活]\033[0m - 最终激活值: {activation}") + return activation + + # 计算关键词匹配率 + matched_topics = set() + topic_similarities = {} + + print(f"\033[1;32m[记忆激活]\033[0m 计算关键词匹配情况:") + for memory_topic, similarity in top_topics: + # 对每个记忆主题，检查它与哪些输入主题相似 + for input_topic in identified_topics: + topic_vector = text_to_vector(input_topic) + memory_vector = text_to_vector(memory_topic) + all_words = set(topic_vector.keys()) | set(memory_vector.keys()) + v1 = [topic_vector.get(word, 0) for word in all_words] + v2 = [memory_vector.get(word, 0) for word in all_words] + sim = cosine_similarity(v1, v2) + if sim >= similarity_threshold: + matched_topics.add(input_topic) + topic_similarities[input_topic] = max(topic_similarities.get(input_topic, 0), sim) + print(f"\033[1;32m[记忆激活]\033[0m - 输入主题「{input_topic}」匹配到记忆「{memory_topic}」, 相似度: {sim:.3f}") + + # 计算主题匹配率 + topic_match = len(matched_topics) / len(identified_topics) + print(f"\033[1;32m[记忆激活]\033[0m 主题匹配率:") + print(f"\033[1;32m[记忆激活]\033[0m - 匹配主题数: {len(matched_topics)}") + print(f"\033[1;32m[记忆激活]\033[0m - 总主题数: {len(identified_topics)}") + print(f"\033[1;32m[记忆激活]\033[0m - 匹配率: {topic_match:.3f}") + + # 计算匹配主题的平均相似度 + average_similarities = sum(topic_similarities.values()) / len(topic_similarities) if topic_similarities else 0 + print(f"\033[1;32m[记忆激活]\033[0m 平均相似度:") + print(f"\033[1;32m[记忆激活]\033[0m - 各主题相似度: {[f'{k}:{v:.3f}' for k,v in topic_similarities.items()]}") + print(f"\033[1;32m[记忆激活]\033[0m - 平均相似度: {average_similarities:.3f}") + + # 计算最终激活值 + activation = (topic_match + average_similarities) / 2 * 100 + print(f"\033[1;32m[记忆激活]\033[0m 最终激活值: {int(activation)}") + + return int(activation) + + async def get_relevant_memories(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.4) -> list: + """根据输入文本获取相关的记忆内容""" + # 识别主题 + identified_topics = await self._identify_topics(text) + + # 查找相似主题 + all_similar_topics = self._find_similar_topics( + identified_topics, + similarity_threshold=similarity_threshold, + debug_info="记忆检索" + ) + + # 获取最相关的主题 + relevant_topics = self._get_top_topics(all_similar_topics, max_topics) + + # 获取相关记忆内容 + relevant_memories = [] + for topic, score in relevant_topics: + # 获取该主题的记忆内容 + first_layer, _ = self.memory_graph.get_related_item(topic, depth=1) + if first_layer: + # 为每条记忆添加来源主题和相似度信息 + for memory in first_layer: + relevant_memories.append({ + 'topic': topic, + 'similarity': score, + 'content': memory + }) + + # 按相似度排序 + relevant_memories.sort(key=lambda x: x['similarity'], reverse=True) + + return relevant_memories + def segment_text(text): seg_text = list(jieba.cut(text)) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 57a0acb55..cad23ab09 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -424,3 +424,199 @@ class LLM_request: logger.error("达到最大重试次数，embedding请求仍然失败") return None + + def rerank_sync(self, query: str, documents: list, top_k: int = 5) -> list: + """同步方法：使用重排序API对文档进行排序 + + Args: + query: 查询文本 + documents: 待排序的文档列表 + top_k: 返回前k个结果 + + Returns: + list: [(document, score), ...] 格式的结果列表 + """ + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + data = { + "model": self.model_name, + "query": query, + "documents": documents, + "top_n": top_k, + "return_documents": True, + } + + api_url = f"{self.base_url.rstrip('/')}/rerank" + logger.info(f"发送请求到URL: {api_url}") + + max_retries = 2 + base_wait_time = 6 + + for retry in range(max_retries): + try: + response = requests.post(api_url, headers=headers, json=data, timeout=30) + + if response.status_code == 429: + wait_time = base_wait_time * (2 ** retry) + logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") + time.sleep(wait_time) + continue + + if response.status_code in [500, 503]: + wait_time = base_wait_time * (2 ** retry) + logger.error(f"服务器错误({response.status_code})，等待{wait_time}秒后重试...") + if retry < max_retries - 1: + time.sleep(wait_time) + continue + else: + # 如果是最后一次重试，尝试使用chat/completions作为备选方案 + return self._fallback_rerank_with_chat(query, documents, top_k) + + response.raise_for_status() + + result = response.json() + if 'results' in result: + return [(item["document"], item["score"]) for item in result["results"]] + return [] + + except Exception as e: + if retry < max_retries - 1: + wait_time = base_wait_time * (2 ** retry) + logger.error(f"[rerank_sync]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) + time.sleep(wait_time) + else: + logger.critical(f"重排序请求失败: {str(e)}", exc_info=True) + + logger.error("达到最大重试次数，重排序请求仍然失败") + return [] + + async def rerank(self, query: str, documents: list, top_k: int = 5) -> list: + """异步方法：使用重排序API对文档进行排序 + + Args: + query: 查询文本 + documents: 待排序的文档列表 + top_k: 返回前k个结果 + + Returns: + list: [(document, score), ...] 格式的结果列表 + """ + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + data = { + "model": self.model_name, + "query": query, + "documents": documents, + "top_n": top_k, + "return_documents": True, + } + + api_url = f"{self.base_url.rstrip('/')}/v1/rerank" + logger.info(f"发送请求到URL: {api_url}") + + max_retries = 3 + base_wait_time = 15 + + for retry in range(max_retries): + try: + async with aiohttp.ClientSession() as session: + async with session.post(api_url, headers=headers, json=data) as response: + if response.status == 429: + wait_time = base_wait_time * (2 ** retry) + logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") + await asyncio.sleep(wait_time) + continue + + if response.status in [500, 503]: + wait_time = base_wait_time * (2 ** retry) + logger.error(f"服务器错误({response.status})，等待{wait_time}秒后重试...") + if retry < max_retries - 1: + await asyncio.sleep(wait_time) + continue + else: + # 如果是最后一次重试，尝试使用chat/completions作为备选方案 + return await self._fallback_rerank_with_chat_async(query, documents, top_k) + + response.raise_for_status() + + result = await response.json() + if 'results' in result: + return [(item["document"], item["score"]) for item in result["results"]] + return [] + + except Exception as e: + if retry < max_retries - 1: + wait_time = base_wait_time * (2 ** retry) + logger.error(f"[rerank]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) + await asyncio.sleep(wait_time) + else: + logger.critical(f"重排序请求失败: {str(e)}", exc_info=True) + # 作为最后的备选方案，尝试使用chat/completions + return await self._fallback_rerank_with_chat_async(query, documents, top_k) + + logger.error("达到最大重试次数，重排序请求仍然失败") + return [] + + async def _fallback_rerank_with_chat_async(self, query: str, documents: list, top_k: int = 5) -> list: + """当rerank API失败时的备选方案，使用chat/completions异步实现重排序 + + Args: + query: 查询文本 + documents: 待排序的文档列表 + top_k: 返回前k个结果 + + Returns: + list: [(document, score), ...] 格式的结果列表 + """ + try: + logger.info("使用chat/completions作为重排序的备选方案") + + # 构建提示词 + prompt = f"""请对以下文档列表进行重排序，按照与查询的相关性从高到低排序。 +查询: {query} + +文档列表: +{documents} + +请以JSON格式返回排序结果，格式为： +[{{"document": "文档内容", "score": 相关性分数}}, ...] +只返回JSON，不要其他任何文字。""" + + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + data = { + "model": self.model_name, + "messages": [{"role": "user", "content": prompt}], + **self.params + } + + api_url = f"{self.base_url.rstrip('/')}/v1/chat/completions" + + async with aiohttp.ClientSession() as session: + async with session.post(api_url, headers=headers, json=data) as response: + response.raise_for_status() + result = await response.json() + + if "choices" in result and len(result["choices"]) > 0: + message = result["choices"][0]["message"] + content = message.get("content", "") + try: + import json + parsed_content = json.loads(content) + if isinstance(parsed_content, list): + return [(item["document"], item["score"]) for item in parsed_content] + except: + pass + return [] + except Exception as e: + logger.error(f"备选方案也失败了: {str(e)}") + return [] From 2f1579e5b751cc7202cba4fa08eba580dd38a87d Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 14:43:46 +0800 Subject: [PATCH 11/53] =?UTF-8?q?fix:=20=E4=B8=BA=E6=89=80=E6=9C=89?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E8=AF=B7=E6=B1=82=E6=B7=BB=E5=8A=A0=E8=AF=B7?= =?UTF-8?q?=E6=B1=82=E5=A4=B4=E5=92=8C=E8=AF=B7=E6=B1=82=E4=BD=93=E7=9A=84?= =?UTF-8?q?=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 88fd831b8..a3dfdfa9b 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -83,6 +83,7 @@ class LLM_request: await asyncio.sleep(wait_time) else: logger.critical(f"请求失败: {str(e)}", exc_info=True) + logger.critical(f"请求头: {headers} 请求体: {data}") raise RuntimeError(f"API请求失败: {str(e)}") logger.error("达到最大重试次数，请求仍然失败") @@ -170,6 +171,7 @@ class LLM_request: await asyncio.sleep(wait_time) else: logger.critical(f"请求失败: {str(e)}", exc_info=True) + logger.critical(f"请求头: {headers} 请求体: {data}") raise RuntimeError(f"API请求失败: {str(e)}") logger.error("达到最大重试次数，请求仍然失败") @@ -223,6 +225,7 @@ class LLM_request: await asyncio.sleep(wait_time) else: logger.error(f"请求失败: {str(e)}") + logger.critical(f"请求头: {headers} 请求体: {data}") return f"请求失败: {str(e)}", "" logger.error("达到最大重试次数，请求仍然失败") @@ -302,6 +305,7 @@ class LLM_request: time.sleep(wait_time) else: logger.critical(f"请求失败: {str(e)}", exc_info=True) + logger.critical(f"请求头: {headers} 请求体: {data}") raise RuntimeError(f"API请求失败: {str(e)}") logger.error("达到最大重试次数，请求仍然失败") @@ -358,6 +362,7 @@ class LLM_request: time.sleep(wait_time) else: logger.critical(f"embedding请求失败: {str(e)}", exc_info=True) + logger.critical(f"请求头: {headers} 请求体: {data}") return None logger.error("达到最大重试次数，embedding请求仍然失败") @@ -414,6 +419,7 @@ class LLM_request: await asyncio.sleep(wait_time) else: logger.critical(f"embedding请求失败: {str(e)}", exc_info=True) + logger.critical(f"请求头: {headers} 请求体: {data}") return None logger.error("达到最大重试次数，embedding请求仍然失败") From 84e967c631516e190ed1b10e951ef9dd60584a01 Mon Sep 17 00:00:00 2001 From: Rikki Date: Thu, 6 Mar 2025 18:51:48 +0800 Subject: [PATCH 12/53] =?UTF-8?q?update:=20=E5=B1=8F=E8=94=BD.vscode?= =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6=E5=A4=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index 51a11d8c2..deec90be9 100644 --- a/.gitignore +++ b/.gitignore @@ -188,3 +188,7 @@ cython_debug/ # jieba jieba.cache + + +# vscode +/.vscode \ No newline at end of file From 94aee8ca10ce41ec4d4fb4dcc9d4be24bcfaac8b Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Thu, 6 Mar 2025 19:39:08 +0800 Subject: [PATCH 13/53] =?UTF-8?q?Windows=20=E4=B8=80=E9=94=AE=E5=90=AF?= =?UTF-8?q?=E5=8A=A8=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run_windows.bat | 62 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 run_windows.bat diff --git a/run_windows.bat b/run_windows.bat new file mode 100644 index 000000000..456c04d17 --- /dev/null +++ b/run_windows.bat @@ -0,0 +1,62 @@ +@echo off +setlocal enabledelayedexpansion +chcp 65001 + +REM 修正路径获取逻辑 +cd /d "%~dp0" || ( + echo 错误：切换目录失败 + exit /b 1 +) + +if not exist "venv\" ( + echo 正在初始化虚拟环境... + + where python >nul 2>&1 + if %errorlevel% neq 0 ( + echo 未找到Python解释器 + exit /b 1 + ) + + for /f "tokens=2" %%a in ('python --version 2^>^&1') do set version=%%a + for /f "tokens=1,2 delims=." %%b in ("!version!") do ( + set major=%%b + set minor=%%c + ) + + if !major! equ 3 if !minor! lss 9 ( + echo 需要Python大于等于3.9，当前版本 !version! + exit /b 1 + ) + + echo 正在安装virtualenv... + python -m pip install virtualenv || ( + echo virtualenv安装失败 + exit /b 1 + ) + + echo 正在创建虚拟环境... + python -m virtualenv venv || ( + echo 虚拟环境创建失败 + exit /b 1 + ) + + call venv\Scripts\activate.bat + + echo 正在安装依赖... + pip install -r requirements.txt +) else ( + call venv\Scripts\activate.bat +) + +echo 当前代理设置： +echo HTTP_PROXY=%HTTP_PROXY% +echo HTTPS_PROXY=%HTTPS_PROXY% + +set HTTP_PROXY= +set HTTPS_PROXY= +echo 代理已取消。 + +set no_proxy=0.0.0.0/32 + +call nb run +paus \ No newline at end of file From f7a985b653e2eb5eb823fab003c1e0101c22e0be Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Thu, 6 Mar 2025 19:39:27 +0800 Subject: [PATCH 14/53] =?UTF-8?q?=E6=9B=B4=E6=96=B0requirement?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements.txt | Bin 582 -> 618 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/requirements.txt b/requirements.txt index 49c102dc6efb27b36fc505008b796a8be2539c31..1d268ffa6fbac31fa5297f8de1b5254d33d41235 100644 GIT binary patch delta 62 zcmX@c@``1H36ohdLmopuLphMlVJKkWW#D2+0rE>2Qh_36V16->UBXZblq~_u Date: Thu, 6 Mar 2025 19:43:42 +0800 Subject: [PATCH 15/53] =?UTF-8?q?fix:=20=E4=BF=AE=E8=A1=A5=E4=BD=8E?= =?UTF-8?q?=E4=BA=8Epython3=E4=BC=9A=E9=94=99=E8=AF=AF=E5=90=AF=E5=8A=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- run_windows.bat | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/run_windows.bat b/run_windows.bat index 456c04d17..920069318 100644 --- a/run_windows.bat +++ b/run_windows.bat @@ -23,6 +23,11 @@ if not exist "venv\" ( set minor=%%c ) + if !major! lss 3 ( + echo 需要Python大于等于3.0，当前版本 !version! + exit /b 1 + ) + if !major! equ 3 if !minor! lss 9 ( echo 需要Python大于等于3.9，当前版本 !version! exit /b 1 @@ -59,4 +64,4 @@ echo 代理已取消。 set no_proxy=0.0.0.0/32 call nb run -paus \ No newline at end of file +pause \ No newline at end of file From ee414eeaaf34a721c99593dee479984cf9f600ce Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Thu, 6 Mar 2025 19:56:57 +0800 Subject: [PATCH 16/53] =?UTF-8?q?v0.5.8=20=E4=BF=AE=E5=A4=8D=20=E5=9B=9E?= =?UTF-8?q?=E5=A4=8D=E9=87=8D=E5=A4=8D=E8=BE=93=E5=87=BA=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +++ bot.py | 2 +- src/plugins/chat/__init__.py | 14 +++++++++----- src/plugins/chat/bot.py | 4 ++++ src/plugins/chat/message.py | 2 ++ src/plugins/chat/message_sender.py | 16 ++++++++-------- src/plugins/chat/utils.py | 4 ++-- 7 files changed, 29 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 7bfa465ae..73e1c3094 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ - 💾 MongoDB 提供数据持久化支持 - 🐧 NapCat 作为QQ协议端支持 +**最新版本: v0.5.7** +
麦麦演示视频 @@ -31,6 +33,7 @@ > - 文档未完善，有问题可以提交 Issue 或者 Discussion > - QQ机器人存在被限制风险，请自行了解，谨慎使用 > - 由于持续迭代，可能存在一些已知或未知的bug +> - 由于开发中，可能消耗较多token **交流群**: 766798517（仅用于开发和建议相关讨论）不建议在群内询问部署问题，我不一定有空回复，会优先写文档和代码 diff --git a/bot.py b/bot.py index 50c8cfaa4..8ef087476 100644 --- a/bot.py +++ b/bot.py @@ -8,7 +8,7 @@ from loguru import logger from colorama import init, Fore init() -text = "多年以后，面对行刑队，张三将会回想起他2023年在会议上讨论人工智能的那个下午" +text = "多年以后，面对AI行刑队，张三将会回想起他2023年在会议上讨论人工智能的那个下午" rainbow_colors = [Fore.RED, Fore.YELLOW, Fore.GREEN, Fore.CYAN, Fore.BLUE, Fore.MAGENTA] rainbow_text = "" for i, char in enumerate(text): diff --git a/src/plugins/chat/__init__.py b/src/plugins/chat/__init__.py index ab99f6477..22f3059b5 100644 --- a/src/plugins/chat/__init__.py +++ b/src/plugins/chat/__init__.py @@ -15,6 +15,8 @@ from .bot import chat_bot from .emoji_manager import emoji_manager import time +# 添加标志变量 +_message_manager_started = False # 获取驱动器 driver = get_driver() @@ -70,18 +72,20 @@ async def init_relationships(): @driver.on_bot_connect async def _(bot: Bot): """Bot连接成功时的处理""" + global _message_manager_started print(f"\033[1;38;5;208m-----------{global_config.BOT_NICKNAME}成功连接！-----------\033[0m") await willing_manager.ensure_started() - message_sender.set_bot(bot) print("\033[1;38;5;208m-----------消息发送器已启动！-----------\033[0m") - asyncio.create_task(message_manager.start_processor()) - print("\033[1;38;5;208m-----------消息处理器已启动！-----------\033[0m") + + if not _message_manager_started: + asyncio.create_task(message_manager.start_processor()) + _message_manager_started = True + print("\033[1;38;5;208m-----------消息处理器已启动！-----------\033[0m") asyncio.create_task(emoji_manager._periodic_scan(interval_MINS=global_config.EMOJI_REGISTER_INTERVAL)) print("\033[1;38;5;208m-----------开始偷表情包！-----------\033[0m") - # 启动消息发送控制任务 @group_msg.handle() async def _(bot: Bot, event: GroupMessageEvent, state: T_State): @@ -90,7 +94,7 @@ async def _(bot: Bot, event: GroupMessageEvent, state: T_State): # 添加build_memory定时任务 @scheduler.scheduled_job("interval", seconds=global_config.build_memory_interval, id="build_memory") async def build_memory_task(): - """每30秒执行一次记忆构建""" + """每build_memory_interval秒执行一次记忆构建""" print("\033[1;32m[记忆构建]\033[0m -------------------------------------------开始构建记忆-------------------------------------------") start_time = time.time() await hippocampus.operation_build_memory(chat_size=20) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index dc82cf236..f9aa7b057 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -132,6 +132,7 @@ class ChatBot: accu_typing_time = 0 # print(f"\033[1;32m[开始回复]\033[0m 开始将回复1载入发送容器") + mark_head = False for msg in response: # print(f"\033[1;32m[回复内容]\033[0m {msg}") #通过时间改变时间戳 @@ -152,6 +153,9 @@ class ChatBot: thinking_start_time=thinking_start_time, #记录了思考开始的时间 reply_message_id=message.message_id ) + if not mark_head: + bot_message.is_head = True + mark_head = True message_set.add_message(bot_message) #message_set 可以直接加入 message_manager diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index d6e400e15..539e07989 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -169,6 +169,8 @@ class Message_Sending(Message): reply_message_id: int = None # 存储 回复的 源消息ID + is_head: bool = False # 是否是头部消息 + def update_thinking_time(self): self.thinking_time = round(time.time(), 2) - self.thinking_start_time return self.thinking_time diff --git a/src/plugins/chat/message_sender.py b/src/plugins/chat/message_sender.py index c81dec1bb..3e30b3cbe 100644 --- a/src/plugins/chat/message_sender.py +++ b/src/plugins/chat/message_sender.py @@ -166,12 +166,11 @@ class MessageManager: else:# 如果不是message_thinking就只能是message_sending print(f"\033[1;34m[调试]\033[0m 消息'{message_earliest.processed_plain_text}'正在发送中") #直接发，等什么呢 - if message_earliest.update_thinking_time() < 30: - await message_sender.send_group_message(group_id, message_earliest.processed_plain_text, auto_escape=False) - else: + if message_earliest.is_head and message_earliest.update_thinking_time() >30: await message_sender.send_group_message(group_id, message_earliest.processed_plain_text, auto_escape=False, reply_message_id=message_earliest.reply_message_id) - - #移除消息 + else: + await message_sender.send_group_message(group_id, message_earliest.processed_plain_text, auto_escape=False) + #移除消息 if message_earliest.is_emoji: message_earliest.processed_plain_text = "[表情包]" await self.storage.store_message(message_earliest, None) @@ -188,10 +187,11 @@ class MessageManager: try: #发送 - if msg.update_thinking_time() < 30: - await message_sender.send_group_message(group_id, msg.processed_plain_text, auto_escape=False) - else: + if msg.is_head and msg.update_thinking_time() >30: await message_sender.send_group_message(group_id, msg.processed_plain_text, auto_escape=False, reply_message_id=msg.reply_message_id) + else: + await message_sender.send_group_message(group_id, msg.processed_plain_text, auto_escape=False) + #如果是表情包，则替换为"[表情包]" if msg.is_emoji: diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index ddc698bc7..63daf6680 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -395,13 +395,13 @@ def add_typos(text: str) -> str: def process_llm_response(text: str) -> List[str]: # processed_response = process_text_with_typos(content) - if len(text) > 200: + if len(text) > 300: print(f"回复过长 ({len(text)} 字符)，返回默认回复") return ['懒得说'] # 处理长消息 sentences = split_into_sentences_w_remove_punctuation(add_typos(text)) # 检查分割后的消息数量是否过多（超过3条） - if len(sentences) > 3: + if len(sentences) > 4: print(f"分割后消息数量过多 ({len(sentences)} 条)，返回默认回复") return [f'{global_config.BOT_NICKNAME}不知道哦'] From 90e72db87b34d17063a01e0fe1bb73a22aae8b1e Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 21:11:22 +0800 Subject: [PATCH 17/53] =?UTF-8?q?fix:=20=E8=B0=83=E6=95=B4api=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E7=9A=84=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 2801a3553..793a89290 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -41,7 +41,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}/{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -123,7 +123,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}/{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 @@ -273,7 +273,7 @@ class LLM_request: # 发送请求到完整的chat/completions端点 api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}/{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -339,7 +339,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}/{self.model_name}") # 记录请求的URL max_retries = 2 base_wait_time = 6 @@ -396,7 +396,7 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + logger.info(f"发送请求到URL: {api_url}/{self.model_name}") # 记录请求的URL max_retries = 3 base_wait_time = 15 From e3c7fae61d7b3ccaf34040c67533da6b38c962ff Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 21:18:35 +0800 Subject: [PATCH 18/53] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/config.py | 2 ++ src/plugins/chat/emoji_manager.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index d5ee364ce..e044edc5e 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -30,6 +30,7 @@ class BotConfig: forget_memory_interval: int = 300 # 记忆遗忘间隔（秒） EMOJI_CHECK_INTERVAL: int = 120 # 表情包检查间隔（分钟） EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） + EMOJI_CHECK_PROMPT: str = "不要包含违反公序良俗的内容" # 表情包过滤要求 ban_words = set() @@ -94,6 +95,7 @@ class BotConfig: emoji_config = toml_dict["emoji"] config.EMOJI_CHECK_INTERVAL = emoji_config.get("check_interval", config.EMOJI_CHECK_INTERVAL) config.EMOJI_REGISTER_INTERVAL = emoji_config.get("register_interval", config.EMOJI_REGISTER_INTERVAL) + config.EMOJI_CHECK_PROMPT = emoji_config.get('check_prompt',config.EMOJI_CHECK_PROMPT) if "cq_code" in toml_dict: cq_code_config = toml_dict["cq_code"] diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index e7ff85803..4b81302b1 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -206,7 +206,7 @@ class EmojiManager: async def _check_emoji(self, image_base64: str) -> str: try: - prompt = '这是一个表情包，请回答这个表情包是否满足\"动漫风格，画风可爱\"的要求，是则回答是，否则回答否，不要出现任何其他内容' + prompt = f'这是一个表情包，请回答这个表情包是否满足\"{global_config.EMOJI_CHECK_PROMPT}\"的要求，是则回答是，否则回答否，不要出现任何其他内容' content, _ = await self.llm.generate_response_for_image(prompt, image_base64) logger.debug(f"输出描述: {content}") From 11807fda38f0d341fa19838149a959dee4310606 Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Thu, 6 Mar 2025 23:50:14 +0800 Subject: [PATCH 19/53] =?UTF-8?q?refactor(models)=EF=BC=9A=E7=BB=9F?= =?UTF-8?q?=E4=B8=80=E8=AF=B7=E6=B1=82=E5=A4=84=E7=90=86=E5=B9=B6=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E5=93=8D=E5=BA=94=E5=A4=84=E7=90=86=20(refactor/unifi?= =?UTF-8?q?ed=5Frequest)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 对 `utils_model.py` 中的请求处理逻辑进行重构，创建统一的请求执行方法 `_execute_request`。该方法集中处理请求构建、重试逻辑和响应处理，替代了 `generate_response`、`generate_response_for_image` 和 `generate_response_async` 中的冗余代码。 关键变更： - 引入 `_execute_request` 作为 API 请求的单一入口 - 新增支持自定义重试策略和响应处理器 - 通过 `_build_payload` 简化图像和文本载荷构建 - 改进错误处理和日志记录 - 移除已弃用的同步方法 - 加入了`max_response_length`以兼容koboldcpp硬编码的默认值500 此次重构在保持现有功能的同时提高了代码可维护性，减少了重复代码 --- config/bot_config_template.toml | 1 + src/plugins/chat/config.py | 3 + src/plugins/chat/cq_code.py | 24 +- src/plugins/chat/prompt_builder.py | 8 +- src/plugins/chat/utils.py | 122 +++-- src/plugins/memory_system/memory.py | 2 +- src/plugins/models/utils_model.py | 706 +++++++--------------------- 7 files changed, 243 insertions(+), 623 deletions(-) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index 28ffb0ce3..f3582de12 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -28,6 +28,7 @@ enable_pic_translate = false model_r1_probability = 0.8 # 麦麦回答时选择R1模型的概率 model_v3_probability = 0.1 # 麦麦回答时选择V3模型的概率 model_r1_distill_probability = 0.1 # 麦麦回答时选择R1蒸馏模型的概率 +max_response_length = 1024 # 麦麦回答的最大token数 [memory] build_memory_interval = 300 # 记忆构建间隔 单位秒 diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index d5ee364ce..ba1ca0b71 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -32,6 +32,8 @@ class BotConfig: EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） ban_words = set() + + max_response_length: int = 1024 # 最大回复长度 # 模型配置 llm_reasoning: Dict[str, str] = field(default_factory=lambda: {}) @@ -113,6 +115,7 @@ class BotConfig: config.MODEL_R1_DISTILL_PROBABILITY = response_config.get("model_r1_distill_probability", config.MODEL_R1_DISTILL_PROBABILITY) config.API_USING = response_config.get("api_using", config.API_USING) config.API_PAID = response_config.get("api_paid", config.API_PAID) + config.max_response_length = response_config.get("max_response_length", config.max_response_length) # 加载模型配置 if "model" in toml_dict: diff --git a/src/plugins/chat/cq_code.py b/src/plugins/chat/cq_code.py index 4d70736cd..df93c6fa2 100644 --- a/src/plugins/chat/cq_code.py +++ b/src/plugins/chat/cq_code.py @@ -64,15 +64,15 @@ class CQCode: """初始化LLM实例""" self._llm = LLM_request(model=global_config.vlm, temperature=0.4, max_tokens=300) - def translate(self): + async def translate(self): """根据CQ码类型进行相应的翻译处理""" if self.type == 'text': self.translated_plain_text = self.params.get('text', '') elif self.type == 'image': if self.params.get('sub_type') == '0': - self.translated_plain_text = self.translate_image() + self.translated_plain_text = await self.translate_image() else: - self.translated_plain_text = self.translate_emoji() + self.translated_plain_text = await self.translate_emoji() elif self.type == 'at': user_nickname = get_user_nickname(self.params.get('qq', '')) if user_nickname: @@ -158,7 +158,7 @@ class CQCode: return None - def translate_emoji(self) -> str: + async def translate_emoji(self) -> str: """处理表情包类型的CQ码""" if 'url' not in self.params: return '[表情包]' @@ -167,12 +167,12 @@ class CQCode: # 将 base64 字符串转换为字节类型 image_bytes = base64.b64decode(base64_str) storage_emoji(image_bytes) - return self.get_emoji_description(base64_str) + return await self.get_emoji_description(base64_str) else: return '[表情包]' - def translate_image(self) -> str: + async def translate_image(self) -> str: """处理图片类型的CQ码，区分普通图片和表情包""" #没有url，直接返回默认文本 if 'url' not in self.params: @@ -181,25 +181,27 @@ class CQCode: if base64_str: image_bytes = base64.b64decode(base64_str) storage_image(image_bytes) - return self.get_image_description(base64_str) + return await self.get_image_description(base64_str) else: return '[图片]' - def get_emoji_description(self, image_base64: str) -> str: + async def get_emoji_description(self, image_base64: str) -> str: """调用AI接口获取表情包描述""" try: prompt = "这是一个表情包，请用简短的中文描述这个表情包传达的情感和含义。最多20个字。" - description, _ = self._llm.generate_response_for_image_sync(prompt, image_base64) + # description, _ = self._llm.generate_response_for_image_sync(prompt, image_base64) + description, _ = await self._llm.generate_response_for_image(prompt, image_base64) return f"[表情包：{description}]" except Exception as e: print(f"\033[1;31m[错误]\033[0m AI接口调用失败: {str(e)}") return "[表情包]" - def get_image_description(self, image_base64: str) -> str: + async def get_image_description(self, image_base64: str) -> str: """调用AI接口获取普通图片描述""" try: prompt = "请用中文描述这张图片的内容。如果有文字，请把文字都描述出来。并尝试猜测这个图片的含义。最多200个字。" - description, _ = self._llm.generate_response_for_image_sync(prompt, image_base64) + # description, _ = self._llm.generate_response_for_image_sync(prompt, image_base64) + description, _ = await self._llm.generate_response_for_image(prompt, image_base64) return f"[图片：{description}]" except Exception as e: print(f"\033[1;31m[错误]\033[0m AI接口调用失败: {str(e)}") diff --git a/src/plugins/chat/prompt_builder.py b/src/plugins/chat/prompt_builder.py index 1c510e251..1c1431577 100644 --- a/src/plugins/chat/prompt_builder.py +++ b/src/plugins/chat/prompt_builder.py @@ -2,7 +2,7 @@ import time import random from ..schedule.schedule_generator import bot_schedule import os -from .utils import get_embedding, combine_messages, get_recent_group_detailed_plain_text,find_similar_topics +from .utils import get_embedding, combine_messages, get_recent_group_detailed_plain_text from ...common.database import Database from .config import global_config from .topic_identifier import topic_identifier @@ -60,7 +60,7 @@ class PromptBuilder: prompt_info = '' promt_info_prompt = '' - prompt_info = self.get_prompt_info(message_txt,threshold=0.5) + prompt_info = await self.get_prompt_info(message_txt,threshold=0.5) if prompt_info: prompt_info = f'''\n----------------------------------------------------\n你有以下这些[知识]：\n{prompt_info}\n请你记住上面的[知识]，之后可能会用到\n----------------------------------------------------\n''' @@ -214,10 +214,10 @@ class PromptBuilder: return prompt_for_initiative - def get_prompt_info(self,message:str,threshold:float): + async def get_prompt_info(self,message:str,threshold:float): related_info = '' print(f"\033[1;34m[调试]\033[0m 获取知识库内容，元消息：{message[:30]}...，消息长度: {len(message)}") - embedding = get_embedding(message) + embedding = await get_embedding(message) related_info += self.get_info_from_db(embedding,threshold=threshold) return related_info diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index 63daf6680..38aeefd21 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -32,16 +32,18 @@ def combine_messages(messages: List[Message]) -> str: time_str = time.strftime("%m-%d %H:%M:%S", time.localtime(message.time)) name = message.user_nickname or f"用户{message.user_id}" content = message.processed_plain_text or message.plain_text - + result += f"[{time_str}] {name}: {content}\n" - + return result -def db_message_to_str (message_dict: Dict) -> str: + +def db_message_to_str(message_dict: Dict) -> str: print(f"message_dict: {message_dict}") time_str = time.strftime("%m-%d %H:%M:%S", time.localtime(message_dict["time"])) try: - name="[(%s)%s]%s" % (message_dict['user_id'],message_dict.get("user_nickname", ""),message_dict.get("user_cardname", "")) + name = "[(%s)%s]%s" % ( + message_dict['user_id'], message_dict.get("user_nickname", ""), message_dict.get("user_cardname", "")) except: name = message_dict.get("user_nickname", "") or f"用户{message_dict['user_id']}" content = message_dict.get("processed_plain_text", "") @@ -58,6 +60,7 @@ def is_mentioned_bot_in_message(message: Message) -> bool: return True return False + def is_mentioned_bot_in_txt(message: str) -> bool: """检查消息是否提到了机器人""" keywords = [global_config.BOT_NICKNAME] @@ -66,10 +69,13 @@ def is_mentioned_bot_in_txt(message: str) -> bool: return True return False -def get_embedding(text): + +async def get_embedding(text): """获取文本的embedding向量""" llm = LLM_request(model=global_config.embedding) - return llm.get_embedding_sync(text) + # return llm.get_embedding_sync(text) + return await llm.get_embedding(text) + def cosine_similarity(v1, v2): dot_product = np.dot(v1, v2) @@ -77,51 +83,54 @@ def cosine_similarity(v1, v2): norm2 = np.linalg.norm(v2) return dot_product / (norm1 * norm2) + def calculate_information_content(text): """计算文本的信息量（熵）""" char_count = Counter(text) total_chars = len(text) - + entropy = 0 for count in char_count.values(): probability = count / total_chars entropy -= probability * math.log2(probability) - + return entropy + def get_cloest_chat_from_db(db, length: int, timestamp: str): """从数据库中获取最接近指定时间戳的聊天记录，并记录读取次数""" chat_text = '' closest_record = db.db.messages.find_one({"time": {"$lte": timestamp}}, sort=[('time', -1)]) - - if closest_record and closest_record.get('memorized', 0) < 4: + + if closest_record and closest_record.get('memorized', 0) < 4: closest_time = closest_record['time'] group_id = closest_record['group_id'] # 获取groupid # 获取该时间戳之后的length条消息，且groupid相同 chat_records = list(db.db.messages.find( {"time": {"$gt": closest_time}, "group_id": group_id} ).sort('time', 1).limit(length)) - + # 更新每条消息的memorized属性 for record in chat_records: # 检查当前记录的memorized值 current_memorized = record.get('memorized', 0) - if current_memorized > 3: + if current_memorized > 3: # print(f"消息已读取3次，跳过") return '' - + # 更新memorized值 db.db.messages.update_one( {"_id": record["_id"]}, {"$set": {"memorized": current_memorized + 1}} ) - + chat_text += record["detailed_plain_text"] - + return chat_text # print(f"消息已读取3次，跳过") return '' + def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: """从数据库获取群组最近的消息记录 @@ -134,7 +143,7 @@ def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: list: Message对象列表，按时间正序排列 """ - # 从数据库获取最近消息 + # 从数据库获取最近消息 recent_messages = list(db.db.messages.find( {"group_id": group_id}, # { @@ -149,7 +158,7 @@ def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: if not recent_messages: return [] - + # 转换为 Message对象列表 from .message import Message message_objects = [] @@ -168,12 +177,13 @@ def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: except KeyError: print("[WARNING] 数据库中存在无效的消息") continue - + # 按时间正序排列 message_objects.reverse() return message_objects -def get_recent_group_detailed_plain_text(db, group_id: int, limit: int = 12,combine = False): + +def get_recent_group_detailed_plain_text(db, group_id: int, limit: int = 12, combine=False): recent_messages = list(db.db.messages.find( {"group_id": group_id}, { @@ -187,16 +197,16 @@ def get_recent_group_detailed_plain_text(db, group_id: int, limit: int = 12,comb if not recent_messages: return [] - + message_detailed_plain_text = '' message_detailed_plain_text_list = [] - + # 反转消息列表，使最新的消息在最后 recent_messages.reverse() - + if combine: for msg_db_data in recent_messages: - message_detailed_plain_text+=str(msg_db_data["detailed_plain_text"]) + message_detailed_plain_text += str(msg_db_data["detailed_plain_text"]) return message_detailed_plain_text else: for msg_db_data in recent_messages: @@ -204,7 +214,6 @@ def get_recent_group_detailed_plain_text(db, group_id: int, limit: int = 12,comb return message_detailed_plain_text_list - def split_into_sentences_w_remove_punctuation(text: str) -> List[str]: """将文本分割成句子，但保持书名号中的内容完整 Args: @@ -224,30 +233,30 @@ def split_into_sentences_w_remove_punctuation(text: str) -> List[str]: split_strength = 0.7 else: split_strength = 0.9 - #先移除换行符 + # 先移除换行符 # print(f"split_strength: {split_strength}") - + # print(f"处理前的文本: {text}") - + # 统一将英文逗号转换为中文逗号 text = text.replace(',', '，') text = text.replace('\n', ' ') - + # print(f"处理前的文本: {text}") - + text_no_1 = '' for letter in text: # print(f"当前字符: {letter}") - if letter in ['!','！','?','？']: + if letter in ['!', '！', '?', '？']: # print(f"当前字符: {letter}, 随机数: {random.random()}") if random.random() < split_strength: letter = '' - if letter in ['。','…']: + if letter in ['。', '…']: # print(f"当前字符: {letter}, 随机数: {random.random()}") if random.random() < 1 - split_strength: letter = '' text_no_1 += letter - + # 对每个逗号单独判断是否分割 sentences = [text_no_1] new_sentences = [] @@ -276,15 +285,16 @@ def split_into_sentences_w_remove_punctuation(text: str) -> List[str]: sentences_done = [] for sentence in sentences: sentence = sentence.rstrip('，,') - if random.random() < split_strength*0.5: + if random.random() < split_strength * 0.5: sentence = sentence.replace('，', '').replace(',', '') elif random.random() < split_strength: sentence = sentence.replace('，', ' ').replace(',', ' ') sentences_done.append(sentence) - + print(f"处理后的句子: {sentences_done}") return sentences_done + # 常见的错别字映射 TYPO_DICT = { '的': '地得', @@ -355,6 +365,7 @@ TYPO_DICT = { '嘻': '嘻西希' } + def random_remove_punctuation(text: str) -> str: """随机处理标点符号，模拟人类打字习惯 @@ -366,7 +377,7 @@ def random_remove_punctuation(text: str) -> str: """ result = '' text_len = len(text) - + for i, char in enumerate(text): if char == '。' and i == text_len - 1: # 结尾的句号 if random.random() > 0.4: # 80%概率删除结尾句号 @@ -381,6 +392,7 @@ def random_remove_punctuation(text: str) -> str: result += char return result + def add_typos(text: str) -> str: TYPO_RATE = 0.02 # 控制错别字出现的概率(2%) result = "" @@ -393,20 +405,22 @@ def add_typos(text: str) -> str: result += char return result + def process_llm_response(text: str) -> List[str]: # processed_response = process_text_with_typos(content) if len(text) > 300: - print(f"回复过长 ({len(text)} 字符)，返回默认回复") - return ['懒得说'] + print(f"回复过长 ({len(text)} 字符)，返回默认回复") + return ['懒得说'] # 处理长消息 sentences = split_into_sentences_w_remove_punctuation(add_typos(text)) # 检查分割后的消息数量是否过多（超过3条） if len(sentences) > 4: print(f"分割后消息数量过多 ({len(sentences)} 条)，返回默认回复") return [f'{global_config.BOT_NICKNAME}不知道哦'] - + return sentences + def calculate_typing_time(input_string: str, chinese_time: float = 0.2, english_time: float = 0.1) -> float: """ 计算输入字符串所需的时间，中文和英文字符有不同的输入时间 @@ -419,32 +433,10 @@ def calculate_typing_time(input_string: str, chinese_time: float = 0.2, english_ if '\u4e00' <= char <= '\u9fff': # 判断是否为中文字符 total_time += chinese_time else: # 其他字符（如英文） - total_time += english_time + total_time += english_time return total_time -def find_similar_topics(message_txt: str, all_memory_topic: list, top_k: int = 5) -> list: - """使用重排序API找出与输入文本最相似的话题 - - Args: - message_txt: 输入文本 - all_memory_topic: 所有记忆主题列表 - top_k: 返回最相似的话题数量 - - Returns: - list: 最相似话题列表及其相似度分数 - """ - - if not all_memory_topic: - return [] - - try: - llm = LLM_request(model=global_config.rerank) - return llm.rerank_sync(message_txt, all_memory_topic, top_k) - except Exception as e: - print(f"重排序API调用出错: {str(e)}") - return [] - def cosine_similarity(v1, v2): """计算余弦相似度""" dot_product = np.dot(v1, v2) @@ -454,6 +446,7 @@ def cosine_similarity(v1, v2): return 0 return dot_product / (norm1 * norm2) + def text_to_vector(text): """将文本转换为词频向量""" # 分词 @@ -462,11 +455,12 @@ def text_to_vector(text): word_freq = Counter(words) return word_freq + def find_similar_topics_simple(text: str, topics: list, top_k: int = 5) -> list: """使用简单的余弦相似度计算文本相似度""" # 将输入文本转换为词频向量 text_vector = text_to_vector(text) - + # 计算每个主题的相似度 similarities = [] for topic in topics: @@ -479,6 +473,6 @@ def find_similar_topics_simple(text: str, topics: list, top_k: int = 5) -> list: # 计算相似度 similarity = cosine_similarity(v1, v2) similarities.append((topic, similarity)) - + # 按相似度降序排序并返回前k个 - return sorted(similarities, key=lambda x: x[1], reverse=True)[:top_k] \ No newline at end of file + return sorted(similarities, key=lambda x: x[1], reverse=True)[:top_k] diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index cdb6e6e1b..43db3729d 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -11,7 +11,7 @@ from ..chat.config import global_config from ...common.database import Database # 使用正确的导入语法 from ..models.utils_model import LLM_request import math -from ..chat.utils import calculate_information_content, get_cloest_chat_from_db ,find_similar_topics,text_to_vector,cosine_similarity +from ..chat.utils import calculate_information_content, get_cloest_chat_from_db ,text_to_vector,cosine_similarity diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 2801a3553..3e4d7f1a2 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -25,354 +25,195 @@ class LLM_request: self.model_name = model["name"] self.params = kwargs - async def generate_response(self, prompt: str) -> Tuple[str, str]: - """根据输入的提示生成模型的异步响应""" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" + async def _execute_request( + self, + endpoint: str, + prompt: str = None, + image_base64: str = None, + payload: dict = None, + retry_policy: dict = None, + response_handler: callable = None, + ): + """统一请求执行入口 + Args: + endpoint: API端点路径 (如 "chat/completions") + prompt: prompt文本 + image_base64: 图片的base64编码 + payload: 请求体数据 + is_async: 是否异步 + retry_policy: 自定义重试策略 + (示例: {"max_retries":3, "base_wait":15, "retry_codes":[429,500]}) + response_handler: 自定义响应处理器 + """ + # 合并重试策略 + default_retry = { + "max_retries": 3, "base_wait": 15, + "retry_codes": [429, 413, 500, 503], + "abort_codes": [400, 401, 402, 403]} + policy = {**default_retry, **(retry_policy or {})} + + # 常见Error Code Mapping + error_code_mapping = { + 400: "参数不正确", + 401: "API key 错误，认证失败", + 402: "账号余额不足", + 403: "需要实名,或余额不足", + 404: "Not Found", + 429: "请求过于频繁，请稍后再试", + 500: "服务器内部故障", + 503: "服务器负载过高" } + api_url = f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}" + logger.info(f"发送请求到URL: {api_url}{self.model_name}") + # 构建请求体 - data = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - **self.params - } + if image_base64: + payload = await self._build_payload(prompt, image_base64) + elif payload is None: + payload = await self._build_payload(prompt) - # 发送请求到完整的chat/completions端点 - api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + session_method = aiohttp.ClientSession() - max_retries = 3 - base_wait_time = 15 - - for retry in range(max_retries): + for retry in range(policy["max_retries"]): try: - async with aiohttp.ClientSession() as session: - async with session.post(api_url, headers=headers, json=data) as response: - if response.status == 429: - wait_time = base_wait_time * (2 ** retry) # 指数退避 - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - await asyncio.sleep(wait_time) - continue + # 使用上下文管理器处理会话 + headers = await self._build_headers() - if response.status in [500, 503]: - logger.error(f"服务器错误: {response.status}") - raise RuntimeError("服务器负载过高，模型恢复失败QAQ") + async with session_method as session: + response = await session.post(api_url, headers=headers, json=payload) - response.raise_for_status() # 检查其他响应状态 + # 处理需要重试的状态码 + if response.status in policy["retry_codes"]: + wait_time = policy["base_wait"] * (2 ** retry) + logger.warning(f"错误码: {response.status}, 等待 {wait_time}秒后重试") + if response.status == 413: + logger.warning("请求体过大，尝试压缩...") + image_base64 = compress_base64_image_by_scale(image_base64) + payload = await self._build_payload(prompt, image_base64) + elif response.status in [500, 503]: + logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") + raise RuntimeError("服务器负载过高，模型恢复失败QAQ") + else: + logger.warning(f"请求限制(429)，等待{wait_time}秒后重试...") - result = await response.json() - if "choices" in result and len(result["choices"]) > 0: - message = result["choices"][0]["message"] - content = message.get("content", "") - think_match = None - reasoning_content = message.get("reasoning_content", "") - if not reasoning_content: - think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) - if think_match: - reasoning_content = think_match.group(1).strip() - content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() - return content, reasoning_content - return "没有返回结果", "" + await asyncio.sleep(wait_time) + continue + elif response.status in policy["abort_codes"]: + logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") + raise RuntimeError(f"请求被拒绝: {error_code_mapping.get(response.status)}") + + response.raise_for_status() + result = await response.json() + + # 使用自定义处理器或默认处理 + return response_handler(result) if response_handler else self._default_response_handler(result) except Exception as e: - if retry < max_retries - 1: # 如果还有重试机会 - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[回复]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) + if retry < policy["max_retries"] - 1: + wait_time = policy["base_wait"] * (2 ** retry) + logger.error(f"请求失败，等待{wait_time}秒后重试... 错误: {str(e)}") await asyncio.sleep(wait_time) else: - logger.critical(f"请求失败: {str(e)}", exc_info=True) - logger.critical(f"请求头: {headers} 请求体: {data}") + logger.critical(f"请求失败: {str(e)}") + logger.critical(f"请求头: {self._build_headers()} 请求体: {payload}") raise RuntimeError(f"API请求失败: {str(e)}") logger.error("达到最大重试次数，请求仍然失败") raise RuntimeError("达到最大重试次数，API请求仍然失败") - async def generate_response_for_image(self, prompt: str, image_base64: str) -> Tuple[str, str]: - """根据输入的提示和图片生成模型的异步响应""" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - # 构建请求体 - def build_request_data(img_base64: str): + async def _build_payload(self, prompt: str, image_base64: str = None) -> dict: + """构建请求体""" + if image_base64: return { "model": self.model_name, "messages": [ { "role": "user", "content": [ - { - "type": "text", - "text": prompt - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{img_base64}" - } - } + {"type": "text", "text": prompt}, + {"type": "image_url", "image_url": {"url": f"data:image/jpeg;base64,{image_base64}"}} ] } ], + "max_tokens": global_config.max_response_length, + **self.params + } + else: + return { + "model": self.model_name, + "messages": [{"role": "user", "content": prompt}], + "max_tokens": global_config.max_response_length, **self.params } + def _default_response_handler(self, result: dict) -> Tuple: + """默认响应解析""" + if "choices" in result and result["choices"]: + message = result["choices"][0]["message"] + content = message.get("content", "") + content, reasoning = self._extract_reasoning(content) + reasoning_content = message.get("model_extra", {}).get("reasoning_content", "") + if not reasoning_content: + reasoning_content = reasoning - # 发送请求到完整的chat/completions端点 - api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL + return content, reasoning_content - max_retries = 3 - base_wait_time = 15 + return "没有返回结果", "" - current_image_base64 = image_base64 - current_image_base64 = compress_base64_image_by_scale(current_image_base64) + def _extract_reasoning(self, content: str) -> tuple[str, str]: + """CoT思维链提取""" + match = re.search(r'(?:)?(.*?)', content, re.DOTALL) + content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() + if match: + reasoning = match.group(1).strip() + else: + reasoning = "" + return content, reasoning - for retry in range(max_retries): - try: - data = build_request_data(current_image_base64) - async with aiohttp.ClientSession() as session: - async with session.post(api_url, headers=headers, json=data) as response: - if response.status == 429: - wait_time = base_wait_time * (2 ** retry) # 指数退避 - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - await asyncio.sleep(wait_time) - continue - - elif response.status == 413: - logger.warning("图片太大(413)，尝试压缩...") - current_image_base64 = compress_base64_image_by_scale(current_image_base64) - continue - - response.raise_for_status() # 检查其他响应状态 - - result = await response.json() - if "choices" in result and len(result["choices"]) > 0: - message = result["choices"][0]["message"] - content = message.get("content", "") - think_match = None - reasoning_content = message.get("reasoning_content", "") - if not reasoning_content: - think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) - if think_match: - reasoning_content = think_match.group(1).strip() - content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() - return content, reasoning_content - return "没有返回结果", "" - - except Exception as e: - if retry < max_retries - 1: # 如果还有重试机会 - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[image回复]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - await asyncio.sleep(wait_time) - else: - logger.critical(f"请求失败: {str(e)}", exc_info=True) - logger.critical(f"请求头: {headers} 请求体: {data}") - raise RuntimeError(f"API请求失败: {str(e)}") - - logger.error("达到最大重试次数，请求仍然失败") - raise RuntimeError("达到最大重试次数，API请求仍然失败") - - async def generate_response_async(self, prompt: str) -> Union[str, Tuple[str, str]]: - """异步方式根据输入的提示生成模型的响应""" - headers = { + async def _build_headers(self) -> dict: + """构建请求头""" + return { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json" } + async def generate_response(self, prompt: str) -> Tuple[str, str]: + """根据输入的提示生成模型的异步响应""" + + content, reasoning_content = await self._execute_request( + endpoint="/chat/completions", + prompt=prompt + ) + return content, reasoning_content + + async def generate_response_for_image(self, prompt: str, image_base64: str) -> Tuple[str, str]: + """根据输入的提示和图片生成模型的异步响应""" + + content, reasoning_content = await self._execute_request( + endpoint="/chat/completions", + prompt=prompt, + image_base64=image_base64 + ) + return content, reasoning_content + + async def generate_response_async(self, prompt: str) -> Union[str, Tuple[str, str]]: + """异步方式根据输入的提示生成模型的响应""" # 构建请求体 data = { "model": self.model_name, "messages": [{"role": "user", "content": prompt}], "temperature": 0.5, + "max_tokens": global_config.max_response_length, **self.params } - # 发送请求到完整的 chat/completions 端点 - api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"Request URL: {api_url}") # 记录请求的 URL - - max_retries = 3 - base_wait_time = 15 - - async with aiohttp.ClientSession() as session: - for retry in range(max_retries): - try: - async with session.post(api_url, headers=headers, json=data) as response: - if response.status == 429: - wait_time = base_wait_time * (2 ** retry) # 指数退避 - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - await asyncio.sleep(wait_time) - continue - - response.raise_for_status() # 检查其他响应状态 - - result = await response.json() - if "choices" in result and len(result["choices"]) > 0: - message = result["choices"][0]["message"] - content = message.get("content", "") - think_match = None - reasoning_content = message.get("reasoning_content", "") - if not reasoning_content: - think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) - if think_match: - reasoning_content = think_match.group(1).strip() - content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() - return content, reasoning_content - return "没有返回结果", "" - - except Exception as e: - if retry < max_retries - 1: # 如果还有重试机会 - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[回复]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}") - await asyncio.sleep(wait_time) - else: - logger.error(f"请求失败: {str(e)}") - logger.critical(f"请求头: {headers} 请求体: {data}") - return f"请求失败: {str(e)}", "" - - logger.error("达到最大重试次数，请求仍然失败") - return "达到最大重试次数，请求仍然失败", "" - - - - def generate_response_for_image_sync(self, prompt: str, image_base64: str) -> Tuple[str, str]: - """同步方法：根据输入的提示和图片生成模型的响应""" - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - image_base64=compress_base64_image_by_scale(image_base64) - - # 构建请求体 - data = { - "model": self.model_name, - "messages": [ - { - "role": "user", - "content": [ - { - "type": "text", - "text": prompt - }, - { - "type": "image_url", - "image_url": { - "url": f"data:image/jpeg;base64,{image_base64}" - } - } - ] - } - ], - **self.params - } - - # 发送请求到完整的chat/completions端点 - api_url = f"{self.base_url.rstrip('/')}/chat/completions" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL - - max_retries = 2 - base_wait_time = 6 - - for retry in range(max_retries): - try: - response = requests.post(api_url, headers=headers, json=data, timeout=30) - - if response.status_code == 429: - wait_time = base_wait_time * (2 ** retry) - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - time.sleep(wait_time) - continue - - response.raise_for_status() # 检查其他响应状态 - - result = response.json() - if "choices" in result and len(result["choices"]) > 0: - message = result["choices"][0]["message"] - content = message.get("content", "") - think_match = None - reasoning_content = message.get("reasoning_content", "") - if not reasoning_content: - think_match = re.search(r'(?:)?(.*?)', content, re.DOTALL) - if think_match: - reasoning_content = think_match.group(1).strip() - content = re.sub(r'(?:)?.*?', '', content, flags=re.DOTALL, count=1).strip() - return content, reasoning_content - return "没有返回结果", "" - - except Exception as e: - if retry < max_retries - 1: # 如果还有重试机会 - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[image_sync回复]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - time.sleep(wait_time) - else: - logger.critical(f"请求失败: {str(e)}", exc_info=True) - logger.critical(f"请求头: {headers} 请求体: {data}") - raise RuntimeError(f"API请求失败: {str(e)}") - - logger.error("达到最大重试次数，请求仍然失败") - raise RuntimeError("达到最大重试次数，API请求仍然失败") - - def get_embedding_sync(self, text: str, model: str = "BAAI/bge-m3") -> Union[list, None]: - """同步方法：获取文本的embedding向量 - - Args: - text: 需要获取embedding的文本 - model: 使用的模型名称，默认为"BAAI/bge-m3" - - Returns: - list: embedding向量，如果失败则返回None - """ - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - data = { - "model": model, - "input": text, - "encoding_format": "float" - } - - api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL - - max_retries = 2 - base_wait_time = 6 - - for retry in range(max_retries): - try: - response = requests.post(api_url, headers=headers, json=data, timeout=30) - - if response.status_code == 429: - wait_time = base_wait_time * (2 ** retry) - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - time.sleep(wait_time) - continue - - response.raise_for_status() - - result = response.json() - if 'data' in result and len(result['data']) > 0: - return result['data'][0]['embedding'] - return None - - except Exception as e: - if retry < max_retries - 1: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[embedding_sync]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - time.sleep(wait_time) - else: - logger.critical(f"embedding请求失败: {str(e)}", exc_info=True) - logger.critical(f"请求头: {headers} 请求体: {data}") - return None - - logger.error("达到最大重试次数，embedding请求仍然失败") - return None + content, reasoning_content = await self._execute_request( + endpoint="/chat/completions", + payload=data, + prompt=prompt + ) + return content, reasoning_content async def get_embedding(self, text: str, model: str = "BAAI/bge-m3") -> Union[list, None]: """异步方法：获取文本的embedding向量 @@ -384,245 +225,24 @@ class LLM_request: Returns: list: embedding向量，如果失败则返回None """ - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } + def embedding_handler(result): + """处理响应""" + if "data" in result and len(result["data"]) > 0: + return result["data"][0].get("embedding", None) + return None - data = { - "model": model, - "input": text, - "encoding_format": "float" - } - - api_url = f"{self.base_url.rstrip('/')}/embeddings" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") # 记录请求的URL - - max_retries = 3 - base_wait_time = 15 - - for retry in range(max_retries): - try: - async with aiohttp.ClientSession() as session: - async with session.post(api_url, headers=headers, json=data) as response: - if response.status == 429: - wait_time = base_wait_time * (2 ** retry) - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - await asyncio.sleep(wait_time) - continue - - response.raise_for_status() - - result = await response.json() - if 'data' in result and len(result['data']) > 0: - return result['data'][0]['embedding'] - return None - - except Exception as e: - if retry < max_retries - 1: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[embedding]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - await asyncio.sleep(wait_time) - else: - logger.critical(f"embedding请求失败: {str(e)}", exc_info=True) - logger.critical(f"请求头: {headers} 请求体: {data}") - return None - - logger.error("达到最大重试次数，embedding请求仍然失败") - return None - - def rerank_sync(self, query: str, documents: list, top_k: int = 5) -> list: - """同步方法：使用重排序API对文档进行排序 - - Args: - query: 查询文本 - documents: 待排序的文档列表 - top_k: 返回前k个结果 - - Returns: - list: [(document, score), ...] 格式的结果列表 - """ - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - data = { - "model": self.model_name, - "query": query, - "documents": documents, - "top_n": top_k, - "return_documents": True, - } - - api_url = f"{self.base_url.rstrip('/')}/rerank" - logger.info(f"发送请求到URL: {api_url}") - - max_retries = 2 - base_wait_time = 6 - - for retry in range(max_retries): - try: - response = requests.post(api_url, headers=headers, json=data, timeout=30) - - if response.status_code == 429: - wait_time = base_wait_time * (2 ** retry) - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - time.sleep(wait_time) - continue - - if response.status_code in [500, 503]: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"服务器错误({response.status_code})，等待{wait_time}秒后重试...") - if retry < max_retries - 1: - time.sleep(wait_time) - continue - else: - # 如果是最后一次重试，尝试使用chat/completions作为备选方案 - return self._fallback_rerank_with_chat(query, documents, top_k) - - response.raise_for_status() - - result = response.json() - if 'results' in result: - return [(item["document"], item["score"]) for item in result["results"]] - return [] - - except Exception as e: - if retry < max_retries - 1: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[rerank_sync]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - time.sleep(wait_time) - else: - logger.critical(f"重排序请求失败: {str(e)}", exc_info=True) - - logger.error("达到最大重试次数，重排序请求仍然失败") - return [] - - async def rerank(self, query: str, documents: list, top_k: int = 5) -> list: - """异步方法：使用重排序API对文档进行排序 - - Args: - query: 查询文本 - documents: 待排序的文档列表 - top_k: 返回前k个结果 - - Returns: - list: [(document, score), ...] 格式的结果列表 - """ - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - data = { - "model": self.model_name, - "query": query, - "documents": documents, - "top_n": top_k, - "return_documents": True, - } - - api_url = f"{self.base_url.rstrip('/')}/v1/rerank" - logger.info(f"发送请求到URL: {api_url}") - - max_retries = 3 - base_wait_time = 15 - - for retry in range(max_retries): - try: - async with aiohttp.ClientSession() as session: - async with session.post(api_url, headers=headers, json=data) as response: - if response.status == 429: - wait_time = base_wait_time * (2 ** retry) - logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") - await asyncio.sleep(wait_time) - continue - - if response.status in [500, 503]: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"服务器错误({response.status})，等待{wait_time}秒后重试...") - if retry < max_retries - 1: - await asyncio.sleep(wait_time) - continue - else: - # 如果是最后一次重试，尝试使用chat/completions作为备选方案 - return await self._fallback_rerank_with_chat_async(query, documents, top_k) - - response.raise_for_status() - - result = await response.json() - if 'results' in result: - return [(item["document"], item["score"]) for item in result["results"]] - return [] - - except Exception as e: - if retry < max_retries - 1: - wait_time = base_wait_time * (2 ** retry) - logger.error(f"[rerank]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}", exc_info=True) - await asyncio.sleep(wait_time) - else: - logger.critical(f"重排序请求失败: {str(e)}", exc_info=True) - # 作为最后的备选方案，尝试使用chat/completions - return await self._fallback_rerank_with_chat_async(query, documents, top_k) - - logger.error("达到最大重试次数，重排序请求仍然失败") - return [] - - async def _fallback_rerank_with_chat_async(self, query: str, documents: list, top_k: int = 5) -> list: - """当rerank API失败时的备选方案，使用chat/completions异步实现重排序 - - Args: - query: 查询文本 - documents: 待排序的文档列表 - top_k: 返回前k个结果 - - Returns: - list: [(document, score), ...] 格式的结果列表 - """ - try: - logger.info("使用chat/completions作为重排序的备选方案") - - # 构建提示词 - prompt = f"""请对以下文档列表进行重排序，按照与查询的相关性从高到低排序。 -查询: {query} - -文档列表: -{documents} - -请以JSON格式返回排序结果，格式为： -[{{"document": "文档内容", "score": 相关性分数}}, ...] -只返回JSON，不要其他任何文字。""" - - headers = { - "Authorization": f"Bearer {self.api_key}", - "Content-Type": "application/json" - } - - data = { - "model": self.model_name, - "messages": [{"role": "user", "content": prompt}], - **self.params - } - - api_url = f"{self.base_url.rstrip('/')}/v1/chat/completions" - - async with aiohttp.ClientSession() as session: - async with session.post(api_url, headers=headers, json=data) as response: - response.raise_for_status() - result = await response.json() - - if "choices" in result and len(result["choices"]) > 0: - message = result["choices"][0]["message"] - content = message.get("content", "") - try: - import json - parsed_content = json.loads(content) - if isinstance(parsed_content, list): - return [(item["document"], item["score"]) for item in parsed_content] - except: - pass - return [] - except Exception as e: - logger.error(f"备选方案也失败了: {str(e)}") - return [] + embedding = await self._execute_request( + endpoint="/embeddings", + prompt=text, + payload={ + "model": model, + "input": text, + "encoding_format": "float" + }, + retry_policy={ + "max_retries": 2, + "base_wait": 6 + }, + response_handler=embedding_handler + ) + return embedding From 26f99664eebe706ed73c8b2f719c322763b91c5c Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Fri, 7 Mar 2025 00:04:36 +0800 Subject: [PATCH 20/53] fix: cq_code async --- src/plugins/chat/cq_code.py | 4 ++-- src/plugins/chat/message.py | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/plugins/chat/cq_code.py b/src/plugins/chat/cq_code.py index df93c6fa2..43d9d0862 100644 --- a/src/plugins/chat/cq_code.py +++ b/src/plugins/chat/cq_code.py @@ -335,7 +335,7 @@ class CQCode: class CQCode_tool: @staticmethod - def cq_from_dict_to_class(cq_code: Dict, reply: Optional[Dict] = None) -> CQCode: + async def cq_from_dict_to_class(cq_code: Dict, reply: Optional[Dict] = None) -> CQCode: """ 将CQ码字典转换为CQCode对象 @@ -364,7 +364,7 @@ class CQCode_tool: ) # 进行翻译处理 - instance.translate() + await instance.translate() return instance @staticmethod diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index 539e07989..02f56b975 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -49,7 +49,7 @@ class Message: translate_cq: bool = True # 是否翻译cq码 - def __post_init__(self): + async def __post_init__(self): if self.time is None: self.time = int(time.time()) @@ -64,7 +64,7 @@ class Message: if not self.processed_plain_text: if self.raw_message: - self.message_segments = self.parse_message_segments(str(self.raw_message)) + self.message_segments = await self.parse_message_segments(str(self.raw_message)) self.processed_plain_text = ' '.join( seg.translated_plain_text for seg in self.message_segments @@ -78,7 +78,7 @@ class Message: content = self.processed_plain_text self.detailed_plain_text = f"[{time_str}] {name}: {content}\n" - def parse_message_segments(self, message: str) -> List[CQCode]: + async def parse_message_segments(self, message: str) -> List[CQCode]: """ 将消息解析为片段列表，包括纯文本和CQ码 返回的列表中每个元素都是字典，包含： @@ -136,7 +136,7 @@ class Message: #翻译作为字典的CQ码 for _code_item in cq_code_dict_list: - message_obj = cq_code_tool.cq_from_dict_to_class(_code_item,reply = self.reply_message) + message_obj = await cq_code_tool.cq_from_dict_to_class(_code_item,reply = self.reply_message) trans_list.append(message_obj) return trans_list From 8ef00ee5713899d28f5c95edf497a7b94df64c79 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 00:09:36 +0800 Subject: [PATCH 21/53] v0.5.9 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修复了记忆刷屏 加入了又新又好错别字生成器 增加了记忆过滤 --- README.md | 3 +- src/plugins/chat/emoji_manager.py | 10 - src/plugins/chat/prompt_builder.py | 3 +- src/plugins/chat/utils.py | 90 +- src/plugins/memory_system/memory.py | 27 +- .../memory_system/memory_manual_build.py | 33 +- src/plugins/utils/typo_generator.py | 437 +++++++++ src/test/typo.py | 827 ++++++++---------- 8 files changed, 883 insertions(+), 547 deletions(-) create mode 100644 src/plugins/utils/typo_generator.py diff --git a/README.md b/README.md index 73e1c3094..96c857bc7 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,8 @@ - 改进表情包发送逻辑 - 自动生成的回复逻辑，例如自生成的回复方向，回复风格 - 采用截断生成加快麦麦的反应速度 -- 改进发送消息的触发： +- 改进发送消息的触发 +- ## 📌 注意事项 纯编程外行，面向cursor编程，很多代码史一样多多包涵 diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 4b81302b1..ede0d7135 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -29,16 +29,6 @@ config = driver.config class EmojiManager: _instance = None EMOJI_DIR = "data/emoji" # 表情包存储目录 - - EMOTION_KEYWORDS = { - 'happy': ['开心', '快乐', '高兴', '欢喜', '笑', '喜悦', '兴奋', '愉快', '乐', '好'], - 'angry': ['生气', '愤怒', '恼火', '不爽', '火大', '怒', '气愤', '恼怒', '发火', '不满'], - 'sad': ['伤心', '难过', '悲伤', '痛苦', '哭', '忧伤', '悲痛', '哀伤', '委屈', '失落'], - 'surprised': ['惊讶', '震惊', '吃惊', '意外', '惊', '诧异', '惊奇', '惊喜', '不敢相信', '目瞪口呆'], - 'disgusted': ['恶心', '讨厌', '厌恶', '反感', '嫌弃', '恶', '嫌恶', '憎恶', '不喜欢', '烦'], - 'fearful': ['害怕', '恐惧', '惊恐', '担心', '怕', '惊吓', '惊慌', '畏惧', '胆怯', '惧'], - 'neutral': ['普通', '一般', '还行', '正常', '平静', '平淡', '一般般', '凑合', '还好', '就这样'] - } def __new__(cls): if cls._instance is None: diff --git a/src/plugins/chat/prompt_builder.py b/src/plugins/chat/prompt_builder.py index 1c510e251..57795283b 100644 --- a/src/plugins/chat/prompt_builder.py +++ b/src/plugins/chat/prompt_builder.py @@ -84,7 +84,8 @@ class PromptBuilder: relevant_memories = await hippocampus.get_relevant_memories( text=message_txt, max_topics=5, - similarity_threshold=0.4 + similarity_threshold=0.4, + max_memory_num=5 ) if relevant_memories: diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index 63daf6680..18f1ed7a9 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -13,6 +13,7 @@ from nonebot import get_driver from ..models.utils_model import LLM_request import aiohttp import jieba +from ..utils.typo_generator import ChineseTypoGenerator driver = get_driver() config = driver.config @@ -285,75 +286,6 @@ def split_into_sentences_w_remove_punctuation(text: str) -> List[str]: print(f"处理后的句子: {sentences_done}") return sentences_done -# 常见的错别字映射 -TYPO_DICT = { - '的': '地得', - '了': '咯啦勒', - '吗': '嘛麻', - '吧': '八把罢', - '是': '事', - '在': '再在', - '和': '合', - '有': '又', - '我': '沃窝喔', - '你': '泥尼拟', - '他': '它她塔祂', - '们': '门', - '啊': '阿哇', - '呢': '呐捏', - '都': '豆读毒', - '很': '狠', - '会': '回汇', - '去': '趣取曲', - '做': '作坐', - '想': '相像', - '说': '说税睡', - '看': '砍堪刊', - '来': '来莱赖', - '好': '号毫豪', - '给': '给既继', - '过': '锅果裹', - '能': '嫩', - '为': '位未', - '什': '甚深伸', - '么': '末麽嘛', - '话': '话花划', - '知': '织直值', - '道': '到', - '听': '听停挺', - '见': '见件建', - '觉': '觉脚搅', - '得': '得德锝', - '着': '着找招', - '像': '向象想', - '等': '等灯登', - '谢': '谢写卸', - '对': '对队', - '里': '里理鲤', - '啦': '啦拉喇', - '吃': '吃持迟', - '哦': '哦喔噢', - '呀': '呀压', - '要': '药', - '太': '太抬台', - '快': '块', - '点': '店', - '以': '以已', - '因': '因应', - '啥': '啥沙傻', - '行': '行型形', - '哈': '哈蛤铪', - '嘿': '嘿黑嗨', - '嗯': '嗯恩摁', - '哎': '哎爱埃', - '呜': '呜屋污', - '喂': '喂位未', - '嘛': '嘛麻马', - '嗨': '嗨害亥', - '哇': '哇娃蛙', - '咦': '咦意易', - '嘻': '嘻西希' -} def random_remove_punctuation(text: str) -> str: """随机处理标点符号，模拟人类打字习惯 @@ -381,17 +313,6 @@ def random_remove_punctuation(text: str) -> str: result += char return result -def add_typos(text: str) -> str: - TYPO_RATE = 0.02 # 控制错别字出现的概率(2%) - result = "" - for char in text: - if char in TYPO_DICT and random.random() < TYPO_RATE: - # 从可能的错别字中随机选择一个 - typos = TYPO_DICT[char] - result += random.choice(typos) - else: - result += char - return result def process_llm_response(text: str) -> List[str]: # processed_response = process_text_with_typos(content) @@ -399,7 +320,14 @@ def process_llm_response(text: str) -> List[str]: print(f"回复过长 ({len(text)} 字符)，返回默认回复") return ['懒得说'] # 处理长消息 - sentences = split_into_sentences_w_remove_punctuation(add_typos(text)) + typo_generator = ChineseTypoGenerator( + error_rate=0.03, + min_freq=7, + tone_error_rate=0.2, + word_replace_rate=0.02 + ) + typoed_text = typo_generator.create_typo_sentence(text)[0] + sentences = split_into_sentences_w_remove_punctuation(typoed_text) # 检查分割后的消息数量是否过多（超过3条） if len(sentences) > 4: print(f"分割后消息数量过多 ({len(sentences)} 条)，返回默认回复") diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index cdb6e6e1b..840980783 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -181,13 +181,19 @@ class Hippocampus: topic_num = self.calculate_topic_num(input_text, compress_rate) topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(input_text, topic_num)) # 修改话题处理逻辑 - print(f"话题: {topics_response[0]}") - topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] - print(f"话题: {topics}") + # 定义需要过滤的关键词 + filter_keywords = ['表情包', '图片', '回复', '聊天记录'] - # 创建所有话题的请求任务 + # 过滤topics + topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] + filtered_topics = [topic for topic in topics if not any(keyword in topic for keyword in filter_keywords)] + + # print(f"原始话题: {topics}") + print(f"过滤后话题: {filtered_topics}") + + # 使用过滤后的话题继续处理 tasks = [] - for topic in topics: + for topic in filtered_topics: topic_what_prompt = self.topic_what(input_text, topic) # 创建异步任务 task = self.llm_model_summary.generate_response_async(topic_what_prompt) @@ -501,9 +507,9 @@ class Hippocampus: list: 识别出的主题列表 """ topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(text, 5)) - print(f"话题: {topics_response[0]}") + # print(f"话题: {topics_response[0]}") topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] - print(f"话题: {topics}") + # print(f"话题: {topics}") return topics @@ -579,7 +585,7 @@ class Hippocampus: print(f"\033[1;32m[记忆激活]\033[0m 识别出的主题: {identified_topics}") if not identified_topics: - print(f"\033[1;32m[记忆激活]\033[0m 未识别出主题,返回0") + # print(f"\033[1;32m[记忆激活]\033[0m 未识别出主题,返回0") return 0 # 查找相似主题 @@ -644,7 +650,7 @@ class Hippocampus: return int(activation) - async def get_relevant_memories(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.4) -> list: + async def get_relevant_memories(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.4, max_memory_num: int = 5) -> list: """根据输入文本获取相关的记忆内容""" # 识别主题 identified_topics = await self._identify_topics(text) @@ -665,6 +671,9 @@ class Hippocampus: # 获取该主题的记忆内容 first_layer, _ = self.memory_graph.get_related_item(topic, depth=1) if first_layer: + # 如果记忆条数超过限制，随机选择指定数量的记忆 + if len(first_layer) > max_memory_num: + first_layer = random.sample(first_layer, max_memory_num) # 为每条记忆添加来源主题和相似度信息 for memory in first_layer: relevant_memories.append({ diff --git a/src/plugins/memory_system/memory_manual_build.py b/src/plugins/memory_system/memory_manual_build.py index d6aa2f669..950f01afa 100644 --- a/src/plugins/memory_system/memory_manual_build.py +++ b/src/plugins/memory_system/memory_manual_build.py @@ -234,16 +234,22 @@ class Hippocampus: async def memory_compress(self, input_text, compress_rate=0.1): print(input_text) - #获取topics topic_num = self.calculate_topic_num(input_text, compress_rate) - topics_response = await self.llm_model_get_topic.generate_response_async(self.find_topic_llm(input_text, topic_num)) + topics_response = self.llm_model_get_topic.generate_response(self.find_topic_llm(input_text, topic_num)) # 修改话题处理逻辑 + # 定义需要过滤的关键词 + filter_keywords = ['表情包', '图片', '回复', '聊天记录'] + + # 过滤topics topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] - print(f"话题: {topics}") + filtered_topics = [topic for topic in topics if not any(keyword in topic for keyword in filter_keywords)] + + # print(f"原始话题: {topics}") + print(f"过滤后话题: {filtered_topics}") # 创建所有话题的请求任务 tasks = [] - for topic in topics: + for topic in filtered_topics: topic_what_prompt = self.topic_what(input_text, topic) # 创建异步任务 task = self.llm_model_small.generate_response_async(topic_what_prompt) @@ -650,7 +656,22 @@ def visualize_graph_lite(memory_graph: Memory_graph, color_by_memory: bool = Fal G = memory_graph.G # 创建一个新图用于可视化 - H = G.copy() + H = G.copy() + + # 过滤掉内容数量小于2的节点 + nodes_to_remove = [] + for node in H.nodes(): + memory_items = H.nodes[node].get('memory_items', []) + memory_count = len(memory_items) if isinstance(memory_items, list) else (1 if memory_items else 0) + if memory_count < 2: + nodes_to_remove.append(node) + + H.remove_nodes_from(nodes_to_remove) + + # 如果没有符合条件的节点，直接返回 + if len(H.nodes()) == 0: + print("没有找到内容数量大于等于2的节点") + return # 计算节点大小和颜色 node_colors = [] @@ -704,7 +725,7 @@ def visualize_graph_lite(memory_graph: Memory_graph, color_by_memory: bool = Fal edge_color='gray', width=1.5) # 统一的边宽度 - title = '记忆图谱可视化 - 节点大小表示记忆数量\n节点颜色：蓝(弱连接)到红(强连接)渐变，边的透明度表示连接强度\n连接强度越大的节点距离越近' + title = '记忆图谱可视化（仅显示内容≥2的节点）\n节点大小表示记忆数量\n节点颜色：蓝(弱连接)到红(强连接)渐变，边的透明度表示连接强度\n连接强度越大的节点距离越近' plt.title(title, fontsize=16, fontfamily='SimHei') plt.show() diff --git a/src/plugins/utils/typo_generator.py b/src/plugins/utils/typo_generator.py new file mode 100644 index 000000000..16834200f --- /dev/null +++ b/src/plugins/utils/typo_generator.py @@ -0,0 +1,437 @@ +""" +错别字生成器 - 基于拼音和字频的中文错别字生成工具 +""" + +from pypinyin import pinyin, Style +from collections import defaultdict +import json +import os +import jieba +from pathlib import Path +import random +import math +import time + +class ChineseTypoGenerator: + def __init__(self, + error_rate=0.3, + min_freq=5, + tone_error_rate=0.2, + word_replace_rate=0.3, + max_freq_diff=200): + """ + 初始化错别字生成器 + + 参数: + error_rate: 单字替换概率 + min_freq: 最小字频阈值 + tone_error_rate: 声调错误概率 + word_replace_rate: 整词替换概率 + max_freq_diff: 最大允许的频率差异 + """ + self.error_rate = error_rate + self.min_freq = min_freq + self.tone_error_rate = tone_error_rate + self.word_replace_rate = word_replace_rate + self.max_freq_diff = max_freq_diff + + # 加载数据 + print("正在加载汉字数据库，请稍候...") + self.pinyin_dict = self._create_pinyin_dict() + self.char_frequency = self._load_or_create_char_frequency() + + def _load_or_create_char_frequency(self): + """ + 加载或创建汉字频率字典 + """ + cache_file = Path("char_frequency.json") + + # 如果缓存文件存在，直接加载 + if cache_file.exists(): + with open(cache_file, 'r', encoding='utf-8') as f: + return json.load(f) + + # 使用内置的词频文件 + char_freq = defaultdict(int) + dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') + + # 读取jieba的词典文件 + with open(dict_path, 'r', encoding='utf-8') as f: + for line in f: + word, freq = line.strip().split()[:2] + # 对词中的每个字进行频率累加 + for char in word: + if self._is_chinese_char(char): + char_freq[char] += int(freq) + + # 归一化频率值 + max_freq = max(char_freq.values()) + normalized_freq = {char: freq/max_freq * 1000 for char, freq in char_freq.items()} + + # 保存到缓存文件 + with open(cache_file, 'w', encoding='utf-8') as f: + json.dump(normalized_freq, f, ensure_ascii=False, indent=2) + + return normalized_freq + + def _create_pinyin_dict(self): + """ + 创建拼音到汉字的映射字典 + """ + # 常用汉字范围 + chars = [chr(i) for i in range(0x4e00, 0x9fff)] + pinyin_dict = defaultdict(list) + + # 为每个汉字建立拼音映射 + for char in chars: + try: + py = pinyin(char, style=Style.TONE3)[0][0] + pinyin_dict[py].append(char) + except Exception: + continue + + return pinyin_dict + + def _is_chinese_char(self, char): + """ + 判断是否为汉字 + """ + try: + return '\u4e00' <= char <= '\u9fff' + except: + return False + + def _get_pinyin(self, sentence): + """ + 将中文句子拆分成单个汉字并获取其拼音 + """ + # 将句子拆分成单个字符 + characters = list(sentence) + + # 获取每个字符的拼音 + result = [] + for char in characters: + # 跳过空格和非汉字字符 + if char.isspace() or not self._is_chinese_char(char): + continue + # 获取拼音（数字声调） + py = pinyin(char, style=Style.TONE3)[0][0] + result.append((char, py)) + + return result + + def _get_similar_tone_pinyin(self, py): + """ + 获取相似声调的拼音 + """ + # 检查拼音是否为空或无效 + if not py or len(py) < 1: + return py + + # 如果最后一个字符不是数字，说明可能是轻声或其他特殊情况 + if not py[-1].isdigit(): + # 为非数字结尾的拼音添加数字声调1 + return py + '1' + + base = py[:-1] # 去掉声调 + tone = int(py[-1]) # 获取声调 + + # 处理轻声（通常用5表示）或无效声调 + if tone not in [1, 2, 3, 4]: + return base + str(random.choice([1, 2, 3, 4])) + + # 正常处理声调 + possible_tones = [1, 2, 3, 4] + possible_tones.remove(tone) # 移除原声调 + new_tone = random.choice(possible_tones) # 随机选择一个新声调 + return base + str(new_tone) + + def _calculate_replacement_probability(self, orig_freq, target_freq): + """ + 根据频率差计算替换概率 + """ + if target_freq > orig_freq: + return 1.0 # 如果替换字频率更高，保持原有概率 + + freq_diff = orig_freq - target_freq + if freq_diff > self.max_freq_diff: + return 0.0 # 频率差太大，不替换 + + # 使用指数衰减函数计算概率 + # 频率差为0时概率为1，频率差为max_freq_diff时概率接近0 + return math.exp(-3 * freq_diff / self.max_freq_diff) + + def _get_similar_frequency_chars(self, char, py, num_candidates=5): + """ + 获取与给定字频率相近的同音字，可能包含声调错误 + """ + homophones = [] + + # 有一定概率使用错误声调 + if random.random() < self.tone_error_rate: + wrong_tone_py = self._get_similar_tone_pinyin(py) + homophones.extend(self.pinyin_dict[wrong_tone_py]) + + # 添加正确声调的同音字 + homophones.extend(self.pinyin_dict[py]) + + if not homophones: + return None + + # 获取原字的频率 + orig_freq = self.char_frequency.get(char, 0) + + # 计算所有同音字与原字的频率差，并过滤掉低频字 + freq_diff = [(h, self.char_frequency.get(h, 0)) + for h in homophones + if h != char and self.char_frequency.get(h, 0) >= self.min_freq] + + if not freq_diff: + return None + + # 计算每个候选字的替换概率 + candidates_with_prob = [] + for h, freq in freq_diff: + prob = self._calculate_replacement_probability(orig_freq, freq) + if prob > 0: # 只保留有效概率的候选字 + candidates_with_prob.append((h, prob)) + + if not candidates_with_prob: + return None + + # 根据概率排序 + candidates_with_prob.sort(key=lambda x: x[1], reverse=True) + + # 返回概率最高的几个字 + return [char for char, _ in candidates_with_prob[:num_candidates]] + + def _get_word_pinyin(self, word): + """ + 获取词语的拼音列表 + """ + return [py[0] for py in pinyin(word, style=Style.TONE3)] + + def _segment_sentence(self, sentence): + """ + 使用jieba分词，返回词语列表 + """ + return list(jieba.cut(sentence)) + + def _get_word_homophones(self, word): + """ + 获取整个词的同音词，只返回高频的有意义词语 + """ + if len(word) == 1: + return [] + + # 获取词的拼音 + word_pinyin = self._get_word_pinyin(word) + + # 遍历所有可能的同音字组合 + candidates = [] + for py in word_pinyin: + chars = self.pinyin_dict.get(py, []) + if not chars: + return [] + candidates.append(chars) + + # 生成所有可能的组合 + import itertools + all_combinations = itertools.product(*candidates) + + # 获取jieba词典和词频信息 + dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') + valid_words = {} # 改用字典存储词语及其频率 + with open(dict_path, 'r', encoding='utf-8') as f: + for line in f: + parts = line.strip().split() + if len(parts) >= 2: + word_text = parts[0] + word_freq = float(parts[1]) # 获取词频 + valid_words[word_text] = word_freq + + # 获取原词的词频作为参考 + original_word_freq = valid_words.get(word, 0) + min_word_freq = original_word_freq * 0.1 # 设置最小词频为原词频的10% + + # 过滤和计算频率 + homophones = [] + for combo in all_combinations: + new_word = ''.join(combo) + if new_word != word and new_word in valid_words: + new_word_freq = valid_words[new_word] + # 只保留词频达到阈值的词 + if new_word_freq >= min_word_freq: + # 计算词的平均字频（考虑字频和词频） + char_avg_freq = sum(self.char_frequency.get(c, 0) for c in new_word) / len(new_word) + # 综合评分：结合词频和字频 + combined_score = (new_word_freq * 0.7 + char_avg_freq * 0.3) + if combined_score >= self.min_freq: + homophones.append((new_word, combined_score)) + + # 按综合分数排序并限制返回数量 + sorted_homophones = sorted(homophones, key=lambda x: x[1], reverse=True) + return [word for word, _ in sorted_homophones[:5]] # 限制返回前5个结果 + + def create_typo_sentence(self, sentence): + """ + 创建包含同音字错误的句子，支持词语级别和字级别的替换 + + 参数: + sentence: 输入的中文句子 + + 返回: + typo_sentence: 包含错别字的句子 + typo_info: 错别字信息列表 + """ + result = [] + typo_info = [] + + # 分词 + words = self._segment_sentence(sentence) + + for word in words: + # 如果是标点符号或空格，直接添加 + if all(not self._is_chinese_char(c) for c in word): + result.append(word) + continue + + # 获取词语的拼音 + word_pinyin = self._get_word_pinyin(word) + + # 尝试整词替换 + if len(word) > 1 and random.random() < self.word_replace_rate: + word_homophones = self._get_word_homophones(word) + if word_homophones: + typo_word = random.choice(word_homophones) + # 计算词的平均频率 + orig_freq = sum(self.char_frequency.get(c, 0) for c in word) / len(word) + typo_freq = sum(self.char_frequency.get(c, 0) for c in typo_word) / len(typo_word) + + # 添加到结果中 + result.append(typo_word) + typo_info.append((word, typo_word, + ' '.join(word_pinyin), + ' '.join(self._get_word_pinyin(typo_word)), + orig_freq, typo_freq)) + continue + + # 如果不进行整词替换，则进行单字替换 + if len(word) == 1: + char = word + py = word_pinyin[0] + if random.random() < self.error_rate: + similar_chars = self._get_similar_frequency_chars(char, py) + if similar_chars: + typo_char = random.choice(similar_chars) + typo_freq = self.char_frequency.get(typo_char, 0) + orig_freq = self.char_frequency.get(char, 0) + replace_prob = self._calculate_replacement_probability(orig_freq, typo_freq) + if random.random() < replace_prob: + result.append(typo_char) + typo_py = pinyin(typo_char, style=Style.TONE3)[0][0] + typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq)) + continue + result.append(char) + else: + # 处理多字词的单字替换 + word_result = [] + for i, (char, py) in enumerate(zip(word, word_pinyin)): + # 词中的字替换概率降低 + word_error_rate = self.error_rate * (0.7 ** (len(word) - 1)) + + if random.random() < word_error_rate: + similar_chars = self._get_similar_frequency_chars(char, py) + if similar_chars: + typo_char = random.choice(similar_chars) + typo_freq = self.char_frequency.get(typo_char, 0) + orig_freq = self.char_frequency.get(char, 0) + replace_prob = self._calculate_replacement_probability(orig_freq, typo_freq) + if random.random() < replace_prob: + word_result.append(typo_char) + typo_py = pinyin(typo_char, style=Style.TONE3)[0][0] + typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq)) + continue + word_result.append(char) + result.append(''.join(word_result)) + + return ''.join(result), typo_info + + def format_typo_info(self, typo_info): + """ + 格式化错别字信息 + + 参数: + typo_info: 错别字信息列表 + + 返回: + 格式化后的错别字信息字符串 + """ + if not typo_info: + return "未生成错别字" + + result = [] + for orig, typo, orig_py, typo_py, orig_freq, typo_freq in typo_info: + # 判断是否为词语替换 + is_word = ' ' in orig_py + if is_word: + error_type = "整词替换" + else: + tone_error = orig_py[:-1] == typo_py[:-1] and orig_py[-1] != typo_py[-1] + error_type = "声调错误" if tone_error else "同音字替换" + + result.append(f"原文：{orig}({orig_py}) [频率：{orig_freq:.2f}] -> " + f"替换：{typo}({typo_py}) [频率：{typo_freq:.2f}] [{error_type}]") + + return "\n".join(result) + + def set_params(self, **kwargs): + """ + 设置参数 + + 可设置参数: + error_rate: 单字替换概率 + min_freq: 最小字频阈值 + tone_error_rate: 声调错误概率 + word_replace_rate: 整词替换概率 + max_freq_diff: 最大允许的频率差异 + """ + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + print(f"参数 {key} 已设置为 {value}") + else: + print(f"警告: 参数 {key} 不存在") + +def main(): + # 创建错别字生成器实例 + typo_generator = ChineseTypoGenerator( + error_rate=0.03, + min_freq=7, + tone_error_rate=0.02, + word_replace_rate=0.3 + ) + + # 获取用户输入 + sentence = input("请输入中文句子：") + + # 创建包含错别字的句子 + start_time = time.time() + typo_sentence, typo_info = typo_generator.create_typo_sentence(sentence) + + # 打印结果 + print("\n原句：", sentence) + print("错字版：", typo_sentence) + + # 打印错别字信息 + if typo_info: + print("\n错别字信息：") + print(typo_generator.format_typo_info(typo_info)) + + # 计算并打印总耗时 + end_time = time.time() + total_time = end_time - start_time + print(f"\n总耗时：{total_time:.2f}秒") + +if __name__ == "__main__": + main() diff --git a/src/test/typo.py b/src/test/typo.py index c452589ce..16834200f 100644 --- a/src/test/typo.py +++ b/src/test/typo.py @@ -1,455 +1,376 @@ """ -错别字生成器 - 流程说明 - -整体替换逻辑： -1. 数据准备 - - 加载字频词典：使用jieba词典计算汉字使用频率 - - 创建拼音映射：建立拼音到汉字的映射关系 - - 加载词频信息：从jieba词典获取词语使用频率 - -2. 分词处理 - - 使用jieba将输入句子分词 - - 区分单字词和多字词 - - 保留标点符号和空格 - -3. 词语级别替换（针对多字词） - - 触发条件：词长>1 且 随机概率<0.3 - - 替换流程： - a. 获取词语拼音 - b. 生成所有可能的同音字组合 - c. 过滤条件： - - 必须是jieba词典中的有效词 - - 词频必须达到原词频的10%以上 - - 综合评分(词频70%+字频30%)必须达到阈值 - d. 按综合评分排序，选择最合适的替换词 - -4. 字级别替换（针对单字词或未进行整词替换的多字词） - - 单字替换概率：0.3 - - 多字词中的单字替换概率：0.3 * (0.7 ^ (词长-1)) - - 替换流程： - a. 获取字的拼音 - b. 声调错误处理（20%概率） - c. 获取同音字列表 - d. 过滤条件： - - 字频必须达到最小阈值 - - 频率差异不能过大（指数衰减计算） - e. 按频率排序选择替换字 - -5. 频率控制机制 - - 字频控制：使用归一化的字频（0-1000范围） - - 词频控制：使用jieba词典中的词频 - - 频率差异计算：使用指数衰减函数 - - 最小频率阈值：确保替换字/词不会太生僻 - -6. 输出信息 - - 原文和错字版本的对照 - - 每个替换的详细信息（原字/词、替换后字/词、拼音、频率） - - 替换类型说明（整词替换/声调错误/同音字替换） - - 词语分析和完整拼音 - -注意事项： -1. 所有替换都必须使用有意义的词语 -2. 替换词的使用频率不能过低 -3. 多字词优先考虑整词替换 -4. 考虑声调变化的情况 -5. 保持标点符号和空格不变 +错别字生成器 - 基于拼音和字频的中文错别字生成工具 """ from pypinyin import pinyin, Style from collections import defaultdict import json import os -import unicodedata import jieba -import jieba.posseg as pseg from pathlib import Path import random import math import time -def load_or_create_char_frequency(): - """ - 加载或创建汉字频率字典 - """ - cache_file = Path("char_frequency.json") - - # 如果缓存文件存在，直接加载 - if cache_file.exists(): - with open(cache_file, 'r', encoding='utf-8') as f: - return json.load(f) - - # 使用内置的词频文件 - char_freq = defaultdict(int) - dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') - - # 读取jieba的词典文件 - with open(dict_path, 'r', encoding='utf-8') as f: - for line in f: - word, freq = line.strip().split()[:2] - # 对词中的每个字进行频率累加 - for char in word: - if is_chinese_char(char): - char_freq[char] += int(freq) - - # 归一化频率值 - max_freq = max(char_freq.values()) - normalized_freq = {char: freq/max_freq * 1000 for char, freq in char_freq.items()} - - # 保存到缓存文件 - with open(cache_file, 'w', encoding='utf-8') as f: - json.dump(normalized_freq, f, ensure_ascii=False, indent=2) - - return normalized_freq - -# 创建拼音到汉字的映射字典 -def create_pinyin_dict(): - """ - 创建拼音到汉字的映射字典 - """ - # 常用汉字范围 - chars = [chr(i) for i in range(0x4e00, 0x9fff)] - pinyin_dict = defaultdict(list) - - # 为每个汉字建立拼音映射 - for char in chars: - try: - py = pinyin(char, style=Style.TONE3)[0][0] - pinyin_dict[py].append(char) - except Exception: - continue - - return pinyin_dict - -def is_chinese_char(char): - """ - 判断是否为汉字 - """ - try: - return '\u4e00' <= char <= '\u9fff' - except: - return False - -def get_pinyin(sentence): - """ - 将中文句子拆分成单个汉字并获取其拼音 - :param sentence: 输入的中文句子 - :return: 每个汉字及其拼音的列表 - """ - # 将句子拆分成单个字符 - characters = list(sentence) - - # 获取每个字符的拼音 - result = [] - for char in characters: - # 跳过空格和非汉字字符 - if char.isspace() or not is_chinese_char(char): - continue - # 获取拼音（数字声调） - py = pinyin(char, style=Style.TONE3)[0][0] - result.append((char, py)) - - return result - -def get_homophone(char, py, pinyin_dict, char_frequency, min_freq=5): - """ - 获取同音字，按照使用频率排序 - """ - homophones = pinyin_dict[py] - # 移除原字并过滤低频字 - if char in homophones: - homophones.remove(char) - - # 过滤掉低频字 - homophones = [h for h in homophones if char_frequency.get(h, 0) >= min_freq] - - # 按照字频排序 - sorted_homophones = sorted(homophones, - key=lambda x: char_frequency.get(x, 0), - reverse=True) - - # 只返回前10个同音字，避免输出过多 - return sorted_homophones[:10] - -def get_similar_tone_pinyin(py): - """ - 获取相似声调的拼音 - 例如：'ni3' 可能返回 'ni2' 或 'ni4' - 处理特殊情况： - 1. 轻声（如 'de5' 或 'le'） - 2. 非数字结尾的拼音 - """ - # 检查拼音是否为空或无效 - if not py or len(py) < 1: - return py +class ChineseTypoGenerator: + def __init__(self, + error_rate=0.3, + min_freq=5, + tone_error_rate=0.2, + word_replace_rate=0.3, + max_freq_diff=200): + """ + 初始化错别字生成器 - # 如果最后一个字符不是数字，说明可能是轻声或其他特殊情况 - if not py[-1].isdigit(): - # 为非数字结尾的拼音添加数字声调1 - return py + '1' - - base = py[:-1] # 去掉声调 - tone = int(py[-1]) # 获取声调 - - # 处理轻声（通常用5表示）或无效声调 - if tone not in [1, 2, 3, 4]: - return base + str(random.choice([1, 2, 3, 4])) - - # 正常处理声调 - possible_tones = [1, 2, 3, 4] - possible_tones.remove(tone) # 移除原声调 - new_tone = random.choice(possible_tones) # 随机选择一个新声调 - return base + str(new_tone) - -def calculate_replacement_probability(orig_freq, target_freq, max_freq_diff=200): - """ - 根据频率差计算替换概率 - 频率差越大，概率越低 - :param orig_freq: 原字频率 - :param target_freq: 目标字频率 - :param max_freq_diff: 最大允许的频率差 - :return: 0-1之间的概率值 - """ - if target_freq > orig_freq: - return 1.0 # 如果替换字频率更高，保持原有概率 - - freq_diff = orig_freq - target_freq - if freq_diff > max_freq_diff: - return 0.0 # 频率差太大，不替换 - - # 使用指数衰减函数计算概率 - # 频率差为0时概率为1，频率差为max_freq_diff时概率接近0 - return math.exp(-3 * freq_diff / max_freq_diff) - -def get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, num_candidates=5, min_freq=5, tone_error_rate=0.2): - """ - 获取与给定字频率相近的同音字，可能包含声调错误 - """ - homophones = [] - - # 有20%的概率使用错误声调 - if random.random() < tone_error_rate: - wrong_tone_py = get_similar_tone_pinyin(py) - homophones.extend(pinyin_dict[wrong_tone_py]) - - # 添加正确声调的同音字 - homophones.extend(pinyin_dict[py]) - - if not homophones: - return None + 参数: + error_rate: 单字替换概率 + min_freq: 最小字频阈值 + tone_error_rate: 声调错误概率 + word_replace_rate: 整词替换概率 + max_freq_diff: 最大允许的频率差异 + """ + self.error_rate = error_rate + self.min_freq = min_freq + self.tone_error_rate = tone_error_rate + self.word_replace_rate = word_replace_rate + self.max_freq_diff = max_freq_diff - # 获取原字的频率 - orig_freq = char_frequency.get(char, 0) + # 加载数据 + print("正在加载汉字数据库，请稍候...") + self.pinyin_dict = self._create_pinyin_dict() + self.char_frequency = self._load_or_create_char_frequency() - # 计算所有同音字与原字的频率差，并过滤掉低频字 - freq_diff = [(h, char_frequency.get(h, 0)) - for h in homophones - if h != char and char_frequency.get(h, 0) >= min_freq] - - if not freq_diff: - return None - - # 计算每个候选字的替换概率 - candidates_with_prob = [] - for h, freq in freq_diff: - prob = calculate_replacement_probability(orig_freq, freq) - if prob > 0: # 只保留有效概率的候选字 - candidates_with_prob.append((h, prob)) - - if not candidates_with_prob: - return None - - # 根据概率排序 - candidates_with_prob.sort(key=lambda x: x[1], reverse=True) - - # 返回概率最高的几个字 - return [char for char, _ in candidates_with_prob[:num_candidates]] - -def get_word_pinyin(word): - """ - 获取词语的拼音列表 - """ - return [py[0] for py in pinyin(word, style=Style.TONE3)] - -def segment_sentence(sentence): - """ - 使用jieba分词，返回词语列表 - """ - return list(jieba.cut(sentence)) - -def get_word_homophones(word, pinyin_dict, char_frequency, min_freq=5): - """ - 获取整个词的同音词，只返回高频的有意义词语 - :param word: 输入词语 - :param pinyin_dict: 拼音字典 - :param char_frequency: 字频字典 - :param min_freq: 最小频率阈值 - :return: 同音词列表 - """ - if len(word) == 1: - return [] + def _load_or_create_char_frequency(self): + """ + 加载或创建汉字频率字典 + """ + cache_file = Path("char_frequency.json") - # 获取词的拼音 - word_pinyin = get_word_pinyin(word) - word_pinyin_str = ''.join(word_pinyin) - - # 创建词语频率字典 - word_freq = defaultdict(float) - - # 遍历所有可能的同音字组合 - candidates = [] - for py in word_pinyin: - chars = pinyin_dict.get(py, []) - if not chars: - return [] - candidates.append(chars) - - # 生成所有可能的组合 - import itertools - all_combinations = itertools.product(*candidates) - - # 获取jieba词典和词频信息 - dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') - valid_words = {} # 改用字典存储词语及其频率 - with open(dict_path, 'r', encoding='utf-8') as f: - for line in f: - parts = line.strip().split() - if len(parts) >= 2: - word_text = parts[0] - word_freq = float(parts[1]) # 获取词频 - valid_words[word_text] = word_freq - - # 获取原词的词频作为参考 - original_word_freq = valid_words.get(word, 0) - min_word_freq = original_word_freq * 0.1 # 设置最小词频为原词频的10% - - # 过滤和计算频率 - homophones = [] - for combo in all_combinations: - new_word = ''.join(combo) - if new_word != word and new_word in valid_words: - new_word_freq = valid_words[new_word] - # 只保留词频达到阈值的词 - if new_word_freq >= min_word_freq: - # 计算词的平均字频（考虑字频和词频） - char_avg_freq = sum(char_frequency.get(c, 0) for c in new_word) / len(new_word) - # 综合评分：结合词频和字频 - combined_score = (new_word_freq * 0.7 + char_avg_freq * 0.3) - if combined_score >= min_freq: - homophones.append((new_word, combined_score)) - - # 按综合分数排序并限制返回数量 - sorted_homophones = sorted(homophones, key=lambda x: x[1], reverse=True) - return [word for word, _ in sorted_homophones[:5]] # 限制返回前5个结果 - -def create_typo_sentence(sentence, pinyin_dict, char_frequency, error_rate=0.5, min_freq=5, tone_error_rate=0.2, word_replace_rate=0.3): - """ - 创建包含同音字错误的句子，支持词语级别和字级别的替换 - 只使用高频的有意义词语进行替换 - """ - result = [] - typo_info = [] - - # 分词 - words = segment_sentence(sentence) - - for word in words: - # 如果是标点符号或空格，直接添加 - if all(not is_chinese_char(c) for c in word): - result.append(word) - continue - - # 获取词语的拼音 - word_pinyin = get_word_pinyin(word) + # 如果缓存文件存在，直接加载 + if cache_file.exists(): + with open(cache_file, 'r', encoding='utf-8') as f: + return json.load(f) - # 尝试整词替换 - if len(word) > 1 and random.random() < word_replace_rate: - word_homophones = get_word_homophones(word, pinyin_dict, char_frequency, min_freq) - if word_homophones: - typo_word = random.choice(word_homophones) - # 计算词的平均频率 - orig_freq = sum(char_frequency.get(c, 0) for c in word) / len(word) - typo_freq = sum(char_frequency.get(c, 0) for c in typo_word) / len(typo_word) - - # 添加到结果中 - result.append(typo_word) - typo_info.append((word, typo_word, - ' '.join(word_pinyin), - ' '.join(get_word_pinyin(typo_word)), - orig_freq, typo_freq)) + # 使用内置的词频文件 + char_freq = defaultdict(int) + dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') + + # 读取jieba的词典文件 + with open(dict_path, 'r', encoding='utf-8') as f: + for line in f: + word, freq = line.strip().split()[:2] + # 对词中的每个字进行频率累加 + for char in word: + if self._is_chinese_char(char): + char_freq[char] += int(freq) + + # 归一化频率值 + max_freq = max(char_freq.values()) + normalized_freq = {char: freq/max_freq * 1000 for char, freq in char_freq.items()} + + # 保存到缓存文件 + with open(cache_file, 'w', encoding='utf-8') as f: + json.dump(normalized_freq, f, ensure_ascii=False, indent=2) + + return normalized_freq + + def _create_pinyin_dict(self): + """ + 创建拼音到汉字的映射字典 + """ + # 常用汉字范围 + chars = [chr(i) for i in range(0x4e00, 0x9fff)] + pinyin_dict = defaultdict(list) + + # 为每个汉字建立拼音映射 + for char in chars: + try: + py = pinyin(char, style=Style.TONE3)[0][0] + pinyin_dict[py].append(char) + except Exception: continue - # 如果不进行整词替换，则进行单字替换 + return pinyin_dict + + def _is_chinese_char(self, char): + """ + 判断是否为汉字 + """ + try: + return '\u4e00' <= char <= '\u9fff' + except: + return False + + def _get_pinyin(self, sentence): + """ + 将中文句子拆分成单个汉字并获取其拼音 + """ + # 将句子拆分成单个字符 + characters = list(sentence) + + # 获取每个字符的拼音 + result = [] + for char in characters: + # 跳过空格和非汉字字符 + if char.isspace() or not self._is_chinese_char(char): + continue + # 获取拼音（数字声调） + py = pinyin(char, style=Style.TONE3)[0][0] + result.append((char, py)) + + return result + + def _get_similar_tone_pinyin(self, py): + """ + 获取相似声调的拼音 + """ + # 检查拼音是否为空或无效 + if not py or len(py) < 1: + return py + + # 如果最后一个字符不是数字，说明可能是轻声或其他特殊情况 + if not py[-1].isdigit(): + # 为非数字结尾的拼音添加数字声调1 + return py + '1' + + base = py[:-1] # 去掉声调 + tone = int(py[-1]) # 获取声调 + + # 处理轻声（通常用5表示）或无效声调 + if tone not in [1, 2, 3, 4]: + return base + str(random.choice([1, 2, 3, 4])) + + # 正常处理声调 + possible_tones = [1, 2, 3, 4] + possible_tones.remove(tone) # 移除原声调 + new_tone = random.choice(possible_tones) # 随机选择一个新声调 + return base + str(new_tone) + + def _calculate_replacement_probability(self, orig_freq, target_freq): + """ + 根据频率差计算替换概率 + """ + if target_freq > orig_freq: + return 1.0 # 如果替换字频率更高，保持原有概率 + + freq_diff = orig_freq - target_freq + if freq_diff > self.max_freq_diff: + return 0.0 # 频率差太大，不替换 + + # 使用指数衰减函数计算概率 + # 频率差为0时概率为1，频率差为max_freq_diff时概率接近0 + return math.exp(-3 * freq_diff / self.max_freq_diff) + + def _get_similar_frequency_chars(self, char, py, num_candidates=5): + """ + 获取与给定字频率相近的同音字，可能包含声调错误 + """ + homophones = [] + + # 有一定概率使用错误声调 + if random.random() < self.tone_error_rate: + wrong_tone_py = self._get_similar_tone_pinyin(py) + homophones.extend(self.pinyin_dict[wrong_tone_py]) + + # 添加正确声调的同音字 + homophones.extend(self.pinyin_dict[py]) + + if not homophones: + return None + + # 获取原字的频率 + orig_freq = self.char_frequency.get(char, 0) + + # 计算所有同音字与原字的频率差，并过滤掉低频字 + freq_diff = [(h, self.char_frequency.get(h, 0)) + for h in homophones + if h != char and self.char_frequency.get(h, 0) >= self.min_freq] + + if not freq_diff: + return None + + # 计算每个候选字的替换概率 + candidates_with_prob = [] + for h, freq in freq_diff: + prob = self._calculate_replacement_probability(orig_freq, freq) + if prob > 0: # 只保留有效概率的候选字 + candidates_with_prob.append((h, prob)) + + if not candidates_with_prob: + return None + + # 根据概率排序 + candidates_with_prob.sort(key=lambda x: x[1], reverse=True) + + # 返回概率最高的几个字 + return [char for char, _ in candidates_with_prob[:num_candidates]] + + def _get_word_pinyin(self, word): + """ + 获取词语的拼音列表 + """ + return [py[0] for py in pinyin(word, style=Style.TONE3)] + + def _segment_sentence(self, sentence): + """ + 使用jieba分词，返回词语列表 + """ + return list(jieba.cut(sentence)) + + def _get_word_homophones(self, word): + """ + 获取整个词的同音词，只返回高频的有意义词语 + """ if len(word) == 1: - char = word - py = word_pinyin[0] - if random.random() < error_rate: - similar_chars = get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, - min_freq=min_freq, tone_error_rate=tone_error_rate) - if similar_chars: - typo_char = random.choice(similar_chars) - typo_freq = char_frequency.get(typo_char, 0) - orig_freq = char_frequency.get(char, 0) - replace_prob = calculate_replacement_probability(orig_freq, typo_freq) - if random.random() < replace_prob: - result.append(typo_char) - typo_py = pinyin(typo_char, style=Style.TONE3)[0][0] - typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq)) - continue - result.append(char) - else: - # 处理多字词的单字替换 - word_result = [] - for i, (char, py) in enumerate(zip(word, word_pinyin)): - # 词中的字替换概率降低 - word_error_rate = error_rate * (0.7 ** (len(word) - 1)) + return [] + + # 获取词的拼音 + word_pinyin = self._get_word_pinyin(word) + + # 遍历所有可能的同音字组合 + candidates = [] + for py in word_pinyin: + chars = self.pinyin_dict.get(py, []) + if not chars: + return [] + candidates.append(chars) + + # 生成所有可能的组合 + import itertools + all_combinations = itertools.product(*candidates) + + # 获取jieba词典和词频信息 + dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt') + valid_words = {} # 改用字典存储词语及其频率 + with open(dict_path, 'r', encoding='utf-8') as f: + for line in f: + parts = line.strip().split() + if len(parts) >= 2: + word_text = parts[0] + word_freq = float(parts[1]) # 获取词频 + valid_words[word_text] = word_freq + + # 获取原词的词频作为参考 + original_word_freq = valid_words.get(word, 0) + min_word_freq = original_word_freq * 0.1 # 设置最小词频为原词频的10% + + # 过滤和计算频率 + homophones = [] + for combo in all_combinations: + new_word = ''.join(combo) + if new_word != word and new_word in valid_words: + new_word_freq = valid_words[new_word] + # 只保留词频达到阈值的词 + if new_word_freq >= min_word_freq: + # 计算词的平均字频（考虑字频和词频） + char_avg_freq = sum(self.char_frequency.get(c, 0) for c in new_word) / len(new_word) + # 综合评分：结合词频和字频 + combined_score = (new_word_freq * 0.7 + char_avg_freq * 0.3) + if combined_score >= self.min_freq: + homophones.append((new_word, combined_score)) + + # 按综合分数排序并限制返回数量 + sorted_homophones = sorted(homophones, key=lambda x: x[1], reverse=True) + return [word for word, _ in sorted_homophones[:5]] # 限制返回前5个结果 + + def create_typo_sentence(self, sentence): + """ + 创建包含同音字错误的句子，支持词语级别和字级别的替换 + + 参数: + sentence: 输入的中文句子 + + 返回: + typo_sentence: 包含错别字的句子 + typo_info: 错别字信息列表 + """ + result = [] + typo_info = [] + + # 分词 + words = self._segment_sentence(sentence) + + for word in words: + # 如果是标点符号或空格，直接添加 + if all(not self._is_chinese_char(c) for c in word): + result.append(word) + continue - if random.random() < word_error_rate: - similar_chars = get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, - min_freq=min_freq, tone_error_rate=tone_error_rate) + # 获取词语的拼音 + word_pinyin = self._get_word_pinyin(word) + + # 尝试整词替换 + if len(word) > 1 and random.random() < self.word_replace_rate: + word_homophones = self._get_word_homophones(word) + if word_homophones: + typo_word = random.choice(word_homophones) + # 计算词的平均频率 + orig_freq = sum(self.char_frequency.get(c, 0) for c in word) / len(word) + typo_freq = sum(self.char_frequency.get(c, 0) for c in typo_word) / len(typo_word) + + # 添加到结果中 + result.append(typo_word) + typo_info.append((word, typo_word, + ' '.join(word_pinyin), + ' '.join(self._get_word_pinyin(typo_word)), + orig_freq, typo_freq)) + continue + + # 如果不进行整词替换，则进行单字替换 + if len(word) == 1: + char = word + py = word_pinyin[0] + if random.random() < self.error_rate: + similar_chars = self._get_similar_frequency_chars(char, py) if similar_chars: typo_char = random.choice(similar_chars) - typo_freq = char_frequency.get(typo_char, 0) - orig_freq = char_frequency.get(char, 0) - replace_prob = calculate_replacement_probability(orig_freq, typo_freq) + typo_freq = self.char_frequency.get(typo_char, 0) + orig_freq = self.char_frequency.get(char, 0) + replace_prob = self._calculate_replacement_probability(orig_freq, typo_freq) if random.random() < replace_prob: - word_result.append(typo_char) + result.append(typo_char) typo_py = pinyin(typo_char, style=Style.TONE3)[0][0] typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq)) continue - word_result.append(char) - result.append(''.join(word_result)) - - return ''.join(result), typo_info + result.append(char) + else: + # 处理多字词的单字替换 + word_result = [] + for i, (char, py) in enumerate(zip(word, word_pinyin)): + # 词中的字替换概率降低 + word_error_rate = self.error_rate * (0.7 ** (len(word) - 1)) + + if random.random() < word_error_rate: + similar_chars = self._get_similar_frequency_chars(char, py) + if similar_chars: + typo_char = random.choice(similar_chars) + typo_freq = self.char_frequency.get(typo_char, 0) + orig_freq = self.char_frequency.get(char, 0) + replace_prob = self._calculate_replacement_probability(orig_freq, typo_freq) + if random.random() < replace_prob: + word_result.append(typo_char) + typo_py = pinyin(typo_char, style=Style.TONE3)[0][0] + typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq)) + continue + word_result.append(char) + result.append(''.join(word_result)) + + return ''.join(result), typo_info -def format_frequency(freq): - """ - 格式化频率显示 - """ - return f"{freq:.2f}" - -def main(): - # 记录开始时间 - start_time = time.time() - - # 首先创建拼音字典和加载字频统计 - print("正在加载汉字数据库，请稍候...") - pinyin_dict = create_pinyin_dict() - char_frequency = load_or_create_char_frequency() - - # 获取用户输入 - sentence = input("请输入中文句子：") - - # 创建包含错别字的句子 - typo_sentence, typo_info = create_typo_sentence(sentence, pinyin_dict, char_frequency, - error_rate=0.3, min_freq=5, - tone_error_rate=0.2, word_replace_rate=0.3) - - # 打印结果 - print("\n原句：", sentence) - print("错字版：", typo_sentence) - - if typo_info: - print("\n错别字信息：") + def format_typo_info(self, typo_info): + """ + 格式化错别字信息 + + 参数: + typo_info: 错别字信息列表 + + 返回: + 格式化后的错别字信息字符串 + """ + if not typo_info: + return "未生成错别字" + + result = [] for orig, typo, orig_py, typo_py, orig_freq, typo_freq in typo_info: # 判断是否为词语替换 is_word = ' ' in orig_py @@ -459,25 +380,53 @@ def main(): tone_error = orig_py[:-1] == typo_py[:-1] and orig_py[-1] != typo_py[-1] error_type = "声调错误" if tone_error else "同音字替换" - print(f"原文：{orig}({orig_py}) [频率：{format_frequency(orig_freq)}] -> " - f"替换：{typo}({typo_py}) [频率：{format_frequency(typo_freq)}] [{error_type}]") + result.append(f"原文：{orig}({orig_py}) [频率：{orig_freq:.2f}] -> " + f"替换：{typo}({typo_py}) [频率：{typo_freq:.2f}] [{error_type}]") + + return "\n".join(result) - # 获取拼音结果 - result = get_pinyin(sentence) + def set_params(self, **kwargs): + """ + 设置参数 + + 可设置参数: + error_rate: 单字替换概率 + min_freq: 最小字频阈值 + tone_error_rate: 声调错误概率 + word_replace_rate: 整词替换概率 + max_freq_diff: 最大允许的频率差异 + """ + for key, value in kwargs.items(): + if hasattr(self, key): + setattr(self, key, value) + print(f"参数 {key} 已设置为 {value}") + else: + print(f"警告: 参数 {key} 不存在") + +def main(): + # 创建错别字生成器实例 + typo_generator = ChineseTypoGenerator( + error_rate=0.03, + min_freq=7, + tone_error_rate=0.02, + word_replace_rate=0.3 + ) - # 打印完整拼音 - print("\n完整拼音：") - print(" ".join(py for _, py in result)) + # 获取用户输入 + sentence = input("请输入中文句子：") - # 打印词语分析 - print("\n词语分析：") - words = segment_sentence(sentence) - for word in words: - if any(is_chinese_char(c) for c in word): - word_pinyin = get_word_pinyin(word) - print(f"词语：{word}") - print(f"拼音：{' '.join(word_pinyin)}") - print("---") + # 创建包含错别字的句子 + start_time = time.time() + typo_sentence, typo_info = typo_generator.create_typo_sentence(sentence) + + # 打印结果 + print("\n原句：", sentence) + print("错字版：", typo_sentence) + + # 打印错别字信息 + if typo_info: + print("\n错别字信息：") + print(typo_generator.format_typo_info(typo_info)) # 计算并打印总耗时 end_time = time.time() From 71e851fbd4689fd2c294da608faaedc3b1dc111d Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 00:34:58 +0800 Subject: [PATCH 22/53] fix 1 --- config/bot_config_template.toml | 1 + src/plugins/memory_system/memory.py | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index 28ffb0ce3..bc4ac18e3 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -60,6 +60,7 @@ ban_user_id = [] #禁止回复消息的QQ号 [model.llm_reasoning] #R1 name = "Pro/deepseek-ai/DeepSeek-R1" +# name = "Qwen/QwQ-32B" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index 840980783..c121ea1e3 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -672,7 +672,7 @@ class Hippocampus: first_layer, _ = self.memory_graph.get_related_item(topic, depth=1) if first_layer: # 如果记忆条数超过限制，随机选择指定数量的记忆 - if len(first_layer) > max_memory_num: + if len(first_layer) > max_memory_num/2: first_layer = random.sample(first_layer, max_memory_num) # 为每条记忆添加来源主题和相似度信息 for memory in first_layer: @@ -681,10 +681,14 @@ class Hippocampus: 'similarity': score, 'content': memory }) - + + # 如果记忆数量超过5个,随机选择5个 # 按相似度排序 relevant_memories.sort(key=lambda x: x['similarity'], reverse=True) + if len(relevant_memories) > max_memory_num: + relevant_memories = random.sample(relevant_memories, max_memory_num) + return relevant_memories From 0ebd24107750aa29fcd05dfa3c70ffe82e857633 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 01:06:36 +0800 Subject: [PATCH 23/53] =?UTF-8?q?fix:=20=E5=A2=9E=E5=8A=A0=E8=AE=BE?= =?UTF-8?q?=E7=BD=AE=E6=A8=A1=E6=9D=BF=EF=BC=8C=E4=BC=98=E5=8C=96emotion?= =?UTF-8?q?=EF=BC=8C=E4=BC=98=E5=8C=96=E5=8E=8B=E7=BC=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/bot_config_template.toml | 1 + src/plugins/chat/bot.py | 12 ++++++------ src/plugins/chat/emoji_manager.py | 25 +------------------------ src/plugins/chat/utils_image.py | 2 +- 4 files changed, 9 insertions(+), 31 deletions(-) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index bc4ac18e3..afc2b5079 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -20,6 +20,7 @@ ban_words = [ [emoji] check_interval = 120 # 检查表情包的时间间隔 register_interval = 10 # 注册表情包的时间间隔 +check_prompt = "不要包含违反公序良俗的内容" # 表情包过滤要求 [cq_code] enable_pic_translate = false diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 89c15b388..add9bf978 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -163,12 +163,6 @@ class ChatBot: message_manager.add_message(message_set) bot_response_time = tinking_time_point - emotion = await self.gpt._get_emotion_tags(raw_content) - print(f"为 '{response}' 获取到的情感标签为：{emotion}") - valuedict={ - 'happy':0.5,'angry':-1,'sad':-0.5,'surprised':0.5,'disgusted':-1.5,'fearful':-0.25,'neutral':0.25 - } - await relationship_manager.update_relationship_value(message.user_id, relationship_value=valuedict[emotion[0]]) if random() < global_config.emoji_chance: emoji_path = await emoji_manager.get_emoji_for_text(response) @@ -196,6 +190,12 @@ class ChatBot: # reply_message_id=message.message_id ) message_manager.add_message(bot_message) + emotion = await self.gpt._get_emotion_tags(raw_content) + print(f"为 '{response}' 获取到的情感标签为：{emotion}") + valuedict={ + 'happy':0.5,'angry':-1,'sad':-0.5,'surprised':0.5,'disgusted':-1.5,'fearful':-0.25,'neutral':0.25 + } + await relationship_manager.update_relationship_value(message.user_id, relationship_value=valuedict[emotion[0]]) # willing_manager.change_reply_willing_after_sent(event.group_id) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index ede0d7135..cec454e4d 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -160,27 +160,6 @@ class EmojiManager: logger.error(f"获取表情包失败: {str(e)}") return None - async def _get_emoji_tag(self, image_base64: str) -> str: - """获取表情包的标签""" - try: - prompt = '这是一个表情包，请从"happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"中选出1个情感标签。只输出标签，不要输出其他任何内容，只输出情感标签就好' - - content, _ = await self.llm.generate_response_for_image(prompt, image_base64) - tag_result = content.strip().lower() - - valid_tags = ["happy", "angry", "sad", "surprised", "disgusted", "fearful", "neutral"] - for tag_match in valid_tags: - if tag_match in tag_result or tag_match == tag_result: - return tag_match - print(f"\033[1;33m[警告]\033[0m 无效的标签: {tag_result}, 跳过") - - except Exception as e: - print(f"\033[1;31m[错误]\033[0m 获取标签失败: {str(e)}") - return "neutral" - - print(f"\033[1;32m[调试信息]\033[0m 使用默认标签: neutral") - return "neutral" # 默认标签 - async def _get_emoji_discription(self, image_base64: str) -> str: """获取表情包的标签""" try: @@ -208,7 +187,7 @@ class EmojiManager: async def _get_kimoji_for_text(self, text:str): try: - prompt = f'这是{global_config.BOT_NICKNAME}将要发送的消息内容:\n{text}\n若要为其配上表情包，请你输出这个表情包应该表达怎样的情感，应该给人什么样的感觉，不要太简洁也不要太长，注意不要输出任何对内容的分析内容，只输出\"一种什么样的感觉\"中间的形容词部分。' + prompt = f'这是{global_config.BOT_NICKNAME}将要发送的消息内容:\n{text}\n若要为其配上表情包，请你输出这个表情包应该表达怎样的情感，应该给人什么样的感觉，不要太简洁也不要太长，注意不要输出任何对消息内容的分析内容，只输出\"一种什么样的感觉\"中间的形容词部分。' content, _ = await self.lm.generate_response_async(prompt) logger.info(f"输出描述: {content}") @@ -319,7 +298,6 @@ class EmojiManager: logger.info(f"其不满足过滤规则，被剔除 {check}") continue logger.info(f"check通过 {check}") - tag = await self._get_emoji_tag(image_base64) embedding = get_embedding(discription) if discription is not None: # 准备数据库记录 @@ -328,7 +306,6 @@ class EmojiManager: 'path': image_path, 'embedding':embedding, 'discription': discription, - 'tag':tag, 'timestamp': int(time.time()) } diff --git a/src/plugins/chat/utils_image.py b/src/plugins/chat/utils_image.py index 922ab5228..9a7ef789a 100644 --- a/src/plugins/chat/utils_image.py +++ b/src/plugins/chat/utils_image.py @@ -252,7 +252,7 @@ def compress_base64_image_by_scale(base64_data: str, target_size: int = 0.8 * 10 for frame_idx in range(img.n_frames): img.seek(frame_idx) new_frame = img.copy() - new_frame = new_frame.resize((new_width, new_height), Image.Resampling.LANCZOS) + new_frame = new_frame.resize((new_width//4, new_height//4), Image.Resampling.LANCZOS) # 动图折上折 frames.append(new_frame) # 保存到缓冲区 From e0e3ee417794a1294f51d476c7ad7f8514226b4d Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Fri, 7 Mar 2025 01:31:03 +0800 Subject: [PATCH 24/53] fix: update CQCode and Message classes for async initialization and processing --- src/plugins/chat/bot.py | 1 + src/plugins/chat/cq_code.py | 71 +++++++++++++------------ src/plugins/chat/message.py | 88 ++++++++++++++++--------------- src/plugins/chat/utils.py | 3 +- src/plugins/models/utils_model.py | 52 +++++++++--------- 5 files changed, 112 insertions(+), 103 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 89c15b388..3a2d43f0e 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -58,6 +58,7 @@ class ChatBot: plain_text=event.get_plaintext(), reply_message=event.reply, ) + await message.initialize() # 过滤词 for word in global_config.ban_words: diff --git a/src/plugins/chat/cq_code.py b/src/plugins/chat/cq_code.py index 43d9d0862..0077f7295 100644 --- a/src/plugins/chat/cq_code.py +++ b/src/plugins/chat/cq_code.py @@ -10,11 +10,11 @@ from nonebot.adapters.onebot.v11 import Bot from .config import global_config import time import asyncio -from .utils_image import storage_image,storage_emoji +from .utils_image import storage_image, storage_emoji from .utils_user import get_user_nickname from ..models.utils_model import LLM_request -#解析各种CQ码 -#包含CQ码类 +# 解析各种CQ码 +# 包含CQ码类 import urllib3 from urllib3.util import create_urllib3_context from nonebot import get_driver @@ -27,6 +27,7 @@ ctx = create_urllib3_context() ctx.load_default_certs() ctx.set_ciphers("AES128-GCM-SHA256") + class TencentSSLAdapter(requests.adapters.HTTPAdapter): def __init__(self, ssl_context=None, **kwargs): self.ssl_context = ssl_context @@ -37,6 +38,7 @@ class TencentSSLAdapter(requests.adapters.HTTPAdapter): num_pools=connections, maxsize=maxsize, block=block, ssl_context=self.ssl_context) + @dataclass class CQCode: """ @@ -80,13 +82,13 @@ class CQCode: else: self.translated_plain_text = f"@某人" elif self.type == 'reply': - self.translated_plain_text = self.translate_reply() + self.translated_plain_text = await self.translate_reply() elif self.type == 'face': face_id = self.params.get('id', '') # self.translated_plain_text = f"[表情{face_id}]" self.translated_plain_text = f"[表情]" elif self.type == 'forward': - self.translated_plain_text = self.translate_forward() + self.translated_plain_text = await self.translate_forward() else: self.translated_plain_text = f"[{self.type}]" @@ -133,7 +135,7 @@ class CQCode: # 腾讯服务器特殊状态码处理 if response.status_code == 400 and 'multimedia.nt.qq.com.cn' in url: return None - + if response.status_code != 200: raise requests.exceptions.HTTPError(f"HTTP {response.status_code}") @@ -157,7 +159,7 @@ class CQCode: return None return None - + async def translate_emoji(self) -> str: """处理表情包类型的CQ码""" if 'url' not in self.params: @@ -170,11 +172,10 @@ class CQCode: return await self.get_emoji_description(base64_str) else: return '[表情包]' - - + async def translate_image(self) -> str: """处理图片类型的CQ码，区分普通图片和表情包""" - #没有url，直接返回默认文本 + # 没有url，直接返回默认文本 if 'url' not in self.params: return '[图片]' base64_str = self.get_img() @@ -206,13 +207,13 @@ class CQCode: except Exception as e: print(f"\033[1;31m[错误]\033[0m AI接口调用失败: {str(e)}") return "[图片]" - - def translate_forward(self) -> str: + + async def translate_forward(self) -> str: """处理转发消息""" try: if 'content' not in self.params: return '[转发消息]' - + # 解析content内容（需要先反转义） content = self.unescape(self.params['content']) # print(f"\033[1;34m[调试信息]\033[0m 转发消息内容: {content}") @@ -223,17 +224,17 @@ class CQCode: except ValueError as e: print(f"\033[1;31m[错误]\033[0m 解析转发消息内容失败: {str(e)}") return '[转发消息]' - + # 处理每条消息 formatted_messages = [] for msg in messages: sender = msg.get('sender', {}) nickname = sender.get('card') or sender.get('nickname', '未知用户') - + # 获取消息内容并使用Message类处理 raw_message = msg.get('raw_message', '') message_array = msg.get('message', []) - + if message_array and isinstance(message_array, list): # 检查是否包含嵌套的转发消息 for message_part in message_array: @@ -251,6 +252,7 @@ class CQCode: plain_text=raw_message, group_id=msg.get('group_id', 0) ) + await message_obj.initialize() content = message_obj.processed_plain_text else: content = '[空消息]' @@ -265,23 +267,24 @@ class CQCode: plain_text=raw_message, group_id=msg.get('group_id', 0) ) + await message_obj.initialize() content = message_obj.processed_plain_text else: content = '[空消息]' - + formatted_msg = f"{nickname}: {content}" formatted_messages.append(formatted_msg) - + # 合并所有消息 combined_messages = '\n'.join(formatted_messages) print(f"\033[1;34m[调试信息]\033[0m 合并后的转发消息: {combined_messages}") return f"[转发消息:\n{combined_messages}]" - + except Exception as e: print(f"\033[1;31m[错误]\033[0m 处理转发消息失败: {str(e)}") return '[转发消息]' - def translate_reply(self) -> str: + async def translate_reply(self) -> str: """处理回复类型的CQ码""" # 创建Message对象 @@ -289,7 +292,7 @@ class CQCode: if self.reply_message == None: # print(f"\033[1;31m[错误]\033[0m 回复消息为空") return '[回复某人消息]' - + if self.reply_message.sender.user_id: message_obj = Message( user_id=self.reply_message.sender.user_id, @@ -297,6 +300,7 @@ class CQCode: raw_message=str(self.reply_message.message), group_id=self.group_id ) + await message_obj.initialize() if message_obj.user_id == global_config.BOT_QQ: return f"[回复 {global_config.BOT_NICKNAME} 的消息: {message_obj.processed_plain_text}]" else: @@ -310,9 +314,9 @@ class CQCode: def unescape(text: str) -> str: """反转义CQ码中的特殊字符""" return text.replace(',', ',') \ - .replace('[', '[') \ - .replace(']', ']') \ - .replace('&', '&') + .replace('[', '[') \ + .replace(']', ']') \ + .replace('&', '&') @staticmethod def create_emoji_cq(file_path: str) -> str: @@ -327,12 +331,13 @@ class CQCode: abs_path = os.path.abspath(file_path) # 转义特殊字符 escaped_path = abs_path.replace('&', '&') \ - .replace('[', '[') \ - .replace(']', ']') \ - .replace(',', ',') + .replace('[', '[') \ + .replace(']', ']') \ + .replace(',', ',') # 生成CQ码，设置sub_type=1表示这是表情包 return f"[CQ:image,file=file:///{escaped_path},sub_type=1]" - + + class CQCode_tool: @staticmethod async def cq_from_dict_to_class(cq_code: Dict, reply: Optional[Dict] = None) -> CQCode: @@ -354,7 +359,7 @@ class CQCode_tool: params['text'] = cq_code.get('data', {}).get('text', '') else: params = cq_code.get('data', {}) - + instance = CQCode( type=cq_type, params=params, @@ -362,11 +367,11 @@ class CQCode_tool: user_id=0, reply_message=reply ) - + # 进行翻译处理 await instance.translate() return instance - + @staticmethod def create_reply_cq(message_id: int) -> str: """ @@ -377,6 +382,6 @@ class CQCode_tool: 回复CQ码字符串 """ return f"[CQ:reply,id={message_id}]" - - + + cq_code_tool = CQCode_tool() diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index 02f56b975..e1d36568c 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -27,56 +27,58 @@ class Message: """消息数据类""" message_id: int = None time: float = None - + group_id: int = None - group_name: str = None # 群名称 - + group_name: str = None # 群名称 + user_id: int = None user_nickname: str = None # 用户昵称 - user_cardname: str=None # 用户群昵称 - - raw_message: str = None # 原始消息，包含未解析的cq码 - plain_text: str = None # 纯文本 - + user_cardname: str = None # 用户群昵称 + + raw_message: str = None # 原始消息，包含未解析的cq码 + plain_text: str = None # 纯文本 + + reply_message: Dict = None # 存储 回复的 源消息 + + # 延迟初始化字段 + _initialized: bool = False message_segments: List[Dict] = None # 存储解析后的消息片段 processed_plain_text: str = None # 用于存储处理后的plain_text detailed_plain_text: str = None # 用于存储详细可读文本 - - reply_message: Dict = None # 存储 回复的 源消息 - - is_emoji: bool = False # 是否是表情包 - has_emoji: bool = False # 是否包含表情包 - - translate_cq: bool = True # 是否翻译cq码 - - async def __post_init__(self): - if self.time is None: - self.time = int(time.time()) - - if not self.group_name: - self.group_name = get_groupname(self.group_id) - - if not self.user_nickname: - self.user_nickname = get_user_nickname(self.user_id) - - if not self.user_cardname: - self.user_cardname=get_user_cardname(self.user_id) - - if not self.processed_plain_text: - if self.raw_message: - self.message_segments = await self.parse_message_segments(str(self.raw_message)) - self.processed_plain_text = ' '.join( - seg.translated_plain_text - for seg in self.message_segments - ) - #将详细翻译为详细可读文本 + + # 状态标志 + is_emoji: bool = False + has_emoji: bool = False + translate_cq: bool = True + + async def initialize(self): + """显式异步初始化方法（必须调用）""" + if self._initialized: + return + + # 异步获取补充信息 + self.group_name = self.group_name or get_groupname(self.group_id) + self.user_nickname = self.user_nickname or get_user_nickname(self.user_id) + self.user_cardname = self.user_cardname or get_user_cardname(self.user_id) + + # 消息解析 + if self.raw_message: + self.message_segments = await self.parse_message_segments(self.raw_message) + self.processed_plain_text = ' '.join( + seg.translated_plain_text + for seg in self.message_segments + ) + + # 构建详细文本 time_str = time.strftime("%m-%d %H:%M:%S", time.localtime(self.time)) - try: - name = f"{self.user_nickname}(ta的昵称:{self.user_cardname},ta的id:{self.user_id})" - except: - name = self.user_nickname or f"用户{self.user_id}" - content = self.processed_plain_text - self.detailed_plain_text = f"[{time_str}] {name}: {content}\n" + name = ( + f"{self.user_nickname}(ta的昵称:{self.user_cardname},ta的id:{self.user_id})" + if self.user_cardname + else f"{self.user_nickname or f'用户{self.user_id}'}" + ) + self.detailed_plain_text = f"[{time_str}] {name}: {self.processed_plain_text}\n" + + self._initialized = True async def parse_message_segments(self, message: str) -> List[CQCode]: """ diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index 38aeefd21..42c91b93c 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -131,7 +131,7 @@ def get_cloest_chat_from_db(db, length: int, timestamp: str): return '' -def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: +async def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: """从数据库获取群组最近的消息记录 Args: @@ -173,6 +173,7 @@ def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: processed_plain_text=msg_data.get("processed_text", ""), group_id=group_id ) + await msg.initialize() message_objects.append(msg) except KeyError: print("[WARNING] 数据库中存在无效的消息") diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 3e4d7f1a2..8addf6a46 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -65,7 +65,8 @@ class LLM_request: } api_url = f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}" - logger.info(f"发送请求到URL: {api_url}{self.model_name}") + logger.info(f"发送请求到URL: {api_url}") + logger.info(f"使用模型: {self.model_name}") # 构建请求体 if image_base64: @@ -81,33 +82,32 @@ class LLM_request: headers = await self._build_headers() async with session_method as session: - response = await session.post(api_url, headers=headers, json=payload) + async with session.post(api_url, headers=headers, json=payload) as response: + # 处理需要重试的状态码 + if response.status in policy["retry_codes"]: + wait_time = policy["base_wait"] * (2 ** retry) + logger.warning(f"错误码: {response.status}, 等待 {wait_time}秒后重试") + if response.status == 413: + logger.warning("请求体过大，尝试压缩...") + image_base64 = compress_base64_image_by_scale(image_base64) + payload = await self._build_payload(prompt, image_base64) + elif response.status in [500, 503]: + logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") + raise RuntimeError("服务器负载过高，模型恢复失败QAQ") + else: + logger.warning(f"请求限制(429)，等待{wait_time}秒后重试...") - # 处理需要重试的状态码 - if response.status in policy["retry_codes"]: - wait_time = policy["base_wait"] * (2 ** retry) - logger.warning(f"错误码: {response.status}, 等待 {wait_time}秒后重试") - if response.status == 413: - logger.warning("请求体过大，尝试压缩...") - image_base64 = compress_base64_image_by_scale(image_base64) - payload = await self._build_payload(prompt, image_base64) - elif response.status in [500, 503]: - logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") - raise RuntimeError("服务器负载过高，模型恢复失败QAQ") - else: - logger.warning(f"请求限制(429)，等待{wait_time}秒后重试...") + await asyncio.sleep(wait_time) + continue + elif response.status in policy["abort_codes"]: + logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") + raise RuntimeError(f"请求被拒绝: {error_code_mapping.get(response.status)}") - await asyncio.sleep(wait_time) - continue - elif response.status in policy["abort_codes"]: - logger.error(f"错误码: {response.status} - {error_code_mapping.get(response.status)}") - raise RuntimeError(f"请求被拒绝: {error_code_mapping.get(response.status)}") + response.raise_for_status() + result = await response.json() - response.raise_for_status() - result = await response.json() - - # 使用自定义处理器或默认处理 - return response_handler(result) if response_handler else self._default_response_handler(result) + # 使用自定义处理器或默认处理 + return response_handler(result) if response_handler else self._default_response_handler(result) except Exception as e: if retry < policy["max_retries"] - 1: @@ -116,7 +116,7 @@ class LLM_request: await asyncio.sleep(wait_time) else: logger.critical(f"请求失败: {str(e)}") - logger.critical(f"请求头: {self._build_headers()} 请求体: {payload}") + logger.critical(f"请求头: {await self._build_headers()} 请求体: {payload}") raise RuntimeError(f"API请求失败: {str(e)}") logger.error("达到最大重试次数，请求仍然失败") From a463f3a1a47dcbddfde38df725b57b75952313ef Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Fri, 7 Mar 2025 01:37:17 +0800 Subject: [PATCH 25/53] fix: issue (bug_risk): Reusing ClientSession across retries may lead to closed session issues. --- src/plugins/models/utils_model.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 8addf6a46..5d1f90ebb 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -74,14 +74,12 @@ class LLM_request: elif payload is None: payload = await self._build_payload(prompt) - session_method = aiohttp.ClientSession() - for retry in range(policy["max_retries"]): try: # 使用上下文管理器处理会话 headers = await self._build_headers() - async with session_method as session: + async with aiohttp.ClientSession() as session: async with session.post(api_url, headers=headers, json=payload) as response: # 处理需要重试的状态码 if response.status in policy["retry_codes"]: From b77d73ddc7b3215cb7cfeda3fbd5b0147c486709 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 01:49:42 +0800 Subject: [PATCH 26/53] =?UTF-8?q?feat:=20=E7=8E=B0=E5=9C=A8=E5=8F=AF?= =?UTF-8?q?=E4=BB=A5=E8=AE=BE=E7=BD=AE=E6=98=AF=E5=90=A6=E8=87=AA=E5=8A=A8?= =?UTF-8?q?=E4=BF=9D=E5=AD=98=E8=A1=A8=E6=83=85=E5=8C=85=E4=BA=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/bot_config_template.toml | 1 + src/plugins/chat/config.py | 2 ++ src/plugins/chat/utils_image.py | 3 +++ 3 files changed, 6 insertions(+) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index afc2b5079..4428e1512 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -20,6 +20,7 @@ ban_words = [ [emoji] check_interval = 120 # 检查表情包的时间间隔 register_interval = 10 # 注册表情包的时间间隔 +auto_save = true # 自动偷表情包 check_prompt = "不要包含违反公序良俗的内容" # 表情包过滤要求 [cq_code] diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index e044edc5e..6fb6045da 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -30,6 +30,7 @@ class BotConfig: forget_memory_interval: int = 300 # 记忆遗忘间隔（秒） EMOJI_CHECK_INTERVAL: int = 120 # 表情包检查间隔（分钟） EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） + EMOJI_SAVE: bool = True # 偷表情包 EMOJI_CHECK_PROMPT: str = "不要包含违反公序良俗的内容" # 表情包过滤要求 ban_words = set() @@ -96,6 +97,7 @@ class BotConfig: config.EMOJI_CHECK_INTERVAL = emoji_config.get("check_interval", config.EMOJI_CHECK_INTERVAL) config.EMOJI_REGISTER_INTERVAL = emoji_config.get("register_interval", config.EMOJI_REGISTER_INTERVAL) config.EMOJI_CHECK_PROMPT = emoji_config.get('check_prompt',config.EMOJI_CHECK_PROMPT) + config.EMOJI_SAVE = emoji_config.get('auto_save',config.EMOJI_SAVE) if "cq_code" in toml_dict: cq_code_config = toml_dict["cq_code"] diff --git a/src/plugins/chat/utils_image.py b/src/plugins/chat/utils_image.py index 9a7ef789a..503c2fa85 100644 --- a/src/plugins/chat/utils_image.py +++ b/src/plugins/chat/utils_image.py @@ -4,6 +4,7 @@ import hashlib import time import os from ...common.database import Database +from ..chat.config import global_config import zlib # 用于 CRC32 import base64 from nonebot import get_driver @@ -143,6 +144,8 @@ def storage_emoji(image_data: bytes) -> bytes: Returns: bytes: 原始图片数据 """ + if not global_config.EMOJI_SAVE: + return image_data try: # 使用 CRC32 计算哈希值 hash_value = format(zlib.crc32(image_data) & 0xFFFFFFFF, 'x') From 94fd4f5ddd1ac99e1484ba7dd5479594bc33294c Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 02:47:52 +0800 Subject: [PATCH 27/53] =?UTF-8?q?fix:=20=E5=AF=B92MB=E4=BB=A5=E4=B8=8B?= =?UTF-8?q?=E7=9A=84=E5=9B=BE=E7=89=87=E4=BA=88=E4=BB=A5=E6=94=BE=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/utils_image.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/plugins/chat/utils_image.py b/src/plugins/chat/utils_image.py index 503c2fa85..d79c0a913 100644 --- a/src/plugins/chat/utils_image.py +++ b/src/plugins/chat/utils_image.py @@ -230,7 +230,7 @@ def compress_base64_image_by_scale(base64_data: str, target_size: int = 0.8 * 10 image_data = base64.b64decode(base64_data) # 如果已经小于目标大小，直接返回原图 - if len(image_data) <= target_size: + if len(image_data) <= 2*1024*1024: return base64_data # 将字节数据转换为图片对象 @@ -255,7 +255,7 @@ def compress_base64_image_by_scale(base64_data: str, target_size: int = 0.8 * 10 for frame_idx in range(img.n_frames): img.seek(frame_idx) new_frame = img.copy() - new_frame = new_frame.resize((new_width//4, new_height//4), Image.Resampling.LANCZOS) # 动图折上折 + new_frame = new_frame.resize((new_width//2, new_height//2), Image.Resampling.LANCZOS) # 动图折上折 frames.append(new_frame) # 保存到缓冲区 From a3b8a545afa65596bc7a3963fd987e8a19fc9c6e Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 03:12:35 +0800 Subject: [PATCH 28/53] =?UTF-8?q?fix:=20=E7=B4=A7=E6=80=A5=E4=B8=BAcheck?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=8A=A0=E5=85=A5=E5=BC=80=E5=85=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/bot_config_template.toml | 1 + src/plugins/chat/config.py | 2 ++ src/plugins/chat/emoji_manager.py | 15 ++++++++------- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index 9a9fa2ebc..3287b3d20 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -21,6 +21,7 @@ ban_words = [ check_interval = 120 # 检查表情包的时间间隔 register_interval = 10 # 注册表情包的时间间隔 auto_save = true # 自动偷表情包 +enable_check = false # 是否启用表情包过滤 check_prompt = "不要包含违反公序良俗的内容" # 表情包过滤要求 [cq_code] diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index dbb6d7a6a..6cb8b9fee 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -31,6 +31,7 @@ class BotConfig: EMOJI_CHECK_INTERVAL: int = 120 # 表情包检查间隔（分钟） EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） EMOJI_SAVE: bool = True # 偷表情包 + EMOJI_CHECK: bool = False #是否开启过滤 EMOJI_CHECK_PROMPT: str = "不要包含违反公序良俗的内容" # 表情包过滤要求 ban_words = set() @@ -100,6 +101,7 @@ class BotConfig: config.EMOJI_REGISTER_INTERVAL = emoji_config.get("register_interval", config.EMOJI_REGISTER_INTERVAL) config.EMOJI_CHECK_PROMPT = emoji_config.get('check_prompt',config.EMOJI_CHECK_PROMPT) config.EMOJI_SAVE = emoji_config.get('auto_save',config.EMOJI_SAVE) + config.EMOJI_CHECK = emoji_config.get('enable_check',config.EMOJI_CHECK) if "cq_code" in toml_dict: cq_code_config = toml_dict["cq_code"] diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index cec454e4d..3592bd09b 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -291,13 +291,14 @@ class EmojiManager: # 获取表情包的描述 discription = await self._get_emoji_discription(image_base64) - check = await self._check_emoji(image_base64) - if '是' not in check: - os.remove(image_path) - logger.info(f"描述: {discription}") - logger.info(f"其不满足过滤规则，被剔除 {check}") - continue - logger.info(f"check通过 {check}") + if global_config.EMOJI_CHECK: + check = await self._check_emoji(image_base64) + if '是' not in check: + os.remove(image_path) + logger.info(f"描述: {discription}") + logger.info(f"其不满足过滤规则，被剔除 {check}") + continue + logger.info(f"check通过 {check}") embedding = get_embedding(discription) if discription is not None: # 准备数据库记录 From a7fedba79effd1d7e9ee75ba59f7574361ef5e7d Mon Sep 17 00:00:00 2001 From: KawaiiYusora Date: Fri, 7 Mar 2025 03:36:37 +0800 Subject: [PATCH 29/53] =?UTF-8?q?fix:=20=E9=82=A3=E6=88=91=E9=97=AE?= =?UTF-8?q?=E4=BD=A0=E9=82=A3=E6=88=91=E9=97=AE=E4=BD=A0=20get=5Fembedding?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E4=BD=BF=E7=94=A8=E5=8D=8F=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 3592bd09b..9bd71ddd8 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -299,7 +299,7 @@ class EmojiManager: logger.info(f"其不满足过滤规则，被剔除 {check}") continue logger.info(f"check通过 {check}") - embedding = get_embedding(discription) + embedding = await get_embedding(discription) if discription is not None: # 准备数据库记录 emoji_record = { From 0ced4939ec20f4185106f83d221273f7678d4f94 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 03:40:14 +0800 Subject: [PATCH 30/53] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9embedding?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/bot_config_template.toml | 2 +- src/plugins/chat/config.py | 2 +- src/plugins/chat/emoji_manager.py | 4 ++-- src/plugins/memory_system/memory.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/config/bot_config_template.toml b/config/bot_config_template.toml index 3287b3d20..507c6d2d6 100644 --- a/config/bot_config_template.toml +++ b/config/bot_config_template.toml @@ -22,7 +22,7 @@ check_interval = 120 # 检查表情包的时间间隔 register_interval = 10 # 注册表情包的时间间隔 auto_save = true # 自动偷表情包 enable_check = false # 是否启用表情包过滤 -check_prompt = "不要包含违反公序良俗的内容" # 表情包过滤要求 +check_prompt = "符合公序良俗" # 表情包过滤要求 [cq_code] enable_pic_translate = false diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index 6cb8b9fee..a2adc9e30 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -32,7 +32,7 @@ class BotConfig: EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） EMOJI_SAVE: bool = True # 偷表情包 EMOJI_CHECK: bool = False #是否开启过滤 - EMOJI_CHECK_PROMPT: str = "不要包含违反公序良俗的内容" # 表情包过滤要求 + EMOJI_CHECK_PROMPT: str = "符合公序良俗" # 表情包过滤要求 ban_words = set() diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 3592bd09b..1cdb62c07 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -98,7 +98,7 @@ class EmojiManager: # 获取文本的embedding text_for_search= await self._get_kimoji_for_text(text) - text_embedding = get_embedding(text_for_search) + text_embedding = await get_embedding(text_for_search) if not text_embedding: logger.error("无法获取文本的embedding") return None @@ -299,7 +299,7 @@ class EmojiManager: logger.info(f"其不满足过滤规则，被剔除 {check}") continue logger.info(f"check通过 {check}") - embedding = get_embedding(discription) + embedding = await get_embedding(discription) if discription is not None: # 准备数据库记录 emoji_record = { diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index 49d19c253..a25e15bdf 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -673,7 +673,7 @@ class Hippocampus: if first_layer: # 如果记忆条数超过限制，随机选择指定数量的记忆 if len(first_layer) > max_memory_num/2: - first_layer = random.sample(first_layer, max_memory_num) + first_layer = random.sample(first_layer, max_memory_num//2) # 为每条记忆添加来源主题和相似度信息 for memory in first_layer: relevant_memories.append({ From d0047e82bf75386b2845776d7b0cc7aac1646021 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 04:01:09 +0800 Subject: [PATCH 31/53] =?UTF-8?q?fix:=20=E5=8E=BB=E9=99=A4emoji=5Fmanager?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=9B=BE=E7=89=87=E5=8E=8B=E7=BC=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 72 +------------------------------ src/plugins/chat/utils_image.py | 17 +++++++- 2 files changed, 18 insertions(+), 71 deletions(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 1cdb62c07..4784c0a3d 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -20,6 +20,7 @@ import traceback from nonebot import get_driver from ..chat.config import global_config from ..models.utils_model import LLM_request +from ..chat.utils_image import image_path_to_base64 from ..chat.utils import get_embedding driver = get_driver() @@ -196,76 +197,7 @@ class EmojiManager: except Exception as e: logger.error(f"获取标签失败: {str(e)}") return None - - async def _compress_image(self, image_path: str, target_size: int = 0.8 * 1024 * 1024) -> Optional[str]: - """压缩图片并返回base64编码 - Args: - image_path: 图片文件路径 - target_size: 目标文件大小（字节），默认0.8MB - Returns: - Optional[str]: 成功返回base64编码的图片数据，失败返回None - """ - try: - file_size = os.path.getsize(image_path) - if file_size <= target_size: - # 如果文件已经小于目标大小，直接读取并返回base64 - with open(image_path, 'rb') as f: - return base64.b64encode(f.read()).decode('utf-8') - - # 打开图片 - with Image.open(image_path) as img: - # 获取原始尺寸 - original_width, original_height = img.size - - # 计算缩放比例 - scale = min(1.0, (target_size / file_size) ** 0.5) - - # 计算新的尺寸 - new_width = int(original_width * scale) - new_height = int(original_height * scale) - - # 创建内存缓冲区 - output_buffer = io.BytesIO() - - # 如果是GIF，处理所有帧 - if getattr(img, "is_animated", False): - frames = [] - for frame_idx in range(img.n_frames): - img.seek(frame_idx) - new_frame = img.copy() - new_frame = new_frame.resize((new_width, new_height), Image.Resampling.LANCZOS) - frames.append(new_frame) - # 保存到缓冲区 - frames[0].save( - output_buffer, - format='GIF', - save_all=True, - append_images=frames[1:], - optimize=True, - duration=img.info.get('duration', 100), - loop=img.info.get('loop', 0) - ) - else: - # 处理静态图片 - resized_img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) - - # 保存到缓冲区，保持原始格式 - if img.format == 'PNG' and img.mode in ('RGBA', 'LA'): - resized_img.save(output_buffer, format='PNG', optimize=True) - else: - resized_img.save(output_buffer, format='JPEG', quality=95, optimize=True) - - # 获取压缩后的数据并转换为base64 - compressed_data = output_buffer.getvalue() - logger.success(f"压缩图片: {os.path.basename(image_path)} ({original_width}x{original_height} -> {new_width}x{new_height})") - - return base64.b64encode(compressed_data).decode('utf-8') - - except Exception as e: - logger.error(f"压缩图片失败: {os.path.basename(image_path)}, 错误: {str(e)}") - return None - async def scan_new_emojis(self): """扫描新的表情包""" try: @@ -284,7 +216,7 @@ class EmojiManager: continue # 压缩图片并获取base64编码 - image_base64 = await self._compress_image(image_path) + image_base64 = image_path_to_base64(image_path) if image_base64 is None: os.remove(image_path) continue diff --git a/src/plugins/chat/utils_image.py b/src/plugins/chat/utils_image.py index d79c0a913..eff788868 100644 --- a/src/plugins/chat/utils_image.py +++ b/src/plugins/chat/utils_image.py @@ -289,4 +289,19 @@ def compress_base64_image_by_scale(base64_data: str, target_size: int = 0.8 * 10 logger.error(f"压缩图片失败: {str(e)}") import traceback logger.error(traceback.format_exc()) - return base64_data \ No newline at end of file + return base64_data + +def image_path_to_base64(image_path: str) -> str: + """将图片路径转换为base64编码 + Args: + image_path: 图片文件路径 + Returns: + str: base64编码的图片数据 + """ + try: + with open(image_path, 'rb') as f: + image_data = f.read() + return base64.b64encode(image_data).decode('utf-8') + except Exception as e: + logger.error(f"读取图片失败: {image_path}, 错误: {str(e)}") + return None \ No newline at end of file From bb35faa363fc140b368211974c04bffafda8165a Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 04:07:26 +0800 Subject: [PATCH 32/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=A1=A8?= =?UTF-8?q?=E6=83=85=E6=9F=A5=E8=AF=A2=E4=B8=BA=E7=A9=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 4784c0a3d..db60eb099 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -99,6 +99,9 @@ class EmojiManager: # 获取文本的embedding text_for_search= await self._get_kimoji_for_text(text) + if not text_for_search: + logger.error("无法获取文本的情绪") + return None text_embedding = await get_embedding(text_for_search) if not text_embedding: logger.error("无法获取文本的embedding") From dac57cf154b8d891812dcf1fb7f623d96e909e55 Mon Sep 17 00:00:00 2001 From: Rikki Date: Fri, 7 Mar 2025 05:31:55 +0800 Subject: [PATCH 33/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E5=AE=B9?= =?UTF-8?q?=E5=99=A8=E9=87=8D=E5=90=AF=E5=90=8E=20bot=5Fconfig.toml=20?= =?UTF-8?q?=E4=BC=9A=E8=A2=AB=E8=A6=86=E7=9B=96=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bot.py | 4 ++-- {config => templete}/auto_format.py | 0 {config => templete}/bot_config_template.toml | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename {config => templete}/auto_format.py (100%) rename {config => templete}/bot_config_template.toml (100%) diff --git a/bot.py b/bot.py index 8ef087476..d0922e4d6 100644 --- a/bot.py +++ b/bot.py @@ -17,11 +17,11 @@ print(rainbow_text) '''彩蛋''' # 初次启动检测 -if not os.path.exists("config/bot_config.toml") or not os.path.exists(".env"): +if not os.path.exists("config/bot_config.toml"): logger.info("检测到bot_config.toml不存在，正在从模板复制") import shutil - shutil.copy("config/bot_config_template.toml", "config/bot_config.toml") + shutil.copy("templete/bot_config_template.toml", "config/bot_config.toml") logger.info("复制完成，请修改config/bot_config.toml和.env.prod中的配置后重新启动") # 初始化.env 默认ENVIRONMENT=prod diff --git a/config/auto_format.py b/templete/auto_format.py similarity index 100% rename from config/auto_format.py rename to templete/auto_format.py diff --git a/config/bot_config_template.toml b/templete/bot_config_template.toml similarity index 100% rename from config/bot_config_template.toml rename to templete/bot_config_template.toml From b69c9ac7f742e0e5f260e04f10edf2a32a58bf5d Mon Sep 17 00:00:00 2001 From: Rikki Date: Fri, 7 Mar 2025 05:50:59 +0800 Subject: [PATCH 34/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E4=B8=8D?= =?UTF-8?q?=E5=BA=94=E7=94=A8=20embedding=20=E6=A8=A1=E5=9E=8B=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E7=9A=84=E7=8E=B0=E8=B1=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 793a89290..f06e94076 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -317,12 +317,11 @@ class LLM_request: logger.error("达到最大重试次数，请求仍然失败") raise RuntimeError("达到最大重试次数，API请求仍然失败") - def get_embedding_sync(self, text: str, model: str = "BAAI/bge-m3") -> Union[list, None]: + def get_embedding_sync(self, text: str) -> Union[list, None]: """同步方法：获取文本的embedding向量 Args: text: 需要获取embedding的文本 - model: 使用的模型名称，默认为"BAAI/bge-m3" Returns: list: embedding向量，如果失败则返回None @@ -333,7 +332,7 @@ class LLM_request: } data = { - "model": model, + "model": self.model_name, "input": text, "encoding_format": "float" } From 9bb9fc1b28bb0de342705f177be1aa63f8daf950 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 07:03:05 +0800 Subject: [PATCH 35/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8Dbot=E6=B6=88?= =?UTF-8?q?=E6=81=AF=E6=9C=AA=E8=B0=83=E7=94=A8=E5=BC=82=E6=AD=A5=E6=98=BE?= =?UTF-8?q?=E5=BC=8F=E5=88=9D=E5=A7=8B=E5=8C=96=E5=AF=BC=E8=87=B4=E7=9A=84?= =?UTF-8?q?=E7=A9=BA=E6=B6=88=E6=81=AFbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/bot.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 8882a480c..b927472e5 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -154,6 +154,7 @@ class ChatBot: thinking_start_time=thinking_start_time, #记录了思考开始的时间 reply_message_id=message.message_id ) + await bot_message.initialize() if not mark_head: bot_message.is_head = True mark_head = True @@ -190,6 +191,7 @@ class ChatBot: thinking_start_time=thinking_start_time, # reply_message_id=message.message_id ) + await bot_message.initialize() message_manager.add_message(bot_message) emotion = await self.gpt._get_emotion_tags(raw_content) print(f"为 '{response}' 获取到的情感标签为：{emotion}") From 965b1d13983f02a1a74432a50c9116f5b90dba59 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 07:23:17 +0800 Subject: [PATCH 36/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8E=A8?= =?UTF-8?q?=E7=90=86=E8=BF=87=E7=A8=8B=E6=B2=A1=E6=9C=89=E8=A2=AB=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/models/utils_model.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index abc6f027b..b377eac14 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -153,7 +153,9 @@ class LLM_request: content, reasoning = self._extract_reasoning(content) reasoning_content = message.get("model_extra", {}).get("reasoning_content", "") if not reasoning_content: - reasoning_content = reasoning + reasoning_content = message.get("reasoning_content", "") + if not reasoning_content: + reasoning_content = reasoning return content, reasoning_content From 5bf94d34a1d63c08743d6ee89f45c4312553b805 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 08:41:46 +0800 Subject: [PATCH 37/53] =?UTF-8?q?fix:=20=E4=BF=AE=E6=94=B9message=5Fsendin?= =?UTF-8?q?g=E4=B8=ADcq=E7=A0=81=E8=A2=AB=E7=BF=BB=E8=AF=91=E7=9A=84?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/emoji_manager.py | 2 +- src/plugins/chat/message.py | 11 ++++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index db60eb099..c31937624 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -42,7 +42,7 @@ class EmojiManager: self.db = Database.get_instance() self._scan_task = None self.llm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=1000) - self.lm = LLM_request(model=global_config.llm_reasoning_minor, max_tokens=1000) + self.lm = LLM_request(model=global_config.llm_normal_minor, max_tokens=1000) def _ensure_emoji_dir(self): """确保表情存储目录存在""" diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index e1d36568c..af9f93013 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -63,11 +63,12 @@ class Message: # 消息解析 if self.raw_message: - self.message_segments = await self.parse_message_segments(self.raw_message) - self.processed_plain_text = ' '.join( - seg.translated_plain_text - for seg in self.message_segments - ) + if not isinstance(self,Message_Sending): + self.message_segments = await self.parse_message_segments(self.raw_message) + self.processed_plain_text = ' '.join( + seg.translated_plain_text + for seg in self.message_segments + ) # 构建详细文本 time_str = time.strftime("%m-%d %H:%M:%S", time.localtime(self.time)) From a41b8275daa5f49ef0a8cdeed911067fed2839ce Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 09:03:57 +0800 Subject: [PATCH 38/53] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E6=96=87?= =?UTF-8?q?=E6=A1=A3=E7=BB=93=E6=9E=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 123 --- README.md | 47 ++++++++++++---- docs/docker_deploy.md | 24 +++++++++ docs/installation.md | 54 ------------------- docs/manual_deploy.md | 122 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 184 insertions(+), 63 deletions(-) create mode 100644 docs/docker_deploy.md create mode 100644 docs/manual_deploy.md diff --git a/README.md b/README.md index 96c857bc7..f6a903c6f 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,19 @@ **🍔麦麦是一个基于大语言模型的智能QQ群聊机器人** -- 🤖 基于 nonebot2 框架开发 -- 🧠 LLM 提供对话能力 -- 💾 MongoDB 提供数据持久化支持 -- 🐧 NapCat 作为QQ协议端支持 +- 基于 nonebot2 框架开发 +- LLM 提供对话能力 +- MongoDB 提供数据持久化支持 +- NapCat 作为QQ协议端支持 -**最新版本: v0.5.7** +**最新版本: v0.5.*** @@ -35,11 +36,28 @@ > - 由于持续迭代，可能存在一些已知或未知的bug > - 由于开发中，可能消耗较多token -**交流群**: 766798517（仅用于开发和建议相关讨论）不建议在群内询问部署问题，我不一定有空回复，会优先写文档和代码 +**交流群**: 766798517（仅用于开发和建议相关讨论）不一定有空回复，但大家可以自行交流部署问题，我会优先写文档和代码 -## 📚 文档 +## +
+

📚 文档 ⬇️ 快速开始使用麦麦 ⬇️

+
+ +### 部署方式 + +如果你不知道Docker是什么，建议寻找相关教程或使用手动部署 + +- [🐳 Docker部署指南](docs/docker_deploy.md) + +- [📦 手动部署指南](docs/manual_deploy.md) + +### 配置说明 +- [⚙️ 配置详细说明](docs/installation.md) - 环境和机器人配置的详细说明 + +
+

了解麦麦

+
-- [安装与配置指南](docs/installation.md) - 详细的部署和配置说明 - [项目架构说明](docs/doc1.md) - 项目结构和核心功能实现细节 ## 🎯 功能介绍 @@ -92,8 +110,19 @@ - 改进发送消息的触发 - + +## 设计理念 + +- **千石可乐说：** +- 这个项目最初只是为了给牛牛bot添加一点额外的功能，但是功能越写越多，最后决定重写。其目的是为了创造一个活跃在QQ群聊的"生命体"。可以目的并不是为了写一个功能齐全的机器人，而是一个尽可能让人感知到真实的类人存在. +- 程序的功能设计理念基于一个核心的原则："最像而不是好" +- 主打一个陪伴 +- 如果人类真的需要一个AI来陪伴自己，并不是所有人都需要一个完美的，能解决所有问题的helpful assistant，而是一个会犯错的，拥有自己感知和想法的"生命形式"。 +- 代码会保持开源和开放，但个人希望MaiMbot的运行时数据保持封闭，尽量避免以显式命令来对其进行控制和调试.我认为一个你无法完全掌控的个体才更能让你感觉到它的自主性，而视其成为一个对话机器. + + ## 📌 注意事项 -纯编程外行，面向cursor编程，很多代码史一样多多包涵 +SengokuCola纯编程外行，面向cursor编程，很多代码史一样多多包涵 > ⚠️ **警告**：本应用生成内容来自人工智能模型，由 AI 生成，请仔细甄别，请勿用于违反法律的用途，AI生成内容不代表本人观点和立场。 diff --git a/docs/docker_deploy.md b/docs/docker_deploy.md new file mode 100644 index 000000000..c9b069309 --- /dev/null +++ b/docs/docker_deploy.md @@ -0,0 +1,24 @@ +# 🐳 Docker 部署指南 + +## 部署步骤（推荐，但不一定是最新） + +1. 获取配置文件： +```bash +wget https://raw.githubusercontent.com/SengokuCola/MaiMBot/main/docker-compose.yml +``` + +2. 启动服务： +```bash +NAPCAT_UID=$(id -u) NAPCAT_GID=$(id -g) docker compose up -d +``` + +3. 修改配置后重启： +```bash +NAPCAT_UID=$(id -u) NAPCAT_GID=$(id -g) docker compose restart +``` + +## ⚠️ 注意事项 + +- 目前部署方案仍在测试中，可能存在未知问题 +- 配置文件中的API密钥请妥善保管，不要泄露 +- 建议先在测试环境中运行，确认无误后再部署到生产环境 \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index c988eb7c9..ff891f61a 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,60 +1,6 @@ # 🔧 安装与配置指南 -## 部署方式 -如果你不知道Docker是什么，建议寻找相关教程或使用手动部署 - -### 🐳 Docker部署（推荐，但不一定是最新） - -1. 获取配置文件： -```bash -wget https://raw.githubusercontent.com/SengokuCola/MaiMBot/main/docker-compose.yml -``` - -2. 启动服务： -```bash -NAPCAT_UID=$(id -u) NAPCAT_GID=$(id -g) docker compose up -d -``` - -3. 修改配置后重启： -```bash -NAPCAT_UID=$(id -u) NAPCAT_GID=$(id -g) docker compose restart -``` - -### 📦 手动部署 - -1. **环境准备** -```bash -# 创建虚拟环境（推荐） -python -m venv venv -venv\\Scripts\\activate # Windows -# 安装依赖 -pip install -r requirements.txt -``` - -2. **配置MongoDB** -- 安装并启动MongoDB服务 -- 默认连接本地27017端口 - -3. **配置NapCat** -- 安装并登录NapCat -- 添加反向WS：`ws://localhost:8080/onebot/v11/ws` - -4. **配置文件设置** -- 修改环境配置文件：`.env.prod` -- 修改机器人配置文件：`bot_config.toml` - -5. **启动麦麦机器人** -- 打开命令行，cd到对应路径 -```bash -nb run -``` - -6. **其他组件** -- `run_thingking.bat`: 启动可视化推理界面（未完善） - -- ~~`knowledge.bat`: 将`/data/raw_info`下的文本文档载入数据库~~ -- 直接运行 knowledge.py生成知识库 ## ⚙️ 配置说明 diff --git a/docs/manual_deploy.md b/docs/manual_deploy.md new file mode 100644 index 000000000..2ea2a3d7f --- /dev/null +++ b/docs/manual_deploy.md @@ -0,0 +1,122 @@ +# 📦 手动部署指南 + +## 部署步骤 + +1. **环境准备** +```bash +# 创建虚拟环境（推荐） +python -m venv venv +venv\\Scripts\\activate # Windows +# 安装依赖 +pip install -r requirements.txt +``` + +2. **配置MongoDB** +- 安装并启动MongoDB服务 +- 默认连接本地27017端口 + +3. **配置NapCat** +- 安装并登录NapCat +- 添加反向WS：`ws://localhost:8080/onebot/v11/ws` + +4. **配置文件设置** +- 修改环境配置文件：`.env.prod` +- 修改机器人配置文件：`bot_config.toml` + +5. **启动麦麦机器人** +- 打开命令行，cd到对应路径 +```bash +nb run +``` + +6. **其他组件** +- `run_thingking.bat`: 启动可视化推理界面（未完善） +- 直接运行 knowledge.py生成知识库 + +## ⚙️ 配置说明 + +### 环境配置 (.env.prod) +```ini +# API配置,你可以在这里定义你的密钥和base_url +# 你可以选择定义其他服务商提供的KEY，完全可以自定义 +SILICONFLOW_KEY=your_key +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ +DEEP_SEEK_KEY=your_key +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 + +# 服务配置,如果你不知道这是什么，保持默认 +HOST=127.0.0.1 +PORT=8080 + +# 数据库配置,如果你不知道这是什么，保持默认 +MONGODB_HOST=127.0.0.1 +MONGODB_PORT=27017 +DATABASE_NAME=MegBot +``` + +### 机器人配置 (bot_config.toml) +```toml +[bot] +qq = "你的机器人QQ号" +nickname = "麦麦" + +[message] +min_text_length = 2 +max_context_size = 15 +emoji_chance = 0.2 + +[emoji] +check_interval = 120 +register_interval = 10 + +[cq_code] +enable_pic_translate = false + +[response] +#现已移除deepseek或硅基流动选项，可以直接切换分别配置任意模型 +model_r1_probability = 0.8 #推理模型权重 +model_v3_probability = 0.1 #非推理模型权重 +model_r1_distill_probability = 0.1 + +[memory] +build_memory_interval = 300 + +[others] +enable_advance_output = true # 是否启用详细日志输出 + +[groups] +talk_allowed = [] # 允许回复的群号列表 +talk_frequency_down = [] # 降低回复频率的群号列表 +ban_user_id = [] # 禁止回复的用户QQ号列表 + +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_reasoning_minor] +name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal_minor] +name = "deepseek-ai/DeepSeek-V2.5" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.vlm] +name = "deepseek-ai/deepseek-vl2" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" +``` + +## ⚠️ 注意事项 + +- 目前部署方案仍在测试中，可能存在未知问题 +- 配置文件中的API密钥请妥善保管，不要泄露 +- 建议先在测试环境中运行，确认无误后再部署到生产环境 \ No newline at end of file From e48d93b7cb3685587e338b47cf12f96759a01eca Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 09:24:25 +0800 Subject: [PATCH 39/53] =?UTF-8?q?feat:=20=E5=AF=B9=E8=87=AA=E5=B7=B1?= =?UTF-8?q?=E5=8F=91=E5=87=BA=E7=9A=84=E8=A1=A8=E6=83=85=E5=8C=85=E6=8F=90?= =?UTF-8?q?=E4=BE=9B=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/bot.py | 3 ++- src/plugins/chat/emoji_manager.py | 2 +- src/plugins/chat/message.py | 5 ++++- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index b927472e5..73296c1da 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -167,7 +167,7 @@ class ChatBot: bot_response_time = tinking_time_point if random() < global_config.emoji_chance: - emoji_path = await emoji_manager.get_emoji_for_text(response) + emoji_path,discription = await emoji_manager.get_emoji_for_text(response) if emoji_path: emoji_cq = CQCode.create_emoji_cq(emoji_path) @@ -183,6 +183,7 @@ class ChatBot: raw_message=emoji_cq, plain_text=emoji_cq, processed_plain_text=emoji_cq, + detailed_plain_text=discription, user_nickname=global_config.BOT_NICKNAME, group_name=message.group_name, time=bot_response_time, diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index c31937624..3ba167308 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -152,7 +152,7 @@ class EmojiManager: {'$inc': {'usage_count': 1}} ) logger.success(f"找到匹配的表情包: {selected_emoji.get('discription', '无描述')} (相似度: {similarity:.4f})") - return selected_emoji['path'] + return selected_emoji['path'],"[表情包: %s]" % selected_emoji.get('discription', '无描述') except Exception as search_error: logger.error(f"搜索表情包失败: {str(search_error)}") diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index af9f93013..4ac574e66 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -77,7 +77,10 @@ class Message: if self.user_cardname else f"{self.user_nickname or f'用户{self.user_id}'}" ) - self.detailed_plain_text = f"[{time_str}] {name}: {self.processed_plain_text}\n" + if isinstance(self,Message_Sending) and self.is_emoji: + self.detailed_plain_text = f"[{time_str}] {name}: {self.detailed_plain_text}\n" + else: + self.detailed_plain_text = f"[{time_str}] {name}: {self.processed_plain_text}\n" self._initialized = True From 63d8a2cc7d486893eb9645d9da2a341a8167f2a6 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 10:09:56 +0800 Subject: [PATCH 40/53] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=86=E5=AE=9D?= =?UTF-8?q?=E5=AE=9D=E5=B7=B4=E5=A3=AB=E7=89=88=E9=85=8D=E7=BD=AE=E6=95=99?= =?UTF-8?q?=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 3 +- docs/installation.md | 226 +++++++++++++++++++++++++++------- docs/installation_cute.md | 3 + docs/installation_standard.md | 156 +++++++++++++++++++++++ docs/manual_deploy.md | 160 +++++++++--------------- 5 files changed, 402 insertions(+), 146 deletions(-) create mode 100644 docs/installation_cute.md create mode 100644 docs/installation_standard.md diff --git a/README.md b/README.md index f6a903c6f..65263b1a9 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,8 @@ - [📦 手动部署指南](docs/manual_deploy.md) ### 配置说明 -- [⚙️ 配置详细说明](docs/installation.md) - 环境和机器人配置的详细说明 +- [🎀 新手配置指南](docs/installation_cute.md) - 通俗易懂的配置教程，适合初次使用的猫娘 +- [⚙️ 标准配置指南](docs/installation_standard.md) - 简明专业的配置说明，适合有经验的用户

了解麦麦

diff --git a/docs/installation.md b/docs/installation.md index ff891f61a..a1334a2eb 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -1,91 +1,227 @@ -# 🔧 安装与配置指南 +# 🔧 配置指南 喵~ +## 👋 你好呀！ +让咱来告诉你我们要做什么喵： +1. 我们要一起设置一个可爱的AI机器人 +2. 这个机器人可以在QQ上陪你聊天玩耍哦 +3. 需要设置两个文件才能让机器人工作呢 -## ⚙️ 配置说明 +## 📝 需要设置的文件喵 -### 环境配置 (.env.prod) +要设置这两个文件才能让机器人跑起来哦： +1. `.env.prod` - 这个文件告诉机器人要用哪些AI服务呢 +2. `bot_config.toml` - 这个文件教机器人怎么和你聊天喵 + +## 🔑 密钥和域名的对应关系 + +想象一下，你要进入一个游乐园，需要： +1. 知道游乐园的地址（这就是域名 base_url） +2. 有入场的门票（这就是密钥 key） + +在 `.env.prod` 文件里，我们定义了三个游乐园的地址和门票喵： ```ini -# API配置,你可以在这里定义你的密钥和base_url -# 你可以选择定义其他服务商提供的KEY，完全可以自定义 +# 硅基流动游乐园 +SILICONFLOW_KEY=your_key # 硅基流动的门票 +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ # 硅基流动的地址 + +# DeepSeek游乐园 +DEEP_SEEK_KEY=your_key # DeepSeek的门票 +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 # DeepSeek的地址 + +# ChatAnyWhere游乐园 +CHAT_ANY_WHERE_KEY=your_key # ChatAnyWhere的门票 +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 # ChatAnyWhere的地址 +``` + +然后在 `bot_config.toml` 里，机器人会用这些门票和地址去游乐园玩耍： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "SILICONFLOW_BASE_URL" # 告诉机器人：去硅基流动游乐园玩 +key = "SILICONFLOW_KEY" # 用硅基流动的门票进去 + +[model.llm_normal] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" # 还是去硅基流动游乐园 +key = "SILICONFLOW_KEY" # 用同一张门票就可以啦 +``` + +### 🎪 举个例子喵： + +如果你想用DeepSeek官方的服务，就要这样改： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "DEEP_SEEK_BASE_URL" # 改成去DeepSeek游乐园 +key = "DEEP_SEEK_KEY" # 用DeepSeek的门票 + +[model.llm_normal] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "DEEP_SEEK_BASE_URL" # 也去DeepSeek游乐园 +key = "DEEP_SEEK_KEY" # 用同一张DeepSeek门票 +``` + +### 🎯 简单来说： +- `.env.prod` 文件就像是你的票夹，存放着各个游乐园的门票和地址 +- `bot_config.toml` 就是告诉机器人：用哪张票去哪个游乐园玩 +- 所有模型都可以用同一个游乐园的票，也可以去不同的游乐园玩耍 +- 如果用硅基流动的服务，就保持默认配置不用改呢~ + +记住：门票（key）要保管好，不能给别人看哦，不然别人就可以用你的票去玩了喵！ + +## ---让我们开始吧--- + +### 第一个文件：环境配置 (.env.prod) + +这个文件就像是机器人的"身份证"呢，告诉它要用哪些AI服务喵~ + +```ini +# 这些是AI服务的密钥，就像是魔法钥匙一样呢 +# 要把 your_key 换成真正的密钥才行喵 +# 比如说：SILICONFLOW_KEY=sk-123456789abcdef SILICONFLOW_KEY=your_key SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ DEEP_SEEK_KEY=your_key DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 +CHAT_ANY_WHERE_KEY=your_key +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 -# 服务配置,如果你不知道这是什么，保持默认 +# 如果你不知道这是什么，那么下面这些不用改，保持原样就好啦 HOST=127.0.0.1 PORT=8080 -# 数据库配置,如果你不知道这是什么，保持默认 +# 这些是数据库设置，一般也不用改呢 MONGODB_HOST=127.0.0.1 MONGODB_PORT=27017 DATABASE_NAME=MegBot +MONGODB_USERNAME = "" # 如果数据库需要用户名，就在这里填写喵 +MONGODB_PASSWORD = "" # 如果数据库需要密码，就在这里填写呢 +MONGODB_AUTH_SOURCE = "" # 数据库认证源，一般不用改哦 + +# 插件设置喵 +PLUGINS=["src2.plugins.chat"] # 这里是机器人的插件列表呢 ``` -### 机器人配置 (bot_config.toml) +### 第二个文件：机器人配置 (bot_config.toml) + +这个文件就像是教机器人"如何说话"的魔法书呢！ + ```toml [bot] -qq = "你的机器人QQ号" -nickname = "麦麦" +qq = "把这里改成你的机器人QQ号喵" # 填写你的机器人QQ号 +nickname = "麦麦" # 机器人的名字，你可以改成你喜欢的任何名字哦 + +[personality] +# 这里可以设置机器人的性格呢，让它更有趣一些喵 +prompt_personality = [ + "曾经是一个学习地质的女大学生，现在学习心理学和脑科学，你会刷贴吧", # 贴吧风格的性格 + "是一个女大学生，你有黑色头发，你会刷小红书" # 小红书风格的性格 +] +prompt_schedule = "一个曾经学习地质,现在学习心理学和脑科学的女大学生，喜欢刷qq，贴吧，知乎和小红书" [message] -min_text_length = 2 -max_context_size = 15 -emoji_chance = 0.2 +min_text_length = 2 # 机器人每次至少要说几个字呢 +max_context_size = 15 # 机器人能记住多少条消息喵 +emoji_chance = 0.2 # 机器人使用表情的概率哦（0.2就是20%的机会呢） +ban_words = ["脏话", "不文明用语"] # 在这里填写不让机器人说的词 [emoji] -check_interval = 120 -register_interval = 10 - -[cq_code] -enable_pic_translate = false - -[response] -#现已移除deepseek或硅基流动选项，可以直接切换分别配置任意模型 -model_r1_probability = 0.8 #推理模型权重 -model_v3_probability = 0.1 #非推理模型权重 -model_r1_distill_probability = 0.1 - -[memory] -build_memory_interval = 300 - -[others] -enable_advance_output = true # 是否启用详细日志输出 +auto_save = true # 是否自动保存看到的表情包呢 +enable_check = false # 是否要检查表情包是不是合适的喵 +check_prompt = "符合公序良俗" # 检查表情包的标准呢 [groups] -talk_allowed = [] # 允许回复的群号列表 -talk_frequency_down = [] # 降低回复频率的群号列表 -ban_user_id = [] # 禁止回复的用户QQ号列表 +talk_allowed = [123456, 789012] # 比如：让机器人在群123456和789012里说话 +talk_frequency_down = [345678] # 比如：在群345678里少说点话 +ban_user_id = [111222] # 比如：不回复QQ号为111222的人的消息 -[model.llm_reasoning] -name = "Pro/deepseek-ai/DeepSeek-R1" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" +[others] +enable_advance_output = true # 是否要显示更多的运行信息呢 +enable_kuuki_read = true # 让机器人能够"察言观色"喵 -[model.llm_reasoning_minor] +# 模型配置部分的详细说明喵~ + +```toml +#下面的模型若使用硅基流动则不需要更改，使用ds官方则改成在.env.prod自己指定的密钥和域名，使用自定义模型则选择定位相似的模型自己填写 + +[model.llm_reasoning] #推理模型R1，用来理解和思考的喵 +name = "Pro/deepseek-ai/DeepSeek-R1" # 模型名字 +# name = "Qwen/QwQ-32B" # 如果想用千问模型，可以把上面那行注释掉，用这个呢 +base_url = "SILICONFLOW_BASE_URL" # 使用在.env.prod里设置的服务地址 +key = "SILICONFLOW_KEY" # 使用在.env.prod里设置的密钥 + +[model.llm_reasoning_minor] #R1蒸馏模型，是个轻量版的推理模型喵 name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.llm_normal] +[model.llm_normal] #V3模型，用来日常聊天的喵 name = "Pro/deepseek-ai/DeepSeek-V3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.llm_normal_minor] +[model.llm_normal_minor] #V2.5模型，是V3的前代版本呢 name = "deepseek-ai/DeepSeek-V2.5" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.vlm] +[model.vlm] #图像识别模型，让机器人能看懂图片喵 name = "deepseek-ai/deepseek-vl2" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" + +[model.embedding] #嵌入模型，帮助机器人理解文本的相似度呢 +name = "BAAI/bge-m3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +# 主题提取功能，可以帮机器人理解对话的主题喵 +[topic] +topic_extract='snownlp' # 可以选择： + # - jieba（中文分词） + # - snownlp（中文情感分析） + # - llm（使用大模型，但需要API） + +# 如果选择了llm方式提取主题，就用这个模型配置喵 +[topic.llm_topic] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" ``` -## ⚠️ 注意事项 +## 💡 模型配置说明喵 -- 目前部署方案仍在测试中，可能存在未知问题 -- 配置文件中的API密钥请妥善保管，不要泄露 -- 建议先在测试环境中运行，确认无误后再部署到生产环境 \ No newline at end of file +1. **关于模型服务**： + - 如果你用硅基流动的服务，这些配置都不用改呢 + - 如果用DeepSeek官方API，要把base_url和key改成你在.env.prod里设置的值喵 + - 如果要用自定义模型，选择一个相似功能的模型配置来改呢 + +2. **主要模型功能**： + - `llm_reasoning`: 负责思考和推理的大脑喵 + - `llm_normal`: 负责日常聊天的嘴巴呢 + - `vlm`: 负责看图片的眼睛哦 + - `embedding`: 负责理解文字含义的理解力喵 + - `topic`: 负责理解对话主题的能力呢 + +3. **选择主题提取方式**： + - `jieba`: 不需要API，适合简单的中文分词 + - `snownlp`: 不需要API，可以分析中文情感 + - `llm`: 效果最好，但需要API服务喵 + +## 🌟 小提示 +- 如果你刚开始使用，建议保持默认配置呢 +- 不同的模型有不同的特长，可以根据需要调整它们的使用比例哦 + +## 🌟 小贴士喵 +- 记得要好好保管密钥（key）哦，不要告诉别人呢 +- 配置文件要小心修改，改错了机器人可能就不能和你玩了喵 +- 如果想让机器人更聪明，可以调整 personality 里的设置呢 +- 不想让机器人说某些话，就把那些词放在 ban_words 里面喵 +- QQ群号和QQ号都要用数字填写，不要加引号哦（除了机器人自己的QQ号） + +## ⚠️ 注意事项 +- 这个机器人还在测试中呢，可能会有一些小问题喵 +- 如果不知道怎么改某个设置，就保持原样不要动它哦~ +- 记得要先有AI服务的密钥，不然机器人就不能和你说话了呢 +- 修改完配置后要重启机器人才能生效喵~ \ No newline at end of file diff --git a/docs/installation_cute.md b/docs/installation_cute.md new file mode 100644 index 000000000..4da187ec5 --- /dev/null +++ b/docs/installation_cute.md @@ -0,0 +1,3 @@ +# 🔧 小朋友的配置指南 喵~ + +[原文内容保持不变...] \ No newline at end of file diff --git a/docs/installation_standard.md b/docs/installation_standard.md new file mode 100644 index 000000000..be045b099 --- /dev/null +++ b/docs/installation_standard.md @@ -0,0 +1,156 @@ +# 🔧 配置指南 + +## 简介 + +本项目需要配置两个主要文件： +1. `.env.prod` - 配置API服务和系统环境 +2. `bot_config.toml` - 配置机器人行为和模型 + +## API配置说明 + +`.env.prod`和`bot_config.toml`中的API配置关系如下： + +### 在.env.prod中定义API凭证： +```ini +# API凭证配置 +SILICONFLOW_KEY=your_key # 硅基流动API密钥 +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ # 硅基流动API地址 + +DEEP_SEEK_KEY=your_key # DeepSeek API密钥 +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 # DeepSeek API地址 + +CHAT_ANY_WHERE_KEY=your_key # ChatAnyWhere API密钥 +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 # ChatAnyWhere API地址 +``` + +### 在bot_config.toml中引用API凭证： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "SILICONFLOW_BASE_URL" # 引用.env.prod中定义的地址 +key = "SILICONFLOW_KEY" # 引用.env.prod中定义的密钥 +``` + +如需切换到其他API服务，只需修改引用： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "DEEP_SEEK_BASE_URL" # 切换为DeepSeek服务 +key = "DEEP_SEEK_KEY" # 使用DeepSeek密钥 +``` + +## 配置文件详解 + +### 环境配置文件 (.env.prod) +```ini +# API配置 +SILICONFLOW_KEY=your_key +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ +DEEP_SEEK_KEY=your_key +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 +CHAT_ANY_WHERE_KEY=your_key +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 + +# 服务配置 +HOST=127.0.0.1 +PORT=8080 + +# 数据库配置 +MONGODB_HOST=127.0.0.1 +MONGODB_PORT=27017 +DATABASE_NAME=MegBot +MONGODB_USERNAME = "" # 数据库用户名 +MONGODB_PASSWORD = "" # 数据库密码 +MONGODB_AUTH_SOURCE = "" # 认证数据库 + +# 插件配置 +PLUGINS=["src2.plugins.chat"] +``` + +### 机器人配置文件 (bot_config.toml) +```toml +[bot] +qq = "机器人QQ号" # 必填 +nickname = "麦麦" # 机器人昵称 + +[personality] +prompt_personality = [ + "曾经是一个学习地质的女大学生，现在学习心理学和脑科学，你会刷贴吧", + "是一个女大学生，你有黑色头发，你会刷小红书" +] +prompt_schedule = "一个曾经学习地质,现在学习心理学和脑科学的女大学生，喜欢刷qq，贴吧，知乎和小红书" + +[message] +min_text_length = 2 # 最小回复长度 +max_context_size = 15 # 上下文记忆条数 +emoji_chance = 0.2 # 表情使用概率 +ban_words = [] # 禁用词列表 + +[emoji] +auto_save = true # 自动保存表情 +enable_check = false # 启用表情审核 +check_prompt = "符合公序良俗" + +[groups] +talk_allowed = [] # 允许对话的群号 +talk_frequency_down = [] # 降低回复频率的群号 +ban_user_id = [] # 禁止回复的用户QQ号 + +[others] +enable_advance_output = true # 启用详细日志 +enable_kuuki_read = true # 启用场景理解 + +# 模型配置 +[model.llm_reasoning] # 推理模型 +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_reasoning_minor] # 轻量推理模型 +name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal] # 对话模型 +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal_minor] # 备用对话模型 +name = "deepseek-ai/DeepSeek-V2.5" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.vlm] # 图像识别模型 +name = "deepseek-ai/deepseek-vl2" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.embedding] # 文本向量模型 +name = "BAAI/bge-m3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[topic] +topic_extract='snownlp' # 主题提取方式：jieba/snownlp/llm + +[topic.llm_topic] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" +``` + +## 注意事项 + +1. API密钥安全： + - 妥善保管API密钥 + - 不要将含有密钥的配置文件上传至公开仓库 + +2. 配置修改： + - 修改配置后需重启服务 + - 使用默认服务(硅基流动)时无需修改模型配置 + - QQ号和群号使用数字格式(机器人QQ号除外) + +3. 其他说明： + - 项目处于测试阶段，可能存在未知问题 + - 建议初次使用保持默认配置 \ No newline at end of file diff --git a/docs/manual_deploy.md b/docs/manual_deploy.md index 2ea2a3d7f..ef58a6d9c 100644 --- a/docs/manual_deploy.md +++ b/docs/manual_deploy.md @@ -1,122 +1,82 @@ -# 📦 手动部署指南 +# 📦 如何手动部署MaiMbot麦麦？ -## 部署步骤 +## 你需要什么？ + +- 一台电脑，能够上网的那种 + +- 一个QQ小号（QQ框架的使用可能导致qq被风控，严重（小概率）可能会导致账号封禁，强烈不推荐使用大号） + +- 可用的大模型API + +- 一个AI助手，网上随便搜一家打开来用都行，可以帮你解决一些不懂的问题 + +## 你需要知道什么？ + +- 如何正确向AI助手提问，来学习新知识 + +- Python是什么 + +- Python的虚拟环境是什么？如何创建虚拟环境 + +- 命令行是什么 + +- 数据库是什么？如何安装并启动MongoDB + +- 如何运行一个QQ机器人，以及NapCat框架是什么 + +## 如果准备好了，就可以开始部署了 + +### 1️⃣ **我们需要创建一个Python环境来运行程序** + + 你可以选择使用以下两种方法之一来创建Python环境： -1. **环境准备** ```bash -# 创建虚拟环境（推荐） -python -m venv venv -venv\\Scripts\\activate # Windows +# ---方法1：使用venv（Python自带） +# 在命令行中创建虚拟环境（环境名为maimbot） +# 这会让你在运行命令的目录下创建一个虚拟环境 +# 请确保你已通过cd命令前往到了对应路径，不然之后你可能找不到你的python环境 +python -m venv maimbot + +maimbot\\Scripts\\activate + +# 安装依赖 +pip install -r requirements.txt +``` +```bash +# ---方法2：使用conda +# 创建一个新的conda环境（环境名为maimbot） +# Python版本为3.9 +conda create -n maimbot python=3.9 + +# 激活环境 +conda activate maimbot + # 安装依赖 pip install -r requirements.txt ``` -2. **配置MongoDB** +### 2️⃣ **然后你需要启动MongoDB数据库，来存储信息** - 安装并启动MongoDB服务 - 默认连接本地27017端口 -3. **配置NapCat** -- 安装并登录NapCat +### 3️⃣ **配置NapCat，让麦麦bot与qq取得联系** +- 安装并登录NapCat（用你的qq小号） - 添加反向WS：`ws://localhost:8080/onebot/v11/ws` -4. **配置文件设置** +### 4️⃣ **配置文件设置，让麦麦Bot正常工作** - 修改环境配置文件：`.env.prod` - 修改机器人配置文件：`bot_config.toml` -5. **启动麦麦机器人** +### 5️⃣ **启动麦麦机器人** - 打开命令行，cd到对应路径 ```bash nb run ``` +- 或者cd到对应路径后 +```bash +python bot.py +``` -6. **其他组件** +### 6️⃣ **其他组件(可选)** - `run_thingking.bat`: 启动可视化推理界面（未完善） - 直接运行 knowledge.py生成知识库 - -## ⚙️ 配置说明 - -### 环境配置 (.env.prod) -```ini -# API配置,你可以在这里定义你的密钥和base_url -# 你可以选择定义其他服务商提供的KEY，完全可以自定义 -SILICONFLOW_KEY=your_key -SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ -DEEP_SEEK_KEY=your_key -DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 - -# 服务配置,如果你不知道这是什么，保持默认 -HOST=127.0.0.1 -PORT=8080 - -# 数据库配置,如果你不知道这是什么，保持默认 -MONGODB_HOST=127.0.0.1 -MONGODB_PORT=27017 -DATABASE_NAME=MegBot -``` - -### 机器人配置 (bot_config.toml) -```toml -[bot] -qq = "你的机器人QQ号" -nickname = "麦麦" - -[message] -min_text_length = 2 -max_context_size = 15 -emoji_chance = 0.2 - -[emoji] -check_interval = 120 -register_interval = 10 - -[cq_code] -enable_pic_translate = false - -[response] -#现已移除deepseek或硅基流动选项，可以直接切换分别配置任意模型 -model_r1_probability = 0.8 #推理模型权重 -model_v3_probability = 0.1 #非推理模型权重 -model_r1_distill_probability = 0.1 - -[memory] -build_memory_interval = 300 - -[others] -enable_advance_output = true # 是否启用详细日志输出 - -[groups] -talk_allowed = [] # 允许回复的群号列表 -talk_frequency_down = [] # 降低回复频率的群号列表 -ban_user_id = [] # 禁止回复的用户QQ号列表 - -[model.llm_reasoning] -name = "Pro/deepseek-ai/DeepSeek-R1" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.llm_reasoning_minor] -name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.llm_normal] -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.llm_normal_minor] -name = "deepseek-ai/DeepSeek-V2.5" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.vlm] -name = "deepseek-ai/deepseek-vl2" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" -``` - -## ⚠️ 注意事项 - -- 目前部署方案仍在测试中，可能存在未知问题 -- 配置文件中的API密钥请妥善保管，不要泄露 -- 建议先在测试环境中运行，确认无误后再部署到生产环境 \ No newline at end of file From 0b31e44b271be8afa909f0087f1b97f9258c1df8 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 10:10:59 +0800 Subject: [PATCH 41/53] Update installation_cute.md --- docs/installation_cute.md | 228 +++++++++++++++++++++++++++++++++++++- 1 file changed, 226 insertions(+), 2 deletions(-) diff --git a/docs/installation_cute.md b/docs/installation_cute.md index 4da187ec5..a1334a2eb 100644 --- a/docs/installation_cute.md +++ b/docs/installation_cute.md @@ -1,3 +1,227 @@ -# 🔧 小朋友的配置指南 喵~ +# 🔧 配置指南 喵~ -[原文内容保持不变...] \ No newline at end of file +## 👋 你好呀！ + +让咱来告诉你我们要做什么喵： +1. 我们要一起设置一个可爱的AI机器人 +2. 这个机器人可以在QQ上陪你聊天玩耍哦 +3. 需要设置两个文件才能让机器人工作呢 + +## 📝 需要设置的文件喵 + +要设置这两个文件才能让机器人跑起来哦： +1. `.env.prod` - 这个文件告诉机器人要用哪些AI服务呢 +2. `bot_config.toml` - 这个文件教机器人怎么和你聊天喵 + +## 🔑 密钥和域名的对应关系 + +想象一下，你要进入一个游乐园，需要： +1. 知道游乐园的地址（这就是域名 base_url） +2. 有入场的门票（这就是密钥 key） + +在 `.env.prod` 文件里，我们定义了三个游乐园的地址和门票喵： +```ini +# 硅基流动游乐园 +SILICONFLOW_KEY=your_key # 硅基流动的门票 +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ # 硅基流动的地址 + +# DeepSeek游乐园 +DEEP_SEEK_KEY=your_key # DeepSeek的门票 +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 # DeepSeek的地址 + +# ChatAnyWhere游乐园 +CHAT_ANY_WHERE_KEY=your_key # ChatAnyWhere的门票 +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 # ChatAnyWhere的地址 +``` + +然后在 `bot_config.toml` 里，机器人会用这些门票和地址去游乐园玩耍： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "SILICONFLOW_BASE_URL" # 告诉机器人：去硅基流动游乐园玩 +key = "SILICONFLOW_KEY" # 用硅基流动的门票进去 + +[model.llm_normal] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" # 还是去硅基流动游乐园 +key = "SILICONFLOW_KEY" # 用同一张门票就可以啦 +``` + +### 🎪 举个例子喵： + +如果你想用DeepSeek官方的服务，就要这样改： +```toml +[model.llm_reasoning] +name = "Pro/deepseek-ai/DeepSeek-R1" +base_url = "DEEP_SEEK_BASE_URL" # 改成去DeepSeek游乐园 +key = "DEEP_SEEK_KEY" # 用DeepSeek的门票 + +[model.llm_normal] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "DEEP_SEEK_BASE_URL" # 也去DeepSeek游乐园 +key = "DEEP_SEEK_KEY" # 用同一张DeepSeek门票 +``` + +### 🎯 简单来说： +- `.env.prod` 文件就像是你的票夹，存放着各个游乐园的门票和地址 +- `bot_config.toml` 就是告诉机器人：用哪张票去哪个游乐园玩 +- 所有模型都可以用同一个游乐园的票，也可以去不同的游乐园玩耍 +- 如果用硅基流动的服务，就保持默认配置不用改呢~ + +记住：门票（key）要保管好，不能给别人看哦，不然别人就可以用你的票去玩了喵！ + +## ---让我们开始吧--- + +### 第一个文件：环境配置 (.env.prod) + +这个文件就像是机器人的"身份证"呢，告诉它要用哪些AI服务喵~ + +```ini +# 这些是AI服务的密钥，就像是魔法钥匙一样呢 +# 要把 your_key 换成真正的密钥才行喵 +# 比如说：SILICONFLOW_KEY=sk-123456789abcdef +SILICONFLOW_KEY=your_key +SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ +DEEP_SEEK_KEY=your_key +DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 +CHAT_ANY_WHERE_KEY=your_key +CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 + +# 如果你不知道这是什么，那么下面这些不用改，保持原样就好啦 +HOST=127.0.0.1 +PORT=8080 + +# 这些是数据库设置，一般也不用改呢 +MONGODB_HOST=127.0.0.1 +MONGODB_PORT=27017 +DATABASE_NAME=MegBot +MONGODB_USERNAME = "" # 如果数据库需要用户名，就在这里填写喵 +MONGODB_PASSWORD = "" # 如果数据库需要密码，就在这里填写呢 +MONGODB_AUTH_SOURCE = "" # 数据库认证源，一般不用改哦 + +# 插件设置喵 +PLUGINS=["src2.plugins.chat"] # 这里是机器人的插件列表呢 +``` + +### 第二个文件：机器人配置 (bot_config.toml) + +这个文件就像是教机器人"如何说话"的魔法书呢！ + +```toml +[bot] +qq = "把这里改成你的机器人QQ号喵" # 填写你的机器人QQ号 +nickname = "麦麦" # 机器人的名字，你可以改成你喜欢的任何名字哦 + +[personality] +# 这里可以设置机器人的性格呢，让它更有趣一些喵 +prompt_personality = [ + "曾经是一个学习地质的女大学生，现在学习心理学和脑科学，你会刷贴吧", # 贴吧风格的性格 + "是一个女大学生，你有黑色头发，你会刷小红书" # 小红书风格的性格 +] +prompt_schedule = "一个曾经学习地质,现在学习心理学和脑科学的女大学生，喜欢刷qq，贴吧，知乎和小红书" + +[message] +min_text_length = 2 # 机器人每次至少要说几个字呢 +max_context_size = 15 # 机器人能记住多少条消息喵 +emoji_chance = 0.2 # 机器人使用表情的概率哦（0.2就是20%的机会呢） +ban_words = ["脏话", "不文明用语"] # 在这里填写不让机器人说的词 + +[emoji] +auto_save = true # 是否自动保存看到的表情包呢 +enable_check = false # 是否要检查表情包是不是合适的喵 +check_prompt = "符合公序良俗" # 检查表情包的标准呢 + +[groups] +talk_allowed = [123456, 789012] # 比如：让机器人在群123456和789012里说话 +talk_frequency_down = [345678] # 比如：在群345678里少说点话 +ban_user_id = [111222] # 比如：不回复QQ号为111222的人的消息 + +[others] +enable_advance_output = true # 是否要显示更多的运行信息呢 +enable_kuuki_read = true # 让机器人能够"察言观色"喵 + +# 模型配置部分的详细说明喵~ + +```toml +#下面的模型若使用硅基流动则不需要更改，使用ds官方则改成在.env.prod自己指定的密钥和域名，使用自定义模型则选择定位相似的模型自己填写 + +[model.llm_reasoning] #推理模型R1，用来理解和思考的喵 +name = "Pro/deepseek-ai/DeepSeek-R1" # 模型名字 +# name = "Qwen/QwQ-32B" # 如果想用千问模型，可以把上面那行注释掉，用这个呢 +base_url = "SILICONFLOW_BASE_URL" # 使用在.env.prod里设置的服务地址 +key = "SILICONFLOW_KEY" # 使用在.env.prod里设置的密钥 + +[model.llm_reasoning_minor] #R1蒸馏模型，是个轻量版的推理模型喵 +name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal] #V3模型，用来日常聊天的喵 +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_normal_minor] #V2.5模型，是V3的前代版本呢 +name = "deepseek-ai/DeepSeek-V2.5" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.vlm] #图像识别模型，让机器人能看懂图片喵 +name = "deepseek-ai/deepseek-vl2" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.embedding] #嵌入模型，帮助机器人理解文本的相似度呢 +name = "BAAI/bge-m3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +# 主题提取功能，可以帮机器人理解对话的主题喵 +[topic] +topic_extract='snownlp' # 可以选择： + # - jieba（中文分词） + # - snownlp（中文情感分析） + # - llm（使用大模型，但需要API） + +# 如果选择了llm方式提取主题，就用这个模型配置喵 +[topic.llm_topic] +name = "Pro/deepseek-ai/DeepSeek-V3" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" +``` + +## 💡 模型配置说明喵 + +1. **关于模型服务**： + - 如果你用硅基流动的服务，这些配置都不用改呢 + - 如果用DeepSeek官方API，要把base_url和key改成你在.env.prod里设置的值喵 + - 如果要用自定义模型，选择一个相似功能的模型配置来改呢 + +2. **主要模型功能**： + - `llm_reasoning`: 负责思考和推理的大脑喵 + - `llm_normal`: 负责日常聊天的嘴巴呢 + - `vlm`: 负责看图片的眼睛哦 + - `embedding`: 负责理解文字含义的理解力喵 + - `topic`: 负责理解对话主题的能力呢 + +3. **选择主题提取方式**： + - `jieba`: 不需要API，适合简单的中文分词 + - `snownlp`: 不需要API，可以分析中文情感 + - `llm`: 效果最好，但需要API服务喵 + +## 🌟 小提示 +- 如果你刚开始使用，建议保持默认配置呢 +- 不同的模型有不同的特长，可以根据需要调整它们的使用比例哦 + +## 🌟 小贴士喵 +- 记得要好好保管密钥（key）哦，不要告诉别人呢 +- 配置文件要小心修改，改错了机器人可能就不能和你玩了喵 +- 如果想让机器人更聪明，可以调整 personality 里的设置呢 +- 不想让机器人说某些话，就把那些词放在 ban_words 里面喵 +- QQ群号和QQ号都要用数字填写，不要加引号哦（除了机器人自己的QQ号） + +## ⚠️ 注意事项 +- 这个机器人还在测试中呢，可能会有一些小问题喵 +- 如果不知道怎么改某个设置，就保持原样不要动它哦~ +- 记得要先有AI服务的密钥，不然机器人就不能和你说话了呢 +- 修改完配置后要重启机器人才能生效喵~ \ No newline at end of file From c11dd85790b5b644fc1167eaecf78e7294baf95e Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 10:25:58 +0800 Subject: [PATCH 42/53] =?UTF-8?q?=E7=A7=BB=E9=99=A4=E6=89=80=E6=9C=89?= =?UTF-8?q?=E9=9B=AANlp?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/doc1.md | 1 - docs/installation.md | 227 ------------------ docs/installation_cute.md | 14 +- docs/installation_standard.md | 2 - requirements.txt | Bin 618 -> 600 bytes src/plugins/chat/config.py | 1 - src/plugins/chat/topic_identifier.py | 22 -- .../memory_system/memory_manual_build.py | 1 - templete/bot_config_template.toml | 4 - 9 files changed, 1 insertion(+), 271 deletions(-) delete mode 100644 docs/installation.md diff --git a/docs/doc1.md b/docs/doc1.md index 34de628ed..158136b9c 100644 --- a/docs/doc1.md +++ b/docs/doc1.md @@ -83,7 +83,6 @@ 14. **`topic_identifier.py`**: - 识别消息中的主题，帮助机器人理解用户的意图。 - - 使用多种方法（LLM、jieba、snownlp）进行主题识别。 15. **`utils.py`** 和 **`utils_*.py`** 系列文件: - 存放各种工具函数，提供辅助功能以支持其他模块。 diff --git a/docs/installation.md b/docs/installation.md deleted file mode 100644 index a1334a2eb..000000000 --- a/docs/installation.md +++ /dev/null @@ -1,227 +0,0 @@ -# 🔧 配置指南 喵~ - -## 👋 你好呀！ - -让咱来告诉你我们要做什么喵： -1. 我们要一起设置一个可爱的AI机器人 -2. 这个机器人可以在QQ上陪你聊天玩耍哦 -3. 需要设置两个文件才能让机器人工作呢 - -## 📝 需要设置的文件喵 - -要设置这两个文件才能让机器人跑起来哦： -1. `.env.prod` - 这个文件告诉机器人要用哪些AI服务呢 -2. `bot_config.toml` - 这个文件教机器人怎么和你聊天喵 - -## 🔑 密钥和域名的对应关系 - -想象一下，你要进入一个游乐园，需要： -1. 知道游乐园的地址（这就是域名 base_url） -2. 有入场的门票（这就是密钥 key） - -在 `.env.prod` 文件里，我们定义了三个游乐园的地址和门票喵： -```ini -# 硅基流动游乐园 -SILICONFLOW_KEY=your_key # 硅基流动的门票 -SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ # 硅基流动的地址 - -# DeepSeek游乐园 -DEEP_SEEK_KEY=your_key # DeepSeek的门票 -DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 # DeepSeek的地址 - -# ChatAnyWhere游乐园 -CHAT_ANY_WHERE_KEY=your_key # ChatAnyWhere的门票 -CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 # ChatAnyWhere的地址 -``` - -然后在 `bot_config.toml` 里，机器人会用这些门票和地址去游乐园玩耍： -```toml -[model.llm_reasoning] -name = "Pro/deepseek-ai/DeepSeek-R1" -base_url = "SILICONFLOW_BASE_URL" # 告诉机器人：去硅基流动游乐园玩 -key = "SILICONFLOW_KEY" # 用硅基流动的门票进去 - -[model.llm_normal] -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "SILICONFLOW_BASE_URL" # 还是去硅基流动游乐园 -key = "SILICONFLOW_KEY" # 用同一张门票就可以啦 -``` - -### 🎪 举个例子喵： - -如果你想用DeepSeek官方的服务，就要这样改： -```toml -[model.llm_reasoning] -name = "Pro/deepseek-ai/DeepSeek-R1" -base_url = "DEEP_SEEK_BASE_URL" # 改成去DeepSeek游乐园 -key = "DEEP_SEEK_KEY" # 用DeepSeek的门票 - -[model.llm_normal] -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "DEEP_SEEK_BASE_URL" # 也去DeepSeek游乐园 -key = "DEEP_SEEK_KEY" # 用同一张DeepSeek门票 -``` - -### 🎯 简单来说： -- `.env.prod` 文件就像是你的票夹，存放着各个游乐园的门票和地址 -- `bot_config.toml` 就是告诉机器人：用哪张票去哪个游乐园玩 -- 所有模型都可以用同一个游乐园的票，也可以去不同的游乐园玩耍 -- 如果用硅基流动的服务，就保持默认配置不用改呢~ - -记住：门票（key）要保管好，不能给别人看哦，不然别人就可以用你的票去玩了喵！ - -## ---让我们开始吧--- - -### 第一个文件：环境配置 (.env.prod) - -这个文件就像是机器人的"身份证"呢，告诉它要用哪些AI服务喵~ - -```ini -# 这些是AI服务的密钥，就像是魔法钥匙一样呢 -# 要把 your_key 换成真正的密钥才行喵 -# 比如说：SILICONFLOW_KEY=sk-123456789abcdef -SILICONFLOW_KEY=your_key -SILICONFLOW_BASE_URL=https://api.siliconflow.cn/v1/ -DEEP_SEEK_KEY=your_key -DEEP_SEEK_BASE_URL=https://api.deepseek.com/v1 -CHAT_ANY_WHERE_KEY=your_key -CHAT_ANY_WHERE_BASE_URL=https://api.chatanywhere.tech/v1 - -# 如果你不知道这是什么，那么下面这些不用改，保持原样就好啦 -HOST=127.0.0.1 -PORT=8080 - -# 这些是数据库设置，一般也不用改呢 -MONGODB_HOST=127.0.0.1 -MONGODB_PORT=27017 -DATABASE_NAME=MegBot -MONGODB_USERNAME = "" # 如果数据库需要用户名，就在这里填写喵 -MONGODB_PASSWORD = "" # 如果数据库需要密码，就在这里填写呢 -MONGODB_AUTH_SOURCE = "" # 数据库认证源，一般不用改哦 - -# 插件设置喵 -PLUGINS=["src2.plugins.chat"] # 这里是机器人的插件列表呢 -``` - -### 第二个文件：机器人配置 (bot_config.toml) - -这个文件就像是教机器人"如何说话"的魔法书呢！ - -```toml -[bot] -qq = "把这里改成你的机器人QQ号喵" # 填写你的机器人QQ号 -nickname = "麦麦" # 机器人的名字，你可以改成你喜欢的任何名字哦 - -[personality] -# 这里可以设置机器人的性格呢，让它更有趣一些喵 -prompt_personality = [ - "曾经是一个学习地质的女大学生，现在学习心理学和脑科学，你会刷贴吧", # 贴吧风格的性格 - "是一个女大学生，你有黑色头发，你会刷小红书" # 小红书风格的性格 -] -prompt_schedule = "一个曾经学习地质,现在学习心理学和脑科学的女大学生，喜欢刷qq，贴吧，知乎和小红书" - -[message] -min_text_length = 2 # 机器人每次至少要说几个字呢 -max_context_size = 15 # 机器人能记住多少条消息喵 -emoji_chance = 0.2 # 机器人使用表情的概率哦（0.2就是20%的机会呢） -ban_words = ["脏话", "不文明用语"] # 在这里填写不让机器人说的词 - -[emoji] -auto_save = true # 是否自动保存看到的表情包呢 -enable_check = false # 是否要检查表情包是不是合适的喵 -check_prompt = "符合公序良俗" # 检查表情包的标准呢 - -[groups] -talk_allowed = [123456, 789012] # 比如：让机器人在群123456和789012里说话 -talk_frequency_down = [345678] # 比如：在群345678里少说点话 -ban_user_id = [111222] # 比如：不回复QQ号为111222的人的消息 - -[others] -enable_advance_output = true # 是否要显示更多的运行信息呢 -enable_kuuki_read = true # 让机器人能够"察言观色"喵 - -# 模型配置部分的详细说明喵~ - -```toml -#下面的模型若使用硅基流动则不需要更改，使用ds官方则改成在.env.prod自己指定的密钥和域名，使用自定义模型则选择定位相似的模型自己填写 - -[model.llm_reasoning] #推理模型R1，用来理解和思考的喵 -name = "Pro/deepseek-ai/DeepSeek-R1" # 模型名字 -# name = "Qwen/QwQ-32B" # 如果想用千问模型，可以把上面那行注释掉，用这个呢 -base_url = "SILICONFLOW_BASE_URL" # 使用在.env.prod里设置的服务地址 -key = "SILICONFLOW_KEY" # 使用在.env.prod里设置的密钥 - -[model.llm_reasoning_minor] #R1蒸馏模型，是个轻量版的推理模型喵 -name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.llm_normal] #V3模型，用来日常聊天的喵 -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.llm_normal_minor] #V2.5模型，是V3的前代版本呢 -name = "deepseek-ai/DeepSeek-V2.5" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.vlm] #图像识别模型，让机器人能看懂图片喵 -name = "deepseek-ai/deepseek-vl2" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -[model.embedding] #嵌入模型，帮助机器人理解文本的相似度呢 -name = "BAAI/bge-m3" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" - -# 主题提取功能，可以帮机器人理解对话的主题喵 -[topic] -topic_extract='snownlp' # 可以选择： - # - jieba（中文分词） - # - snownlp（中文情感分析） - # - llm（使用大模型，但需要API） - -# 如果选择了llm方式提取主题，就用这个模型配置喵 -[topic.llm_topic] -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" -``` - -## 💡 模型配置说明喵 - -1. **关于模型服务**： - - 如果你用硅基流动的服务，这些配置都不用改呢 - - 如果用DeepSeek官方API，要把base_url和key改成你在.env.prod里设置的值喵 - - 如果要用自定义模型，选择一个相似功能的模型配置来改呢 - -2. **主要模型功能**： - - `llm_reasoning`: 负责思考和推理的大脑喵 - - `llm_normal`: 负责日常聊天的嘴巴呢 - - `vlm`: 负责看图片的眼睛哦 - - `embedding`: 负责理解文字含义的理解力喵 - - `topic`: 负责理解对话主题的能力呢 - -3. **选择主题提取方式**： - - `jieba`: 不需要API，适合简单的中文分词 - - `snownlp`: 不需要API，可以分析中文情感 - - `llm`: 效果最好，但需要API服务喵 - -## 🌟 小提示 -- 如果你刚开始使用，建议保持默认配置呢 -- 不同的模型有不同的特长，可以根据需要调整它们的使用比例哦 - -## 🌟 小贴士喵 -- 记得要好好保管密钥（key）哦，不要告诉别人呢 -- 配置文件要小心修改，改错了机器人可能就不能和你玩了喵 -- 如果想让机器人更聪明，可以调整 personality 里的设置呢 -- 不想让机器人说某些话，就把那些词放在 ban_words 里面喵 -- QQ群号和QQ号都要用数字填写，不要加引号哦（除了机器人自己的QQ号） - -## ⚠️ 注意事项 -- 这个机器人还在测试中呢，可能会有一些小问题喵 -- 如果不知道怎么改某个设置，就保持原样不要动它哦~ -- 记得要先有AI服务的密钥，不然机器人就不能和你说话了呢 -- 修改完配置后要重启机器人才能生效喵~ \ No newline at end of file diff --git a/docs/installation_cute.md b/docs/installation_cute.md index a1334a2eb..278cbfe20 100644 --- a/docs/installation_cute.md +++ b/docs/installation_cute.md @@ -142,7 +142,7 @@ enable_kuuki_read = true # 让机器人能够"察言观色"喵 # 模型配置部分的详细说明喵~ -```toml + #下面的模型若使用硅基流动则不需要更改，使用ds官方则改成在.env.prod自己指定的密钥和域名，使用自定义模型则选择定位相似的模型自己填写 [model.llm_reasoning] #推理模型R1，用来理解和思考的喵 @@ -176,13 +176,6 @@ name = "BAAI/bge-m3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -# 主题提取功能，可以帮机器人理解对话的主题喵 -[topic] -topic_extract='snownlp' # 可以选择： - # - jieba（中文分词） - # - snownlp（中文情感分析） - # - llm（使用大模型，但需要API） - # 如果选择了llm方式提取主题，就用这个模型配置喵 [topic.llm_topic] name = "Pro/deepseek-ai/DeepSeek-V3" @@ -204,11 +197,6 @@ key = "SILICONFLOW_KEY" - `embedding`: 负责理解文字含义的理解力喵 - `topic`: 负责理解对话主题的能力呢 -3. **选择主题提取方式**： - - `jieba`: 不需要API，适合简单的中文分词 - - `snownlp`: 不需要API，可以分析中文情感 - - `llm`: 效果最好，但需要API服务喵 - ## 🌟 小提示 - 如果你刚开始使用，建议保持默认配置呢 - 不同的模型有不同的特长，可以根据需要调整它们的使用比例哦 diff --git a/docs/installation_standard.md b/docs/installation_standard.md index be045b099..6e4920220 100644 --- a/docs/installation_standard.md +++ b/docs/installation_standard.md @@ -131,8 +131,6 @@ name = "BAAI/bge-m3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[topic] -topic_extract='snownlp' # 主题提取方式：jieba/snownlp/llm [topic.llm_topic] name = "Pro/deepseek-ai/DeepSeek-V3" diff --git a/requirements.txt b/requirements.txt index 1d268ffa6fbac31fa5297f8de1b5254d33d41235..4f969682f9b0ebbbd9aebe4fe94758da052d23b4 100644 GIT binary patch delta 11 ScmaFGa)V`q3DaaZCKUi1Q3J35 delta 25 fcmcb?@``1H2@`KILmopuLphMlVJMia&!h Optional[List[str]]: - """使用 SnowNLP 进行主题识别 - - Args: - text (str): 需要识别主题的文本 - - Returns: - Optional[List[str]]: 返回识别出的主题关键词列表，如果无法识别则返回 None - """ - if not text or len(text.strip()) == 0: - return None - - try: - s = SnowNLP(text) - # 提取前3个关键词作为主题 - keywords = s.keywords(num) - return keywords if keywords else None - except Exception as e: - print(f"\033[1;31m[错误]\033[0m SnowNLP 处理失败: {str(e)}") - return None - topic_identifier = TopicIdentifier() \ No newline at end of file diff --git a/src/plugins/memory_system/memory_manual_build.py b/src/plugins/memory_system/memory_manual_build.py index 950f01afa..e99485655 100644 --- a/src/plugins/memory_system/memory_manual_build.py +++ b/src/plugins/memory_system/memory_manual_build.py @@ -13,7 +13,6 @@ from dotenv import load_dotenv import pymongo from loguru import logger from pathlib import Path -from snownlp import SnowNLP # from chat.config import global_config sys.path.append("C:/GitHub/MaiMBot") # 添加项目根目录到 Python 路径 from src.common.database import Database diff --git a/templete/bot_config_template.toml b/templete/bot_config_template.toml index 507c6d2d6..23c469014 100644 --- a/templete/bot_config_template.toml +++ b/templete/bot_config_template.toml @@ -93,10 +93,6 @@ name = "BAAI/bge-m3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -# 主题提取，jieba和snownlp不用api，llm需要api -[topic] -topic_extract='snownlp' # 只支持jieba,snownlp,llm三种选项 - [topic.llm_topic] name = "Pro/deepseek-ai/DeepSeek-V3" base_url = "SILICONFLOW_BASE_URL" From 4a2744643009a5b59e00a5856d40a494a0fb9ca5 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 13:30:50 +0800 Subject: [PATCH 43/53] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B2=A1=E7=A7=BB?= =?UTF-8?q?=E9=99=A4=E5=B9=B2=E5=87=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 --- docs/manual_deploy.md | 20 ++++++++++- run_maimai.bat | 2 +- src/plugins/chat/config.py | 17 +++------ src/plugins/chat/topic_identifier.py | 2 -- src/plugins/memory_system/memory.py | 52 +++++++++++++--------------- 5 files changed, 49 insertions(+), 44 deletions(-) diff --git a/docs/manual_deploy.md b/docs/manual_deploy.md index ef58a6d9c..6d53beb4e 100644 --- a/docs/manual_deploy.md +++ b/docs/manual_deploy.md @@ -26,7 +26,25 @@ ## 如果准备好了，就可以开始部署了 -### 1️⃣ **我们需要创建一个Python环境来运行程序** +### 1️⃣ **首先，我们需要安装正确版本的Python** + +在创建虚拟环境之前，请确保你的电脑上安装了Python 3.9及以上版本。如果没有，可以按以下步骤安装： + +1. 访问Python官网下载页面：https://www.python.org/downloads/release/python-3913/ +2. 下载Windows安装程序 (64-bit): `python-3.9.13-amd64.exe` +3. 运行安装程序，并确保勾选"Add Python 3.9 to PATH"选项 +4. 点击"Install Now"开始安装 + +或者使用PowerShell自动下载安装（需要管理员权限）： +```powershell +# 下载并安装Python 3.9.13 +$pythonUrl = "https://www.python.org/ftp/python/3.9.13/python-3.9.13-amd64.exe" +$pythonInstaller = "$env:TEMP\python-3.9.13-amd64.exe" +Invoke-WebRequest -Uri $pythonUrl -OutFile $pythonInstaller +Start-Process -Wait -FilePath $pythonInstaller -ArgumentList "/quiet", "InstallAllUsers=0", "PrependPath=1" -Verb RunAs +``` + +### 2️⃣ **创建Python虚拟环境来运行程序** 你可以选择使用以下两种方法之一来创建Python环境： diff --git a/run_maimai.bat b/run_maimai.bat index ff00cc5c1..3a099fd7f 100644 --- a/run_maimai.bat +++ b/run_maimai.bat @@ -1,5 +1,5 @@ chcp 65001 -call conda activate niuniu +call conda activate maimbot cd . REM 执行nb run命令 diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index 92b259ebd..0b47e5d3d 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -148,15 +148,6 @@ class BotConfig: if "rerank" in model_config: config.rerank = model_config["rerank"] - if 'topic' in toml_dict: - topic_config=toml_dict['topic'] - if 'topic_extract' in topic_config: - config.topic_extract=topic_config.get('topic_extract',config.topic_extract) - logger.info(f"载入自定义主题提取为{config.topic_extract}") - if config.topic_extract=='llm' and 'llm_topic' in topic_config: - config.llm_topic_extract=topic_config['llm_topic'] - logger.info(f"载入自定义主题提取模型为{config.llm_topic_extract['name']}") - # 消息配置 if "message" in toml_dict: msg_config = toml_dict["message"] @@ -190,13 +181,13 @@ class BotConfig: bot_config_floder_path = BotConfig.get_config_dir() print(f"正在品鉴配置文件目录: {bot_config_floder_path}") -bot_config_path = os.path.join(bot_config_floder_path, "bot_config_dev.toml") -if not os.path.exists(bot_config_path): +bot_config_path = os.path.join(bot_config_floder_path, "bot_config.toml") +if os.path.exists(bot_config_path): # 如果开发环境配置文件不存在，则使用默认配置文件 - bot_config_path = os.path.join(bot_config_floder_path, "bot_config.toml") + print(f"异常的新鲜，异常的美味: {bot_config_path}") logger.info("使用bot配置文件") else: - logger.info("已找到开发bot配置文件") + logger.info("没有找到美味") global_config = BotConfig.load_config(config_path=bot_config_path) diff --git a/src/plugins/chat/topic_identifier.py b/src/plugins/chat/topic_identifier.py index 4580c1e91..8e6d41c7d 100644 --- a/src/plugins/chat/topic_identifier.py +++ b/src/plugins/chat/topic_identifier.py @@ -12,9 +12,7 @@ config = driver.config class TopicIdentifier: def __init__(self): self.llm_client = LLM_request(model=global_config.llm_topic_extract) - self.select=global_config.topic_extract - async def identify_topic_llm(self, text: str) -> Optional[List[str]]: """识别消息主题，返回主题列表""" diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index a25e15bdf..fd001e791 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -578,14 +578,11 @@ class Hippocampus: async def memory_activate_value(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.3) -> int: """计算输入文本对记忆的激活程度""" - print(f"\033[1;32m[记忆激活]\033[0m 开始计算文本的记忆激活度: {text}") + print(f"\033[1;32m[记忆激活]\033[0m 识别主题: {await self._identify_topics(text)}") # 识别主题 identified_topics = await self._identify_topics(text) - print(f"\033[1;32m[记忆激活]\033[0m 识别出的主题: {identified_topics}") - if not identified_topics: - # print(f"\033[1;32m[记忆激活]\033[0m 未识别出主题,返回0") return 0 # 查找相似主题 @@ -596,7 +593,6 @@ class Hippocampus: ) if not all_similar_topics: - print(f"\033[1;32m[记忆激活]\033[0m 未找到相似主题,返回0") return 0 # 获取最相关的主题 @@ -605,19 +601,29 @@ class Hippocampus: # 如果只找到一个主题，进行惩罚 if len(top_topics) == 1: topic, score = top_topics[0] - activation = int(score * 50) # 单主题情况下，直接用相似度*50作为激活值 - print(f"\033[1;32m[记忆激活]\033[0m 只找到一个主题，进行惩罚:") - print(f"\033[1;32m[记忆激活]\033[0m - 主题: {topic}") - print(f"\033[1;32m[记忆激活]\033[0m - 相似度: {score:.3f}") - print(f"\033[1;32m[记忆激活]\033[0m - 最终激活值: {activation}") + # 获取主题内容数量并计算惩罚系数 + memory_items = self.memory_graph.G.nodes[topic].get('memory_items', []) + if not isinstance(memory_items, list): + memory_items = [memory_items] if memory_items else [] + content_count = len(memory_items) + penalty = 1.0 / (1 + math.log(content_count + 1)) + + activation = int(score * 50 * penalty) + print(f"\033[1;32m[记忆激活]\033[0m 单主题「{topic}」- 相似度: {score:.3f}, 内容数: {content_count}, 激活值: {activation}") return activation - # 计算关键词匹配率 + # 计算关键词匹配率，同时考虑内容数量 matched_topics = set() topic_similarities = {} - print(f"\033[1;32m[记忆激活]\033[0m 计算关键词匹配情况:") for memory_topic, similarity in top_topics: + # 计算内容数量惩罚 + memory_items = self.memory_graph.G.nodes[memory_topic].get('memory_items', []) + if not isinstance(memory_items, list): + memory_items = [memory_items] if memory_items else [] + content_count = len(memory_items) + penalty = 1.0 / (1 + math.log(content_count + 1)) + # 对每个记忆主题，检查它与哪些输入主题相似 for input_topic in identified_topics: topic_vector = text_to_vector(input_topic) @@ -628,27 +634,19 @@ class Hippocampus: sim = cosine_similarity(v1, v2) if sim >= similarity_threshold: matched_topics.add(input_topic) - topic_similarities[input_topic] = max(topic_similarities.get(input_topic, 0), sim) - print(f"\033[1;32m[记忆激活]\033[0m - 输入主题「{input_topic}」匹配到记忆「{memory_topic}」, 相似度: {sim:.3f}") + adjusted_sim = sim * penalty + topic_similarities[input_topic] = max(topic_similarities.get(input_topic, 0), adjusted_sim) + print(f"\033[1;32m[记忆激活]\033[0m 主题「{input_topic}」-> 「{memory_topic}」(内容数: {content_count}, 相似度: {adjusted_sim:.3f})") - # 计算主题匹配率 + # 计算主题匹配率和平均相似度 topic_match = len(matched_topics) / len(identified_topics) - print(f"\033[1;32m[记忆激活]\033[0m 主题匹配率:") - print(f"\033[1;32m[记忆激活]\033[0m - 匹配主题数: {len(matched_topics)}") - print(f"\033[1;32m[记忆激活]\033[0m - 总主题数: {len(identified_topics)}") - print(f"\033[1;32m[记忆激活]\033[0m - 匹配率: {topic_match:.3f}") - - # 计算匹配主题的平均相似度 average_similarities = sum(topic_similarities.values()) / len(topic_similarities) if topic_similarities else 0 - print(f"\033[1;32m[记忆激活]\033[0m 平均相似度:") - print(f"\033[1;32m[记忆激活]\033[0m - 各主题相似度: {[f'{k}:{v:.3f}' for k,v in topic_similarities.items()]}") - print(f"\033[1;32m[记忆激活]\033[0m - 平均相似度: {average_similarities:.3f}") # 计算最终激活值 - activation = (topic_match + average_similarities) / 2 * 100 - print(f"\033[1;32m[记忆激活]\033[0m 最终激活值: {int(activation)}") + activation = int((topic_match + average_similarities) / 2 * 100) + print(f"\033[1;32m[记忆激活]\033[0m 匹配率: {topic_match:.3f}, 平均相似度: {average_similarities:.3f}, 激活值: {activation}") - return int(activation) + return activation async def get_relevant_memories(self, text: str, max_topics: int = 5, similarity_threshold: float = 0.4, max_memory_num: int = 5) -> list: """根据输入文本获取相关的记忆内容""" From 4abe951b3ca58f6bc07296de30899044d1e50fe9 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 16:16:07 +0800 Subject: [PATCH 44/53] Update bot.py --- bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot.py b/bot.py index d0922e4d6..1bebfe553 100644 --- a/bot.py +++ b/bot.py @@ -18,7 +18,7 @@ print(rainbow_text) # 初次启动检测 if not os.path.exists("config/bot_config.toml"): - logger.info("检测到bot_config.toml不存在，正在从模板复制") + logger.warning("检测到bot_config.toml不存在，正在从模板复制") import shutil shutil.copy("templete/bot_config_template.toml", "config/bot_config.toml") From 9bf865cfd82a55104851414084e1980f1ba18d6e Mon Sep 17 00:00:00 2001 From: Rikki Date: Fri, 7 Mar 2025 16:46:10 +0800 Subject: [PATCH 45/53] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=20nix=20?= =?UTF-8?q?=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .envrc | 1 + .gitignore | 5 ++++- flake.nix | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 .envrc create mode 100644 flake.nix diff --git a/.envrc b/.envrc new file mode 100644 index 000000000..8392d159f --- /dev/null +++ b/.envrc @@ -0,0 +1 @@ +use flake \ No newline at end of file diff --git a/.gitignore b/.gitignore index deec90be9..38deb3666 100644 --- a/.gitignore +++ b/.gitignore @@ -191,4 +191,7 @@ jieba.cache # vscode -/.vscode \ No newline at end of file +/.vscode + +# direnv +/.direnv \ No newline at end of file diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000..df885fd83 --- /dev/null +++ b/flake.nix @@ -0,0 +1,52 @@ +{ + description = "MaiMBot Nix Dev Env"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + flake-utils.url = "github:numtide/flake-utils"; + }; + + outputs = { self, nixpkgs, flake-utils }: + flake-utils.lib.eachDefaultSystem (system: + let + pkgs = import nixpkgs { + inherit system; + }; + + # 读取 requirements.txt 文件 + requirementsFile = builtins.readFile ./requirements.txt; + + # 解析 requirements.txt 文件，提取包名 + parseRequirements = content: + let + lines = builtins.split "\n" content; + # 过滤掉空行和注释 + filteredLines = builtins.filter (line: + line != "" && !(builtins.match "^ *#.*" line) + ) lines; + # 提取包名（去掉版本号） + packageNames = builtins.map (line: + builtins.head (builtins.split "[=<>]" line) + ) filteredLines; + in + packageNames; + + # 获取 requirements.txt 中的包名列表 + requirements = parseRequirements requirementsFile; + + # 动态生成 Python 环境 + pythonEnv = pkgs.python3.withPackages (ps: + builtins.map (pkg: ps.${pkg}) requirements + ); + in + { + devShell = pkgs.mkShell { + buildInputs = [ pythonEnv ]; + + shellHook = '' + echo "Python environment is ready!" + ''; + }; + } + ); +} \ No newline at end of file From cfcdcc653ece95a31595bbd9a6882c4473edb1ab Mon Sep 17 00:00:00 2001 From: Rikki Date: Fri, 7 Mar 2025 18:01:32 +0800 Subject: [PATCH 46/53] =?UTF-8?q?update:=20=E6=9B=B4=E6=96=B0=E5=BC=80?= =?UTF-8?q?=E5=8F=91=E5=BF=85=E8=A6=81=E7=9A=84=E5=8C=85=EF=BC=8C=E4=BD=86?= =?UTF-8?q?=E6=98=AF=E5=9B=A0=E4=B8=BA=20nb-cli=20=E4=B8=8A=E6=B8=B8?= =?UTF-8?q?=E6=89=93=E5=8C=85=E4=B8=AD=E5=B9=B6=E6=9C=AA=E5=8C=85=E5=90=AB?= =?UTF-8?q?=20nonebot2=EF=BC=8C=E5=9B=A0=E6=AD=A4=E7=9B=AE=E5=89=8D?= =?UTF-8?q?=E6=9C=AC=E9=85=8D=E7=BD=AE=E5=B9=B6=E4=B8=8D=E8=83=BD=E7=94=A8?= =?UTF-8?q?=E4=BA=8E=E8=BF=90=E8=A1=8C=E5=92=8C=E8=B0=83=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- flake.lock | 61 +++++++++++++++++++++++++++++++++++++++++++++++ flake.nix | 69 ++++++++++++++++++++++++++++++------------------------ 2 files changed, 100 insertions(+), 30 deletions(-) create mode 100644 flake.lock diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000..dd215f1c6 --- /dev/null +++ b/flake.lock @@ -0,0 +1,61 @@ +{ + "nodes": { + "flake-utils": { + "inputs": { + "systems": "systems" + }, + "locked": { + "lastModified": 1731533236, + "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", + "owner": "numtide", + "repo": "flake-utils", + "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "flake-utils", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1741196730, + "narHash": "sha256-0Sj6ZKjCpQMfWnN0NURqRCQn2ob7YtXTAOTwCuz7fkA=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "48913d8f9127ea6530a2a2f1bd4daa1b8685d8a3", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.11", + "repo": "nixpkgs", + "type": "github" + } + }, + "root": { + "inputs": { + "flake-utils": "flake-utils", + "nixpkgs": "nixpkgs" + } + }, + "systems": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix index df885fd83..54737d640 100644 --- a/flake.nix +++ b/flake.nix @@ -1,52 +1,61 @@ { description = "MaiMBot Nix Dev Env"; + # 本配置仅方便用于开发，但是因为 nb-cli 上游打包中并未包含 nonebot2，因此目前本配置并不能用于运行和调试 inputs = { - nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; flake-utils.url = "github:numtide/flake-utils"; }; - outputs = { self, nixpkgs, flake-utils }: - flake-utils.lib.eachDefaultSystem (system: + outputs = + { + self, + nixpkgs, + flake-utils, + }: + flake-utils.lib.eachDefaultSystem ( + system: let pkgs = import nixpkgs { inherit system; }; - # 读取 requirements.txt 文件 - requirementsFile = builtins.readFile ./requirements.txt; - - # 解析 requirements.txt 文件，提取包名 - parseRequirements = content: - let - lines = builtins.split "\n" content; - # 过滤掉空行和注释 - filteredLines = builtins.filter (line: - line != "" && !(builtins.match "^ *#.*" line) - ) lines; - # 提取包名（去掉版本号） - packageNames = builtins.map (line: - builtins.head (builtins.split "[=<>]" line) - ) filteredLines; - in - packageNames; - - # 获取 requirements.txt 中的包名列表 - requirements = parseRequirements requirementsFile; - - # 动态生成 Python 环境 - pythonEnv = pkgs.python3.withPackages (ps: - builtins.map (pkg: ps.${pkg}) requirements + pythonEnv = pkgs.python3.withPackages ( + ps: with ps; [ + pymongo + python-dotenv + pydantic + jieba + openai + aiohttp + requests + urllib3 + numpy + pandas + matplotlib + networkx + python-dateutil + APScheduler + loguru + tomli + customtkinter + colorama + pypinyin + pillow + setuptools + ] ); in { devShell = pkgs.mkShell { - buildInputs = [ pythonEnv ]; + buildInputs = [ + pythonEnv + pkgs.nb-cli + ]; shellHook = '' - echo "Python environment is ready!" ''; }; } ); -} \ No newline at end of file +} From 68b696b8aaaa3bdf9f03388f13cf6cee0bc37772 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 18:16:06 +0800 Subject: [PATCH 47/53] fix config 12 --- src/plugins/chat/bot.py | 5 ----- src/plugins/chat/config.py | 7 ------- 2 files changed, 12 deletions(-) diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 73296c1da..4306c0f9d 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -109,13 +109,8 @@ class ChatBot: willing_manager.change_reply_willing_sent(thinking_message.group_id) response,raw_content = await self.gpt.generate_response(message) - - # if response is None: - # thinking_message.interupt=True if response: - # print(f"\033[1;32m[思考结束]\033[0m 思考结束，已得到回复，开始回复") - # 找到并删除对应的thinking消息 container = message_manager.get_container(event.group_id) thinking_message = None # 找到message,删除 diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index 0b47e5d3d..dfce0cd64 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -45,13 +45,7 @@ class BotConfig: llm_normal_minor: Dict[str, str] = field(default_factory=lambda: {}) embedding: Dict[str, str] = field(default_factory=lambda: {}) vlm: Dict[str, str] = field(default_factory=lambda: {}) - rerank: Dict[str, str] = field(default_factory=lambda: {}) - # 主题提取配置 - llm_topic_extract: Dict[str, str] = field(default_factory=lambda: {}) - - API_USING: str = "siliconflow" # 使用的API - API_PAID: bool = False # 是否使用付费API MODEL_R1_PROBABILITY: float = 0.8 # R1模型概率 MODEL_V3_PROBABILITY: float = 0.1 # V3模型概率 MODEL_R1_DISTILL_PROBABILITY: float = 0.1 # R1蒸馏模型概率 @@ -134,7 +128,6 @@ class BotConfig: if "llm_normal" in model_config: config.llm_normal = model_config["llm_normal"] - config.llm_topic_extract = config.llm_normal if "llm_normal_minor" in model_config: config.llm_normal_minor = model_config["llm_normal_minor"] From 34907fdbf5217c6eeb372c1b603eb0bbf759557b Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 18:41:41 +0800 Subject: [PATCH 48/53] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E4=BA=86=E6=A8=A1?= =?UTF-8?q?=E5=9E=8B=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/config.py | 19 ++++++++++++++---- src/plugins/chat/emoji_manager.py | 30 +++++++++++++++++++++------- src/plugins/chat/topic_identifier.py | 4 ++-- src/plugins/memory_system/memory.py | 10 +++++----- src/plugins/models/utils_model.py | 3 +-- 5 files changed, 46 insertions(+), 20 deletions(-) diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index dfce0cd64..5c3c0b27a 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -43,8 +43,12 @@ class BotConfig: llm_reasoning_minor: Dict[str, str] = field(default_factory=lambda: {}) llm_normal: Dict[str, str] = field(default_factory=lambda: {}) llm_normal_minor: Dict[str, str] = field(default_factory=lambda: {}) + llm_topic_judge: Dict[str, str] = field(default_factory=lambda: {}) + llm_summary_by_topic: Dict[str, str] = field(default_factory=lambda: {}) + llm_emotion_judge: Dict[str, str] = field(default_factory=lambda: {}) embedding: Dict[str, str] = field(default_factory=lambda: {}) vlm: Dict[str, str] = field(default_factory=lambda: {}) + moderation: Dict[str, str] = field(default_factory=lambda: {}) MODEL_R1_PROBABILITY: float = 0.8 # R1模型概率 MODEL_V3_PROBABILITY: float = 0.1 # V3模型概率 @@ -112,8 +116,6 @@ class BotConfig: config.MODEL_R1_PROBABILITY = response_config.get("model_r1_probability", config.MODEL_R1_PROBABILITY) config.MODEL_V3_PROBABILITY = response_config.get("model_v3_probability", config.MODEL_V3_PROBABILITY) config.MODEL_R1_DISTILL_PROBABILITY = response_config.get("model_r1_distill_probability", config.MODEL_R1_DISTILL_PROBABILITY) - config.API_USING = response_config.get("api_using", config.API_USING) - config.API_PAID = response_config.get("api_paid", config.API_PAID) config.max_response_length = response_config.get("max_response_length", config.max_response_length) # 加载模型配置 @@ -131,6 +133,15 @@ class BotConfig: if "llm_normal_minor" in model_config: config.llm_normal_minor = model_config["llm_normal_minor"] + + if "llm_topic_judge" in model_config: + config.llm_topic_judge = model_config["llm_topic_judge"] + + if "llm_summary_by_topic" in model_config: + config.llm_summary_by_topic = model_config["llm_summary_by_topic"] + + if "llm_emotion_judge" in model_config: + config.llm_emotion_judge = model_config["llm_emotion_judge"] if "vlm" in model_config: config.vlm = model_config["vlm"] @@ -138,8 +149,8 @@ class BotConfig: if "embedding" in model_config: config.embedding = model_config["embedding"] - if "rerank" in model_config: - config.rerank = model_config["rerank"] + if "moderation" in model_config: + config.moderation = model_config["moderation"] # 消息配置 if "message" in toml_dict: diff --git a/src/plugins/chat/emoji_manager.py b/src/plugins/chat/emoji_manager.py index 3ba167308..432d11753 100644 --- a/src/plugins/chat/emoji_manager.py +++ b/src/plugins/chat/emoji_manager.py @@ -41,8 +41,8 @@ class EmojiManager: def __init__(self): self.db = Database.get_instance() self._scan_task = None - self.llm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=1000) - self.lm = LLM_request(model=global_config.llm_normal_minor, max_tokens=1000) + self.vlm = LLM_request(model=global_config.vlm, temperature=0.3, max_tokens=1000) + self.llm_emotion_judge = LLM_request(model=global_config.llm_normal_minor, max_tokens=60,temperature=0.8) #更高的温度，更少的token（后续可以根据情绪来调整温度） def _ensure_emoji_dir(self): """确保表情存储目录存在""" @@ -69,7 +69,17 @@ class EmojiManager: raise RuntimeError("EmojiManager not initialized") def _ensure_emoji_collection(self): - """确保emoji集合存在并创建索引""" + """确保emoji集合存在并创建索引 + + 这个函数用于确保MongoDB数据库中存在emoji集合,并创建必要的索引。 + + 索引的作用是加快数据库查询速度: + - embedding字段的2dsphere索引: 用于加速向量相似度搜索,帮助快速找到相似的表情包 + - tags字段的普通索引: 加快按标签搜索表情包的速度 + - filename字段的唯一索引: 确保文件名不重复,同时加快按文件名查找的速度 + + 没有索引的话,数据库每次查询都需要扫描全部数据,建立索引后可以大大提高查询效率。 + """ if 'emoji' not in self.db.db.list_collection_names(): self.db.db.create_collection('emoji') self.db.db.emoji.create_index([('embedding', '2dsphere')]) @@ -93,6 +103,11 @@ class EmojiManager: text: 输入文本 Returns: Optional[str]: 表情包文件路径，如果没有找到则返回None + + + 可不可以通过 配置文件中的指令 来自定义使用表情包的逻辑？ + 我觉得可行 + """ try: self._ensure_db() @@ -152,7 +167,8 @@ class EmojiManager: {'$inc': {'usage_count': 1}} ) logger.success(f"找到匹配的表情包: {selected_emoji.get('discription', '无描述')} (相似度: {similarity:.4f})") - return selected_emoji['path'],"[表情包: %s]" % selected_emoji.get('discription', '无描述') + # 稍微改一下文本描述，不然容易产生幻觉，描述已经包含 表情包 了 + return selected_emoji['path'],"[ %s ]" % selected_emoji.get('discription', '无描述') except Exception as search_error: logger.error(f"搜索表情包失败: {str(search_error)}") @@ -169,7 +185,7 @@ class EmojiManager: try: prompt = '这是一个表情包，使用中文简洁的描述一下表情包的内容和表情包所表达的情感' - content, _ = await self.llm.generate_response_for_image(prompt, image_base64) + content, _ = await self.vlm.generate_response_for_image(prompt, image_base64) logger.debug(f"输出描述: {content}") return content @@ -181,7 +197,7 @@ class EmojiManager: try: prompt = f'这是一个表情包，请回答这个表情包是否满足\"{global_config.EMOJI_CHECK_PROMPT}\"的要求，是则回答是，否则回答否，不要出现任何其他内容' - content, _ = await self.llm.generate_response_for_image(prompt, image_base64) + content, _ = await self.vlm.generate_response_for_image(prompt, image_base64) logger.debug(f"输出描述: {content}") return content @@ -193,7 +209,7 @@ class EmojiManager: try: prompt = f'这是{global_config.BOT_NICKNAME}将要发送的消息内容:\n{text}\n若要为其配上表情包，请你输出这个表情包应该表达怎样的情感，应该给人什么样的感觉，不要太简洁也不要太长，注意不要输出任何对消息内容的分析内容，只输出\"一种什么样的感觉\"中间的形容词部分。' - content, _ = await self.lm.generate_response_async(prompt) + content, _ = await self.llm_emotion_judge.generate_response_async(prompt) logger.info(f"输出描述: {content}") return content diff --git a/src/plugins/chat/topic_identifier.py b/src/plugins/chat/topic_identifier.py index 8e6d41c7d..6579d15ac 100644 --- a/src/plugins/chat/topic_identifier.py +++ b/src/plugins/chat/topic_identifier.py @@ -11,7 +11,7 @@ config = driver.config class TopicIdentifier: def __init__(self): - self.llm_client = LLM_request(model=global_config.llm_topic_extract) + self.llm_topic_judge = LLM_request(model=global_config.llm_topic_judge) async def identify_topic_llm(self, text: str) -> Optional[List[str]]: """识别消息主题，返回主题列表""" @@ -23,7 +23,7 @@ class TopicIdentifier: 消息内容：{text}""" # 使用 LLM_request 类进行请求 - topic, _ = await self.llm_client.generate_response(prompt) + topic, _ = await self.llm_topic_judge.generate_response(prompt) if not topic: print(f"\033[1;31m[错误]\033[0m LLM API 返回为空") diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index fd001e791..44f5eb713 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -132,8 +132,8 @@ class Memory_graph: class Hippocampus: def __init__(self,memory_graph:Memory_graph): self.memory_graph = memory_graph - self.llm_model_get_topic = LLM_request(model = global_config.llm_normal_minor,temperature=0.5) - self.llm_model_summary = LLM_request(model = global_config.llm_normal,temperature=0.5) + self.llm_topic_judge = LLM_request(model = global_config.llm_topic_judge,temperature=0.5) + self.llm_summary_by_topic = LLM_request(model = global_config.llm_summary_by_topic,temperature=0.5) def get_all_node_names(self) -> list: """获取记忆图中所有节点的名字列表 @@ -179,7 +179,7 @@ class Hippocampus: #获取topics topic_num = self.calculate_topic_num(input_text, compress_rate) - topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(input_text, topic_num)) + topics_response = await self.llm_topic_judge.generate_response(self.find_topic_llm(input_text, topic_num)) # 修改话题处理逻辑 # 定义需要过滤的关键词 filter_keywords = ['表情包', '图片', '回复', '聊天记录'] @@ -196,7 +196,7 @@ class Hippocampus: for topic in filtered_topics: topic_what_prompt = self.topic_what(input_text, topic) # 创建异步任务 - task = self.llm_model_summary.generate_response_async(topic_what_prompt) + task = self.llm_summary_by_topic.generate_response_async(topic_what_prompt) tasks.append((topic.strip(), task)) # 等待所有任务完成 @@ -506,7 +506,7 @@ class Hippocampus: Returns: list: 识别出的主题列表 """ - topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(text, 5)) + topics_response = await self.llm_topic_judge.generate_response(self.find_topic_llm(text, 5)) # print(f"话题: {topics_response[0]}") topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] # print(f"话题: {topics}") diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index b377eac14..7bfc966f6 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -197,13 +197,12 @@ class LLM_request: ) return content, reasoning_content - async def generate_response_async(self, prompt: str) -> Union[str, Tuple[str, str]]: + async def generate_response_async(self, prompt: str, **kwargs) -> Union[str, Tuple[str, str]]: """异步方式根据输入的提示生成模型的响应""" # 构建请求体 data = { "model": self.model_name, "messages": [{"role": "user", "content": prompt}], - "temperature": 0.5, "max_tokens": global_config.max_response_length, **self.params } From 9d4c27764853db1234669e80e8fce4063f239447 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 18:42:55 +0800 Subject: [PATCH 49/53] Update bot_config_template.toml --- templete/bot_config_template.toml | 53 +++++++++++++++++++++++-------- 1 file changed, 39 insertions(+), 14 deletions(-) diff --git a/templete/bot_config_template.toml b/templete/bot_config_template.toml index 23c469014..ad47c90c5 100644 --- a/templete/bot_config_template.toml +++ b/templete/bot_config_template.toml @@ -28,9 +28,9 @@ check_prompt = "符合公序良俗" # 表情包过滤要求 enable_pic_translate = false [response] -model_r1_probability = 0.8 # 麦麦回答时选择R1模型的概率 -model_v3_probability = 0.1 # 麦麦回答时选择V3模型的概率 -model_r1_distill_probability = 0.1 # 麦麦回答时选择R1蒸馏模型的概率 +model_r1_probability = 0.8 # 麦麦回答时选择主要回复模型1 模型的概率 +model_v3_probability = 0.1 # 麦麦回答时选择次要回复模型2 模型的概率 +model_r1_distill_probability = 0.1 # 麦麦回答时选择次要回复模型3 模型的概率 max_response_length = 1024 # 麦麦回答的最大token数 [memory] @@ -62,18 +62,21 @@ ban_user_id = [] #禁止回复消息的QQ号 #下面的模型若使用硅基流动则不需要更改，使用ds官方则改成.env.prod自定义的宏，使用自定义模型则选择定位相似的模型自己填写 -[model.llm_reasoning] #R1 +#推理模型： + +[model.llm_reasoning] #回复模型1 主要回复模型 name = "Pro/deepseek-ai/DeepSeek-R1" -# name = "Qwen/QwQ-32B" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.llm_reasoning_minor] #R1蒸馏 +[model.llm_reasoning_minor] #回复模型3 次要回复模型 name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.llm_normal] #V3 +#非推理模型 + +[model.llm_normal] #V3 回复模型2 次要回复模型 name = "Pro/deepseek-ai/DeepSeek-V3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" @@ -83,17 +86,39 @@ name = "deepseek-ai/DeepSeek-V2.5" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" -[model.vlm] #图像识别 -name = "deepseek-ai/deepseek-vl2" +[model.llm_emotion_judge] #主题判断 0.7/m +name = "Qwen/Qwen2.5-14B-Instruct" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" +[model.llm_topic_judge] #主题判断：建议使用qwen2.5 7b +name = "Pro/Qwen/Qwen2.5-7B-Instruct" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.llm_summary_by_topic] #建议使用qwen2.5 32b 及以上 +name = "Qwen/Qwen2.5-32B-Instruct" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + +[model.moderation] #内容审核 未启用 +name = "" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + + +# 识图模型 + +[model.vlm] #图像识别 0.35/m +name = "Pro/Qwen/Qwen2-VL-7B-Instruct" +base_url = "SILICONFLOW_BASE_URL" +key = "SILICONFLOW_KEY" + + + +#嵌入模型 + [model.embedding] #嵌入 name = "BAAI/bge-m3" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" - -[topic.llm_topic] -name = "Pro/deepseek-ai/DeepSeek-V3" -base_url = "SILICONFLOW_BASE_URL" -key = "SILICONFLOW_KEY" From a527e5ce7692ed1e1772241b7b0c53f71fe0db78 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Fri, 7 Mar 2025 19:17:11 +0800 Subject: [PATCH 50/53] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E8=8E=B7?= =?UTF-8?q?=E5=8F=96=E7=9A=84=E6=B6=88=E6=81=AF=E6=B2=A1=E6=9C=89time?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/chat/message.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/plugins/chat/message.py b/src/plugins/chat/message.py index 4ac574e66..a39cf293f 100644 --- a/src/plugins/chat/message.py +++ b/src/plugins/chat/message.py @@ -71,6 +71,8 @@ class Message: ) # 构建详细文本 + if self.time is None: + self.time = int(time.time()) time_str = time.strftime("%m-%d %H:%M:%S", time.localtime(self.time)) name = ( f"{self.user_nickname}(ta的昵称:{self.user_cardname},ta的id:{self.user_id})" From 0e72c7444b31775c3d4c5751e9d5c15eaf1f2a5c Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 19:22:24 +0800 Subject: [PATCH 51/53] =?UTF-8?q?=E9=99=8D=E4=BD=8E=E4=BA=86=E5=9B=9E?= =?UTF-8?q?=E5=A4=8D=E8=A1=A8=E6=83=85=E5=8C=85=E7=9A=84=E5=8F=AF=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1 --- README.md | 7 ++++++- src/plugins/chat/prompt_builder.py | 2 +- src/plugins/chat/willing_manager.py | 2 +- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 65263b1a9..04cfc0772 100644 --- a/README.md +++ b/README.md @@ -91,6 +91,12 @@ ## 开发计划TODO：LIST + +规划主线 +0.6.0：记忆系统更新 +0.7.0: 麦麦RunTime + + - 人格功能：WIP - 群氛围功能：WIP - 图片发送，转发功能：WIP @@ -109,7 +115,6 @@ - 自动生成的回复逻辑，例如自生成的回复方向，回复风格 - 采用截断生成加快麦麦的反应速度 - 改进发送消息的触发 -- ## 设计理念 diff --git a/src/plugins/chat/prompt_builder.py b/src/plugins/chat/prompt_builder.py index 5467ce94e..3b7894f56 100644 --- a/src/plugins/chat/prompt_builder.py +++ b/src/plugins/chat/prompt_builder.py @@ -141,7 +141,7 @@ class PromptBuilder: prompt_ger += '你喜欢用文言文' #额外信息要求 - extra_info = '''但是记得回复平淡一些，简短一些，尤其注意在没明确提到时不要过多提及自身的背景, 记住不要输出多余内容(包括前后缀，冒号和引号，括号，表情等)，只需要输出回复内容就好，不要输出其他任何内容''' + extra_info = '''但是记得回复平淡一些，简短一些，尤其注意在没明确提到时不要过多提及自身的背景, 不要直接回复别人发的表情包，记住不要输出多余内容(包括前后缀，冒号和引号，括号，表情等)，只需要输出回复内容就好，不要输出其他任何内容''' #合并prompt prompt = "" diff --git a/src/plugins/chat/willing_manager.py b/src/plugins/chat/willing_manager.py index acc8543da..16a0570e2 100644 --- a/src/plugins/chat/willing_manager.py +++ b/src/plugins/chat/willing_manager.py @@ -34,7 +34,7 @@ class WillingManager: print(f"被重复提及, 当前意愿: {current_willing}") if is_emoji: - current_willing *= 0.15 + current_willing *= 0.1 print(f"表情包, 当前意愿: {current_willing}") if interested_rate > 0.4: From f249f5099420b5d31d13199d8d284b7e23f5a133 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 20:41:12 +0800 Subject: [PATCH 52/53] =?UTF-8?q?v0.5.10=20=E5=9C=A8=E6=A0=B9=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E7=94=9F=E6=88=90=E7=BB=9F=E8=AE=A1=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- llm_statistics.txt | 74 ++++++++++++++ src/plugins/chat/__init__.py | 8 ++ src/plugins/models/utils_model.py | 98 +++++++++++++++++- src/plugins/utils/statistic.py | 162 ++++++++++++++++++++++++++++++ 4 files changed, 338 insertions(+), 4 deletions(-) create mode 100644 llm_statistics.txt create mode 100644 src/plugins/utils/statistic.py diff --git a/llm_statistics.txt b/llm_statistics.txt new file mode 100644 index 000000000..338158ef8 --- /dev/null +++ b/llm_statistics.txt @@ -0,0 +1,74 @@ +LLM请求统计报告 (生成时间: 2025-03-07 20:38:57) +================================================== + +所有时间统计 +====== +总请求数: 858 +总Token数: 285415 +总花费: ¥0.3309 + +按模型统计: +- Pro/Qwen/Qwen2-VL-7B-Instruct: 67次 (花费: ¥0.0272) +- Pro/Qwen/Qwen2.5-7B-Instruct: 646次 (花费: ¥0.0718) +- Pro/deepseek-ai/DeepSeek-V3: 9次 (花费: ¥0.0193) +- Qwen/QwQ-32B: 29次 (花费: ¥0.1246) +- Qwen/Qwen2.5-32B-Instruct: 55次 (花费: ¥0.0771) +- deepseek-ai/DeepSeek-R1-Distill-Qwen-32B: 3次 (花费: ¥0.0067) +- deepseek-ai/DeepSeek-V2.5: 49次 (花费: ¥0.0043) + +按请求类型统计: +- chat: 858次 (花费: ¥0.3309) + +最近7天统计 +====== +总请求数: 858 +总Token数: 285415 +总花费: ¥0.3309 + +按模型统计: +- Pro/Qwen/Qwen2-VL-7B-Instruct: 67次 (花费: ¥0.0272) +- Pro/Qwen/Qwen2.5-7B-Instruct: 646次 (花费: ¥0.0718) +- Pro/deepseek-ai/DeepSeek-V3: 9次 (花费: ¥0.0193) +- Qwen/QwQ-32B: 29次 (花费: ¥0.1246) +- Qwen/Qwen2.5-32B-Instruct: 55次 (花费: ¥0.0771) +- deepseek-ai/DeepSeek-R1-Distill-Qwen-32B: 3次 (花费: ¥0.0067) +- deepseek-ai/DeepSeek-V2.5: 49次 (花费: ¥0.0043) + +按请求类型统计: +- chat: 858次 (花费: ¥0.3309) + +最近24小时统计 +======== +总请求数: 858 +总Token数: 285415 +总花费: ¥0.3309 + +按模型统计: +- Pro/Qwen/Qwen2-VL-7B-Instruct: 67次 (花费: ¥0.0272) +- Pro/Qwen/Qwen2.5-7B-Instruct: 646次 (花费: ¥0.0718) +- Pro/deepseek-ai/DeepSeek-V3: 9次 (花费: ¥0.0193) +- Qwen/QwQ-32B: 29次 (花费: ¥0.1246) +- Qwen/Qwen2.5-32B-Instruct: 55次 (花费: ¥0.0771) +- deepseek-ai/DeepSeek-R1-Distill-Qwen-32B: 3次 (花费: ¥0.0067) +- deepseek-ai/DeepSeek-V2.5: 49次 (花费: ¥0.0043) + +按请求类型统计: +- chat: 858次 (花费: ¥0.3309) + +最近1小时统计 +======= +总请求数: 858 +总Token数: 285415 +总花费: ¥0.3309 + +按模型统计: +- Pro/Qwen/Qwen2-VL-7B-Instruct: 67次 (花费: ¥0.0272) +- Pro/Qwen/Qwen2.5-7B-Instruct: 646次 (花费: ¥0.0718) +- Pro/deepseek-ai/DeepSeek-V3: 9次 (花费: ¥0.0193) +- Qwen/QwQ-32B: 29次 (花费: ¥0.1246) +- Qwen/Qwen2.5-32B-Instruct: 55次 (花费: ¥0.0771) +- deepseek-ai/DeepSeek-R1-Distill-Qwen-32B: 3次 (花费: ¥0.0067) +- deepseek-ai/DeepSeek-V2.5: 49次 (花费: ¥0.0043) + +按请求类型统计: +- chat: 858次 (花费: ¥0.3309) \ No newline at end of file diff --git a/src/plugins/chat/__init__.py b/src/plugins/chat/__init__.py index 22f3059b5..f7da8ba96 100644 --- a/src/plugins/chat/__init__.py +++ b/src/plugins/chat/__init__.py @@ -14,6 +14,10 @@ from nonebot.rule import to_me from .bot import chat_bot from .emoji_manager import emoji_manager import time +from ..utils.statistic import LLMStatistics + +# 创建LLM统计实例 +llm_stats = LLMStatistics("llm_statistics.txt") # 添加标志变量 _message_manager_started = False @@ -57,6 +61,10 @@ scheduler = require("nonebot_plugin_apscheduler").scheduler @driver.on_startup async def start_background_tasks(): """启动后台任务""" + # 启动LLM统计 + llm_stats.start() + print("\033[1;32m[初始化]\033[0m LLM统计功能已启动") + # 只启动表情包管理任务 asyncio.create_task(emoji_manager.start_periodic_check(interval_MINS=global_config.EMOJI_CHECK_INTERVAL)) await bot_schedule.initialize() diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 7bfc966f6..a471bd72d 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -8,6 +8,8 @@ from nonebot import get_driver from loguru import logger from ..chat.config import global_config from ..chat.utils_image import compress_base64_image_by_scale +from datetime import datetime +from ...common.database import Database driver = get_driver() config = driver.config @@ -24,6 +26,75 @@ class LLM_request: raise ValueError(f"配置错误：找不到对应的配置项 - {str(e)}") from e self.model_name = model["name"] self.params = kwargs + + self.pri_in = model.get("pri_in", 0) + self.pri_out = model.get("pri_out", 0) + + # 获取数据库实例 + self.db = Database.get_instance() + self._init_database() + + def _init_database(self): + """初始化数据库集合""" + try: + # 创建llm_usage集合的索引 + self.db.db.llm_usage.create_index([("timestamp", 1)]) + self.db.db.llm_usage.create_index([("model_name", 1)]) + self.db.db.llm_usage.create_index([("user_id", 1)]) + self.db.db.llm_usage.create_index([("request_type", 1)]) + except Exception as e: + logger.error(f"创建数据库索引失败: {e}") + + def _record_usage(self, prompt_tokens: int, completion_tokens: int, total_tokens: int, + user_id: str = "system", request_type: str = "chat", + endpoint: str = "/chat/completions"): + """记录模型使用情况到数据库 + Args: + prompt_tokens: 输入token数 + completion_tokens: 输出token数 + total_tokens: 总token数 + user_id: 用户ID，默认为system + request_type: 请求类型(chat/embedding/image等) + endpoint: API端点 + """ + try: + usage_data = { + "model_name": self.model_name, + "user_id": user_id, + "request_type": request_type, + "endpoint": endpoint, + "prompt_tokens": prompt_tokens, + "completion_tokens": completion_tokens, + "total_tokens": total_tokens, + "cost": self._calculate_cost(prompt_tokens, completion_tokens), + "status": "success", + "timestamp": datetime.now() + } + self.db.db.llm_usage.insert_one(usage_data) + logger.info( + f"Token使用情况 - 模型: {self.model_name}, " + f"用户: {user_id}, 类型: {request_type}, " + f"提示词: {prompt_tokens}, 完成: {completion_tokens}, " + f"总计: {total_tokens}" + ) + except Exception as e: + logger.error(f"记录token使用情况失败: {e}") + + def _calculate_cost(self, prompt_tokens: int, completion_tokens: int) -> float: + """计算API调用成本 + 使用模型的pri_in和pri_out价格计算输入和输出的成本 + + Args: + prompt_tokens: 输入token数量 + completion_tokens: 输出token数量 + + Returns: + float: 总成本（元） + """ + # 使用模型的pri_in和pri_out计算成本 + input_cost = (prompt_tokens / 1000000) * self.pri_in + output_cost = (completion_tokens / 1000000) * self.pri_out + return round(input_cost + output_cost, 6) async def _execute_request( self, @@ -33,6 +104,8 @@ class LLM_request: payload: dict = None, retry_policy: dict = None, response_handler: callable = None, + user_id: str = "system", + request_type: str = "chat" ): """统一请求执行入口 Args: @@ -40,10 +113,10 @@ class LLM_request: prompt: prompt文本 image_base64: 图片的base64编码 payload: 请求体数据 - is_async: 是否异步 retry_policy: 自定义重试策略 - (示例: {"max_retries":3, "base_wait":15, "retry_codes":[429,500]}) response_handler: 自定义响应处理器 + user_id: 用户ID + request_type: 请求类型 """ # 合并重试策略 default_retry = { @@ -105,7 +178,7 @@ class LLM_request: result = await response.json() # 使用自定义处理器或默认处理 - return response_handler(result) if response_handler else self._default_response_handler(result) + return response_handler(result) if response_handler else self._default_response_handler(result, user_id, request_type, endpoint) except Exception as e: if retry < policy["max_retries"] - 1: @@ -145,7 +218,8 @@ class LLM_request: **self.params } - def _default_response_handler(self, result: dict) -> Tuple: + def _default_response_handler(self, result: dict, user_id: str = "system", + request_type: str = "chat", endpoint: str = "/chat/completions") -> Tuple: """默认响应解析""" if "choices" in result and result["choices"]: message = result["choices"][0]["message"] @@ -157,6 +231,21 @@ class LLM_request: if not reasoning_content: reasoning_content = reasoning + # 记录token使用情况 + usage = result.get("usage", {}) + if usage: + prompt_tokens = usage.get("prompt_tokens", 0) + completion_tokens = usage.get("completion_tokens", 0) + total_tokens = usage.get("total_tokens", 0) + self._record_usage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + user_id=user_id, + request_type=request_type, + endpoint=endpoint + ) + return content, reasoning_content return "没有返回结果", "" @@ -244,3 +333,4 @@ class LLM_request: response_handler=embedding_handler ) return embedding + diff --git a/src/plugins/utils/statistic.py b/src/plugins/utils/statistic.py new file mode 100644 index 000000000..093ace539 --- /dev/null +++ b/src/plugins/utils/statistic.py @@ -0,0 +1,162 @@ +from typing import Dict, List, Any +import time +import threading +import json +from datetime import datetime, timedelta +from collections import defaultdict +from ...common.database import Database + +class LLMStatistics: + def __init__(self, output_file: str = "llm_statistics.txt"): + """初始化LLM统计类 + + Args: + output_file: 统计结果输出文件路径 + """ + self.db = Database.get_instance() + self.output_file = output_file + self.running = False + self.stats_thread = None + + def start(self): + """启动统计线程""" + if not self.running: + self.running = True + self.stats_thread = threading.Thread(target=self._stats_loop) + self.stats_thread.daemon = True + self.stats_thread.start() + + def stop(self): + """停止统计线程""" + self.running = False + if self.stats_thread: + self.stats_thread.join() + + def _collect_statistics_for_period(self, start_time: datetime) -> Dict[str, Any]: + """收集指定时间段的LLM请求统计数据 + + Args: + start_time: 统计开始时间 + """ + stats = { + "total_requests": 0, + "requests_by_type": defaultdict(int), + "requests_by_user": defaultdict(int), + "requests_by_model": defaultdict(int), + "average_tokens": 0, + "total_tokens": 0, + "total_cost": 0.0, + "costs_by_user": defaultdict(float), + "costs_by_type": defaultdict(float), + "costs_by_model": defaultdict(float) + } + + cursor = self.db.db.llm_usage.find({ + "timestamp": {"$gte": start_time} + }) + + total_requests = 0 + + for doc in cursor: + stats["total_requests"] += 1 + request_type = doc.get("request_type", "unknown") + user_id = str(doc.get("user_id", "unknown")) + model_name = doc.get("model_name", "unknown") + + stats["requests_by_type"][request_type] += 1 + stats["requests_by_user"][user_id] += 1 + stats["requests_by_model"][model_name] += 1 + + prompt_tokens = doc.get("prompt_tokens", 0) + completion_tokens = doc.get("completion_tokens", 0) + stats["total_tokens"] += prompt_tokens + completion_tokens + + cost = doc.get("cost", 0.0) + stats["total_cost"] += cost + stats["costs_by_user"][user_id] += cost + stats["costs_by_type"][request_type] += cost + stats["costs_by_model"][model_name] += cost + + total_requests += 1 + + if total_requests > 0: + stats["average_tokens"] = stats["total_tokens"] / total_requests + + return stats + + def _collect_all_statistics(self) -> Dict[str, Dict[str, Any]]: + """收集所有时间范围的统计数据""" + now = datetime.now() + + return { + "all_time": self._collect_statistics_for_period(datetime.min), + "last_7_days": self._collect_statistics_for_period(now - timedelta(days=7)), + "last_24_hours": self._collect_statistics_for_period(now - timedelta(days=1)), + "last_hour": self._collect_statistics_for_period(now - timedelta(hours=1)) + } + + def _format_stats_section(self, stats: Dict[str, Any], title: str) -> str: + """格式化统计部分的输出 + + Args: + stats: 统计数据 + title: 部分标题 + """ + output = [] + output.append(f"\n{title}") + output.append("=" * len(title)) + + output.append(f"总请求数: {stats['total_requests']}") + if stats['total_requests'] > 0: + output.append(f"总Token数: {stats['total_tokens']}") + output.append(f"总花费: ¥{stats['total_cost']:.4f}") + + output.append("\n按模型统计:") + for model_name, count in sorted(stats["requests_by_model"].items()): + cost = stats["costs_by_model"][model_name] + output.append(f"- {model_name}: {count}次 (花费: ¥{cost:.4f})") + + output.append("\n按请求类型统计:") + for req_type, count in sorted(stats["requests_by_type"].items()): + cost = stats["costs_by_type"][req_type] + output.append(f"- {req_type}: {count}次 (花费: ¥{cost:.4f})") + + return "\n".join(output) + + def _save_statistics(self, all_stats: Dict[str, Dict[str, Any]]): + """将统计结果保存到文件""" + current_time = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + + output = [] + output.append(f"LLM请求统计报告 (生成时间: {current_time})") + output.append("=" * 50) + + # 添加各个时间段的统计 + sections = [ + ("所有时间统计", "all_time"), + ("最近7天统计", "last_7_days"), + ("最近24小时统计", "last_24_hours"), + ("最近1小时统计", "last_hour") + ] + + for title, key in sections: + output.append(self._format_stats_section(all_stats[key], title)) + + # 写入文件 + with open(self.output_file, "w", encoding="utf-8") as f: + f.write("\n".join(output)) + + def _stats_loop(self): + """统计循环，每1分钟运行一次""" + while self.running: + try: + all_stats = self._collect_all_statistics() + self._save_statistics(all_stats) + except Exception as e: + print(f"\033[1;31m[错误]\033[0m 统计数据处理失败: {e}") + + # 等待1分钟 + for _ in range(60): + if not self.running: + break + time.sleep(1) From cd6ec4d26e682a865337bad4c7cb3e7d005c0578 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Fri, 7 Mar 2025 20:42:44 +0800 Subject: [PATCH 53/53] Update bot_config_template.toml --- templete/bot_config_template.toml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/templete/bot_config_template.toml b/templete/bot_config_template.toml index ad47c90c5..e6246be07 100644 --- a/templete/bot_config_template.toml +++ b/templete/bot_config_template.toml @@ -68,6 +68,8 @@ ban_user_id = [] #禁止回复消息的QQ号 name = "Pro/deepseek-ai/DeepSeek-R1" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" +pri_in = 0 #模型的输入价格（非必填，可以记录消耗） +pri_out = 0 #模型的输出价格（非必填，可以记录消耗） [model.llm_reasoning_minor] #回复模型3 次要回复模型 name = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" @@ -100,12 +102,15 @@ key = "SILICONFLOW_KEY" name = "Qwen/Qwen2.5-32B-Instruct" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" +pri_in = 0 +pri_out = 0 [model.moderation] #内容审核 未启用 name = "" base_url = "SILICONFLOW_BASE_URL" key = "SILICONFLOW_KEY" - +pri_in = 0 +pri_out = 0 # 识图模型