From 19e467febefe78437cff4c3537eeeb568d917dc9 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Wed, 5 Mar 2025 23:13:47 +0800 Subject: [PATCH 1/3] =?UTF-8?q?fix:=20=E8=A2=AB=E7=A6=81=E8=A8=80=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E5=8F=91=E9=80=81=E7=BA=BF=E7=A8=8B=E6=AD=BB=E6=8E=89?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E5=A4=8D=E6=89=80=E6=9C=89=E7=9A=84=E5=8E=8B?= =?UTF-8?q?=E7=BC=A9=EF=BC=8C=E4=BF=AE=E5=A4=8D=E7=A9=BA=E6=97=A5=E7=A8=8B?= =?UTF-8?q?=E7=9A=84key=20error?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 3 + src/plugins/chat/message_sender.py | 16 +++-- src/plugins/chat/utils_image.py | 73 +++++++++++++++------- src/plugins/models/utils_model.py | 3 + src/plugins/schedule/schedule_generator.py | 2 + 5 files changed, 70 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index f704a19ba..51a11d8c2 100644 --- a/.gitignore +++ b/.gitignore @@ -185,3 +185,6 @@ cython_debug/ # PyPI configuration file .pypirc .env + +# jieba +jieba.cache diff --git a/src/plugins/chat/message_sender.py b/src/plugins/chat/message_sender.py index 998586943..970fd3682 100644 --- a/src/plugins/chat/message_sender.py +++ b/src/plugins/chat/message_sender.py @@ -52,12 +52,16 @@ class Message_Sender: await asyncio.sleep(typing_time) # 发送消息 - await self._current_bot.send_group_msg( - group_id=group_id, - message=message, - auto_escape=auto_escape - ) - print(f"\033[1;34m[调试]\033[0m 发送消息{message}成功") + try: + await self._current_bot.send_group_msg( + group_id=group_id, + message=message, + auto_escape=auto_escape + ) + print(f"\033[1;34m[调试]\033[0m 发送消息{message}成功") + except Exception as e: + print(f"发生错误 {e}") + print(f"\033[1;34m[调试]\033[0m 发送消息{message}失败") class MessageContainer: diff --git a/src/plugins/chat/utils_image.py b/src/plugins/chat/utils_image.py index efe2f1c92..922ab5228 100644 --- a/src/plugins/chat/utils_image.py +++ b/src/plugins/chat/utils_image.py @@ -7,6 +7,7 @@ from ...common.database import Database import zlib # 用于 CRC32 import base64 from nonebot import get_driver +from loguru import logger driver = get_driver() config = driver.config @@ -213,11 +214,11 @@ def storage_image(image_data: bytes) -> bytes: print(f"\033[1;31m[错误]\033[0m 保存图片失败: {str(e)}") return image_data -def compress_base64_image_by_scale(base64_data: str, scale: float = 0.5) -> str: - """按比例压缩base64格式的图片 +def compress_base64_image_by_scale(base64_data: str, target_size: int = 0.8 * 1024 * 1024) -> str: + """压缩base64格式的图片到指定大小 Args: base64_data: base64编码的图片数据 - scale: 压缩比例（0-1之间的浮点数） + target_size: 目标文件大小（字节），默认0.8MB Returns: str: 压缩后的base64图片数据 """ @@ -225,34 +226,64 @@ def compress_base64_image_by_scale(base64_data: str, scale: float = 0.5) -> str: # 将base64转换为字节数据 image_data = base64.b64decode(base64_data) + # 如果已经小于目标大小，直接返回原图 + if len(image_data) <= target_size: + return base64_data + # 将字节数据转换为图片对象 img = Image.open(io.BytesIO(image_data)) - # 如果是动图，直接返回原图 - if getattr(img, 'is_animated', False): - return base64_data - + # 获取原始尺寸 + original_width, original_height = img.size + + # 计算缩放比例 + scale = min(1.0, (target_size / len(image_data)) ** 0.5) + # 计算新的尺寸 - new_width = int(img.width * scale) - new_height = int(img.height * scale) + new_width = int(original_width * scale) + new_height = int(original_height * scale) - # 缩放图片 - img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) + # 创建内存缓冲区 + output_buffer = io.BytesIO() - # 转换为RGB模式（去除透明通道） - if img.mode in ('RGBA', 'P'): - img = img.convert('RGB') + # 如果是GIF，处理所有帧 + if getattr(img, "is_animated", False): + frames = [] + for frame_idx in range(img.n_frames): + img.seek(frame_idx) + new_frame = img.copy() + new_frame = new_frame.resize((new_width, new_height), Image.Resampling.LANCZOS) + frames.append(new_frame) + + # 保存到缓冲区 + frames[0].save( + output_buffer, + format='GIF', + save_all=True, + append_images=frames[1:], + optimize=True, + duration=img.info.get('duration', 100), + loop=img.info.get('loop', 0) + ) + else: + # 处理静态图片 + resized_img = img.resize((new_width, new_height), Image.Resampling.LANCZOS) + + # 保存到缓冲区，保持原始格式 + if img.format == 'PNG' and img.mode in ('RGBA', 'LA'): + resized_img.save(output_buffer, format='PNG', optimize=True) + else: + resized_img.save(output_buffer, format='JPEG', quality=95, optimize=True) - # 保存压缩后的图片 - output = io.BytesIO() - img.save(output, format='JPEG', quality=85, optimize=True) - compressed_data = output.getvalue() + # 获取压缩后的数据并转换为base64 + compressed_data = output_buffer.getvalue() + logger.success(f"压缩图片: {original_width}x{original_height} -> {new_width}x{new_height}") + logger.info(f"压缩前大小: {len(image_data)/1024:.1f}KB, 压缩后大小: {len(compressed_data)/1024:.1f}KB") - # 转换回base64 return base64.b64encode(compressed_data).decode('utf-8') except Exception as e: - print(f"\033[1;31m[错误]\033[0m 压缩图片失败: {str(e)}") + logger.error(f"压缩图片失败: {str(e)}") import traceback - print(traceback.format_exc()) + logger.error(traceback.format_exc()) return base64_data \ No newline at end of file diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 3ba873d74..858755cc0 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -124,6 +124,7 @@ class LLM_request: base_wait_time = 15 current_image_base64 = image_base64 + current_image_base64 = compress_base64_image_by_scale(current_image_base64) for retry in range(max_retries): @@ -177,6 +178,8 @@ class LLM_request: "Content-Type": "application/json" } + image_base64=compress_base64_image_by_scale(image_base64) + # 构建请求体 data = { "model": self.model_name, diff --git a/src/plugins/schedule/schedule_generator.py b/src/plugins/schedule/schedule_generator.py index 7afa5e7ba..c9a1c8910 100644 --- a/src/plugins/schedule/schedule_generator.py +++ b/src/plugins/schedule/schedule_generator.py @@ -104,6 +104,8 @@ class ScheduleGenerator: min_diff = float('inf') # 检查今天的日程 + if not self.today_schedule.keys(): + return "摸鱼" for time_str in self.today_schedule.keys(): diff = abs(self._time_diff(current_time, time_str)) if closest_time is None or diff < min_diff: From a70f76c8192254098728ca66c017dfbd63992423 Mon Sep 17 00:00:00 2001 From: SengokuCola <1026294844@qq.com> Date: Wed, 5 Mar 2025 23:58:03 +0800 Subject: [PATCH 2/3] =?UTF-8?q?v0.5.4.0=20=E8=AE=B0=E5=BF=86=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 移除jieba --- src/plugins/chat/__init__.py | 33 ++-- src/plugins/chat/bot.py | 8 +- src/plugins/chat/config.py | 2 +- src/plugins/chat/llm_generator.py | 6 +- src/plugins/chat/prompt_builder.py | 14 +- src/plugins/chat/topic_identifier.py | 62 +------ src/plugins/chat/utils.py | 44 +++-- src/plugins/chat/willing_manager.py | 4 +- src/plugins/knowledege/knowledge_library.py | 2 +- src/plugins/memory_system/memory.py | 123 +++++++------ .../memory_system/memory_manual_build.py | 163 ++++++++---------- src/plugins/models/utils_model.py | 59 +++++++ 12 files changed, 266 insertions(+), 254 deletions(-) diff --git a/src/plugins/chat/__init__.py b/src/plugins/chat/__init__.py index 66824d986..ab99f6477 100644 --- a/src/plugins/chat/__init__.py +++ b/src/plugins/chat/__init__.py @@ -13,6 +13,7 @@ from .willing_manager import willing_manager from nonebot.rule import to_me from .bot import chat_bot from .emoji_manager import emoji_manager +import time # 获取驱动器 @@ -86,31 +87,27 @@ async def _(bot: Bot): async def _(bot: Bot, event: GroupMessageEvent, state: T_State): await chat_bot.handle_message(event, bot) -''' -@scheduler.scheduled_job("interval", seconds=300000, id="monitor_relationships") -async def monitor_relationships(): - """每15秒打印一次关系数据""" - relationship_manager.print_all_relationships() -''' - # 添加build_memory定时任务 @scheduler.scheduled_job("interval", seconds=global_config.build_memory_interval, id="build_memory") async def build_memory_task(): """每30秒执行一次记忆构建""" - print("\033[1;32m[记忆构建]\033[0m 开始构建记忆...") - await hippocampus.operation_build_memory(chat_size=30) - print("\033[1;32m[记忆构建]\033[0m 记忆构建完成") + print("\033[1;32m[记忆构建]\033[0m -------------------------------------------开始构建记忆-------------------------------------------") + start_time = time.time() + await hippocampus.operation_build_memory(chat_size=20) + end_time = time.time() + print(f"\033[1;32m[记忆构建]\033[0m -------------------------------------------记忆构建完成：耗时: {end_time - start_time:.2f} 秒-------------------------------------------") + @scheduler.scheduled_job("interval", seconds=global_config.forget_memory_interval, id="forget_memory") async def forget_memory_task(): """每30秒执行一次记忆构建""" - print("\033[1;32m[记忆遗忘]\033[0m 开始遗忘记忆...") - await hippocampus.operation_forget_topic(percentage=0.1) - print("\033[1;32m[记忆遗忘]\033[0m 记忆遗忘完成") + # print("\033[1;32m[记忆遗忘]\033[0m 开始遗忘记忆...") + # await hippocampus.operation_forget_topic(percentage=0.1) + # print("\033[1;32m[记忆遗忘]\033[0m 记忆遗忘完成") -@scheduler.scheduled_job("interval", seconds=global_config.build_memory_interval + 10, id="build_memory") -async def build_memory_task(): +@scheduler.scheduled_job("interval", seconds=global_config.build_memory_interval + 10, id="merge_memory") +async def merge_memory_task(): """每30秒执行一次记忆构建""" - print("\033[1;32m[记忆整合]\033[0m 开始整合") - await hippocampus.operation_merge_memory(percentage=0.1) - print("\033[1;32m[记忆整合]\033[0m 记忆整合完成") + # print("\033[1;32m[记忆整合]\033[0m 开始整合") + # await hippocampus.operation_merge_memory(percentage=0.1) + # print("\033[1;32m[记忆整合]\033[0m 记忆整合完成") diff --git a/src/plugins/chat/bot.py b/src/plugins/chat/bot.py index 6b0e76db5..e3525b3bb 100644 --- a/src/plugins/chat/bot.py +++ b/src/plugins/chat/bot.py @@ -69,11 +69,9 @@ class ChatBot: current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(message.time)) - identifier=topic_identifier.identify_topic() - if global_config.topic_extract=='llm': - topic=await identifier(message.processed_plain_text) - else: - topic=identifier(message.detailed_plain_text) + + topic=await topic_identifier.identify_topic_llm(message.processed_plain_text) + # topic1 = topic_identifier.identify_topic_jieba(message.processed_plain_text) # topic2 = await topic_identifier.identify_topic_llm(message.processed_plain_text) diff --git a/src/plugins/chat/config.py b/src/plugins/chat/config.py index 298683054..be599f48a 100644 --- a/src/plugins/chat/config.py +++ b/src/plugins/chat/config.py @@ -26,7 +26,7 @@ class BotConfig: talk_frequency_down_groups = set() ban_user_id = set() - build_memory_interval: int = 60 # 记忆构建间隔（秒） + build_memory_interval: int = 30 # 记忆构建间隔（秒） forget_memory_interval: int = 300 # 记忆遗忘间隔（秒） EMOJI_CHECK_INTERVAL: int = 120 # 表情包检查间隔（分钟） EMOJI_REGISTER_INTERVAL: int = 10 # 表情包注册间隔（分钟） diff --git a/src/plugins/chat/llm_generator.py b/src/plugins/chat/llm_generator.py index f380d62f4..04f2e73ad 100644 --- a/src/plugins/chat/llm_generator.py +++ b/src/plugins/chat/llm_generator.py @@ -95,7 +95,11 @@ class ResponseGenerator: # return None # 生成回复 - content, reasoning_content = await model.generate_response(prompt) + try: + content, reasoning_content = await model.generate_response(prompt) + except Exception as e: + print(f"生成回复时出错: {e}") + return None # 保存到数据库 self._save_to_db( diff --git a/src/plugins/chat/prompt_builder.py b/src/plugins/chat/prompt_builder.py index c354631c6..ba22a403d 100644 --- a/src/plugins/chat/prompt_builder.py +++ b/src/plugins/chat/prompt_builder.py @@ -36,7 +36,9 @@ class PromptBuilder: memory_prompt = '' start_time = time.time() # 记录开始时间 - topic = topic_identifier.identify_topic_jieba(message_txt) + # topic = await topic_identifier.identify_topic_llm(message_txt) + topic = topic_identifier.identify_topic_snownlp(message_txt) + # print(f"\033[1;32m[pb主题识别]\033[0m 主题: {topic}") all_first_layer_items = [] # 存储所有第一层记忆 @@ -64,15 +66,7 @@ class PromptBuilder: if overlap: # print(f"\033[1;32m[前额叶]\033[0m 发现主题 '{current_topic}' 和 '{other_topic}' 有共同的第二层记忆: {overlap}") overlapping_second_layer.update(overlap) - - # 合并所有需要的记忆 - # if all_first_layer_items: - # print(f"\033[1;32m[前额叶]\033[0m 合并所有需要的记忆1: {all_first_layer_items}") - # if overlapping_second_layer: - # print(f"\033[1;32m[前额叶]\033[0m 合并所有需要的记忆2: {list(overlapping_second_layer)}") - - # 使用集合去重 - # 从每个来源随机选择2条记忆（如果有的话） + selected_first_layer = random.sample(all_first_layer_items, min(2, len(all_first_layer_items))) if all_first_layer_items else [] selected_second_layer = random.sample(list(overlapping_second_layer), min(2, len(overlapping_second_layer))) if overlapping_second_layer else [] diff --git a/src/plugins/chat/topic_identifier.py b/src/plugins/chat/topic_identifier.py index 07749e837..812d4e321 100644 --- a/src/plugins/chat/topic_identifier.py +++ b/src/plugins/chat/topic_identifier.py @@ -15,16 +15,6 @@ class TopicIdentifier: self.llm_client = LLM_request(model=global_config.llm_topic_extract) self.select=global_config.topic_extract - def identify_topic(self): - if self.select=='jieba': - return self.identify_topic_jieba - elif self.select=='snownlp': - return self.identify_topic_snownlp - elif self.select=='llm': - return self.identify_topic_llm - else: - return self.identify_topic_snownlp - async def identify_topic_llm(self, text: str) -> Optional[List[str]]: """识别消息主题，返回主题列表""" @@ -48,56 +38,10 @@ class TopicIdentifier: # 解析主题字符串为列表 topic_list = [t.strip() for t in topic.split(",") if t.strip()] + + print(f"\033[1;32m[主题识别]\033[0m 主题: {topic_list}") return topic_list if topic_list else None - def identify_topic_jieba(self, text: str) -> Optional[str]: - """使用jieba识别主题""" - words = jieba.lcut(text) - # 去除停用词和标点符号 - stop_words = { - '的', '了', '和', '是', '就', '都', '而', '及', '与', '这', '那', '但', '然', '却', - '因为', '所以', '如果', '虽然', '一个', '我', '你', '他', '她', '它', '我们', '你们', - '他们', '在', '有', '个', '把', '被', '让', '给', '从', '向', '到', '又', '也', '很', - '啊', '吧', '呢', '吗', '呀', '哦', '哈', '么', '嘛', '啦', '哎', '唉', '哇', '嗯', - '哼', '哪', '什么', '怎么', '为什么', '怎样', '如何', '什么样', '这样', '那样', '这么', - '那么', '多少', '几', '谁', '哪里', '哪儿', '什么时候', '何时', '为何', '怎么办', - '怎么样', '这些', '那些', '一些', '一点', '一下', '一直', '一定', '一般', '一样', - '一会儿', '一边', '一起', - # 添加更多量词 - '个', '只', '条', '张', '片', '块', '本', '册', '页', '幅', '面', '篇', '份', - '朵', '颗', '粒', '座', '幢', '栋', '间', '层', '家', '户', '位', '名', '群', - '双', '对', '打', '副', '套', '批', '组', '串', '包', '箱', '袋', '瓶', '罐', - # 添加更多介词 - '按', '按照', '把', '被', '比', '比如', '除', '除了', '当', '对', '对于', - '根据', '关于', '跟', '和', '将', '经', '经过', '靠', '连', '论', '通过', - '同', '往', '为', '为了', '围绕', '于', '由', '由于', '与', '在', '沿', '沿着', - '依', '依照', '以', '因', '因为', '用', '由', '与', '自', '自从' - } - - # 过滤掉停用词和标点符号,只保留名词和动词 - filtered_words = [] - for word in words: - if word not in stop_words and not word.strip() in { - '。', '，', '、', '：', '；', '！', '？', '"', '"', ''', ''', - '（', '）', '【', '】', '《', '》', '…', '—', '·', '、', '~', - '～', '+', '=', '-', '/', '\\', '|', '*', '#', '@', '$', '%', - '^', '&', '[', ']', '{', '}', '<', '>', '`', '_', '.', ',', - ';', ':', '\'', '"', '(', ')', '?', '!', '±', '×', '÷', '≠', - '≈', '∈', '∉', '⊆', '⊇', '⊂', '⊃', '∪', '∩', '∧', '∨' - }: - filtered_words.append(word) - - # 统计词频 - word_freq = {} - for word in filtered_words: - word_freq[word] = word_freq.get(word, 0) + 1 - - # 按词频排序，取前3个 - sorted_words = sorted(word_freq.items(), key=lambda x: x[1], reverse=True) - top_words = [word for word, freq in sorted_words[:3]] - - return top_words if top_words else None - def identify_topic_snownlp(self, text: str) -> Optional[List[str]]: """使用 SnowNLP 进行主题识别 @@ -113,7 +57,7 @@ class TopicIdentifier: try: s = SnowNLP(text) # 提取前3个关键词作为主题 - keywords = s.keywords(3) + keywords = s.keywords(5) return keywords if keywords else None except Exception as e: print(f"\033[1;31m[错误]\033[0m SnowNLP 处理失败: {str(e)}") diff --git a/src/plugins/chat/utils.py b/src/plugins/chat/utils.py index 63151592d..aa16268ef 100644 --- a/src/plugins/chat/utils.py +++ b/src/plugins/chat/utils.py @@ -75,13 +75,11 @@ def cosine_similarity(v1, v2): norm2 = np.linalg.norm(v2) return dot_product / (norm1 * norm2) -def calculate_information_content(text): +def calculate_information_content(text): """计算文本的信息量（熵）""" - # 统计字符频率 char_count = Counter(text) total_chars = len(text) - # 计算熵 entropy = 0 for count in char_count.values(): probability = count / total_chars @@ -90,27 +88,37 @@ def calculate_information_content(text): return entropy def get_cloest_chat_from_db(db, length: int, timestamp: str): - # 从数据库中根据时间戳获取离其最近的聊天记录 + """从数据库中获取最接近指定时间戳的聊天记录，并记录读取次数""" chat_text = '' - closest_record = db.db.messages.find_one({"time": {"$lte": timestamp}}, sort=[('time', -1)]) # 调试输出 - # print(f"距离time最近的消息时间: {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(closest_record['time'])))}") + closest_record = db.db.messages.find_one({"time": {"$lte": timestamp}}, sort=[('time', -1)]) - if closest_record: + if closest_record and closest_record.get('memorized', 0) < 4: closest_time = closest_record['time'] group_id = closest_record['group_id'] # 获取groupid # 获取该时间戳之后的length条消息，且groupid相同 - chat_record = list(db.db.messages.find({"time": {"$gt": closest_time}, "group_id": group_id}).sort('time', 1).limit(length)) - for record in chat_record: - time_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(int(record['time']))) - try: - displayname="[(%s)%s]%s" % (record["user_id"],record["user_nickname"],record["user_cardname"]) - except: - displayname=record["user_nickname"] or "用户" + str(record["user_id"]) - chat_text += f'[{time_str}] {displayname}: {record["processed_plain_text"]}\n' # 添加发送者和时间信息 + chat_records = list(db.db.messages.find( + {"time": {"$gt": closest_time}, "group_id": group_id} + ).sort('time', 1).limit(length)) + + # 更新每条消息的memorized属性 + for record in chat_records: + # 检查当前记录的memorized值 + current_memorized = record.get('memorized', 0) + if current_memorized > 3: + # print(f"消息已读取3次，跳过") + return '' + + # 更新memorized值 + db.db.messages.update_one( + {"_id": record["_id"]}, + {"$set": {"memorized": current_memorized + 1}} + ) + + chat_text += record["detailed_plain_text"] + return chat_text - - return [] # 如果没有找到记录，返回空列表 - + print(f"消息已读取3次，跳过") + return '' def get_recent_group_messages(db, group_id: int, limit: int = 12) -> list: """从数据库获取群组最近的消息记录 diff --git a/src/plugins/chat/willing_manager.py b/src/plugins/chat/willing_manager.py index ab8c5ee25..7559406f9 100644 --- a/src/plugins/chat/willing_manager.py +++ b/src/plugins/chat/willing_manager.py @@ -52,8 +52,8 @@ class WillingManager: reply_probability = reply_probability / 3.5 reply_probability = min(reply_probability, 1) - if reply_probability < 0.1: - reply_probability = 0.1 + if reply_probability < 0: + reply_probability = 0 return reply_probability def change_reply_willing_sent(self, group_id: int): diff --git a/src/plugins/knowledege/knowledge_library.py b/src/plugins/knowledege/knowledge_library.py index cd6122b4c..d7071985e 100644 --- a/src/plugins/knowledege/knowledge_library.py +++ b/src/plugins/knowledege/knowledge_library.py @@ -79,7 +79,7 @@ class KnowledgeLibrary: content = f.read() # 按1024字符分段 - segments = [content[i:i+400] for i in range(0, len(content), 400)] + segments = [content[i:i+600] for i in range(0, len(content), 600)] # 处理每个分段 for segment in segments: diff --git a/src/plugins/memory_system/memory.py b/src/plugins/memory_system/memory.py index 9ad740844..4d20d05a9 100644 --- a/src/plugins/memory_system/memory.py +++ b/src/plugins/memory_system/memory.py @@ -9,8 +9,14 @@ import random import time from ..chat.config import global_config from ...common.database import Database # 使用正确的导入语法 -from ..chat.utils import calculate_information_content, get_cloest_chat_from_db from ..models.utils_model import LLM_request +import math +from ..chat.utils import calculate_information_content, get_cloest_chat_from_db + + + + + class Memory_graph: def __init__(self): self.G = nx.Graph() # 使用 networkx 的图结构 @@ -126,8 +132,8 @@ class Memory_graph: class Hippocampus: def __init__(self,memory_graph:Memory_graph): self.memory_graph = memory_graph - self.llm_model = LLM_request(model = global_config.llm_normal,temperature=0.5) - self.llm_model_small = LLM_request(model = global_config.llm_normal_minor,temperature=0.5) + self.llm_model_get_topic = LLM_request(model = global_config.llm_normal_minor,temperature=0.5) + self.llm_model_summary = LLM_request(model = global_config.llm_normal,temperature=0.5) def calculate_node_hash(self, concept, memory_items): """计算节点的特征值""" @@ -158,54 +164,75 @@ class Hippocampus: random_time = current_timestamp - random.randint(3600*4, 3600*24) # 随机时间 chat_ = get_cloest_chat_from_db(db=self.memory_graph.db, length=chat_size, timestamp=random_time) chat_text.append(chat_) - return chat_text + return [text for text in chat_text if text] - async def memory_compress(self, input_text, rate=1): - information_content = calculate_information_content(input_text) - print(f"文本的信息量（熵）: {information_content:.4f} bits") - topic_num = max(1, min(5, int(information_content * rate / 4))) - topic_prompt = find_topic(input_text, topic_num) - topic_response = await self.llm_model.generate_response(topic_prompt) - # 检查 topic_response 是否为元组 - if isinstance(topic_response, tuple): - topics = topic_response[0].split(",") # 假设第一个元素是我们需要的字符串 - else: - topics = topic_response.split(",") - compressed_memory = set() + async def memory_compress(self, input_text, compress_rate=0.1): + print(input_text) + + #获取topics + topic_num = self.calculate_topic_num(input_text, compress_rate) + topics_response = await self.llm_model_get_topic.generate_response(self.find_topic_llm(input_text, topic_num)) + # 修改话题处理逻辑 + print(f"话题: {topics_response[0]}") + topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] + print(f"话题: {topics}") + + # 创建所有话题的请求任务 + tasks = [] for topic in topics: - topic_what_prompt = topic_what(input_text,topic) - topic_what_response = await self.llm_model_small.generate_response(topic_what_prompt) - compressed_memory.add((topic.strip(), topic_what_response[0])) # 将话题和记忆作为元组存储 + topic_what_prompt = self.topic_what(input_text, topic) + # 创建异步任务 + task = self.llm_model_summary.generate_response_async(topic_what_prompt) + tasks.append((topic.strip(), task)) + + # 等待所有任务完成 + compressed_memory = set() + for topic, task in tasks: + response = await task + if response: + compressed_memory.add((topic, response[0])) + return compressed_memory - async def operation_build_memory(self,chat_size=12): - #最近消息获取频率 - time_frequency = {'near':1,'mid':2,'far':2} + def calculate_topic_num(self,text, compress_rate): + """计算文本的话题数量""" + information_content = calculate_information_content(text) + topic_by_length = text.count('\n')*compress_rate + topic_by_information_content = max(1, min(5, int((information_content-3) * 2))) + topic_num = int((topic_by_length + topic_by_information_content)/2) + print(f"topic_by_length: {topic_by_length}, topic_by_information_content: {topic_by_information_content}, topic_num: {topic_num}") + return topic_num + + async def operation_build_memory(self,chat_size=20): + # 最近消息获取频率 + time_frequency = {'near':2,'mid':4,'far':2} memory_sample = self.get_memory_sample(chat_size,time_frequency) - # print(f"\033[1;32m[记忆构建]\033[0m 获取记忆样本: {memory_sample}") + for i, input_text in enumerate(memory_sample, 1): - #加载进度可视化 + # 加载进度可视化 + all_topics = [] progress = (i / len(memory_sample)) * 100 bar_length = 30 filled_length = int(bar_length * i // len(memory_sample)) bar = '█' * filled_length + '-' * (bar_length - filled_length) print(f"\n进度: [{bar}] {progress:.1f}% ({i}/{len(memory_sample)})") - if input_text: - # 生成压缩后记忆 - first_memory = set() - first_memory = await self.memory_compress(input_text, 2.5) - #将记忆加入到图谱中 - for topic, memory in first_memory: - topics = segment_text(topic) - print(f"\033[1;34m话题\033[0m: {topic},节点: {topics}, 记忆: {memory}") - for split_topic in topics: - self.memory_graph.add_dot(split_topic,memory) - for split_topic in topics: - for other_split_topic in topics: - if split_topic != other_split_topic: - self.memory_graph.connect_dot(split_topic, other_split_topic) - else: - print(f"空消息 跳过") + + # 生成压缩后记忆 ,表现为 (话题,记忆) 的元组 + compressed_memory = set() + compress_rate = 0.1 + compressed_memory = await self.memory_compress(input_text, compress_rate) + print(f"\033[1;33m压缩后记忆数量\033[0m: {len(compressed_memory)}") + + # 将记忆加入到图谱中 + for topic, memory in compressed_memory: + print(f"\033[1;32m添加节点\033[0m: {topic}") + self.memory_graph.add_dot(topic, memory) + all_topics.append(topic) # 收集所有话题 + for i in range(len(all_topics)): + for j in range(i + 1, len(all_topics)): + print(f"\033[1;32m连接节点\033[0m: {all_topics[i]} 和 {all_topics[j]}") + self.memory_graph.connect_dot(all_topics[i], all_topics[j]) + self.sync_memory_to_db() def sync_memory_to_db(self): @@ -448,19 +475,19 @@ class Hippocampus: else: print("\n本次检查没有需要合并的节点") + def find_topic_llm(self,text, topic_num): + prompt = f'这是一段文字：{text}。请你从这段话中总结出{topic_num}个关键的概念，可以是名词，动词，或者特定人物，帮我列出来，用逗号,隔开，尽可能精简。只需要列举{topic_num}个话题就好，不要有序号，不要告诉我其他内容。' + return prompt + + def topic_what(self,text, topic): + prompt = f'这是一段文字：{text}。我想让你基于这段文字来概括"{topic}"这个概念，帮我总结成一句自然的话，可以包含时间和人物，以及具体的观点。只输出这句话就好' + return prompt + def segment_text(text): seg_text = list(jieba.cut(text)) return seg_text -def find_topic(text, topic_num): - prompt = f'这是一段文字：{text}。请你从这段话中总结出{topic_num}个话题，帮我列出来，用逗号隔开，尽可能精简。只需要列举{topic_num}个话题就好，不要告诉我其他内容。' - return prompt - -def topic_what(text, topic): - prompt = f'这是一段文字：{text}。我想知道这记忆里有什么关于{topic}的话题，帮我总结成一句自然的话，可以包含时间和人物。只输出这句话就好' - return prompt - from nonebot import get_driver driver = get_driver() diff --git a/src/plugins/memory_system/memory_manual_build.py b/src/plugins/memory_system/memory_manual_build.py index 66933dd04..d6aa2f669 100644 --- a/src/plugins/memory_system/memory_manual_build.py +++ b/src/plugins/memory_system/memory_manual_build.py @@ -73,8 +73,6 @@ class Database: logger.error(f"初始化MongoDB失败: {str(e)}") raise - - def calculate_information_content(text): """计算文本的信息量（熵）""" char_count = Counter(text) @@ -88,19 +86,36 @@ def calculate_information_content(text): return entropy def get_cloest_chat_from_db(db, length: int, timestamp: str): - """从数据库中获取最接近指定时间戳的聊天记录""" + """从数据库中获取最接近指定时间戳的聊天记录，并记录读取次数""" chat_text = '' closest_record = db.db.messages.find_one({"time": {"$lte": timestamp}}, sort=[('time', -1)]) - if closest_record: + if closest_record and closest_record.get('memorized', 0) < 4: closest_time = closest_record['time'] group_id = closest_record['group_id'] # 获取groupid # 获取该时间戳之后的length条消息，且groupid相同 - chat_record = list(db.db.messages.find({"time": {"$gt": closest_time}, "group_id": group_id}).sort('time', 1).limit(length)) - for record in chat_record: + chat_records = list(db.db.messages.find( + {"time": {"$gt": closest_time}, "group_id": group_id} + ).sort('time', 1).limit(length)) + + # 更新每条消息的memorized属性 + for record in chat_records: + # 检查当前记录的memorized值 + current_memorized = record.get('memorized', 0) + if current_memorized > 3: + print(f"消息已读取3次，跳过") + return '' + + # 更新memorized值 + db.db.messages.update_one( + {"_id": record["_id"]}, + {"$set": {"memorized": current_memorized + 1}} + ) + chat_text += record["detailed_plain_text"] + return chat_text - + print(f"消息已读取3次，跳过") return '' class Memory_graph: @@ -186,24 +201,26 @@ class Hippocampus: self.memory_graph = memory_graph self.llm_model = LLMModel() self.llm_model_small = LLMModel(model_name="deepseek-ai/DeepSeek-V2.5") + self.llm_model_get_topic = LLMModel(model_name="Pro/Qwen/Qwen2.5-7B-Instruct") + self.llm_model_summary = LLMModel(model_name="Qwen/Qwen2.5-32B-Instruct") def get_memory_sample(self, chat_size=20, time_frequency:dict={'near':2,'mid':4,'far':3}): current_timestamp = datetime.datetime.now().timestamp() chat_text = [] #短期：1h 中期：4h 长期：24h for _ in range(time_frequency.get('near')): # 循环10次 - random_time = current_timestamp - random.randint(1, 3600) # 随机时间 + random_time = current_timestamp - random.randint(1, 3600*4) # 随机时间 chat_ = get_cloest_chat_from_db(db=self.memory_graph.db, length=chat_size, timestamp=random_time) chat_text.append(chat_) for _ in range(time_frequency.get('mid')): # 循环10次 - random_time = current_timestamp - random.randint(3600, 3600*4) # 随机时间 + random_time = current_timestamp - random.randint(3600*4, 3600*24) # 随机时间 chat_ = get_cloest_chat_from_db(db=self.memory_graph.db, length=chat_size, timestamp=random_time) chat_text.append(chat_) for _ in range(time_frequency.get('far')): # 循环10次 - random_time = current_timestamp - random.randint(3600*4, 3600*24) # 随机时间 + random_time = current_timestamp - random.randint(3600*24, 3600*24*7) # 随机时间 chat_ = get_cloest_chat_from_db(db=self.memory_graph.db, length=chat_size, timestamp=random_time) chat_text.append(chat_) - return chat_text + return [chat for chat in chat_text if chat] def calculate_topic_num(self,text, compress_rate): """计算文本的话题数量""" @@ -219,8 +236,9 @@ class Hippocampus: #获取topics topic_num = self.calculate_topic_num(input_text, compress_rate) - topics_response = await self.llm_model_small.generate_response_async(self.find_topic_llm(input_text, topic_num)) - topics = topics_response[0].split(",") + topics_response = await self.llm_model_get_topic.generate_response_async(self.find_topic_llm(input_text, topic_num)) + # 修改话题处理逻辑 + topics = [topic.strip() for topic in topics_response[0].replace("，", ",").replace("、", ",").replace(" ", ",").split(",") if topic.strip()] print(f"话题: {topics}") # 创建所有话题的请求任务 @@ -241,57 +259,41 @@ class Hippocampus: return compressed_memory async def operation_build_memory(self, chat_size=12): - #最近消息获取频率 - time_frequency = {'near':1,'mid':2,'far':2} - memory_sample = self.get_memory_sample(chat_size,time_frequency) + # 最近消息获取频率 + time_frequency = {'near': 3, 'mid': 8, 'far': 5} + memory_sample = self.get_memory_sample(chat_size, time_frequency) + all_topics = [] # 用于存储所有话题 + for i, input_text in enumerate(memory_sample, 1): - #加载进度可视化 + # 加载进度可视化 + all_topics = [] progress = (i / len(memory_sample)) * 100 bar_length = 30 filled_length = int(bar_length * i // len(memory_sample)) bar = '█' * filled_length + '-' * (bar_length - filled_length) print(f"\n进度: [{bar}] {progress:.1f}% ({i}/{len(memory_sample)})") + + # 生成压缩后记忆 ,表现为 (话题,记忆) 的元组 + compressed_memory = set() + compress_rate = 0.1 + compressed_memory = await self.memory_compress(input_text, compress_rate) + print(f"\033[1;33m压缩后记忆数量\033[0m: {len(compressed_memory)}") - if input_text: - # 生成压缩后记忆 ,表现为 (话题,记忆) 的元组 - compressed_memory = set() - compress_rate = 0.15 - compressed_memory = await self.memory_compress(input_text,compress_rate) - print(f"\033[1;33m压缩后记忆数量\033[0m: {len(compressed_memory)}") + # 将记忆加入到图谱中 + for topic, memory in compressed_memory: + print(f"\033[1;32m添加节点\033[0m: {topic}") + self.memory_graph.add_dot(topic, memory) + all_topics.append(topic) # 收集所有话题 + for i in range(len(all_topics)): + for j in range(i + 1, len(all_topics)): + print(f"\033[1;32m连接节点\033[0m: {all_topics[i]} 和 {all_topics[j]}") + self.memory_graph.connect_dot(all_topics[i], all_topics[j]) + - #将记忆加入到图谱中 - for topic, memory in compressed_memory: - # 将jieba分词结果转换为列表以便多次使用 - topics = list(jieba.cut(topic)) - print(f"\033[1;34m话题\033[0m: {topic}") - print(f"\033[1;34m分词结果\033[0m: {topics}") - print(f"\033[1;34m记忆\033[0m: {memory}") - - # 如果分词结果少于2个词，跳过连接 - if len(topics) < 2: - print(f"\033[1;31m分词结果少于2个词，跳过连接\033[0m") - # 仍然添加单个节点 - for split_topic in topics: - self.memory_graph.add_dot(split_topic, memory) - continue - - # 先添加所有节点 - for split_topic in topics: - print(f"\033[1;32m添加节点\033[0m: {split_topic}") - self.memory_graph.add_dot(split_topic, memory) - - # 再添加节点之间的连接 - for i, split_topic in enumerate(topics): - for j, other_split_topic in enumerate(topics): - if i < j: # 只连接一次，避免重复连接 - print(f"\033[1;32m连接节点\033[0m: {split_topic} 和 {other_split_topic}") - self.memory_graph.connect_dot(split_topic, other_split_topic) - else: - print(f"空消息 跳过") - # 每处理完一条消息就同步一次到数据库 - self.sync_memory_to_db_2() + + self.sync_memory_to_db() def sync_memory_from_db(self): """ @@ -344,7 +346,7 @@ class Hippocampus: nodes = sorted([source, target]) return hash(f"{nodes[0]}:{nodes[1]}") - def sync_memory_to_db_2(self): + def sync_memory_to_db(self): """ 检查并同步内存中的图结构与数据库 使用特征值(哈希值)快速判断是否需要更新 @@ -448,11 +450,13 @@ class Hippocampus: logger.success("完成记忆图谱与数据库的差异同步") def find_topic_llm(self,text, topic_num): - prompt = f'这是一段文字：{text}。请你从这段话中总结出{topic_num}个话题，帮我列出来，用逗号隔开，尽可能精简。只需要列举{topic_num}个话题就好，不要告诉我其他内容。' + # prompt = f'这是一段文字：{text}。请你从这段话中总结出{topic_num}个话题，帮我列出来，用逗号隔开，尽可能精简。只需要列举{topic_num}个话题就好，不要告诉我其他内容。' + prompt = f'这是一段文字：{text}。请你从这段话中总结出{topic_num}个关键的概念，可以是名词，动词，或者特定人物，帮我列出来，用逗号,隔开，尽可能精简。只需要列举{topic_num}个话题就好，不要有序号，不要告诉我其他内容。' return prompt def topic_what(self,text, topic): - prompt = f'这是一段文字：{text}。我想知道这记忆里有什么关于{topic}的话题，帮我总结成一句自然的话，可以包含时间和人物，以及具体的观点。只输出这句话就好' + # prompt = f'这是一段文字：{text}。我想知道这段文字里有什么关于{topic}的话题，帮我总结成一句自然的话，可以包含时间和人物，以及具体的观点。只输出这句话就好' + prompt = f'这是一段文字：{text}。我想让你基于这段文字来概括"{topic}"这个概念，帮我总结成一句自然的话，可以包含时间和人物，以及具体的观点。只输出这句话就好' return prompt def remove_node_from_db(self, topic): @@ -557,7 +561,7 @@ class Hippocampus: # 同步到数据库 if forgotten_nodes: - self.sync_memory_to_db_2() + self.sync_memory_to_db() logger.info(f"完成遗忘操作，共遗忘 {len(forgotten_nodes)} 个节点的记忆") else: logger.info("本次检查没有节点满足遗忘条件") @@ -632,7 +636,7 @@ class Hippocampus: # 同步到数据库 if merged_nodes: - self.sync_memory_to_db_2() + self.sync_memory_to_db() print(f"\n完成记忆合并操作，共处理 {len(merged_nodes)} 个节点") else: print("\n本次检查没有需要合并的节点") @@ -667,7 +671,7 @@ def visualize_graph_lite(memory_graph: Memory_graph, color_by_memory: bool = Fal memory_count = len(memory_items) if isinstance(memory_items, list) else (1 if memory_items else 0) # 使用指数函数使变化更明显 ratio = memory_count / max_memories - size = 500 + 5000 * (ratio ** 2) # 使用平方函数使差异更明显 + size = 400 + 2000 * (ratio ** 2) # 增大节点大小 node_sizes.append(size) # 计算节点颜色（基于连接数） @@ -682,45 +686,22 @@ def visualize_graph_lite(memory_graph: Memory_graph, color_by_memory: bool = Fal blue = max(0.0, 1.0 - color_ratio) node_colors.append((red, 0, blue)) - # 获取边的权重和透明度 - edge_colors = [] - max_strength = 1 - - # 找出最大强度值 - for (u, v) in H.edges(): - strength = H[u][v].get('strength', 1) - max_strength = max(max_strength, strength) - - # 创建边权重字典用于布局 - edge_weights = {} - - # 计算每条边的透明度和权重 - for (u, v) in H.edges(): - strength = H[u][v].get('strength', 1) - # 将强度映射到透明度范围 [0.05, 0.8] - alpha = 0.02 + 0.55 * (strength / max_strength) - # 使用统一的蓝色，但透明度不同 - edge_colors.append((0, 0, 1, alpha)) - # 设置边的权重（强度越大，权重越大，节点间距离越小） - edge_weights[(u, v)] = strength - # 绘制图形 - plt.figure(figsize=(20, 16)) # 增加图形尺寸 - # 调整弹簧布局参数，使用边权重影响布局 + plt.figure(figsize=(16, 12)) # 减小图形尺寸 pos = nx.spring_layout(H, - k=2.0, # 增加节点间斥力 + k=1, # 调整节点间斥力 iterations=100, # 增加迭代次数 - scale=2.0, # 增加布局尺寸 + scale=1.5, # 减小布局尺寸 weight='strength') # 使用边的strength属性作为权重 nx.draw(H, pos, with_labels=True, node_color=node_colors, node_size=node_sizes, - font_size=8, # 稍微减小字体大小 + font_size=12, # 保持增大的字体大小 font_family='SimHei', font_weight='bold', - edge_color=edge_colors, + edge_color='gray', width=1.5) # 统一的边宽度 title = '记忆图谱可视化 - 节点大小表示记忆数量\n节点颜色：蓝(弱连接)到红(强连接)渐变，边的透明度表示连接强度\n连接强度越大的节点距离越近' @@ -733,7 +714,7 @@ async def main(): db = Database.get_instance() start_time = time.time() - test_pare = {'do_build_memory':False,'do_forget_topic':True,'do_visualize_graph':True,'do_query':False,'do_merge_memory':True} + test_pare = {'do_build_memory':True,'do_forget_topic':False,'do_visualize_graph':True,'do_query':False,'do_merge_memory':False} # 创建记忆图 memory_graph = Memory_graph() @@ -750,11 +731,11 @@ async def main(): # 构建记忆 if test_pare['do_build_memory']: logger.info("开始构建记忆...") - chat_size = 25 + chat_size = 20 await hippocampus.operation_build_memory(chat_size=chat_size) end_time = time.time() - logger.info(f"\033[32m[构建记忆耗时: {end_time - start_time:.2f} 秒,chat_size={chat_size},chat_count = {chat_size}]\033[0m") + logger.info(f"\033[32m[构建记忆耗时: {end_time - start_time:.2f} 秒,chat_size={chat_size},chat_count = 16]\033[0m") if test_pare['do_forget_topic']: logger.info("开始遗忘记忆...") diff --git a/src/plugins/models/utils_model.py b/src/plugins/models/utils_model.py index 3ba873d74..1b015ed57 100644 --- a/src/plugins/models/utils_model.py +++ b/src/plugins/models/utils_model.py @@ -55,6 +55,10 @@ class LLM_request: logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") await asyncio.sleep(wait_time) continue + + if response.status in [500, 503]: + logger.error(f"服务器错误: {response.status}") + raise RuntimeError("服务器负载过高，模型恢复失败QAQ") response.raise_for_status() # 检查其他响应状态 @@ -170,6 +174,61 @@ class LLM_request: logger.error("达到最大重试次数，请求仍然失败") raise RuntimeError("达到最大重试次数，API请求仍然失败") + async def generate_response_async(self, prompt: str) -> Union[str, Tuple[str, str]]: + """异步方式根据输入的提示生成模型的响应""" + headers = { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json" + } + + # 构建请求体 + data = { + "model": self.model_name, + "messages": [{"role": "user", "content": prompt}], + "temperature": 0.5, + **self.params + } + + # 发送请求到完整的 chat/completions 端点 + api_url = f"{self.base_url.rstrip('/')}/chat/completions" + logger.info(f"Request URL: {api_url}") # 记录请求的 URL + + max_retries = 3 + base_wait_time = 15 + + async with aiohttp.ClientSession() as session: + for retry in range(max_retries): + try: + async with session.post(api_url, headers=headers, json=data) as response: + if response.status == 429: + wait_time = base_wait_time * (2 ** retry) # 指数退避 + logger.warning(f"遇到请求限制(429)，等待{wait_time}秒后重试...") + await asyncio.sleep(wait_time) + continue + + response.raise_for_status() # 检查其他响应状态 + + result = await response.json() + if "choices" in result and len(result["choices"]) > 0: + content = result["choices"][0]["message"]["content"] + reasoning_content = result["choices"][0]["message"].get("reasoning_content", "") + return content, reasoning_content + return "没有返回结果", "" + + except Exception as e: + if retry < max_retries - 1: # 如果还有重试机会 + wait_time = base_wait_time * (2 ** retry) + logger.error(f"[回复]请求失败，等待{wait_time}秒后重试... 错误: {str(e)}") + await asyncio.sleep(wait_time) + else: + logger.error(f"请求失败: {str(e)}") + return f"请求失败: {str(e)}", "" + + logger.error("达到最大重试次数，请求仍然失败") + return "达到最大重试次数，请求仍然失败", "" + + + def generate_response_for_image_sync(self, prompt: str, image_base64: str) -> Tuple[str, str]: """同步方法：根据输入的提示和图片生成模型的响应""" headers = { From 8c45e7d2be01f6de3e1aa75f1f824c63dfa5b9a0 Mon Sep 17 00:00:00 2001 From: tcmofashi Date: Thu, 6 Mar 2025 00:40:39 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20=E5=BD=BB=E5=BA=95=E8=A7=A3=E5=86=B3?= =?UTF-8?q?schedule?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/plugins/schedule/schedule_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/plugins/schedule/schedule_generator.py b/src/plugins/schedule/schedule_generator.py index c9a1c8910..f2b11c33f 100644 --- a/src/plugins/schedule/schedule_generator.py +++ b/src/plugins/schedule/schedule_generator.py @@ -104,7 +104,7 @@ class ScheduleGenerator: min_diff = float('inf') # 检查今天的日程 - if not self.today_schedule.keys(): + if not self.today_schedule: return "摸鱼" for time_str in self.today_schedule.keys(): diff = abs(self._time_diff(current_time, time_str))