better：统一化聊天记录获取和格式化接口，修改分割函数

src/heart_flow/heartflow.py

@@ -216,7 +216,7 @@ class Heartflow:
 
         return response
 
-    def create_subheartflow(self, subheartflow_id):
+    async def create_subheartflow(self, subheartflow_id):
         """
         创建一个新的SubHeartflow实例
         添加一个SubHeartflow实例到self._subheartflows字典中
@@ -229,6 +229,7 @@ class Heartflow:
                 # 创建一个观察对象，目前只可以用chat_id创建观察对象
                 logger.debug(f"创建 observation: {subheartflow_id}")
                 observation = ChattingObservation(subheartflow_id)
+                await observation.initialize()
                 subheartflow.add_observation(observation)
                 logger.debug("添加 observation 成功")
                 # 创建异步任务

src/heart_flow/observation.py

@@ -3,10 +3,9 @@
 from datetime import datetime
 from src.plugins.models.utils_model import LLMRequest
 from src.config.config import global_config
-from src.common.database import db
 from src.common.logger import get_module_logger
 import traceback
-import asyncio
+from src.plugins.utils.chat_message_builder import get_raw_msg_before_timestamp_with_chat, build_readable_messages,get_raw_msg_by_timestamp_with_chat,num_new_messages_since
 
 logger = get_module_logger("observation")
 
@@ -37,25 +36,16 @@ class ChattingObservation(Observation):
         self.mid_memorys = []
         self.max_mid_memory_len = global_config.compress_length_limit
         self.mid_memory_info = ""
-        self.now_message_info = ""
-
-        # self._observe_lock = asyncio.Lock()  # 移除锁
-
-        # 初始化时加载最近的10条消息
-        initial_messages_cursor = (
-            db.messages.find({"chat_id": self.chat_id, "time": {"$lt": self.last_observe_time}})
-            .sort("time", -1)  # 按时间倒序
-            .limit(10)  # 获取最多10条
-        )
-        initial_messages = list(initial_messages_cursor)
-        initial_messages.reverse()  # 恢复时间正序
-
-        self.talking_message = initial_messages  # 将这些消息设为初始上下文
-        self.now_message_info = self.translate_message_list_to_str(self.talking_message)  # 更新初始的 now_message_info
 
         self.llm_summary = LLMRequest(
             model=global_config.llm_observation, temperature=0.7, max_tokens=300, request_type="chat_observation"
         )
+    
+    async def initialize(self):
+        initial_messages = get_raw_msg_before_timestamp_with_chat(self.chat_id, self.last_observe_time, 10)
+        self.talking_message = initial_messages  # 将这些消息设为初始上下文
+        self.talking_message_str = await build_readable_messages(self.talking_message)
+
 
     # 进行一次观察 返回观察结果observe_info
     def get_observe_info(self, ids=None):
@@ -76,126 +66,80 @@ class ChattingObservation(Observation):
                 except Exception as e:
                     logger.error(f"获取mid_memory_id失败: {e}")
                     traceback.print_exc()
-                    # print(f"获取mid_memory_id失败: {e}")
-                    return self.now_message_info
+                    return self.talking_message_str
 
-            return mid_memory_str + "现在群里正在聊：\n" + self.now_message_info
+            return mid_memory_str + "现在群里正在聊：\n" + self.talking_message_str
 
         else:
-            return self.now_message_info
+            return self.talking_message_str
 
     async def observe(self):
-        # async with self._observe_lock:  # 移除锁
         # 查找新消息，最多获取 self.max_now_obs_len 条
-        new_messages_cursor = (
-            db.messages.find({"chat_id": self.chat_id, "time": {"$gt": self.last_observe_time}})
-            .sort("time", -1)  # 按时间倒序排序
-            .limit(self.max_now_obs_len)  # 限制数量
+        print("2222222222222222221111111111111111开始观察")
+        new_messages_list = get_raw_msg_by_timestamp_with_chat(
+            chat_id=self.chat_id, 
+            timestamp_start=self.last_observe_time, 
+            timestamp_end=datetime.now().timestamp(), # 使用当前时间作为结束时间戳
+            limit=self.max_now_obs_len, 
+            limit_mode="latest"
         )
-        new_messages = list(new_messages_cursor)
-        new_messages.reverse()  # 反转列表，使消息按时间正序排列
-
-        if not new_messages:
-            # 如果没有获取到限制数量内的较新消息，可能仍然有更早的消息，但我们只关注最近的
-            # 检查是否有任何新消息（即使超出限制），以决定是否更新 last_observe_time
-            # 注意：这里的查询也可能与其他并发 observe 冲突，但锁保护了状态更新
-            # 由于外部已加锁，此处的并发冲突担忧不再需要
-            any_new_message = db.messages.find_one({"chat_id": self.chat_id, "time": {"$gt": self.last_observe_time}})
-            if not any_new_message:
-                return  # 确实没有新消息
-
-            # 如果有超过限制的更早的新消息，仍然需要更新时间戳，防止重复获取旧消息
-            # 但不将它们加入 talking_message
-            latest_message_time_cursor = (
-                db.messages.find({"chat_id": self.chat_id, "time": {"$gt": self.last_observe_time}})
-                .sort("time", -1)
-                .limit(1)
-            )
-            latest_time_doc = next(latest_message_time_cursor, None)
-            if latest_time_doc:
-                # 确保只在严格大于时更新，避免因并发查询导致时间戳回退
-                if latest_time_doc["time"] > self.last_observe_time:
-                    self.last_observe_time = latest_time_doc["time"]
-            return  # 返回，因为我们只关心限制内的最新消息
-
-        self.last_observe_time = new_messages[-1]["time"]
-        self.talking_message.extend(new_messages)
+        print(f"2222222222222222221111111111111111获取到新消息{len(new_messages_list)}条")
+        if new_messages_list:  # 检查列表是否为空
+            self.last_observe_time = new_messages_list[-1]["time"]
+            self.talking_message.extend(new_messages_list)
 
         if len(self.talking_message) > self.max_now_obs_len:
-            try:  # 使用 try...finally 仅用于可能的LLM调用错误处理
-                # 计算需要移除的消息数量，保留最新的 max_now_obs_len 条
-                messages_to_remove_count = len(self.talking_message) - self.max_now_obs_len
-                oldest_messages = self.talking_message[:messages_to_remove_count]
-                self.talking_message = self.talking_message[messages_to_remove_count:]  # 保留后半部分，即最新的
-                oldest_messages_str = "\n".join(
-                    [msg["detailed_plain_text"] for msg in oldest_messages if "detailed_plain_text" in msg]
-                )  # 增加检查
-                oldest_timestamps = [msg["time"] for msg in oldest_messages]
+            # 计算需要移除的消息数量，保留最新的 max_now_obs_len 条
+            messages_to_remove_count = len(self.talking_message) - self.max_now_obs_len
+            oldest_messages = self.talking_message[:messages_to_remove_count]
+            self.talking_message = self.talking_message[messages_to_remove_count:]  # 保留后半部分，即最新的
+            
+            oldest_messages_str = await build_readable_messages(oldest_messages)
 
-                # 调用 LLM 总结主题
-                prompt = f"请总结以下聊天记录的主题：\n{oldest_messages_str}\n主题,用一句话概括包括人物事件和主要信息，不要分点："
-                summary = "无法总结主题"  # 默认值
-                try:
-                    summary_result, _ = await self.llm_summary.generate_response_async(prompt)
-                    if summary_result:  # 确保结果不为空
-                        summary = summary_result
-                except Exception as e:
-                    logger.error(f"总结主题失败 for chat {self.chat_id}: {e}")
-                    # 保留默认总结 "无法总结主题"
+            # 调用 LLM 总结主题
+            prompt = f"请总结以下聊天记录的主题：\n{oldest_messages_str}\n用一句话概括包括人物事件和主要信息，不要分点："
+            summary = "没有主题的闲聊"  # 默认值
+            try:
+                summary_result, _ = await self.llm_summary.generate_response_async(prompt)
+                if summary_result:  # 确保结果不为空
+                    summary = summary_result
+            except Exception as e:
+                logger.error(f"总结主题失败 for chat {self.chat_id}: {e}")
+                # 保留默认总结 "没有主题的闲聊"
 
-                mid_memory = {
-                    "id": str(int(datetime.now().timestamp())),
-                    "theme": summary,
-                    "messages": oldest_messages,  # 存储原始消息对象
-                    "timestamps": oldest_timestamps,
-                    "chat_id": self.chat_id,
-                    "created_at": datetime.now().timestamp(),
-                }
-                # print(f"mid_memory：{mid_memory}")
-                # 存入内存中的 mid_memorys
-                self.mid_memorys.append(mid_memory)
-                if len(self.mid_memorys) > self.max_mid_memory_len:
-                    self.mid_memorys.pop(0)  # 移除最旧的
+            mid_memory = {
+                "id": str(int(datetime.now().timestamp())),
+                "theme": summary,
+                "messages": oldest_messages,  # 存储原始消息对象
+                "readable_messages": oldest_messages_str,
+                # "timestamps": oldest_timestamps,
+                "chat_id": self.chat_id,
+                "created_at": datetime.now().timestamp(),
+            }
 
-                mid_memory_str = "之前聊天的内容概述是：\n"
-                for mid_memory_item in self.mid_memorys:  # 重命名循环变量以示区分
-                    time_diff = int((datetime.now().timestamp() - mid_memory_item["created_at"]) / 60)
-                    mid_memory_str += (
-                        f"距离现在{time_diff}分钟前(聊天记录id:{mid_memory_item['id']})：{mid_memory_item['theme']}\n"
-                    )
-                self.mid_memory_info = mid_memory_str
-            except Exception as e:  # 将异常处理移至此处以覆盖整个总结过程
-                logger.error(f"处理和总结旧消息时出错 for chat {self.chat_id}: {e}")
-                traceback.print_exc()  # 记录详细堆栈
+            self.mid_memorys.append(mid_memory)
+            if len(self.mid_memorys) > self.max_mid_memory_len:
+                self.mid_memorys.pop(0)  # 移除最旧的
 
+            mid_memory_str = "之前聊天的内容概述是：\n"
+            for mid_memory_item in self.mid_memorys:  # 重命名循环变量以示区分
+                time_diff = int((datetime.now().timestamp() - mid_memory_item["created_at"]) / 60)
+                mid_memory_str += (
+                    f"距离现在{time_diff}分钟前(聊天记录id:{mid_memory_item['id']})：{mid_memory_item['theme']}\n"
+                )
+            self.mid_memory_info = mid_memory_str
+        # except Exception as e:  # 将异常处理移至此处以覆盖整个总结过程
+        #     logger.error(f"处理和总结旧消息时出错 for chat {self.chat_id}: {e}")
+        #     traceback.print_exc()  # 记录详细堆栈
             # print(f"处理后self.talking_message：{self.talking_message}")
 
-        now_message_str = ""
-        # 使用 self.translate_message_list_to_str 更新当前聊天内容
-        now_message_str += self.translate_message_list_to_str(talking_message=self.talking_message)
-        self.now_message_info = now_message_str
-
+        self.talking_message_str = await build_readable_messages(self.talking_message)
+        
         logger.trace(
-            f"Chat {self.chat_id} - 压缩早期记忆：{self.mid_memory_info}\n现在聊天内容：{self.now_message_info}"
+            f"Chat {self.chat_id} - 压缩早期记忆：{self.mid_memory_info}\n现在聊天内容：{self.talking_message_str}"
         )
 
     async def has_new_messages_since(self, timestamp: float) -> bool:
         """检查指定时间戳之后是否有新消息"""
-        try:
-            # 只需检查是否存在，不需要获取内容，使用 {"_id": 1} 提高效率
-            new_message = await asyncio.to_thread(
-                db.messages.find_one, {"chat_id": self.chat_id, "time": {"$gt": timestamp}}, {"_id": 1}
-            )
-            # new_message = db.messages.find_one({"chat_id": self.chat_id, "time": {"$gt": timestamp}}, {"_id": 1}) # find_one 不是异步的
-            return new_message is not None
-        except Exception as e:
-            logger.error(f"检查新消息时出错 for chat {self.chat_id} since {timestamp}: {e}")
-            return False
-
-    @staticmethod
-    def translate_message_list_to_str(talking_message):
-        talking_message_str = ""
-        for message in talking_message:
-            talking_message_str += message["detailed_plain_text"]
-
-        return talking_message_str
+        count = num_new_messages_since(chat_id=self.chat_id, timestamp_start=timestamp)
+        return count > 0

src/heart_flow/sub_heartflow.py

@@ -18,7 +18,6 @@ from src.common.logger import get_module_logger, LogConfig, SUB_HEARTFLOW_STYLE_
 # from typing import Union
 from src.individuality.individuality import Individuality
 import random
-from src.plugins.chat.chat_stream import ChatStream
 from src.plugins.person_info.relationship_manager import relationship_manager
 from ..plugins.utils.prompt_builder import Prompt, global_prompt_manager
 
@@ -40,7 +39,7 @@ def init_prompt():
     prompt += "刚刚你的想法是：\n我是{bot_name}，我想，{current_thinking_info}\n"
     prompt += "-----------------------------------\n"
     prompt += "现在是{time_now}，你正在上网，和qq群里的网友们聊天，群里正在聊的话题是：\n{chat_observe_info}\n"
-    prompt += "你现在{mood_info}\n"
+    prompt += "\n你现在{mood_info}\n"
     # prompt += "你注意到{sender_name}刚刚说：{message_txt}\n"
     prompt += "现在请你根据刚刚的想法继续思考，思考时可以想想如何对群聊内容进行回复，要不要对群里的话题进行回复，关注新话题，可以适当转换话题，大家正在说的话才是聊天的主题。\n"
     prompt += "回复的要求是：平淡一些，简短一些，说中文，如果你要回复，最好只回复一个人的一个话题\n"
@@ -90,40 +89,8 @@ class SubHeartflow:
 
         self.running_knowledges = []
 
-        self._thinking_lock = asyncio.Lock()  # 添加思考锁，防止并发思考
-
         self.bot_name = global_config.BOT_NICKNAME
 
-    def add_observation(self, observation: Observation):
-        """添加一个新的observation对象到列表中，如果已存在相同id的observation则不添加"""
-        # 查找是否存在相同id的observation
-        for existing_obs in self.observations:
-            if existing_obs.observe_id == observation.observe_id:
-                # 如果找到相同id的observation，直接返回
-                return
-        # 如果没有找到相同id的observation，则添加新的
-        self.observations.append(observation)
-
-    def remove_observation(self, observation: Observation):
-        """从列表中移除一个observation对象"""
-        if observation in self.observations:
-            self.observations.remove(observation)
-
-    def get_all_observations(self) -> list[Observation]:
-        """获取所有observation对象"""
-        return self.observations
-
-    def clear_observations(self):
-        """清空所有observation对象"""
-        self.observations.clear()
-
-    def _get_primary_observation(self) -> Optional[ChattingObservation]:
-        """获取主要的（通常是第一个）ChattingObservation实例"""
-        if self.observations and isinstance(self.observations[0], ChattingObservation):
-            return self.observations[0]
-        logger.warning(f"SubHeartflow {self.subheartflow_id} 没有找到有效的 ChattingObservation")
-        return None
-
     async def subheartflow_start_working(self):
         while True:
             current_time = time.time()
@@ -154,115 +121,134 @@ class SubHeartflow:
                 logger.error(f"[{self.subheartflow_id}] Error during pre-thinking observation: {e}")
                 logger.error(traceback.format_exc())
 
-    async def do_observe(self):
-        # 现在推荐使用 ensure_observed()，但保留此方法以兼容旧用法（或特定场景）
-        observation = self._get_primary_observation()
-        if observation:
-            await observation.observe()
-        else:
-            logger.error(f"[{self.subheartflow_id}] do_observe called but no valid observation found.")
 
     async def do_thinking_before_reply(
         self,
-        chat_stream: ChatStream,
         extra_info: str,
         obs_id: list[str] = None,  # 修改 obs_id 类型为 list[str]
     ):
-        async with self._thinking_lock:  # 获取思考锁
-            # --- 在思考前确保观察已执行 --- #
-            await self.ensure_observed()
+        # --- 在思考前确保观察已执行 --- #
+        # await self.ensure_observed()
 
-            self.last_active_time = time.time()  # 更新最后激活时间戳
+        self.last_active_time = time.time()  # 更新最后激活时间戳
 
-            current_thinking_info = self.current_mind
-            mood_info = self.current_state.mood
-            observation = self._get_primary_observation()
-            if not observation:
-                logger.error(f"[{self.subheartflow_id}] Cannot perform thinking without observation.")
-                return "", []  # 返回空结果
-
-            # --- 获取观察信息 --- #
-            chat_observe_info = ""
-            if obs_id:
-                try:
-                    chat_observe_info = observation.get_observe_info(obs_id)
-                    logger.debug(f"[{self.subheartflow_id}] Using specific observation IDs: {obs_id}")
-                except Exception as e:
-                    logger.error(
-                        f"[{self.subheartflow_id}] Error getting observe info with IDs {obs_id}: {e}. Falling back."
-                    )
-                    chat_observe_info = observation.get_observe_info()  # 出错时回退到默认观察
-            else:
-                chat_observe_info = observation.get_observe_info()
-                logger.debug(f"[{self.subheartflow_id}] Using default observation info.")
-
-            # --- 构建 Prompt (基本逻辑不变) --- #
-            extra_info_prompt = ""
-            if extra_info:
-                for tool_name, tool_data in extra_info.items():
-                    extra_info_prompt += f"{tool_name} 相关信息:\n"
-                    for item in tool_data:
-                        extra_info_prompt += f"- {item['name']}: {item['content']}\n"
-            else:
-                extra_info_prompt = "无工具信息。\n"  # 提供默认值
-
-            individuality = Individuality.get_instance()
-            prompt_personality = f"你的名字是{self.bot_name}，你"
-            prompt_personality += individuality.personality.personality_core
-
-            # 添加随机性格侧面
-            if individuality.personality.personality_sides:
-                random_side = random.choice(individuality.personality.personality_sides)
-                prompt_personality += f"，{random_side}"
-
-            # 添加随机身份细节
-            if individuality.identity.identity_detail:
-                random_detail = random.choice(individuality.identity.identity_detail)
-                prompt_personality += f"，{random_detail}"
-
-            time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
-
-            prompt = (await global_prompt_manager.get_prompt_async("sub_heartflow_prompt_before")).format(
-                extra_info=extra_info_prompt,
-                # relation_prompt_all=relation_prompt_all,
-                prompt_personality=prompt_personality,
-                bot_name=self.bot_name,
-                current_thinking_info=current_thinking_info,
-                time_now=time_now,
-                chat_observe_info=chat_observe_info,
-                mood_info=mood_info,
-                # sender_name=sender_name_sign,
-                # message_txt=message_txt,
-            )
-
-            prompt = await relationship_manager.convert_all_person_sign_to_person_name(prompt)
-            prompt = parse_text_timestamps(prompt, mode="lite")
-
-            logger.debug(f"[{self.subheartflow_id}] 心流思考prompt:\n{prompt}\n")
+        current_thinking_info = self.current_mind
+        mood_info = self.current_state.mood
+        observation = self._get_primary_observation()
 
+        # --- 获取观察信息 --- #
+        chat_observe_info = ""
+        if obs_id:
             try:
-                response, reasoning_content = await self.llm_model.generate_response_async(prompt)
-
-                logger.debug(f"[{self.subheartflow_id}] 心流思考结果:\n{response}\n")
-
-                if not response:  # 如果 LLM 返回空，给一个默认想法
-                    response = "(不知道该想些什么...)"
-                    logger.warning(f"[{self.subheartflow_id}] LLM 返回空结果，思考失败。")
+                chat_observe_info = observation.get_observe_info(obs_id)
+                logger.debug(f"[{self.subheartflow_id}] Using specific observation IDs: {obs_id}")
             except Exception as e:
-                logger.error(f"[{self.subheartflow_id}] 内心独白获取失败: {e}")
-                response = "(思考时发生错误...)"  # 错误时的默认想法
+                logger.error(
+                    f"[{self.subheartflow_id}] Error getting observe info with IDs {obs_id}: {e}. Falling back."
+                )
+                chat_observe_info = observation.get_observe_info()  # 出错时回退到默认观察
+        else:
+            chat_observe_info = observation.get_observe_info()
+            logger.debug(f"[{self.subheartflow_id}] Using default observation info.")
 
-            self.update_current_mind(response)
+        # --- 构建 Prompt (基本逻辑不变) --- #
+        extra_info_prompt = ""
+        if extra_info:
+            for tool_name, tool_data in extra_info.items():
+                extra_info_prompt += f"{tool_name} 相关信息:\n"
+                for item in tool_data:
+                    extra_info_prompt += f"- {item['name']}: {item['content']}\n"
+        else:
+            extra_info_prompt = "无工具信息。\n"  # 提供默认值
 
-            # self.current_mind 已经在 update_current_mind 中更新
+        individuality = Individuality.get_instance()
+        prompt_personality = f"你的名字是{self.bot_name}，你"
+        prompt_personality += individuality.personality.personality_core
+
+        # 添加随机性格侧面
+        if individuality.personality.personality_sides:
+            random_side = random.choice(individuality.personality.personality_sides)
+            prompt_personality += f"，{random_side}"
+
+        # 添加随机身份细节
+        if individuality.identity.identity_detail:
+            random_detail = random.choice(individuality.identity.identity_detail)
+            prompt_personality += f"，{random_detail}"
+
+        time_now = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+
+        prompt = (await global_prompt_manager.get_prompt_async("sub_heartflow_prompt_before")).format(
+            extra_info=extra_info_prompt,
+            # relation_prompt_all=relation_prompt_all,
+            prompt_personality=prompt_personality,
+            bot_name=self.bot_name,
+            current_thinking_info=current_thinking_info,
+            time_now=time_now,
+            chat_observe_info=chat_observe_info,
+            mood_info=mood_info,
+            # sender_name=sender_name_sign,
+            # message_txt=message_txt,
+        )
+
+        prompt = await relationship_manager.convert_all_person_sign_to_person_name(prompt)
+        prompt = parse_text_timestamps(prompt, mode="lite")
+
+        logger.debug(f"[{self.subheartflow_id}] 心流思考prompt:\n{prompt}\n")
+
+        try:
+            response, reasoning_content = await self.llm_model.generate_response_async(prompt)
+
+            logger.debug(f"[{self.subheartflow_id}] 心流思考结果:\n{response}\n")
+
+            if not response:  # 如果 LLM 返回空，给一个默认想法
+                response = "(不知道该想些什么...)"
+                logger.warning(f"[{self.subheartflow_id}] LLM 返回空结果，思考失败。")
+        except Exception as e:
+            logger.error(f"[{self.subheartflow_id}] 内心独白获取失败: {e}")
+            response = "(思考时发生错误...)"  # 错误时的默认想法
+
+        self.update_current_mind(response)
+
+        # self.current_mind 已经在 update_current_mind 中更新
+
+        # logger.info(f"[{self.subheartflow_id}] 思考前脑内状态：{self.current_mind}")
+        return self.current_mind, self.past_mind
 
-            # logger.info(f"[{self.subheartflow_id}] 思考前脑内状态：{self.current_mind}")
-            return self.current_mind, self.past_mind
 
     def update_current_mind(self, response):
         self.past_mind.append(self.current_mind)
         self.current_mind = response
 
+    def add_observation(self, observation: Observation):
+        """添加一个新的observation对象到列表中，如果已存在相同id的observation则不添加"""
+        # 查找是否存在相同id的observation
+        for existing_obs in self.observations:
+            if existing_obs.observe_id == observation.observe_id:
+                # 如果找到相同id的observation，直接返回
+                return
+        # 如果没有找到相同id的observation，则添加新的
+        self.observations.append(observation)
+
+    def remove_observation(self, observation: Observation):
+        """从列表中移除一个observation对象"""
+        if observation in self.observations:
+            self.observations.remove(observation)
+
+    def get_all_observations(self) -> list[Observation]:
+        """获取所有observation对象"""
+        return self.observations
+
+    def clear_observations(self):
+        """清空所有observation对象"""
+        self.observations.clear()
+
+    def _get_primary_observation(self) -> Optional[ChattingObservation]:
+        """获取主要的（通常是第一个）ChattingObservation实例"""
+        if self.observations and isinstance(self.observations[0], ChattingObservation):
+            return self.observations[0]
+        logger.warning(f"SubHeartflow {self.subheartflow_id} 没有找到有效的 ChattingObservation")
+        return None
+
 
 init_prompt()
 # subheartflow = SubHeartflow()