diff --git a/src/chat/message_receive/message.py b/src/chat/message_receive/message.py index b179a3098..1346e73c5 100644 --- a/src/chat/message_receive/message.py +++ b/src/chat/message_receive/message.py @@ -9,6 +9,7 @@ from maim_message import Seg, UserInfo, BaseMessageInfo, MessageBase from src.common.logger import get_logger from src.chat.utils.utils_image import get_image_manager +from src.chat.utils.utils_voice import get_voice_text from .chat_stream import ChatStream install(extra_lines=3) @@ -106,6 +107,7 @@ class MessageRecv(Message): self.has_emoji = False self.is_picid = False self.has_picid = False + self.is_voice = False self.is_mentioned = None self.is_command = False @@ -153,17 +155,27 @@ class MessageRecv(Message): self.has_emoji = True self.is_emoji = True self.is_picid = False + self.is_voice = False if isinstance(segment.data, str): return await get_image_manager().get_emoji_description(segment.data) return "[发了一个表情包,网卡了加载不出来]" + elif segment.type == "voice": + self.is_picid = False + self.is_emoji = False + self.is_voice = True + if isinstance(segment.data, str): + return await get_voice_text(segment.data) + return "[发了一段语音,网卡了加载不出来]" elif segment.type == "mention_bot": self.is_picid = False self.is_emoji = False + self.is_voice = False self.is_mentioned = float(segment.data) # type: ignore return "" elif segment.type == "priority_info": self.is_picid = False self.is_emoji = False + self.is_voice = False if isinstance(segment.data, dict): # 处理优先级信息 self.priority_mode = "priority" @@ -212,10 +224,12 @@ class MessageRecvS4U(MessageRecv): """ try: if segment.type == "text": + self.is_voice = False self.is_picid = False self.is_emoji = False return segment.data # type: ignore elif segment.type == "image": + self.is_voice = False # 如果是base64图片数据 if isinstance(segment.data, str): self.has_picid = True @@ -233,12 +247,22 @@ class MessageRecvS4U(MessageRecv): if isinstance(segment.data, str): return await get_image_manager().get_emoji_description(segment.data) return "[发了一个表情包,网卡了加载不出来]" + elif segment.type == "voice": + self.has_picid = False + self.is_picid = False + self.is_emoji = False + self.is_voice = True + if isinstance(segment.data, str): + return await get_voice_text(segment.data) + return "[发了一段语音,网卡了加载不出来]" elif segment.type == "mention_bot": + self.is_voice = False self.is_picid = False self.is_emoji = False self.is_mentioned = float(segment.data) # type: ignore return "" elif segment.type == "priority_info": + self.is_voice = False self.is_picid = False self.is_emoji = False if isinstance(segment.data, dict): @@ -253,6 +277,7 @@ class MessageRecvS4U(MessageRecv): """ return "" elif segment.type == "gift": + self.is_voice = False self.is_gift = True # 解析gift_info,格式为"名称:数量" name, count = segment.data.split(":", 1) # type: ignore @@ -343,6 +368,10 @@ class MessageProcessBase(Message): if isinstance(seg.data, str): return await get_image_manager().get_emoji_description(seg.data) return "[表情,网卡了加载不出来]" + elif seg.type == "voice": + if isinstance(seg.data, str): + return await get_voice_text(seg.data) + return "[发了一段语音,网卡了加载不出来]" elif seg.type == "at": return f"[@{seg.data}]" elif seg.type == "reply": diff --git a/src/chat/utils/utils_voice.py b/src/chat/utils/utils_voice.py new file mode 100644 index 000000000..1bc3e7dda --- /dev/null +++ b/src/chat/utils/utils_voice.py @@ -0,0 +1,35 @@ +import base64 + +from src.config.config import global_config +from src.llm_models.utils_model import LLMRequest + +from src.common.logger import get_logger +from rich.traceback import install +install(extra_lines=3) + +logger = get_logger("chat_voice") + +async def get_voice_text(voice_base64: str) -> str: + """获取音频文件描述""" + if not global_config.chat.enable_asr: + logger.warning("语音识别未启用,无法处理语音消息") + return "[语音]" + try: + # 解码base64音频数据 + # 确保base64字符串只包含ASCII字符 + if isinstance(voice_base64, str): + voice_base64 = voice_base64.encode("ascii", errors="ignore").decode("ascii") + voice_bytes = base64.b64decode(voice_base64) + _llm = LLMRequest(model=global_config.model.voice, request_type="voice") + text = await _llm.generate_response_for_voice(voice_bytes) + if text is None: + logger.warning("未能生成语音文本") + return "[语音(文本生成失败)]" + + logger.debug(f"描述是{text}") + + return f"[语音:{text}]" + except Exception as e: + logger.error(f"语音转文字失败: {str(e)}") + return "[语音]" + diff --git a/src/config/official_configs.py b/src/config/official_configs.py index 67b314f7f..be3ac1834 100644 --- a/src/config/official_configs.py +++ b/src/config/official_configs.py @@ -106,6 +106,9 @@ class ChatConfig(ConfigBase): focus_value: float = 1.0 """麦麦的专注思考能力,越低越容易专注,消耗token也越多""" + enable_asr: bool = False + """是否启用语音识别""" + def get_current_talk_frequency(self, chat_stream_id: Optional[str] = None) -> float: """ 根据当前时间和聊天流获取对应的 talk_frequency @@ -630,6 +633,9 @@ class ModelConfig(ConfigBase): vlm: dict[str, Any] = field(default_factory=lambda: {}) """视觉语言模型配置""" + voice: dict[str, Any] = field(default_factory=lambda: {}) + """语音识别模型配置""" + tool_use: dict[str, Any] = field(default_factory=lambda: {}) """专注工具使用模型配置""" diff --git a/src/llm_models/utils_model.py b/src/llm_models/utils_model.py index 1077cfa09..1f90a730a 100644 --- a/src/llm_models/utils_model.py +++ b/src/llm_models/utils_model.py @@ -216,6 +216,8 @@ class LLMRequest: prompt: str = None, image_base64: str = None, image_format: str = None, + file_bytes: bytes = None, + file_format: str = None, payload: dict = None, retry_policy: dict = None, ) -> Dict[str, Any]: @@ -225,6 +227,8 @@ class LLMRequest: prompt: prompt文本 image_base64: 图片的base64编码 image_format: 图片格式 + file_bytes: 文件的二进制数据 + file_format: 文件格式 payload: 请求体数据 retry_policy: 自定义重试策略 request_type: 请求类型 @@ -246,30 +250,33 @@ class LLMRequest: # 构建请求体 if image_base64: payload = await self._build_payload(prompt, image_base64, image_format) + elif file_bytes: + payload = await self._build_formdata_payload(file_bytes, file_format) elif payload is None: payload = await self._build_payload(prompt) - if stream_mode: - payload["stream"] = stream_mode + if not file_bytes: + if stream_mode: + payload["stream"] = stream_mode - if self.temp != 0.7: - payload["temperature"] = self.temp + if self.temp != 0.7: + payload["temperature"] = self.temp - # 添加enable_thinking参数(如果不是默认值False) - if not self.enable_thinking: - payload["enable_thinking"] = False + # 添加enable_thinking参数(如果不是默认值False) + if not self.enable_thinking: + payload["enable_thinking"] = False - if self.thinking_budget != 4096: - payload["thinking_budget"] = self.thinking_budget + if self.thinking_budget != 4096: + payload["thinking_budget"] = self.thinking_budget - if self.max_tokens: - payload["max_tokens"] = self.max_tokens + if self.max_tokens: + payload["max_tokens"] = self.max_tokens - # if "max_tokens" not in payload and "max_completion_tokens" not in payload: - # payload["max_tokens"] = global_config.model.model_max_output_length - # 如果 payload 中依然存在 max_tokens 且需要转换,在这里进行再次检查 - if self.model_name.lower() in self.MODELS_NEEDING_TRANSFORMATION and "max_tokens" in payload: - payload["max_completion_tokens"] = payload.pop("max_tokens") + # if "max_tokens" not in payload and "max_completion_tokens" not in payload: + # payload["max_tokens"] = global_config.model.model_max_output_length + # 如果 payload 中依然存在 max_tokens 且需要转换,在这里进行再次检查 + if self.model_name.lower() in self.MODELS_NEEDING_TRANSFORMATION and "max_tokens" in payload: + payload["max_completion_tokens"] = payload.pop("max_tokens") return { "policy": policy, @@ -278,6 +285,8 @@ class LLMRequest: "stream_mode": stream_mode, "image_base64": image_base64, # 保留必要的exception处理所需的原始数据 "image_format": image_format, + "file_bytes": file_bytes, + "file_format": file_format, "prompt": prompt, } @@ -287,6 +296,8 @@ class LLMRequest: prompt: str = None, image_base64: str = None, image_format: str = None, + file_bytes: bytes = None, + file_format: str = None, payload: dict = None, retry_policy: dict = None, response_handler: callable = None, @@ -299,6 +310,8 @@ class LLMRequest: prompt: prompt文本 image_base64: 图片的base64编码 image_format: 图片格式 + file_bytes: 文件的二进制数据 + file_format: 文件格式 payload: 请求体数据 retry_policy: 自定义重试策略 response_handler: 自定义响应处理器 @@ -307,25 +320,36 @@ class LLMRequest: """ # 获取请求配置 request_content = await self._prepare_request( - endpoint, prompt, image_base64, image_format, payload, retry_policy + endpoint, prompt, image_base64, image_format, file_bytes, file_format, payload, retry_policy ) if request_type is None: request_type = self.request_type for retry in range(request_content["policy"]["max_retries"]): try: # 使用上下文管理器处理会话 - headers = await self._build_headers() + if file_bytes: + headers = await self._build_headers(is_formdata=True) + else: + headers = await self._build_headers(is_formdata=False) # 似乎是openai流式必须要的东西,不过阿里云的qwq-plus加了这个没有影响 if request_content["stream_mode"]: headers["Accept"] = "text/event-stream" async with aiohttp.ClientSession(connector=await get_tcp_connector()) as session: - async with session.post( - request_content["api_url"], headers=headers, json=request_content["payload"] + post_kwargs = {"headers": headers} + #form-data数据上传方式不同 + if file_bytes: + post_kwargs["data"] = request_content["payload"] + else: + post_kwargs["json"] = request_content["payload"] + + async with session.post( + request_content["api_url"], **post_kwargs ) as response: handled_result = await self._handle_response( response, request_content, retry, response_handler, user_id, request_type, endpoint ) - return handled_result + return handled_result + except Exception as e: handled_payload, count_delta = await self._handle_exception(e, retry, request_content) retry += count_delta # 降级不计入重试次数 @@ -640,6 +664,33 @@ class LLMRequest: new_params["max_completion_tokens"] = new_params.pop("max_tokens") return new_params + async def _build_formdata_payload(self, file_bytes: bytes, file_format: str) -> aiohttp.FormData: + """构建form-data请求体""" + # 目前只适配了音频文件 + # 如果后续要支持其他类型的文件,可以在这里添加更多的处理逻辑 + data = aiohttp.FormData() + content_type_list = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "ogg": "audio/ogg", + "flac": "audio/flac", + "aac": "audio/aac", + } + + content_type = content_type_list.get(file_format) + if not content_type: + logger.warning(f"暂不支持的文件类型: {file_format}") + + data.add_field( + "file",io.BytesIO(file_bytes), + filename=f"file.{file_format}", + content_type=f'{content_type}' # 根据实际文件类型设置 + ) + data.add_field( + "model", self.model_name + ) + return data + async def _build_payload(self, prompt: str, image_base64: str = None, image_format: str = None) -> dict: """构建请求体""" # 复制一份参数,避免直接修改 self.params @@ -725,7 +776,8 @@ class LLMRequest: return content, reasoning_content, tool_calls else: return content, reasoning_content - + elif "text" in result and result["text"]: + return result["text"] return "没有返回结果", "" @staticmethod @@ -739,11 +791,15 @@ class LLMRequest: reasoning = "" return content, reasoning - async def _build_headers(self, no_key: bool = False) -> dict: + async def _build_headers(self, no_key: bool = False, is_formdata: bool = False) -> dict: """构建请求头""" if no_key: + if is_formdata: + return {"Authorization": "Bearer **********"} return {"Authorization": "Bearer **********", "Content-Type": "application/json"} else: + if is_formdata: + return {"Authorization": f"Bearer {self.api_key}"} return {"Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json"} # 防止小朋友们截图自己的key @@ -761,6 +817,11 @@ class LLMRequest: content, reasoning_content = response return content, reasoning_content + async def generate_response_for_voice(self, voice_bytes: bytes) -> Tuple: + """根据输入的语音文件生成模型的异步响应""" + response = await self._execute_request(endpoint="/audio/transcriptions",file_bytes=voice_bytes, file_format='wav') + return response + async def generate_response_async(self, prompt: str, **kwargs) -> Union[str, Tuple]: """异步方式根据输入的提示生成模型的响应""" # 构建请求体,不硬编码max_tokens diff --git a/template/bot_config_template.toml b/template/bot_config_template.toml index fbb816621..3b21dae38 100644 --- a/template/bot_config_template.toml +++ b/template/bot_config_template.toml @@ -87,6 +87,7 @@ talk_frequency_adjust = [ # - 时间支持跨天,例如 "00:10,0.3" 表示从凌晨0:10开始使用频率0.3 # - 系统会自动将 "platform:id:type" 转换为内部的哈希chat_id进行匹配 +enable_asr = false # 是否启用语音识别,启用后麦麦可以通过语音输入进行对话,启用该功能需要配置语音识别模型[model.voice] [message_receive] # 以下是消息过滤,可以根据规则过滤特定消息,将不会读取这些消息 @@ -294,6 +295,12 @@ provider = "SILICONFLOW" pri_in = 0.35 pri_out = 0.35 +[model.voice] # 语音识别模型 +name = "FunAudioLLM/SenseVoiceSmall" +provider = "SILICONFLOW" +pri_in = 0 +pri_out = 0 + [model.tool_use] #工具调用模型,需要使用支持工具调用的模型 name = "Qwen/Qwen3-14B" provider = "SILICONFLOW"