gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: llm统计现已记录模型反应时间

Browse Source
This commit is contained in:
SengokuCola
2025-08-11 21:51:59 +08:00
parent 849928a8f3
commit 268b428e8f
13 changed files with 117 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/chat/chat_loop/heartFC_chat.py
View File

@@ -487,7 +487,7 @@ class HeartFChatting:
available_actions=available_actions,
reply_reason=action_info.get("reasoning", ""),
enable_tool=global_config.tool.enable_tool,
request_type="chat.replyer",
request_type="replyer",
from_plugin=False,
)

4
src/chat/express/expression_learner.py
View File

@@ -38,7 +38,7 @@ def init_prompt() -> None:
请从上面这段群聊中概括除了人名为"SELF"之外的人的语言风格
1. 只考虑文字，不要考虑表情包和图片
2. 不要涉及具体的人名，只考虑语言风格，特殊的梗，不要总结自己
2. 不要涉及具体的人名，但是可以涉及具体名词
3. 思考有没有特殊的梗，一并总结成语言风格
4. 例子仅供参考，请严格根据群聊内容总结!!!
注意：总结成如下格式的规律，总结的内容要详细，但具有概括性：
@@ -59,7 +59,7 @@ def init_prompt() -> None:
class ExpressionLearner:
def __init__(self, chat_id: str) -> None:
self.express_learn_model: LLMRequest = LLMRequest(
model_set=model_config.model_task_config.replyer, request_type="expressor.learner"
model_set=model_config.model_task_config.replyer, request_type="expression.learner"
)
self.chat_id = chat_id
self.chat_name = get_chat_manager().get_stream_name(chat_id) or chat_id

16
src/chat/express/expression_selector.py
View File

@@ -25,7 +25,7 @@ def init_prompt():
以下是可选的表达情境：
{all_situations}
请你分析聊天内容的语境、情绪、话题类型，从上述情境中选择最适合当前聊天情境的{min_num}-{max_num}个情境。
请你分析聊天内容的语境、情绪、话题类型，从上述情境中选择最适合当前聊天情境的，最多{max_num}个情境。
考虑因素包括：
1. 聊天的情绪氛围（轻松、严肃、幽默等）
2. 话题类型（日常、技术、游戏、情感等）
@@ -35,7 +35,7 @@ def init_prompt():
请以JSON格式输出，只需要输出选中的情境编号：
例如：
{{
"selected_situations": [2, 3, 5, 7, 19, 22, 25, 38, 39, 45, 48, 64]
"selected_situations": [2, 3, 5, 7, 19]
}}
请严格按照JSON格式输出，不要包含其他内容：
@@ -195,7 +195,6 @@ class ExpressionSelector:
chat_id: str,
chat_info: str,
max_num: int = 10,
min_num: int = 5,
target_message: Optional[str] = None,
) -> List[Dict[str, Any]]:
# sourcery skip: inline-variable, list-comprehension
@@ -206,8 +205,8 @@ class ExpressionSelector:
logger.debug(f"聊天流 {chat_id} 不允许使用表达，返回空列表")
return []
# 1. 获取35个随机表达方式（现在按权重抽取）
style_exprs = self.get_random_expressions(chat_id, 30)
# 1. 获取20个随机表达方式（现在按权重抽取）
style_exprs = self.get_random_expressions(chat_id, 10)
# 2. 构建所有表达方式的索引和情境列表
all_expressions = []
@@ -219,7 +218,7 @@ class ExpressionSelector:
expr_with_type = expr.copy()
expr_with_type["type"] = "style"
all_expressions.append(expr_with_type)
all_situations.append(f"{len(all_expressions)}.{expr['situation']}")
all_situations.append(f"{len(all_expressions)}.当 {expr['situation']} 时，使用 {expr['style']}")
if not all_expressions:
logger.warning("没有找到可用的表达方式")
@@ -239,13 +238,12 @@ class ExpressionSelector:
bot_name=global_config.bot.nickname,
chat_observe_info=chat_info,
all_situations=all_situations_str,
min_num=min_num,
max_num=max_num,
target_message=target_message_str,
target_message_extra_block=target_message_extra_block,
)
# print(prompt)
print(prompt)
# 4. 调用LLM
try:
@@ -255,7 +253,7 @@ class ExpressionSelector:
# logger.info(f"LLM请求时间: {model_name} {time.time() - start_time} \n{prompt}")
# logger.info(f"模型名称: {model_name}")
# logger.info(f"LLM返回结果: {content}")
logger.info(f"LLM返回结果: {content}")
# if reasoning_content:
# logger.info(f"LLM推理: {reasoning_content}")
# else:

2
src/chat/memory_system/Hippocampus.py
View File

@@ -200,7 +200,7 @@ class Hippocampus:
self.parahippocampal_gyrus = ParahippocampalGyrus(self)
# 从数据库加载记忆图
self.entorhinal_cortex.sync_memory_from_db()
self.model_small = LLMRequest(model_set=model_config.model_task_config.utils_small, request_type="memory.small")
self.model_small = LLMRequest(model_set=model_config.model_task_config.utils_small, request_type="memory.modify")
def get_all_node_names(self) -> list:
"""获取记忆图中所有节点的名字列表"""

6
src/chat/replyer/default_generator.py
View File

@@ -117,8 +117,8 @@ def init_prompt():
你现在正在一个QQ群里聊天，以下是正在进行的聊天内容：
{background_dialogue_prompt}
你现在想补充说明你刚刚自己的发言内容：{target}
请你根据聊天内容，组织一条新回复。
你现在想补充说明你刚刚自己的发言内容：{target}，原因是{reason}
请你根据聊天内容，组织一条新回复。注意，{target} 是刚刚你自己的发言，你要在这基础上进一步发言，请按照你自己的角度来继续进行回复。
你现在的心情是：{mood_state}
{reply_style}
{keywords_reaction_prompt}
@@ -331,7 +331,7 @@ class DefaultReplyer:
# 使用从处理器传来的选中表达方式
# LLM模式：调用LLM选择5-10个，然后随机选5个
selected_expressions = await expression_selector.select_suitable_expressions_llm(
self.chat_stream.stream_id, chat_history, max_num=8, min_num=2, target_message=target
self.chat_stream.stream_id, chat_history, max_num=8, target_message=target
)
if selected_expressions:

78
src/chat/utils/statistic.py
View File

@@ -36,6 +36,18 @@ COST_BY_TYPE = "costs_by_type"
COST_BY_USER = "costs_by_user"
COST_BY_MODEL = "costs_by_model"
COST_BY_MODULE = "costs_by_module"
TIME_COST_BY_TYPE = "time_costs_by_type"
TIME_COST_BY_USER = "time_costs_by_user"
TIME_COST_BY_MODEL = "time_costs_by_model"
TIME_COST_BY_MODULE = "time_costs_by_module"
AVG_TIME_COST_BY_TYPE = "avg_time_costs_by_type"
AVG_TIME_COST_BY_USER = "avg_time_costs_by_user"
AVG_TIME_COST_BY_MODEL = "avg_time_costs_by_model"
AVG_TIME_COST_BY_MODULE = "avg_time_costs_by_module"
STD_TIME_COST_BY_TYPE = "std_time_costs_by_type"
STD_TIME_COST_BY_USER = "std_time_costs_by_user"
STD_TIME_COST_BY_MODEL = "std_time_costs_by_model"
STD_TIME_COST_BY_MODULE = "std_time_costs_by_module"
ONLINE_TIME = "online_time"
TOTAL_MSG_CNT = "total_messages"
MSG_CNT_BY_CHAT = "messages_by_chat"
@@ -293,6 +305,18 @@ class StatisticOutputTask(AsyncTask):
COST_BY_USER: defaultdict(float),
COST_BY_MODEL: defaultdict(float),
COST_BY_MODULE: defaultdict(float),
TIME_COST_BY_TYPE: defaultdict(list),
TIME_COST_BY_USER: defaultdict(list),
TIME_COST_BY_MODEL: defaultdict(list),
TIME_COST_BY_MODULE: defaultdict(list),
AVG_TIME_COST_BY_TYPE: defaultdict(float),
AVG_TIME_COST_BY_USER: defaultdict(float),
AVG_TIME_COST_BY_MODEL: defaultdict(float),
AVG_TIME_COST_BY_MODULE: defaultdict(float),
STD_TIME_COST_BY_TYPE: defaultdict(float),
STD_TIME_COST_BY_USER: defaultdict(float),
STD_TIME_COST_BY_MODEL: defaultdict(float),
STD_TIME_COST_BY_MODULE: defaultdict(float),
}
for period_key, _ in collect_period
}
@@ -344,7 +368,41 @@ class StatisticOutputTask(AsyncTask):
stats[period_key][COST_BY_USER][user_id] += cost
stats[period_key][COST_BY_MODEL][model_name] += cost
stats[period_key][COST_BY_MODULE][module_name] += cost
# 收集time_cost数据
time_cost = record.time_cost or 0.0
if time_cost > 0: # 只记录有效的time_cost
stats[period_key][TIME_COST_BY_TYPE][request_type].append(time_cost)
stats[period_key][TIME_COST_BY_USER][user_id].append(time_cost)
stats[period_key][TIME_COST_BY_MODEL][model_name].append(time_cost)
stats[period_key][TIME_COST_BY_MODULE][module_name].append(time_cost)
break
# 计算平均耗时和标准差
for period_key in stats:
for category in [REQ_CNT_BY_TYPE, REQ_CNT_BY_USER, REQ_CNT_BY_MODEL, REQ_CNT_BY_MODULE]:
time_cost_key = f"time_costs_by_{category.split('_')[-1]}"
avg_key = f"avg_time_costs_by_{category.split('_')[-1]}"
std_key = f"std_time_costs_by_{category.split('_')[-1]}"
for item_name in stats[period_key][category]:
time_costs = stats[period_key][time_cost_key].get(item_name, [])
if time_costs:
# 计算平均耗时
avg_time_cost = sum(time_costs) / len(time_costs)
stats[period_key][avg_key][item_name] = round(avg_time_cost, 3)
# 计算标准差
if len(time_costs) > 1:
variance = sum((x - avg_time_cost) ** 2 for x in time_costs) / len(time_costs)
std_time_cost = variance ** 0.5
stats[period_key][std_key][item_name] = round(std_time_cost, 3)
else:
stats[period_key][std_key][item_name] = 0.0
else:
stats[period_key][avg_key][item_name] = 0.0
stats[period_key][std_key][item_name] = 0.0
return stats
@staticmethod
@@ -566,11 +624,11 @@ class StatisticOutputTask(AsyncTask):
"""
if stats[TOTAL_REQ_CNT] <= 0:
return ""
data_fmt = "{:<32} {:>10} {:>12} {:>12} {:>12} {:>9.4f}¥"
data_fmt = "{:<32} {:>10} {:>12} {:>12} {:>12} {:>9.4f}¥ {:>10} {:>10}"
output = [
"按模型分类统计:",
" 模型名称 调用次数 输入Token 输出Token Token总量 累计花费",
" 模型名称 调用次数 输入Token 输出Token Token总量 累计花费 平均耗时(秒) 标准差(秒)",
]
for model_name, count in sorted(stats[REQ_CNT_BY_MODEL].items()):
name = f"{model_name[:29]}..." if len(model_name) > 32 else model_name
@@ -578,7 +636,9 @@ class StatisticOutputTask(AsyncTask):
out_tokens = stats[OUT_TOK_BY_MODEL][model_name]
tokens = stats[TOTAL_TOK_BY_MODEL][model_name]
cost = stats[COST_BY_MODEL][model_name]
output.append(data_fmt.format(name, count, in_tokens, out_tokens, tokens, cost))
avg_time_cost = stats[AVG_TIME_COST_BY_MODEL][model_name]
std_time_cost = stats[STD_TIME_COST_BY_MODEL][model_name]
output.append(data_fmt.format(name, count, in_tokens, out_tokens, tokens, cost, avg_time_cost, std_time_cost))
output.append("")
return "\n".join(output)
@@ -663,6 +723,8 @@ class StatisticOutputTask(AsyncTask):
f"<td>{stat_data[OUT_TOK_BY_MODEL][model_name]}</td>"
f"<td>{stat_data[TOTAL_TOK_BY_MODEL][model_name]}</td>"
f"<td>{stat_data[COST_BY_MODEL][model_name]:.4f} ¥</td>"
f"<td>{stat_data[AVG_TIME_COST_BY_MODEL][model_name]:.3f} 秒</td>"
f"<td>{stat_data[STD_TIME_COST_BY_MODEL][model_name]:.3f} 秒</td>"
f"</tr>"
for model_name, count in sorted(stat_data[REQ_CNT_BY_MODEL].items())
]
@@ -677,6 +739,8 @@ class StatisticOutputTask(AsyncTask):
f"<td>{stat_data[OUT_TOK_BY_TYPE][req_type]}</td>"
f"<td>{stat_data[TOTAL_TOK_BY_TYPE][req_type]}</td>"
f"<td>{stat_data[COST_BY_TYPE][req_type]:.4f} ¥</td>"
f"<td>{stat_data[AVG_TIME_COST_BY_TYPE][req_type]:.3f} 秒</td>"
f"<td>{stat_data[STD_TIME_COST_BY_TYPE][req_type]:.3f} 秒</td>"
f"</tr>"
for req_type, count in sorted(stat_data[REQ_CNT_BY_TYPE].items())
]
@@ -691,6 +755,8 @@ class StatisticOutputTask(AsyncTask):
f"<td>{stat_data[OUT_TOK_BY_MODULE][module_name]}</td>"
f"<td>{stat_data[TOTAL_TOK_BY_MODULE][module_name]}</td>"
f"<td>{stat_data[COST_BY_MODULE][module_name]:.4f} ¥</td>"
f"<td>{stat_data[AVG_TIME_COST_BY_MODULE][module_name]:.3f} 秒</td>"
f"<td>{stat_data[STD_TIME_COST_BY_MODULE][module_name]:.3f} 秒</td>"
f"</tr>"
for module_name, count in sorted(stat_data[REQ_CNT_BY_MODULE].items())
]
@@ -717,7 +783,7 @@ class StatisticOutputTask(AsyncTask):
<h2>按模型分类统计</h2>
<table>
<thead><tr><th>模型名称</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th></tr></thead>
<thead><tr><th>模型名称</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th><th>平均耗时(秒)</th><th>标准差(秒)</th></tr></thead>
<tbody>
{model_rows}
</tbody>
@@ -726,7 +792,7 @@ class StatisticOutputTask(AsyncTask):
<h2>按模块分类统计</h2>
<table>
<thead>
<tr><th>模块名称</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th></tr>
<tr><th>模块名称</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th><th>平均耗时(秒)</th><th>标准差(秒)</th></tr>
</thead>
<tbody>
{module_rows}
@@ -736,7 +802,7 @@ class StatisticOutputTask(AsyncTask):
<h2>按请求类型分类统计</h2>
<table>
<thead>
<tr><th>请求类型</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th></tr>
<tr><th>请求类型</th><th>调用次数</th><th>输入Token</th><th>输出Token</th><th>Token总量</th><th>累计花费</th><th>平均耗时(秒)</th><th>标准差(秒)</th></tr>
</thead>
<tbody>
{type_rows}

3
src/common/database/database_model.py
View File

@@ -79,6 +79,8 @@ class LLMUsage(BaseModel):
"""
model_name = TextField(index=True) # 添加索引
model_assign_name = TextField(null=True) # 添加索引
model_api_provider = TextField(null=True) # 添加索引
user_id = TextField(index=True) # 添加索引
request_type = TextField(index=True) # 添加索引
endpoint = TextField()
@@ -86,6 +88,7 @@ class LLMUsage(BaseModel):
completion_tokens = IntegerField()
total_tokens = IntegerField()
cost = DoubleField()
time_cost = DoubleField(null=True)
status = TextField()
timestamp = DateTimeField(index=True) # 更改为 DateTimeField 并添加索引

17
src/config/config.py
View File

@@ -109,11 +109,18 @@ def get_value_by_path(d, path):
def set_value_by_path(d, path, value):
"""设置嵌套字典中指定路径的值"""
for k in path[:-1]:
if k not in d or not isinstance(d[k], dict):
d[k] = {}
d = d[k]
d[path[-1]] = value
# 使用 tomlkit.item 来保持 TOML 格式
try:
d[path[-1]] = tomlkit.item(value)
except (TypeError, ValueError):
# 如果转换失败，直接赋值
d[path[-1]] = value
def compare_default_values(new, old, path=None, logs=None, changes=None):
@@ -237,6 +244,7 @@ def _update_config_generic(config_name: str, template_name: str):
for log in logs:
logger.info(log)
# 检查旧配置是否等于旧默认值，如果是则更新为新默认值
config_updated = False
for path, old_default, new_default in changes:
old_value = get_value_by_path(old_config, path)
if old_value == old_default:
@@ -244,6 +252,13 @@ def _update_config_generic(config_name: str, template_name: str):
logger.info(
f"已自动将{config_name}配置 {'.'.join(path)} 的值从旧默认值 {old_default} 更新为新默认值 {new_default}"
)
config_updated = True
# 如果配置有更新，立即保存到文件
if config_updated:
with open(old_config_path, "w", encoding="utf-8") as f:
f.write(tomlkit.dumps(old_config))
logger.info(f"已保存更新后的{config_name}配置文件")
else:
logger.info(f"未检测到{config_name}模板默认值变动")

5
src/llm_models/utils.py
View File

@@ -155,7 +155,7 @@ class LLMUsageRecorder:
logger.error(f"创建 LLMUsage 表失败: {str(e)}")
def record_usage_to_database(
self, model_info: ModelInfo, model_usage: UsageRecord, user_id: str, request_type: str, endpoint: str
self, model_info: ModelInfo, model_usage: UsageRecord, user_id: str, request_type: str, endpoint: str, time_cost: float = 0.0
):
input_cost = (model_usage.prompt_tokens / 1000000) * model_info.price_in
output_cost = (model_usage.completion_tokens / 1000000) * model_info.price_out
@@ -164,6 +164,8 @@ class LLMUsageRecorder:
# 使用 Peewee 模型创建记录
LLMUsage.create(
model_name=model_info.model_identifier,
model_assign_name=model_info.name,
model_api_provider=model_info.api_provider,
user_id=user_id,
request_type=request_type,
endpoint=endpoint,
@@ -171,6 +173,7 @@ class LLMUsageRecorder:
completion_tokens=model_usage.completion_tokens or 0,
total_tokens=model_usage.total_tokens or 0,
cost=total_cost or 0.0,
time_cost = round(time_cost or 0.0, 3),
status="success",
timestamp=datetime.now(), # Peewee 会处理 DateTimeField
)

7
src/llm_models/utils_model.py
View File

@@ -71,6 +71,7 @@ class LLMRequest:
(Tuple[str, str, str, Optional[List[ToolCall]]]): 响应内容、推理内容、模型名称、工具调用列表
"""
# 模型选择
start_time = time.time()
model_info, api_provider, client = self._select_model()
# 请求体构建
@@ -105,6 +106,7 @@ class LLMRequest:
user_id="system",
request_type=self.request_type,
endpoint="/chat/completions",
time_cost=time.time() - start_time,
)
return content, (reasoning_content, model_info.name, tool_calls)
@@ -149,8 +151,6 @@ class LLMRequest:
# 请求体构建
start_time = time.time()
message_builder = MessageBuilder()
message_builder.add_text_content(prompt)
messages = [message_builder.build()]
@@ -190,6 +190,7 @@ class LLMRequest:
user_id="system",
request_type=self.request_type,
endpoint="/chat/completions",
time_cost=time.time() - start_time,
)
if not content:
@@ -208,6 +209,7 @@ class LLMRequest:
(Tuple[List[float], str]): (嵌入向量，使用的模型名称)
"""
# 无需构建消息体，直接使用输入文本
start_time = time.time()
model_info, api_provider, client = self._select_model()
# 请求并处理返回值
@@ -228,6 +230,7 @@ class LLMRequest:
user_id="system",
request_type=self.request_type,
endpoint="/embeddings",
time_cost=time.time() - start_time,
)
if not embedding:

2
src/mais4u/mais4u_chat/s4u_prompt.py
View File

@@ -104,7 +104,7 @@ class PromptBuilder:
# 使用从处理器传来的选中表达方式
# LLM模式：调用LLM选择5-10个，然后随机选5个
selected_expressions = await expression_selector.select_suitable_expressions_llm(
chat_stream.stream_id, chat_history, max_num=12, min_num=5, target_message=target
chat_stream.stream_id, chat_history, max_num=12, target_message=target
)
if selected_expressions:

2
src/person_info/group_relationship_manager.py
View File

@@ -22,7 +22,7 @@ logger = get_logger("group_relationship_manager")
class GroupRelationshipManager:
def __init__(self):
self.group_llm = LLMRequest(
model_set=model_config.model_task_config.utils, request_type="group.relationship"
model_set=model_config.model_task_config.utils, request_type="relationship.group"
)
self.last_group_impression_time = 0.0
self.last_group_impression_message_count = 0

2
src/person_info/relationship_manager.py
View File

@@ -20,7 +20,7 @@ logger = get_logger("relation")
class RelationshipManager:
def __init__(self):
self.relationship_llm = LLMRequest(
model_set=model_config.model_task_config.utils, request_type="relationship"
model_set=model_config.model_task_config.utils, request_type="relationship.person"
) # 用于动作规划
@staticmethod