gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/MoFox-Studio/MoFox-Core into dev

Browse Source
This commit is contained in:
Windpicker-owo
2025-12-08 17:19:33 +08:00
parent a1d60ab026 f9b193c86d
commit 01bcfb491a
136 changed files with 2188 additions and 2230 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
src/chat/utils/chat_message_builder.py
View File

@@ -1125,7 +1125,7 @@ async def build_anonymous_messages(messages: list[dict[str, Any]], filter_for_le
"""
构建匿名可读消息，将不同人的名称转为唯一占位符（A、B、C...），bot自己用SELF。
处理 回复<aaa:bbb> 和 @<aaa:bbb> 字段，将bbb映射为匿名占位符。
Args:
messages: 消息列表
filter_for_learning: 是否为表达学习场景进行额外过滤（过滤掉纯回复、纯@、纯图片等无意义内容）
@@ -1162,16 +1162,16 @@ async def build_anonymous_messages(messages: list[dict[str, Any]], filter_for_le
person_map[person_id] = chr(current_char)
current_char += 1
return person_map[person_id]
def is_meaningless_content(content: str, msg: dict) -> bool:
"""
判断消息内容是否无意义（用于表达学习过滤）
"""
if not content or not content.strip():
return True
stripped = content.strip()
# 检查消息标记字段
if msg.get("is_emoji", False):
return True
@@ -1181,32 +1181,32 @@ async def build_anonymous_messages(messages: list[dict[str, Any]], filter_for_le
return True
if msg.get("is_command", False):
return True
# 🔥 检查纯回复消息（只有[回复<xxx>]没有其他内容）
reply_pattern = r"^\s*\[回复[^\]]*\]\s*$"
if re.match(reply_pattern, stripped):
return True
# 🔥 检查纯@消息（只有@xxx没有其他内容）
at_pattern = r"^\s*(@[^\s]+\s*)+$"
if re.match(at_pattern, stripped):
return True
# 🔥 检查纯图片消息
image_pattern = r"^\s*(\[图片\]|\[动画表情\]|\[表情\]|\[picid:[^\]]+\])\s*$"
if re.match(image_pattern, stripped):
return True
# 🔥 移除回复标记、@标记、图片标记后检查是否还有实质内容
clean_content = re.sub(r"\[回复[^\]]*\]", "", stripped)
clean_content = re.sub(r"@[^\s]+", "", clean_content)
clean_content = re.sub(r"\[图片\]|\[动画表情\]|\[表情\]|\[picid:[^\]]+\]", "", clean_content)
clean_content = clean_content.strip()
# 如果移除后内容太短（少于2个字符），认为无意义
if len(clean_content) < 2:
return True
return False
for msg in messages:
@@ -1227,7 +1227,7 @@ async def build_anonymous_messages(messages: list[dict[str, Any]], filter_for_le
# For anonymous messages, we just replace with a placeholder.
content = re.sub(r"\[picid:([^\]]+)\]", "[图片]", content)
# 🔥 表达学习场景：过滤无意义消息
if filter_for_learning and is_meaningless_content(content, msg):
continue

6
src/chat/utils/prompt.py
View File

@@ -1082,7 +1082,7 @@ class Prompt:
[新] 根据用户ID构建关系信息字符串。
"""
from src.person_info.relationship_fetcher import relationship_fetcher_manager
person_info_manager = get_person_info_manager()
person_id = person_info_manager.get_person_id(platform, user_id)
@@ -1091,11 +1091,11 @@ class Prompt:
return f"你似乎还不认识这位用户（ID: {user_id}），这是你们的第一次互动。"
relationship_fetcher = relationship_fetcher_manager.get_fetcher(chat_id)
# 并行构建用户信息和聊天流印象
user_relation_info_task = relationship_fetcher.build_relation_info(person_id, points_num=5)
stream_impression_task = relationship_fetcher.build_chat_stream_impression(chat_id)
user_relation_info, stream_impression = await asyncio.gather(
user_relation_info_task, stream_impression_task
)

2
src/chat/utils/prompt_component_manager.py
View File

@@ -524,7 +524,7 @@ class PromptComponentManager:
is_built_in=False,
)
# 从动态规则中收集并关联其所有注入规则
for target, rules_in_target in self._dynamic_rules.items():
for rules_in_target in self._dynamic_rules.values():
if name in rules_in_target:
rule, _, _ = rules_in_target[name]
dynamic_info.injection_rules.append(rule)

22
src/chat/utils/report_generator.py
View File

@@ -136,7 +136,7 @@ class HTMLReportGenerator:
for chat_id, count in sorted(stat_data[MSG_CNT_BY_CHAT].items())
]
)
# 先计算基础数据
total_tokens = sum(stat_data.get(TOTAL_TOK_BY_MODEL, {}).values())
total_requests = stat_data.get(TOTAL_REQ_CNT, 0)
@@ -144,21 +144,21 @@ class HTMLReportGenerator:
total_messages = stat_data.get(TOTAL_MSG_CNT, 0)
online_seconds = stat_data.get(ONLINE_TIME, 0)
online_hours = online_seconds / 3600 if online_seconds > 0 else 0
# 大模型相关效率指标
avg_cost_per_req = (total_cost / total_requests) if total_requests > 0 else 0
(total_cost / total_requests) if total_requests > 0 else 0
avg_cost_per_msg = (total_cost / total_messages) if total_messages > 0 else 0
avg_tokens_per_msg = (total_tokens / total_messages) if total_messages > 0 else 0
avg_tokens_per_req = (total_tokens / total_requests) if total_requests > 0 else 0
msg_to_req_ratio = (total_messages / total_requests) if total_requests > 0 else 0
cost_per_hour = (total_cost / online_hours) if online_hours > 0 else 0
req_per_hour = (total_requests / online_hours) if online_hours > 0 else 0
# Token效率 (输出/输入比率)
total_in_tokens = sum(stat_data.get(IN_TOK_BY_MODEL, {}).values())
total_out_tokens = sum(stat_data.get(OUT_TOK_BY_MODEL, {}).values())
token_efficiency = (total_out_tokens / total_in_tokens) if total_in_tokens > 0 else 0
# 生成效率指标表格数据
efficiency_data = [
("💸 平均每条消息成本", f"{avg_cost_per_msg:.6f} ¥", "处理每条用户消息的平均AI成本"),
@@ -172,14 +172,14 @@ class HTMLReportGenerator:
("📈 Token/在线小时", f"{(total_tokens / online_hours) if online_hours > 0 else 0:.0f}", "每在线小时处理的Token数"),
("💬 消息/在线小时", f"{(total_messages / online_hours) if online_hours > 0 else 0:.1f}", "每在线小时处理的消息数"),
]
efficiency_rows = "\n".join(
[
f"<tr><td style='font-weight: 500;'>{metric}</td><td style='color: #1976D2; font-weight: 600; font-size: 1.1em;'>{value}</td><td style='color: #546E7A;'>{desc}</td></tr>"
for metric, value, desc in efficiency_data
]
)
# 计算活跃聊天数和最活跃聊天
msg_by_chat = stat_data.get(MSG_CNT_BY_CHAT, {})
active_chats = len(msg_by_chat)
@@ -189,9 +189,9 @@ class HTMLReportGenerator:
most_active_chat = self.name_mapping.get(most_active_id, (most_active_id, 0))[0]
most_active_count = msg_by_chat[most_active_id]
most_active_chat = f"{most_active_chat} ({most_active_count}条)"
avg_msg_per_chat = (total_messages / active_chats) if active_chats > 0 else 0
summary_cards = f"""
<div class="summary-cards">
<div class="card">
@@ -350,8 +350,8 @@ class HTMLReportGenerator:
generation_time=now.strftime("%Y-%m-%d %H:%M:%S"),
tab_list="\n".join(tab_list_html),
tab_content="\n".join(tab_content_html_list),
all_chart_data=json.dumps(chart_data, separators=(',', ':'), ensure_ascii=False),
static_chart_data=json.dumps(static_chart_data, separators=(',', ':'), ensure_ascii=False),
all_chart_data=json.dumps(chart_data, separators=(",", ":"), ensure_ascii=False),
static_chart_data=json.dumps(static_chart_data, separators=(",", ":"), ensure_ascii=False),
report_css=report_css,
report_js=report_js,
)

60
src/chat/utils/statistic.py
View File

@@ -3,8 +3,8 @@ from collections import defaultdict
from datetime import datetime, timedelta
from typing import Any
from src.common.database.compatibility import db_get, db_query
from src.common.database.api.query import QueryBuilder
from src.common.database.compatibility import db_get, db_query
from src.common.database.core.models import LLMUsage, Messages, OnlineTime
from src.common.logger import get_logger
from src.manager.async_task_manager import AsyncTask
@@ -299,21 +299,21 @@ class StatisticOutputTask(AsyncTask):
# 以最早的时间戳为起始时间获取记录
# 🔧 内存优化：使用分批查询代替全量加载
query_start_time = collect_period[-1][1]
query_builder = (
QueryBuilder(LLMUsage)
.no_cache()
.filter(timestamp__gte=query_start_time)
.order_by("-timestamp")
)
total_processed = 0
async for batch in query_builder.iter_batches(batch_size=STAT_BATCH_SIZE, as_dict=True):
for record in batch:
if total_processed >= STAT_MAX_RECORDS:
logger.warning(f"统计处理记录数达到上限 {STAT_MAX_RECORDS}，跳过剩余记录")
break
if not isinstance(record, dict):
continue
@@ -384,11 +384,11 @@ class StatisticOutputTask(AsyncTask):
total_processed += 1
if total_processed % 500 == 0:
await StatisticOutputTask._yield_control(total_processed, interval=1)
# 检查是否达到上限
if total_processed >= STAT_MAX_RECORDS:
break
# 每批处理完后让出控制权
await asyncio.sleep(0)
# -- 计算派生指标 --
@@ -480,7 +480,7 @@ class StatisticOutputTask(AsyncTask):
"labels": [item[0] for item in sorted_models],
"data": [round(item[1], 4) for item in sorted_models],
}
# 1. Token输入输出对比条形图
model_names = list(period_stats[REQ_CNT_BY_MODEL].keys())
if model_names:
@@ -489,7 +489,7 @@ class StatisticOutputTask(AsyncTask):
"input_tokens": [period_stats[IN_TOK_BY_MODEL].get(m, 0) for m in model_names],
"output_tokens": [period_stats[OUT_TOK_BY_MODEL].get(m, 0) for m in model_names],
}
# 2. 响应时间分布散点图数据（限制数据点以提高加载速度）
scatter_data = []
max_points_per_model = 50 # 每个模型最多50个点
@@ -500,7 +500,7 @@ class StatisticOutputTask(AsyncTask):
sampled_costs = time_costs[::step][:max_points_per_model]
else:
sampled_costs = time_costs
for idx, time_cost in enumerate(sampled_costs):
scatter_data.append({
"model": model_name,
@@ -509,7 +509,7 @@ class StatisticOutputTask(AsyncTask):
"tokens": period_stats[TOTAL_TOK_BY_MODEL].get(model_name, 0) // len(time_costs) if time_costs else 0
})
period_stats[SCATTER_CHART_RESPONSE_TIME] = scatter_data
# 3. 模型效率雷达图
if model_names:
# 取前5个最常用的模型
@@ -522,14 +522,14 @@ class StatisticOutputTask(AsyncTask):
avg_time = period_stats[AVG_TIME_COST_BY_MODEL].get(model_name, 0)
cost_per_ktok = period_stats[COST_PER_KTOK_BY_MODEL].get(model_name, 0)
avg_tokens = period_stats[AVG_TOK_BY_MODEL].get(model_name, 0)
# 简单的归一化（反向归一化时间和成本，值越小越好）
max_req = max([period_stats[REQ_CNT_BY_MODEL].get(m[0], 1) for m in top_models])
max_tps = max([period_stats[TPS_BY_MODEL].get(m[0], 1) for m in top_models])
max_time = max([period_stats[AVG_TIME_COST_BY_MODEL].get(m[0], 0.1) for m in top_models])
max_cost = max([period_stats[COST_PER_KTOK_BY_MODEL].get(m[0], 0.001) for m in top_models])
max_tokens = max([period_stats[AVG_TOK_BY_MODEL].get(m[0], 1) for m in top_models])
radar_data.append({
"model": model_name,
"metrics": [
@@ -544,7 +544,7 @@ class StatisticOutputTask(AsyncTask):
"labels": ["请求量", "TPS", "响应速度", "成本效益", "Token容量"],
"datasets": radar_data
}
# 4. 供应商请求占比环形图
provider_requests = period_stats[REQ_CNT_BY_PROVIDER]
if provider_requests:
@@ -553,7 +553,7 @@ class StatisticOutputTask(AsyncTask):
"labels": [item[0] for item in sorted_provider_reqs],
"data": [item[1] for item in sorted_provider_reqs],
}
# 5. 平均响应时间条形图
if model_names:
sorted_by_time = sorted(
@@ -626,7 +626,7 @@ class StatisticOutputTask(AsyncTask):
if overlap_end > overlap_start:
stats[period_key][ONLINE_TIME] += (overlap_end - overlap_start).total_seconds()
break
# 每批处理完后让出控制权
await asyncio.sleep(0)
@@ -666,7 +666,7 @@ class StatisticOutputTask(AsyncTask):
if total_processed >= STAT_MAX_RECORDS:
logger.warning(f"消息统计处理记录数达到上限 {STAT_MAX_RECORDS}，跳过剩余记录")
break
if not isinstance(message, dict):
continue
message_time_ts = message.get("time") # This is a float timestamp
@@ -709,11 +709,11 @@ class StatisticOutputTask(AsyncTask):
total_processed += 1
if total_processed % 500 == 0:
await StatisticOutputTask._yield_control(total_processed, interval=1)
# 检查是否达到上限
if total_processed >= STAT_MAX_RECORDS:
break
# 每批处理完后让出控制权
await asyncio.sleep(0)
@@ -822,10 +822,10 @@ class StatisticOutputTask(AsyncTask):
def _compress_time_cost_lists(self, data: dict[str, Any]) -> dict[str, Any]:
"""🔧 内存优化：将 TIME_COST_BY_* 的 list 压缩为聚合数据
原始格式: {"model_a": [1.2, 2.3, 3.4, ...]} (可能无限增长)
压缩格式: {"model_a": {"sum": 6.9, "count": 3, "sum_sq": 18.29}}
这样合并时只需要累加 sum/count/sum_sq，不会无限增长。
avg = sum / count
std = sqrt(sum_sq / count - (sum / count)^2)
@@ -835,17 +835,17 @@ class StatisticOutputTask(AsyncTask):
TIME_COST_BY_TYPE, TIME_COST_BY_USER, TIME_COST_BY_MODEL,
TIME_COST_BY_MODULE, TIME_COST_BY_PROVIDER
]
result = dict(data) # 浅拷贝
for key in time_cost_keys:
if key not in result:
continue
original = result[key]
if not isinstance(original, dict):
continue
compressed = {}
for sub_key, values in original.items():
if isinstance(values, list):
@@ -863,9 +863,9 @@ class StatisticOutputTask(AsyncTask):
else:
# 未知格式，保留原值
compressed[sub_key] = values
result[key] = compressed
return result
def _convert_defaultdict_to_dict(self, data):
@@ -985,7 +985,7 @@ class StatisticOutputTask(AsyncTask):
.filter(timestamp__gte=start_time)
.order_by("-timestamp")
)
async for batch in llm_query_builder.iter_batches(batch_size=STAT_BATCH_SIZE, as_dict=True):
for record in batch:
if not isinstance(record, dict) or not record.get("timestamp"):
@@ -1010,7 +1010,7 @@ class StatisticOutputTask(AsyncTask):
if module_name not in cost_by_module:
cost_by_module[module_name] = [0.0] * len(time_points)
cost_by_module[module_name][idx] += cost
await asyncio.sleep(0)
# 🔧 内存优化：使用分批查询 Messages
@@ -1020,7 +1020,7 @@ class StatisticOutputTask(AsyncTask):
.filter(time__gte=start_time.timestamp())
.order_by("-time")
)
async for batch in msg_query_builder.iter_batches(batch_size=STAT_BATCH_SIZE, as_dict=True):
for msg in batch:
if not isinstance(msg, dict) or not msg.get("time"):
@@ -1040,7 +1040,7 @@ class StatisticOutputTask(AsyncTask):
if chat_name not in message_by_chat:
message_by_chat[chat_name] = [0] * len(time_points)
message_by_chat[chat_name][idx] += 1
await asyncio.sleep(0)
return {

18
src/chat/utils/typo_generator.py
View File

@@ -36,21 +36,21 @@ def get_typo_generator(
) -> "ChineseTypoGenerator":
"""
获取错别字生成器单例（内存优化）
如果参数与缓存的单例不同，会更新参数但复用拼音字典和字频数据。
参数:
error_rate: 单字替换概率
min_freq: 最小字频阈值
tone_error_rate: 声调错误概率
word_replace_rate: 整词替换概率
max_freq_diff: 最大允许的频率差异
返回:
ChineseTypoGenerator 实例
"""
global _typo_generator_singleton
with _singleton_lock:
if _typo_generator_singleton is None:
_typo_generator_singleton = ChineseTypoGenerator(
@@ -70,7 +70,7 @@ def get_typo_generator(
word_replace_rate=word_replace_rate,
max_freq_diff=max_freq_diff,
)
return _typo_generator_singleton
@@ -87,7 +87,7 @@ class ChineseTypoGenerator:
max_freq_diff: 最大允许的频率差异
"""
global _shared_pinyin_dict, _shared_char_frequency
self.error_rate = error_rate
self.min_freq = min_freq
self.tone_error_rate = tone_error_rate
@@ -99,7 +99,7 @@ class ChineseTypoGenerator:
_shared_pinyin_dict = self._create_pinyin_dict()
logger.debug("拼音字典已创建并缓存")
self.pinyin_dict = _shared_pinyin_dict
if _shared_char_frequency is None:
_shared_char_frequency = self._load_or_create_char_frequency()
logger.debug("字频数据已加载并缓存")
@@ -454,10 +454,10 @@ class ChineseTypoGenerator:
# 50%概率返回纠正建议
if random.random() < 0.5:
if word_typos:
wrong_word, correct_word = random.choice(word_typos)
_wrong_word, correct_word = random.choice(word_typos)
correction_suggestion = correct_word
elif char_typos:
wrong_char, correct_char = random.choice(char_typos)
_wrong_char, correct_char = random.choice(char_typos)
correction_suggestion = correct_char
return "".join(result), correction_suggestion

4
src/chat/utils/utils.py
View File

@@ -9,13 +9,15 @@ from typing import Any
import numpy as np
import rjieba
from src.common.data_models.database_data_model import DatabaseUserInfo
# MessageRecv 已被移除，现在使用 DatabaseMessages
from src.common.logger import get_logger
from src.common.message_repository import count_messages, find_messages
from src.config.config import global_config, model_config
from src.llm_models.utils_model import LLMRequest
from src.person_info.person_info import PersonInfoManager, get_person_info_manager
from src.common.data_models.database_data_model import DatabaseUserInfo
from .typo_generator import get_typo_generator
logger = get_logger("chat_utils")

2
src/chat/utils/utils_image.py
View File

@@ -189,7 +189,7 @@ class ImageManager:
# 4. 如果都未命中，则调用新逻辑生成描述
logger.info(f"[新表情识别] 表情包未注册且无缓存 (Hash: {image_hash[:8]}...)，调用新逻辑生成描述")
full_description, emotions = await emoji_manager.build_emoji_description(image_base64)
full_description, _emotions = await emoji_manager.build_emoji_description(image_base64)
if not full_description:
logger.warning("未能通过新逻辑生成有效描述")