gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(database): 添加MySQL支持并重构数据库配置

Browse Source 
- 新增DataBaseConfig类用于集中管理数据库配置
- 重构数据库初始化逻辑，支持SQLite和MySQL两种数据库类型
- 为数据库表添加表前缀支持，便于多实例部署
- 更新数据库模型字段类型和长度限制
- 在配置模板中添加数据库配置节
This commit is contained in:
cuckoo711
2025-08-07 10:55:48 +08:00
parent 3d98b56c15
commit b6f5831785
5 changed files with 145 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files

57
src/common/database/database.py
View File

@@ -1,9 +1,11 @@
import os
from pymongo import MongoClient
from peewee import SqliteDatabase
from peewee import MySQLDatabase, SqliteDatabase
from pymongo.database import Database
from rich.traceback import install
from src.config.config import global_config
install(extra_lines=3)
_client = None
@@ -57,26 +59,39 @@ class DBWrapper:
return get_db()[key] # type: ignore
def create_peewee_database():
data_base_config = global_config.data_base
if data_base_config.db_type == "mysql":
return MySQLDatabase(
data_base_config.database,
user=data_base_config.username,
password=data_base_config.password,
host=data_base_config.host,
port=int(data_base_config.port),
charset='utf8mb4'
)
elif data_base_config.db_type == "sqlite":
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
_DB_DIR = os.path.join(ROOT_PATH, "data")
_DB_FILE = os.path.join(_DB_DIR, "MaiBot.db")
os.makedirs(_DB_DIR, exist_ok=True)
return SqliteDatabase(
_DB_FILE,
pragmas={
"journal_mode": "wal", # WAL模式提高并发性能
"cache_size": -64 * 1000, # 64MB缓存
"foreign_keys": 1,
"ignore_check_constraints": 0,
"synchronous": 0, # 异步写入提高性能
"busy_timeout": 1000, # 1秒超时而不是3秒
}, )
else:
raise ValueError(f"Unsupported PEEWEE_DB_TYPE: {data_base_config.db_type}")
# 全局数据库访问点
memory_db: Database = DBWrapper() # type: ignore
# 定义数据库文件路径
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
_DB_DIR = os.path.join(ROOT_PATH, "data")
_DB_FILE = os.path.join(_DB_DIR, "MaiBot.db")
# 确保数据库目录存在
os.makedirs(_DB_DIR, exist_ok=True)
memory_db: Database | DBWrapper = DBWrapper()
# 全局 Peewee SQLite 数据库访问点
db = SqliteDatabase(
_DB_FILE,
pragmas={
"journal_mode": "wal", # WAL模式提高并发性能
"cache_size": -64 * 1000, # 64MB缓存
"foreign_keys": 1,
"ignore_check_constraints": 0,
"synchronous": 0, # 异步写入提高性能
"busy_timeout": 1000, # 1秒超时而不是3秒
},
)
db = create_peewee_database()

87
src/common/database/database_model.py
View File

@@ -1,9 +1,16 @@
from peewee import Model, DoubleField, IntegerField, BooleanField, TextField, FloatField, DateTimeField
from .database import db
import datetime
from src.common.logger import get_logger
from peewee import BooleanField, CharField, DateTimeField, DoubleField, FloatField, IntegerField, Model, TextField
from src.common.database.database import db
from src.common.logger import get_logger
from src.config.config import global_config
table_prefix = global_config.data_base.table_prefix
logger = get_logger("database_model")
logger.info(f"正在加载数据库模型...数据库表前缀为: {table_prefix}")
# 请在此处定义您的数据库实例。
# 您需要取消注释并配置适合您的数据库的部分。
# 例如，对于 SQLite:
@@ -34,7 +41,7 @@ class ChatStreams(BaseModel):
# stream_id: "a544edeb1a9b73e3e1d77dff36e41264"
# 假设 stream_id 是唯一的，并为其创建索引以提高查询性能。
stream_id = TextField(unique=True, index=True)
stream_id = CharField(max_length=64, unique=True)
# create_time: 1746096761.4490178 (时间戳，精确到小数点后7位)
# DoubleField 用于存储浮点数，适合此类时间戳。
@@ -70,7 +77,7 @@ class ChatStreams(BaseModel):
# 如果不使用带有数据库实例的 BaseModel，或者想覆盖它，
# 请取消注释并在下面设置数据库实例：
# database = db
table_name = "chat_streams" # 可选：明确指定数据库中的表名
table_name = table_prefix + "chat_streams" # 可选：明确指定数据库中的表名
class LLMUsage(BaseModel):
@@ -78,9 +85,9 @@ class LLMUsage(BaseModel):
用于存储 API 使用日志数据的模型。
"""
model_name = TextField(index=True) # 添加索引
user_id = TextField(index=True) # 添加索引
request_type = TextField(index=True) # 添加索引
model_name = CharField(max_length=64, index=True) # 添加索引
user_id = CharField(max_length=64, index=True) # 添加索引
request_type = CharField(max_length=64, index=True) # 添加索引
endpoint = TextField()
prompt_tokens = IntegerField()
completion_tokens = IntegerField()
@@ -92,15 +99,15 @@ class LLMUsage(BaseModel):
class Meta:
# 如果 BaseModel.Meta.database 已设置，则此模型将继承该数据库配置。
# database = db
table_name = "llm_usage"
table_name = table_prefix + "llm_usage"
class Emoji(BaseModel):
"""表情包"""
full_path = TextField(unique=True, index=True) # 文件的完整路径 (包括文件名)
full_path = CharField(max_length=512, unique=True) # 文件的完整路径 (包括文件名)
format = TextField() # 图片格式
emoji_hash = TextField(index=True) # 表情包的哈希值
emoji_hash = CharField(max_length=64, index=True) # 表情包的哈希值
description = TextField() # 表情包的描述
query_count = IntegerField(default=0) # 查询次数（用于统计表情包被查询描述的次数）
is_registered = BooleanField(default=False) # 是否已注册
@@ -114,7 +121,7 @@ class Emoji(BaseModel):
class Meta:
# database = db # 继承自 BaseModel
table_name = "emoji"
table_name = table_prefix + "emoji"
class Messages(BaseModel):
@@ -122,10 +129,10 @@ class Messages(BaseModel):
用于存储消息数据的模型。
"""
message_id = TextField(index=True) # 消息 ID (更改自 IntegerField)
message_id = CharField(max_length=128, index=True) # 消息 ID (更改自 IntegerField)
time = DoubleField() # 消息时间戳
chat_id = TextField(index=True) # 对应的 ChatStreams stream_id
chat_id = CharField(max_length=128, index=True) # 对应的 ChatStreams stream_id
reply_to = TextField(null=True)
@@ -165,7 +172,7 @@ class Messages(BaseModel):
class Meta:
# database = db # 继承自 BaseModel
table_name = "messages"
table_name = table_prefix + "messages"
class ActionRecords(BaseModel):
@@ -183,13 +190,13 @@ class ActionRecords(BaseModel):
action_build_into_prompt = BooleanField(default=False)
action_prompt_display = TextField()
chat_id = TextField(index=True) # 对应的 ChatStreams stream_id
chat_id = CharField(max_length=128, index=True) # 对应的 ChatStreams stream_id
chat_info_stream_id = TextField()
chat_info_platform = TextField()
class Meta:
# database = db # 继承自 BaseModel
table_name = "action_records"
table_name = table_prefix + "action_records"
class Images(BaseModel):
@@ -198,9 +205,9 @@ class Images(BaseModel):
"""
image_id = TextField(default="") # 图片唯一ID
emoji_hash = TextField(index=True) # 图像的哈希值
emoji_hash = CharField(max_length=64, index=True) # 图像的哈希值
description = TextField(null=True) # 图像的描述
path = TextField(unique=True) # 图像文件的路径
path = CharField(max_length=512, unique=True) # 图像文件的路径
# base64 = TextField() # 图片的base64编码
count = IntegerField(default=1) # 图片被引用的次数
timestamp = FloatField() # 时间戳
@@ -208,7 +215,7 @@ class Images(BaseModel):
vlm_processed = BooleanField(default=False) # 是否已经过VLM处理
class Meta:
table_name = "images"
table_name = table_prefix + "images"
class ImageDescriptions(BaseModel):
@@ -217,13 +224,13 @@ class ImageDescriptions(BaseModel):
"""
type = TextField() # 类型，例如 "emoji"
image_description_hash = TextField(index=True) # 图像的哈希值
image_description_hash = CharField(max_length=64, index=True) # 图像的哈希值
description = TextField() # 图像的描述
timestamp = FloatField() # 时间戳
class Meta:
# database = db # 继承自 BaseModel
table_name = "image_descriptions"
table_name = table_prefix + "image_descriptions"
class OnlineTime(BaseModel):
@@ -232,14 +239,14 @@ class OnlineTime(BaseModel):
"""
# timestamp: "$date": "2025-05-01T18:52:18.191Z" (存储为字符串)
timestamp = TextField(default=datetime.datetime.now) # 时间戳
timestamp = CharField(max_length=64, default=datetime.datetime.now) # 时间戳
duration = IntegerField() # 时长，单位分钟
start_timestamp = DateTimeField(default=datetime.datetime.now)
end_timestamp = DateTimeField(index=True)
class Meta:
# database = db # 继承自 BaseModel
table_name = "online_time"
table_name = table_prefix + "online_time"
class PersonInfo(BaseModel):
@@ -247,11 +254,11 @@ class PersonInfo(BaseModel):
用于存储个人信息数据的模型。
"""
person_id = TextField(unique=True, index=True) # 个人唯一ID
person_id = CharField(max_length=64, unique=True) # 个人唯一ID
person_name = TextField(null=True) # 个人名称 (允许为空)
name_reason = TextField(null=True) # 名称设定的原因
platform = TextField() # 平台
user_id = TextField(index=True) # 用户ID
user_id = CharField(max_length=64, index=True) # 用户ID
nickname = TextField() # 用户昵称
impression = TextField(null=True) # 个人印象
short_impression = TextField(null=True) # 个人印象的简短描述
@@ -266,11 +273,11 @@ class PersonInfo(BaseModel):
class Meta:
# database = db # 继承自 BaseModel
table_name = "person_info"
table_name = table_prefix + "person_info"
class Memory(BaseModel):
memory_id = TextField(index=True)
memory_id = CharField(max_length=128, index=True)
chat_id = TextField(null=True)
memory_text = TextField(null=True)
keywords = TextField(null=True)
@@ -278,7 +285,7 @@ class Memory(BaseModel):
last_view_time = FloatField(null=True)
class Meta:
table_name = "memory"
table_name = table_prefix + "memory"
class Expression(BaseModel):
@@ -290,16 +297,16 @@ class Expression(BaseModel):
style = TextField()
count = FloatField()
last_active_time = FloatField()
chat_id = TextField(index=True)
chat_id = CharField(max_length=128, index=True)
type = TextField()
create_date = FloatField(null=True) # 创建日期，允许为空以兼容老数据
class Meta:
table_name = "expression"
table_name = table_prefix + "expression"
class ThinkingLog(BaseModel):
chat_id = TextField(index=True)
chat_id = CharField(max_length=128, index=True)
trigger_text = TextField(null=True)
response_text = TextField(null=True)
@@ -319,7 +326,7 @@ class ThinkingLog(BaseModel):
created_at = DateTimeField(default=datetime.datetime.now)
class Meta:
table_name = "thinking_logs"
table_name = table_prefix + "thinking_logs"
class GraphNodes(BaseModel):
@@ -327,14 +334,14 @@ class GraphNodes(BaseModel):
用于存储记忆图节点的模型
"""
concept = TextField(unique=True, index=True) # 节点概念
concept = CharField(max_length=128, unique=True) # 节点概念
memory_items = TextField() # JSON格式存储的记忆列表
hash = TextField() # 节点哈希值
created_time = FloatField() # 创建时间戳
last_modified = FloatField() # 最后修改时间戳
class Meta:
table_name = "graph_nodes"
table_name = table_prefix + "graph_nodes"
class GraphEdges(BaseModel):
@@ -342,15 +349,15 @@ class GraphEdges(BaseModel):
用于存储记忆图边的模型
"""
source = TextField(index=True) # 源节点
target = TextField(index=True) # 目标节点
source = CharField(max_length=128, index=True) # 源节点
target = CharField(max_length=128, index=True) # 目标节点
strength = IntegerField() # 连接强度
hash = TextField() # 边哈希值
created_time = FloatField() # 创建时间戳
last_modified = FloatField() # 最后修改时间戳
class Meta:
table_name = "graph_edges"
table_name = table_prefix + "graph_edges"
def create_tables():
@@ -400,7 +407,7 @@ def initialize_database():
GraphEdges,
ActionRecords, # 添加 ActionRecords 到初始化列表
]
del_extra = False # 是否删除多余字段
try:
with db: # 管理 table_exists 检查的连接
for model in models:
@@ -452,6 +459,8 @@ def initialize_database():
logger.error(f"添加字段 '{field_name}' 失败: {e}")
# 检查并删除多余字段（新增逻辑）
if not del_extra:
continue
extra_fields = existing_columns - model_fields
if extra_fields:
logger.warning(f"表 '{table_name}' 存在多余字段: {extra_fields}")

2
src/config/config.py
View File

@@ -14,6 +14,7 @@ from src.common.logger import get_logger
from src.config.config_base import ConfigBase
from src.config.official_configs import (
BotConfig,
DataBaseConfig,
PersonalityConfig,
ExpressionConfig,
ChatConfig,
@@ -348,6 +349,7 @@ class Config(ConfigBase):
debug: DebugConfig
custom_prompt: CustomPromptConfig
voice: VoiceConfig
data_base: DataBaseConfig
@dataclass

38
src/config/official_configs.py
View File

@@ -1,5 +1,4 @@
import re
from dataclasses import dataclass, field
from typing import Literal, Optional
@@ -17,7 +16,7 @@ from src.config.config_base import ConfigBase
@dataclass
class BotConfig(ConfigBase):
"""QQ机器人配置类"""
platform: str
"""平台"""
@@ -68,7 +67,7 @@ class ChatConfig(ConfigBase):
max_context_size: int = 18
"""上下文长度"""
willing_amplifier: float = 1.0
replyer_random_probability: float = 0.5
@@ -272,6 +271,7 @@ class NormalChatConfig(ConfigBase):
willing_mode: str = "classical"
"""意愿模式"""
@dataclass
class ExpressionConfig(ConfigBase):
"""表达配置类"""
@@ -301,7 +301,8 @@ class ToolConfig(ConfigBase):
enable_tool: bool = False
"""是否在聊天中启用工具"""
@dataclass
class VoiceConfig(ConfigBase):
"""语音识别配置类"""
@@ -387,7 +388,7 @@ class MemoryConfig(ConfigBase):
memory_ban_words: list[str] = field(default_factory=lambda: ["表情包", "图片", "回复", "聊天记录"])
"""不允许记忆的词列表"""
enable_instant_memory: bool = True
"""是否启用即时记忆"""
@@ -398,7 +399,7 @@ class MoodConfig(ConfigBase):
enable_mood: bool = False
"""是否启用情绪系统"""
mood_update_threshold: float = 1.0
"""情绪更新阈值,越高，更新越慢"""
@@ -449,6 +450,7 @@ class KeywordReactionConfig(ConfigBase):
if not isinstance(rule, KeywordRuleConfig):
raise ValueError(f"规则必须是KeywordRuleConfig类型，而不是{type(rule).__name__}")
@dataclass
class CustomPromptConfig(ConfigBase):
"""自定义提示词配置类"""
@@ -598,3 +600,27 @@ class LPMMKnowledgeConfig(ConfigBase):
embedding_dimension: int = 1024
"""嵌入向量维度，应该与模型的输出维度一致"""
class DataBaseConfig(ConfigBase):
"""数据库配置类"""
db_type: Literal["sqlite", "mysql"] = "sqlite"
"""数据库类型，支持sqlite、mysql"""
host: str = "127.0.0.1"
"""数据库主机地址"""
port: int = 3306
"""数据库端口号"""
username: str = ""
"""数据库用户名"""
password: str = ""
"""数据库密码"""
database: str = "MaiBot"
"""数据库名称"""
table_prefix: str = ""
"""数据库表前缀"""

44
template/bot_config_template.toml
View File

@@ -1,5 +1,5 @@
[inner]
version = "6.0.0"
version = "6.1.0"
#----以下是给开发人员阅读的，如果你只是部署了麦麦，不需要阅读----
#如果你想要修改配置文件，请在修改后将version的值进行变更
@@ -13,14 +13,14 @@ version = "6.0.0"
#----以上是给开发人员阅读的，如果你只是部署了麦麦，不需要阅读----
[bot]
platform = "qq"
platform = "qq"
qq_account = 1145141919810 # 麦麦的QQ账号
nickname = "麦麦" # 麦麦的昵称
alias_names = ["麦叠", "牢麦"] # 麦麦的别名
[personality]
# 建议50字以内，描述人格的核心特质
personality_core = "是一个积极向上的女大学生"
personality_core = "是一个积极向上的女大学生"
# 人格的细节，描述人格的一些侧面
personality_side = "用一句话或几句话描述人格的侧面特质"
#アイデンティティがない 生まれないらららら
@@ -39,7 +39,7 @@ enable_expression_learning = false # 是否启用表达学习，麦麦会学习
learning_interval = 350 # 学习间隔 单位秒
expression_groups = [
["qq:1919810:private","qq:114514:private","qq:1111111:group"], # 在这里设置互通组，相同组的chat_id会共享学习到的表达方式
["qq:1919810:private", "qq:114514:private", "qq:1111111:group"], # 在这里设置互通组，相同组的chat_id会共享学习到的表达方式
# 格式：["qq:123456:private","qq:654321:group"]
# 注意：如果为群聊，则需要设置为group，如果设置为私聊，则需要设置为private
]
@@ -51,7 +51,7 @@ relation_frequency = 1 # 关系频率，麦麦构建关系的频率
[chat] #麦麦的聊天通用设置
focus_value = 1
focus_value = 1
# 麦麦的专注思考能力，越高越容易专注，可能消耗更多token
# 专注时能更好把握发言时机，能够进行持久的连续对话
@@ -95,7 +95,7 @@ talk_frequency_adjust = [
# 以下是消息过滤，可以根据规则过滤特定消息，将不会读取这些消息
ban_words = [
# "403","张三"
]
]
ban_msgs_regex = [
# 需要过滤的消息（原始消息）匹配的正则表达式，匹配到的消息将被过滤，若不了解正则表达式请勿修改
@@ -139,7 +139,7 @@ consolidation_check_percentage = 0.05 # 检查节点比例
enable_instant_memory = false # 是否启用即时记忆，测试功能，可能存在未知问题
#不希望记忆的词，已经记忆的不会受到影响，需要手动清理
memory_ban_words = [ "表情包", "图片", "回复", "聊天记录" ]
memory_ban_words = ["表情包", "图片", "回复", "聊天记录"]
[voice]
enable_asr = false # 是否启用语音识别，启用后麦麦可以识别语音消息，启用该功能需要配置语音识别模型[model.voice]s
@@ -190,10 +190,10 @@ enable_response_post_process = true # 是否启用回复后处理，包括错别
[chinese_typo]
enable = true # 是否启用中文错别字生成器
error_rate=0.01 # 单字替换概率
min_freq=9 # 最小字频阈值
tone_error_rate=0.1 # 声调错误概率
word_replace_rate=0.006 # 整词替换概率
error_rate = 0.01 # 单字替换概率
min_freq = 9 # 最小字频阈值
tone_error_rate = 0.1 # 声调错误概率
word_replace_rate = 0.006 # 整词替换概率
[response_splitter]
enable = true # 是否启用回复分割器
@@ -210,8 +210,8 @@ console_log_level = "INFO" # 控制台日志级别，可选: DEBUG, INFO, WARNIN
file_log_level = "DEBUG" # 文件日志级别，可选: DEBUG, INFO, WARNING, ERROR, CRITICAL
# 第三方库日志控制
suppress_libraries = ["faiss","httpx", "urllib3", "asyncio", "websockets", "httpcore", "requests", "peewee", "openai","uvicorn","jieba"] # 完全屏蔽的库
library_log_levels = { "aiohttp" = "WARNING"} # 设置特定库的日志级别
suppress_libraries = ["faiss", "httpx", "urllib3", "asyncio", "websockets", "httpcore", "requests", "peewee", "openai", "uvicorn", "jieba"] # 完全屏蔽的库
library_log_levels = { "aiohttp" = "WARNING" } # 设置特定库的日志级别
[debug]
show_prompt = false # 是否显示prompt
@@ -220,9 +220,9 @@ show_prompt = false # 是否显示prompt
auth_token = [] # 认证令牌，用于API验证，为空则不启用验证
# 以下项目若要使用需要打开use_custom，并单独配置maim_message的服务器
use_custom = false # 是否启用自定义的maim_message服务器，注意这需要设置新的端口，不能与.env重复
host="127.0.0.1"
port=8090
mode="ws" # 支持ws和tcp两种模式
host = "127.0.0.1"
port = 8090
mode = "ws" # 支持ws和tcp两种模式
use_wss = false # 是否使用WSS安全连接，只支持ws模式
cert_file = "" # SSL证书文件路径，仅在use_wss=true时有效
key_file = "" # SSL密钥文件路径，仅在use_wss=true时有效
@@ -231,4 +231,14 @@ key_file = "" # SSL密钥文件路径，仅在use_wss=true时有效
enable = true
[experimental] #实验性功能
enable_friend_chat = false # 是否启用好友聊天
enable_friend_chat = false # 是否启用好友聊天
[data_base] #数据库配置
# 数据库类型，可选：sqlite, mysql
db_type = "sqlite" # 数据库类型
host = "" # 数据库主机地址,如果是sqlite则不需要填写
port = 3306 # 数据库端口,如果是sqlite则不需要填写
username = "" # 数据库用户名,如果是sqlite则不需要填写
password = "" # 数据库密码,如果是sqlite则不需要填写
database = "MaiBot" # 数据库名称,如果是sqlite则不需要填写
table_prefix = "" # 数据库表前缀,用于支持多实例部署