Merge remote-tracking branch 'upstream/debug' into feature

src/plugins/chat/config.py

@@ -138,7 +138,7 @@ class BotConfig:
             if "others" in toml_dict:
                 others_config = toml_dict["others"]
                 config.enable_advance_output = others_config.get("enable_advance_output", config.enable_advance_output)
-
+                config.enable_kuuki_read = others_config.get("enable_kuuki_read", config.enable_kuuki_read)
             
             logger.success(f"成功加载配置文件: {config_path}")
                 

src/plugins/chat/prompt_builder.py

@@ -118,7 +118,7 @@ class PromptBuilder:
         prompt_info = self.get_prompt_info(message_txt,threshold=0.5)
         if prompt_info:
             prompt_info = f'''\n----------------------------------------------------\n你有以下这些[知识]：\n{prompt_info}\n请你记住上面的[知识]，之后可能会用到\n----------------------------------------------------\n'''
-            promt_info_prompt = '你有一些[知识]，在上面可以参考。'
+            # promt_info_prompt = '你有一些[知识]，在上面可以参考。'
             
         end_time = time.time()
         print(f"\033[1;32m[知识检索]\033[0m 耗时: {(end_time - start_time):.3f}秒")

src/plugins/chat/utils.py

@@ -196,6 +196,8 @@ def get_recent_group_detailed_plain_text(db, group_id: int, limit: int = 12,comb
     message_detailed_plain_text = ''
     message_detailed_plain_text_list = []
     
+    # 反转消息列表，使最新的消息在最后
+    recent_messages.reverse()
     
     if combine:
         for msg_db_data in recent_messages:

src/plugins/models/utils_model.py

@@ -166,8 +166,8 @@ class LLM_request:
         # 发送请求到完整的chat/completions端点
         api_url = f"{self.base_url.rstrip('/')}/chat/completions"
         
-        max_retries = 3
-        base_wait_time = 15
+        max_retries = 2
+        base_wait_time = 6
         
         for retry in range(max_retries):
             try:

src/plugins/schedule/schedule_generator.py

@@ -128,6 +128,10 @@ class ScheduleGenerator:
     
     def _time_diff(self, time1: str, time2: str) -> int:
         """计算两个时间字符串之间的分钟差"""
+        if time1=="24:00":
+            time1="23:59"
+        if time2=="24:00":
+            time2="23:59"
         t1 = datetime.datetime.strptime(time1, "%H:%M")
         t2 = datetime.datetime.strptime(time2, "%H:%M")
         diff = int((t2 - t1).total_seconds() / 60)
@@ -165,4 +169,4 @@ class ScheduleGenerator:
 # if __name__ == "__main__":
 #     main() 
     
-bot_schedule = ScheduleGenerator()
+bot_schedule = ScheduleGenerator()

src/test/emotion_cal.py

@@ -0,0 +1,70 @@
+from textblob import TextBlob
+import jieba
+from translate import Translator
+
+def analyze_emotion(text):
+    """
+    分析文本的情感,返回情感极性和主观性得分
+    :param text: 输入文本
+    :return: (情感极性, 主观性) 元组
+    情感极性: -1(非常消极) 到 1(非常积极)
+    主观性: 0(客观) 到 1(主观)
+    """
+    try:
+        # 创建翻译器
+        translator = Translator(to_lang="en", from_lang="zh")
+        
+        # 如果是中文文本,先翻译成英文
+        # 因为TextBlob的情感分析主要基于英文
+        translated_text = translator.translate(text)
+        
+        # 创建TextBlob对象
+        blob = TextBlob(translated_text)
+        
+        # 获取情感极性和主观性
+        polarity = blob.sentiment.polarity
+        subjectivity = blob.sentiment.subjectivity
+        
+        return polarity, subjectivity
+        
+    except Exception as e:
+        print(f"分析过程中出现错误: {str(e)}")
+        return None, None
+
+def get_emotion_description(polarity, subjectivity):
+    """
+    根据情感极性和主观性生成描述性文字
+    """
+    if polarity is None or subjectivity is None:
+        return "无法分析情感"
+        
+    # 情感极性描述
+    if polarity > 0.5:
+        emotion = "非常积极"
+    elif polarity > 0:
+        emotion = "较为积极"
+    elif polarity == 0:
+        emotion = "中性"
+    elif polarity > -0.5:
+        emotion = "较为消极"
+    else:
+        emotion = "非常消极"
+        
+    # 主观性描述
+    if subjectivity > 0.7:
+        subj = "非常主观"
+    elif subjectivity > 0.3:
+        subj = "较为主观"
+    else:
+        subj = "较为客观"
+        
+    return f"情感倾向: {emotion}, 表达方式: {subj}"
+
+if __name__ == "__main__":
+    # 测试样例
+    test_text = "今天天气真好,我感到非常开心！"
+    polarity, subjectivity = analyze_emotion(test_text)
+    print(f"测试文本: {test_text}")
+    print(f"情感极性: {polarity:.2f}")
+    print(f"主观性得分: {subjectivity:.2f}")
+    print(get_emotion_description(polarity, subjectivity)) 

src/test/emotion_cal_bert.py

@@ -0,0 +1,74 @@
+from transformers import pipeline, AutoModelForSequenceClassification, AutoTokenizer
+
+def setup_bert_analyzer():
+    """
+    设置中文BERT情感分析器
+    """
+    # 使用专门针对中文情感分析的模型
+    model_name = "uer/roberta-base-finetuned-jd-binary-chinese"
+    
+    try:
+        # 加载模型和分词器
+        tokenizer = AutoTokenizer.from_pretrained(model_name)
+        model = AutoModelForSequenceClassification.from_pretrained(model_name)
+        
+        # 创建情感分析pipeline
+        analyzer = pipeline("sentiment-analysis", 
+                          model=model, 
+                          tokenizer=tokenizer)
+        
+        return analyzer
+    except Exception as e:
+        print(f"模型加载错误: {str(e)}")
+        return None
+
+def analyze_emotion_bert(text, analyzer):
+    """
+    使用BERT模型进行中文情感分析
+    """
+    try:
+        if not analyzer:
+            return None
+            
+        # 进行情感分析
+        result = analyzer(text)[0]
+        
+        return {
+            'label': result['label'],
+            'score': result['score']
+        }
+    except Exception as e:
+        print(f"分析过程中出现错误: {str(e)}")
+        return None
+
+def get_emotion_description_bert(result):
+    """
+    将BERT的情感分析结果转换为描述性文字
+    """
+    if not result:
+        return "无法分析情感"
+    
+    label = "积极" if result['label'] == 'positive' else "消极"
+    confidence = result['score']
+    
+    if confidence > 0.9:
+        strength = "强烈"
+    elif confidence > 0.7:
+        strength = "明显"
+    else:
+        strength = "轻微"
+    
+    return f"{strength}{label}"
+
+if __name__ == "__main__":
+    # 初始化分析器
+    analyzer = setup_bert_analyzer()
+    
+    # 测试样例
+    test_text = "这个产品质量很好，使用起来非常方便，推荐购买！"
+    result = analyze_emotion_bert(test_text, analyzer)
+    
+    print(f"测试文本: {test_text}")
+    if result:
+        print(f"情感倾向: {get_emotion_description_bert(result)}")
+        print(f"置信度: {result['score']:.2f}") 

src/test/emotion_cal_hanlp.py

@@ -0,0 +1,62 @@
+import hanlp
+
+def analyze_emotion_hanlp(text):
+    """
+    使用HanLP进行中文情感分析
+    """
+    try:
+        # 使用更基础的模型
+        tokenizer = hanlp.load('PKU_NAME_MERGED_SIX_MONTHS_CONVSEG')
+        
+        # 分词
+        words = tokenizer(text)
+        
+        # 简单的情感词典方法
+        positive_words = {'好', '棒', '优秀', '喜欢', '开心', '快乐', '美味', '推荐', '优质', '满意'}
+        negative_words = {'差', '糟', '烂', '讨厌', '失望', '难受', '恶心', '不满', '差劲', '垃圾'}
+        
+        # 计算情感得分
+        score = 0
+        for word in words:
+            if word in positive_words:
+                score += 1
+            elif word in negative_words:
+                score -= 1
+                
+        # 归一化得分
+        if score > 0:
+            return 1
+        elif score < 0:
+            return 0
+        else:
+            return 0.5
+            
+    except Exception as e:
+        print(f"分析过程中出现错误: {str(e)}")
+        return None
+
+def get_emotion_description_hanlp(score):
+    """
+    将HanLP的情感分析结果转换为描述性文字
+    """
+    if score is None:
+        return "无法分析情感"
+    elif score == 1:
+        return "积极"
+    elif score == 0:
+        return "消极"
+    else:
+        return "中性"
+
+if __name__ == "__main__":
+    # 测试样例
+    test_texts = [
+        "这家餐厅的服务态度很好，菜品也很美味！",
+        "这个产品质量太差了，一点都不值这个价",
+        "今天天气不错，但是工作很累"
+    ]
+    
+    for test_text in test_texts:
+        result = analyze_emotion_hanlp(test_text)
+        print(f"\n测试文本: {test_text}")
+        print(f"情感倾向: {get_emotion_description_hanlp(result)}") 

src/test/emotion_cal_snownlp.py

@@ -0,0 +1,53 @@
+from snownlp import SnowNLP
+
+def analyze_emotion_snownlp(text):
+    """
+    使用SnowNLP进行中文情感分析
+    :param text: 输入文本
+    :return: 情感得分(0-1之间，越接近1越积极)
+    """
+    try:
+        s = SnowNLP(text)
+        sentiment_score = s.sentiments
+        
+        # 获取文本的关键词
+        keywords = s.keywords(3)
+        
+        return {
+            'sentiment_score': sentiment_score,
+            'keywords': keywords,
+            'summary': s.summary(1)  # 生成文本摘要
+        }
+    except Exception as e:
+        print(f"分析过程中出现错误: {str(e)}")
+        return None
+
+def get_emotion_description_snownlp(score):
+    """
+    将情感得分转换为描述性文字
+    """
+    if score is None:
+        return "无法分析情感"
+    
+    if score > 0.8:
+        return "非常积极"
+    elif score > 0.6:
+        return "较为积极"
+    elif score > 0.4:
+        return "中性偏积极"
+    elif score > 0.2:
+        return "中性偏消极"
+    else:
+        return "消极"
+
+if __name__ == "__main__":
+    # 测试样例
+    test_text = "我们学校有免费的gpt4用"
+    result = analyze_emotion_snownlp(test_text)
+    
+    if result:
+        print(f"测试文本: {test_text}")
+        print(f"情感得分: {result['sentiment_score']:.2f}")
+        print(f"情感倾向: {get_emotion_description_snownlp(result['sentiment_score'])}")
+        print(f"关键词: {', '.join(result['keywords'])}")
+        print(f"文本摘要: {result['summary'][0]}") 

src/test/snownlp_demo.py

@@ -0,0 +1,54 @@
+from snownlp import SnowNLP
+
+def demo_snownlp_features(text):
+    """
+    展示SnowNLP的主要功能
+    :param text: 输入文本
+    """
+    print(f"\n=== SnowNLP功能演示 ===")
+    print(f"输入文本: {text}")
+    
+    # 创建SnowNLP对象
+    s = SnowNLP(text)
+    
+    # 1. 分词
+    print(f"\n1. 分词结果:")
+    print(f"   {' | '.join(s.words)}")
+    
+    # 2. 情感分析
+    print(f"\n2. 情感分析:")
+    sentiment = s.sentiments
+    print(f"   情感得分: {sentiment:.2f}")
+    print(f"   情感倾向: {'积极' if sentiment > 0.5 else '消极' if sentiment < 0.5 else '中性'}")
+    
+    # 3. 关键词提取
+    print(f"\n3. 关键词提取:")
+    print(f"   {', '.join(s.keywords(3))}")
+    
+    # 4. 词性标注
+    print(f"\n4. 词性标注:")
+    print(f"   {' '.join([f'{word}/{tag}' for word, tag in s.tags])}")
+    
+    # 5. 拼音转换
+    print(f"\n5. 拼音:")
+    print(f"   {' '.join(s.pinyin)}")
+    
+    # 6. 文本摘要
+    if len(text) > 100:  # 只对较长文本生成摘要
+        print(f"\n6. 文本摘要:")
+        print(f"   {' '.join(s.summary(3))}")
+
+if __name__ == "__main__":
+    # 测试用例
+    test_texts = [
+        "这家新开的餐厅很不错，菜品种类丰富，味道可口，服务态度也很好，价格实惠，强烈推荐大家来尝试！",
+        "这部电影剧情混乱，演技浮夸，特效粗糙，配乐难听，完全浪费了我的时间和票价。",
+        """人工智能正在改变我们的生活方式。它能够帮助我们完成复杂的计算任务，
+        提供个性化的服务推荐，优化交通路线，辅助医疗诊断。但同时我们也要警惕
+        人工智能带来的问题，比如隐私安全、就业变化等。如何正确认识和利用人工智能，
+        是我们每个人都需要思考的问题。"""
+    ]
+    
+    for text in test_texts:
+        demo_snownlp_features(text)
+        print("\n" + "="*50) 

src/test/typo.py

@@ -0,0 +1,488 @@
+"""
+错别字生成器 - 流程说明
+
+整体替换逻辑：
+1. 数据准备
+   - 加载字频词典：使用jieba词典计算汉字使用频率
+   - 创建拼音映射：建立拼音到汉字的映射关系
+   - 加载词频信息：从jieba词典获取词语使用频率
+
+2. 分词处理
+   - 使用jieba将输入句子分词
+   - 区分单字词和多字词
+   - 保留标点符号和空格
+
+3. 词语级别替换（针对多字词）
+   - 触发条件：词长>1 且 随机概率<0.3
+   - 替换流程：
+     a. 获取词语拼音
+     b. 生成所有可能的同音字组合
+     c. 过滤条件：
+        - 必须是jieba词典中的有效词
+        - 词频必须达到原词频的10%以上
+        - 综合评分(词频70%+字频30%)必须达到阈值
+     d. 按综合评分排序，选择最合适的替换词
+
+4. 字级别替换（针对单字词或未进行整词替换的多字词）
+   - 单字替换概率：0.3
+   - 多字词中的单字替换概率：0.3 * (0.7 ^ (词长-1))
+   - 替换流程：
+     a. 获取字的拼音
+     b. 声调错误处理（20%概率）
+     c. 获取同音字列表
+     d. 过滤条件：
+        - 字频必须达到最小阈值
+        - 频率差异不能过大（指数衰减计算）
+     e. 按频率排序选择替换字
+
+5. 频率控制机制
+   - 字频控制：使用归一化的字频（0-1000范围）
+   - 词频控制：使用jieba词典中的词频
+   - 频率差异计算：使用指数衰减函数
+   - 最小频率阈值：确保替换字/词不会太生僻
+
+6. 输出信息
+   - 原文和错字版本的对照
+   - 每个替换的详细信息（原字/词、替换后字/词、拼音、频率）
+   - 替换类型说明（整词替换/声调错误/同音字替换）
+   - 词语分析和完整拼音
+
+注意事项：
+1. 所有替换都必须使用有意义的词语
+2. 替换词的使用频率不能过低
+3. 多字词优先考虑整词替换
+4. 考虑声调变化的情况
+5. 保持标点符号和空格不变
+"""
+
+from pypinyin import pinyin, Style
+from collections import defaultdict
+import json
+import os
+import unicodedata
+import jieba
+import jieba.posseg as pseg
+from pathlib import Path
+import random
+import math
+import time
+
+def load_or_create_char_frequency():
+    """
+    加载或创建汉字频率字典
+    """
+    cache_file = Path("char_frequency.json")
+    
+    # 如果缓存文件存在，直接加载
+    if cache_file.exists():
+        with open(cache_file, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    # 使用内置的词频文件
+    char_freq = defaultdict(int)
+    dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt')
+    
+    # 读取jieba的词典文件
+    with open(dict_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            word, freq = line.strip().split()[:2]
+            # 对词中的每个字进行频率累加
+            for char in word:
+                if is_chinese_char(char):
+                    char_freq[char] += int(freq)
+    
+    # 归一化频率值
+    max_freq = max(char_freq.values())
+    normalized_freq = {char: freq/max_freq * 1000 for char, freq in char_freq.items()}
+    
+    # 保存到缓存文件
+    with open(cache_file, 'w', encoding='utf-8') as f:
+        json.dump(normalized_freq, f, ensure_ascii=False, indent=2)
+    
+    return normalized_freq
+
+# 创建拼音到汉字的映射字典
+def create_pinyin_dict():
+    """
+    创建拼音到汉字的映射字典
+    """
+    # 常用汉字范围
+    chars = [chr(i) for i in range(0x4e00, 0x9fff)]
+    pinyin_dict = defaultdict(list)
+    
+    # 为每个汉字建立拼音映射
+    for char in chars:
+        try:
+            py = pinyin(char, style=Style.TONE3)[0][0]
+            pinyin_dict[py].append(char)
+        except Exception:
+            continue
+    
+    return pinyin_dict
+
+def is_chinese_char(char):
+    """
+    判断是否为汉字
+    """
+    try:
+        return '\u4e00' <= char <= '\u9fff'
+    except:
+        return False
+
+def get_pinyin(sentence):
+    """
+    将中文句子拆分成单个汉字并获取其拼音
+    :param sentence: 输入的中文句子
+    :return: 每个汉字及其拼音的列表
+    """
+    # 将句子拆分成单个字符
+    characters = list(sentence)
+    
+    # 获取每个字符的拼音
+    result = []
+    for char in characters:
+        # 跳过空格和非汉字字符
+        if char.isspace() or not is_chinese_char(char):
+            continue
+        # 获取拼音（数字声调）
+        py = pinyin(char, style=Style.TONE3)[0][0]
+        result.append((char, py))
+    
+    return result
+
+def get_homophone(char, py, pinyin_dict, char_frequency, min_freq=5):
+    """
+    获取同音字，按照使用频率排序
+    """
+    homophones = pinyin_dict[py]
+    # 移除原字并过滤低频字
+    if char in homophones:
+        homophones.remove(char)
+    
+    # 过滤掉低频字
+    homophones = [h for h in homophones if char_frequency.get(h, 0) >= min_freq]
+    
+    # 按照字频排序
+    sorted_homophones = sorted(homophones, 
+                             key=lambda x: char_frequency.get(x, 0), 
+                             reverse=True)
+    
+    # 只返回前10个同音字，避免输出过多
+    return sorted_homophones[:10]
+
+def get_similar_tone_pinyin(py):
+    """
+    获取相似声调的拼音
+    例如：'ni3' 可能返回 'ni2' 或 'ni4'
+    处理特殊情况：
+    1. 轻声（如 'de5' 或 'le'）
+    2. 非数字结尾的拼音
+    """
+    # 检查拼音是否为空或无效
+    if not py or len(py) < 1:
+        return py
+        
+    # 如果最后一个字符不是数字，说明可能是轻声或其他特殊情况
+    if not py[-1].isdigit():
+        # 为非数字结尾的拼音添加数字声调1
+        return py + '1'
+    
+    base = py[:-1]  # 去掉声调
+    tone = int(py[-1])  # 获取声调
+    
+    # 处理轻声（通常用5表示）或无效声调
+    if tone not in [1, 2, 3, 4]:
+        return base + str(random.choice([1, 2, 3, 4]))
+    
+    # 正常处理声调
+    possible_tones = [1, 2, 3, 4]
+    possible_tones.remove(tone)  # 移除原声调
+    new_tone = random.choice(possible_tones)  # 随机选择一个新声调
+    return base + str(new_tone)
+
+def calculate_replacement_probability(orig_freq, target_freq, max_freq_diff=200):
+    """
+    根据频率差计算替换概率
+    频率差越大，概率越低
+    :param orig_freq: 原字频率
+    :param target_freq: 目标字频率
+    :param max_freq_diff: 最大允许的频率差
+    :return: 0-1之间的概率值
+    """
+    if target_freq > orig_freq:
+        return 1.0  # 如果替换字频率更高，保持原有概率
+    
+    freq_diff = orig_freq - target_freq
+    if freq_diff > max_freq_diff:
+        return 0.0  # 频率差太大，不替换
+    
+    # 使用指数衰减函数计算概率
+    # 频率差为0时概率为1，频率差为max_freq_diff时概率接近0
+    return math.exp(-3 * freq_diff / max_freq_diff)
+
+def get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, num_candidates=5, min_freq=5, tone_error_rate=0.2):
+    """
+    获取与给定字频率相近的同音字，可能包含声调错误
+    """
+    homophones = []
+    
+    # 有20%的概率使用错误声调
+    if random.random() < tone_error_rate:
+        wrong_tone_py = get_similar_tone_pinyin(py)
+        homophones.extend(pinyin_dict[wrong_tone_py])
+    
+    # 添加正确声调的同音字
+    homophones.extend(pinyin_dict[py])
+    
+    if not homophones:
+        return None
+        
+    # 获取原字的频率
+    orig_freq = char_frequency.get(char, 0)
+    
+    # 计算所有同音字与原字的频率差，并过滤掉低频字
+    freq_diff = [(h, char_frequency.get(h, 0)) 
+                for h in homophones 
+                if h != char and char_frequency.get(h, 0) >= min_freq]
+    
+    if not freq_diff:
+        return None
+    
+    # 计算每个候选字的替换概率
+    candidates_with_prob = []
+    for h, freq in freq_diff:
+        prob = calculate_replacement_probability(orig_freq, freq)
+        if prob > 0:  # 只保留有效概率的候选字
+            candidates_with_prob.append((h, prob))
+    
+    if not candidates_with_prob:
+        return None
+    
+    # 根据概率排序
+    candidates_with_prob.sort(key=lambda x: x[1], reverse=True)
+    
+    # 返回概率最高的几个字
+    return [char for char, _ in candidates_with_prob[:num_candidates]]
+
+def get_word_pinyin(word):
+    """
+    获取词语的拼音列表
+    """
+    return [py[0] for py in pinyin(word, style=Style.TONE3)]
+
+def segment_sentence(sentence):
+    """
+    使用jieba分词，返回词语列表
+    """
+    return list(jieba.cut(sentence))
+
+def get_word_homophones(word, pinyin_dict, char_frequency, min_freq=5):
+    """
+    获取整个词的同音词，只返回高频的有意义词语
+    :param word: 输入词语
+    :param pinyin_dict: 拼音字典
+    :param char_frequency: 字频字典
+    :param min_freq: 最小频率阈值
+    :return: 同音词列表
+    """
+    if len(word) == 1:
+        return []
+        
+    # 获取词的拼音
+    word_pinyin = get_word_pinyin(word)
+    word_pinyin_str = ''.join(word_pinyin)
+    
+    # 创建词语频率字典
+    word_freq = defaultdict(float)
+    
+    # 遍历所有可能的同音字组合
+    candidates = []
+    for py in word_pinyin:
+        chars = pinyin_dict.get(py, [])
+        if not chars:
+            return []
+        candidates.append(chars)
+    
+    # 生成所有可能的组合
+    import itertools
+    all_combinations = itertools.product(*candidates)
+    
+    # 获取jieba词典和词频信息
+    dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt')
+    valid_words = {}  # 改用字典存储词语及其频率
+    with open(dict_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            parts = line.strip().split()
+            if len(parts) >= 2:
+                word_text = parts[0]
+                word_freq = float(parts[1])  # 获取词频
+                valid_words[word_text] = word_freq
+    
+    # 获取原词的词频作为参考
+    original_word_freq = valid_words.get(word, 0)
+    min_word_freq = original_word_freq * 0.1  # 设置最小词频为原词频的10%
+    
+    # 过滤和计算频率
+    homophones = []
+    for combo in all_combinations:
+        new_word = ''.join(combo)
+        if new_word != word and new_word in valid_words:
+            new_word_freq = valid_words[new_word]
+            # 只保留词频达到阈值的词
+            if new_word_freq >= min_word_freq:
+                # 计算词的平均字频（考虑字频和词频）
+                char_avg_freq = sum(char_frequency.get(c, 0) for c in new_word) / len(new_word)
+                # 综合评分：结合词频和字频
+                combined_score = (new_word_freq * 0.7 + char_avg_freq * 0.3)
+                if combined_score >= min_freq:
+                    homophones.append((new_word, combined_score))
+    
+    # 按综合分数排序并限制返回数量
+    sorted_homophones = sorted(homophones, key=lambda x: x[1], reverse=True)
+    return [word for word, _ in sorted_homophones[:5]]  # 限制返回前5个结果
+
+def create_typo_sentence(sentence, pinyin_dict, char_frequency, error_rate=0.5, min_freq=5, tone_error_rate=0.2, word_replace_rate=0.3):
+    """
+    创建包含同音字错误的句子，支持词语级别和字级别的替换
+    只使用高频的有意义词语进行替换
+    """
+    result = []
+    typo_info = []
+    
+    # 分词
+    words = segment_sentence(sentence)
+    
+    for word in words:
+        # 如果是标点符号或空格，直接添加
+        if all(not is_chinese_char(c) for c in word):
+            result.append(word)
+            continue
+            
+        # 获取词语的拼音
+        word_pinyin = get_word_pinyin(word)
+        
+        # 尝试整词替换
+        if len(word) > 1 and random.random() < word_replace_rate:
+            word_homophones = get_word_homophones(word, pinyin_dict, char_frequency, min_freq)
+            if word_homophones:
+                typo_word = random.choice(word_homophones)
+                # 计算词的平均频率
+                orig_freq = sum(char_frequency.get(c, 0) for c in word) / len(word)
+                typo_freq = sum(char_frequency.get(c, 0) for c in typo_word) / len(typo_word)
+                
+                # 添加到结果中
+                result.append(typo_word)
+                typo_info.append((word, typo_word, 
+                                ' '.join(word_pinyin), 
+                                ' '.join(get_word_pinyin(typo_word)), 
+                                orig_freq, typo_freq))
+                continue
+        
+        # 如果不进行整词替换，则进行单字替换
+        if len(word) == 1:
+            char = word
+            py = word_pinyin[0]
+            if random.random() < error_rate:
+                similar_chars = get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, 
+                                                         min_freq=min_freq, tone_error_rate=tone_error_rate)
+                if similar_chars:
+                    typo_char = random.choice(similar_chars)
+                    typo_freq = char_frequency.get(typo_char, 0)
+                    orig_freq = char_frequency.get(char, 0)
+                    replace_prob = calculate_replacement_probability(orig_freq, typo_freq)
+                    if random.random() < replace_prob:
+                        result.append(typo_char)
+                        typo_py = pinyin(typo_char, style=Style.TONE3)[0][0]
+                        typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq))
+                        continue
+            result.append(char)
+        else:
+            # 处理多字词的单字替换
+            word_result = []
+            for i, (char, py) in enumerate(zip(word, word_pinyin)):
+                # 词中的字替换概率降低
+                word_error_rate = error_rate * (0.7 ** (len(word) - 1))
+                
+                if random.random() < word_error_rate:
+                    similar_chars = get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, 
+                                                             min_freq=min_freq, tone_error_rate=tone_error_rate)
+                    if similar_chars:
+                        typo_char = random.choice(similar_chars)
+                        typo_freq = char_frequency.get(typo_char, 0)
+                        orig_freq = char_frequency.get(char, 0)
+                        replace_prob = calculate_replacement_probability(orig_freq, typo_freq)
+                        if random.random() < replace_prob:
+                            word_result.append(typo_char)
+                            typo_py = pinyin(typo_char, style=Style.TONE3)[0][0]
+                            typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq))
+                            continue
+                word_result.append(char)
+            result.append(''.join(word_result))
+    
+    return ''.join(result), typo_info
+
+def format_frequency(freq):
+    """
+    格式化频率显示
+    """
+    return f"{freq:.2f}"
+
+def main():
+    # 记录开始时间
+    start_time = time.time()
+    
+    # 首先创建拼音字典和加载字频统计
+    print("正在加载汉字数据库，请稍候...")
+    pinyin_dict = create_pinyin_dict()
+    char_frequency = load_or_create_char_frequency()
+    
+    # 获取用户输入
+    sentence = input("请输入中文句子：")
+    
+    # 创建包含错别字的句子
+    typo_sentence, typo_info = create_typo_sentence(sentence, pinyin_dict, char_frequency, 
+                                                  error_rate=0.3, min_freq=5, 
+                                                  tone_error_rate=0.2, word_replace_rate=0.3)
+    
+    # 打印结果
+    print("\n原句：", sentence)
+    print("错字版：", typo_sentence)
+    
+    if typo_info:
+        print("\n错别字信息：")
+        for orig, typo, orig_py, typo_py, orig_freq, typo_freq in typo_info:
+            # 判断是否为词语替换
+            is_word = ' ' in orig_py
+            if is_word:
+                error_type = "整词替换"
+            else:
+                tone_error = orig_py[:-1] == typo_py[:-1] and orig_py[-1] != typo_py[-1]
+                error_type = "声调错误" if tone_error else "同音字替换"
+            
+            print(f"原文：{orig}({orig_py}) [频率：{format_frequency(orig_freq)}] -> "
+                  f"替换：{typo}({typo_py}) [频率：{format_frequency(typo_freq)}] [{error_type}]")
+    
+    # 获取拼音结果
+    result = get_pinyin(sentence)
+    
+    # 打印完整拼音
+    print("\n完整拼音：")
+    print(" ".join(py for _, py in result))
+    
+    # 打印词语分析
+    print("\n词语分析：")
+    words = segment_sentence(sentence)
+    for word in words:
+        if any(is_chinese_char(c) for c in word):
+            word_pinyin = get_word_pinyin(word)
+            print(f"词语：{word}")
+            print(f"拼音：{' '.join(word_pinyin)}")
+            print("---")
+    
+    # 计算并打印总耗时
+    end_time = time.time()
+    total_time = end_time - start_time
+    print(f"\n总耗时：{total_time:.2f}秒")
+
+if __name__ == "__main__":
+    main()

src/test/typo_word.py

@@ -0,0 +1,301 @@
+from pypinyin import pinyin, Style
+from collections import defaultdict
+import json
+import os
+import unicodedata
+import jieba
+import jieba.posseg as pseg
+from pathlib import Path
+import random
+import math
+
+def load_or_create_char_frequency():
+    """
+    加载或创建汉字频率字典
+    """
+    cache_file = Path("char_frequency.json")
+    
+    # 如果缓存文件存在，直接加载
+    if cache_file.exists():
+        with open(cache_file, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    # 使用内置的词频文件
+    char_freq = defaultdict(int)
+    dict_path = os.path.join(os.path.dirname(jieba.__file__), 'dict.txt')
+    
+    # 读取jieba的词典文件
+    with open(dict_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            word, freq = line.strip().split()[:2]
+            # 对词中的每个字进行频率累加
+            for char in word:
+                if is_chinese_char(char):
+                    char_freq[char] += int(freq)
+    
+    # 归一化频率值
+    max_freq = max(char_freq.values())
+    normalized_freq = {char: freq/max_freq * 1000 for char, freq in char_freq.items()}
+    
+    # 保存到缓存文件
+    with open(cache_file, 'w', encoding='utf-8') as f:
+        json.dump(normalized_freq, f, ensure_ascii=False, indent=2)
+    
+    return normalized_freq
+
+# 创建拼音到汉字的映射字典
+def create_pinyin_dict():
+    """
+    创建拼音到汉字的映射字典
+    """
+    # 常用汉字范围
+    chars = [chr(i) for i in range(0x4e00, 0x9fff)]
+    pinyin_dict = defaultdict(list)
+    
+    # 为每个汉字建立拼音映射
+    for char in chars:
+        try:
+            py = pinyin(char, style=Style.TONE3)[0][0]
+            pinyin_dict[py].append(char)
+        except Exception:
+            continue
+    
+    return pinyin_dict
+
+def is_chinese_char(char):
+    """
+    判断是否为汉字
+    """
+    try:
+        return '\u4e00' <= char <= '\u9fff'
+    except:
+        return False
+
+def get_pinyin(sentence):
+    """
+    将中文句子拆分成单个汉字并获取其拼音
+    :param sentence: 输入的中文句子
+    :return: 每个汉字及其拼音的列表
+    """
+    # 将句子拆分成单个字符
+    characters = list(sentence)
+    
+    # 获取每个字符的拼音
+    result = []
+    for char in characters:
+        # 跳过空格和非汉字字符
+        if char.isspace() or not is_chinese_char(char):
+            continue
+        # 获取拼音（数字声调）
+        py = pinyin(char, style=Style.TONE3)[0][0]
+        result.append((char, py))
+    
+    return result
+
+def get_homophone(char, py, pinyin_dict, char_frequency, min_freq=5):
+    """
+    获取同音字，按照使用频率排序
+    """
+    homophones = pinyin_dict[py]
+    # 移除原字并过滤低频字
+    if char in homophones:
+        homophones.remove(char)
+    
+    # 过滤掉低频字
+    homophones = [h for h in homophones if char_frequency.get(h, 0) >= min_freq]
+    
+    # 按照字频排序
+    sorted_homophones = sorted(homophones, 
+                             key=lambda x: char_frequency.get(x, 0), 
+                             reverse=True)
+    
+    # 只返回前10个同音字，避免输出过多
+    return sorted_homophones[:10]
+
+def get_similar_tone_pinyin(py):
+    """
+    获取相似声调的拼音
+    例如：'ni3' 可能返回 'ni2' 或 'ni4'
+    """
+    base = py[:-1]  # 去掉声调
+    tone = int(py[-1])  # 获取声调
+    possible_tones = [1, 2, 3, 4]
+    possible_tones.remove(tone)  # 移除原声调
+    new_tone = random.choice(possible_tones)  # 随机选择一个新声调
+    return base + str(new_tone)
+
+def calculate_replacement_probability(orig_freq, target_freq, max_freq_diff=200):
+    """
+    根据频率差计算替换概率
+    频率差越大，概率越低
+    :param orig_freq: 原字频率
+    :param target_freq: 目标字频率
+    :param max_freq_diff: 最大允许的频率差
+    :return: 0-1之间的概率值
+    """
+    if target_freq > orig_freq:
+        return 1.0  # 如果替换字频率更高，保持原有概率
+    
+    freq_diff = orig_freq - target_freq
+    if freq_diff > max_freq_diff:
+        return 0.0  # 频率差太大，不替换
+    
+    # 使用指数衰减函数计算概率
+    # 频率差为0时概率为1，频率差为max_freq_diff时概率接近0
+    return math.exp(-3 * freq_diff / max_freq_diff)
+
+def get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, num_candidates=5, min_freq=5, tone_error_rate=0.2):
+    """
+    获取与给定字频率相近的同音字，可能包含声调错误
+    """
+    homophones = []
+    
+    # 有20%的概率使用错误声调
+    if random.random() < tone_error_rate:
+        wrong_tone_py = get_similar_tone_pinyin(py)
+        homophones.extend(pinyin_dict[wrong_tone_py])
+    
+    # 添加正确声调的同音字
+    homophones.extend(pinyin_dict[py])
+    
+    if not homophones:
+        return None
+        
+    # 获取原字的频率
+    orig_freq = char_frequency.get(char, 0)
+    
+    # 计算所有同音字与原字的频率差，并过滤掉低频字
+    freq_diff = [(h, char_frequency.get(h, 0)) 
+                for h in homophones 
+                if h != char and char_frequency.get(h, 0) >= min_freq]
+    
+    if not freq_diff:
+        return None
+    
+    # 计算每个候选字的替换概率
+    candidates_with_prob = []
+    for h, freq in freq_diff:
+        prob = calculate_replacement_probability(orig_freq, freq)
+        if prob > 0:  # 只保留有效概率的候选字
+            candidates_with_prob.append((h, prob))
+    
+    if not candidates_with_prob:
+        return None
+    
+    # 根据概率排序
+    candidates_with_prob.sort(key=lambda x: x[1], reverse=True)
+    
+    # 返回概率最高的几个字
+    return [char for char, _ in candidates_with_prob[:num_candidates]]
+
+def create_typo_sentence(sentence, pinyin_dict, char_frequency, error_rate=0.5, min_freq=5, tone_error_rate=0.2):
+    """
+    创建包含同音字错误的句子，保留原文标点符号
+    """
+    result = []
+    typo_info = []
+    
+    # 获取每个字的拼音
+    chars_with_pinyin = get_pinyin(sentence)
+    
+    # 创建原字到拼音的映射，用于跟踪已处理的字符
+    processed_chars = {char: py for char, py in chars_with_pinyin}
+    
+    # 遍历原句中的每个字符
+    char_index = 0
+    for i, char in enumerate(sentence):
+        if char.isspace():
+            # 保留空格
+            result.append(char)
+        elif char in processed_chars:
+            # 处理汉字
+            py = processed_chars[char]
+            # 基础错误率
+            if random.random() < error_rate:
+                # 获取频率相近的同音字（可能包含声调错误）
+                similar_chars = get_similar_frequency_chars(char, py, pinyin_dict, char_frequency, 
+                                                         min_freq=min_freq, tone_error_rate=tone_error_rate)
+                if similar_chars:
+                    # 随机选择一个替换字
+                    typo_char = random.choice(similar_chars)
+                    # 获取替换字的频率
+                    typo_freq = char_frequency.get(typo_char, 0)
+                    orig_freq = char_frequency.get(char, 0)
+                    
+                    # 计算实际替换概率
+                    replace_prob = calculate_replacement_probability(orig_freq, typo_freq)
+                    
+                    # 根据频率差进行概率替换
+                    if random.random() < replace_prob:
+                        result.append(typo_char)
+                        # 获取替换字的实际拼音
+                        typo_py = pinyin(typo_char, style=Style.TONE3)[0][0]
+                        typo_info.append((char, typo_char, py, typo_py, orig_freq, typo_freq))
+                    else:
+                        result.append(char)
+                else:
+                    result.append(char)
+            else:
+                result.append(char)
+            char_index += 1
+        else:
+            # 保留非汉字字符（标点符号等）
+            result.append(char)
+    
+    return ''.join(result), typo_info
+
+def format_frequency(freq):
+    """
+    格式化频率显示
+    """
+    return f"{freq:.2f}"
+
+def main():
+    # 首先创建拼音字典和加载字频统计
+    print("正在加载汉字数据库，请稍候...")
+    pinyin_dict = create_pinyin_dict()
+    char_frequency = load_or_create_char_frequency()
+    
+    # 获取用户输入
+    sentence = input("请输入中文句子：")
+    
+    # 创建包含错别字的句子
+    typo_sentence, typo_info = create_typo_sentence(sentence, pinyin_dict, char_frequency, 
+                                                  min_freq=5, tone_error_rate=0.2)
+    
+    # 打印结果
+    print("\n原句：", sentence)
+    print("错字版：", typo_sentence)
+    
+    if typo_info:
+        print("\n错别字信息：")
+        for orig, typo, orig_py, typo_py, orig_freq, typo_freq in typo_info:
+            tone_error = orig_py[:-1] == typo_py[:-1] and orig_py[-1] != typo_py[-1]
+            error_type = "声调错误" if tone_error else "同音字替换"
+            print(f"原字：{orig}({orig_py}) [频率：{format_frequency(orig_freq)}] -> "
+                  f"错字：{typo}({typo_py}) [频率：{format_frequency(typo_freq)}] [{error_type}]")
+    
+    # 获取拼音结果
+    result = get_pinyin(sentence)
+    
+    # 打印完整拼音
+    print("\n完整拼音：")
+    print(" ".join(py for _, py in result))
+    
+    # 打印所有可能的同音字
+    print("\n每个字的所有同音字（按频率排序，仅显示频率>=5的字）：")
+    for char, py in result:
+        homophones = get_homophone(char, py, pinyin_dict, char_frequency, min_freq=5)
+        char_freq = char_frequency.get(char, 0)
+        print(f"{char}: {py} [频率：{format_frequency(char_freq)}]")
+        if homophones:
+            homophone_info = []
+            for h in homophones:
+                h_freq = char_frequency.get(h, 0)
+                homophone_info.append(f"{h}[{format_frequency(h_freq)}]")
+            print(f"同音字: {'，'.join(homophone_info)}")
+        else:
+            print("没有找到频率>=5的同音字")
+
+if __name__ == "__main__":
+    main()