优化多线程处理,调整嵌入获取和存储逻辑,增强模型一致性校验

This commit is contained in:
墨梓柒
2025-08-02 23:52:41 +08:00
parent 9b65b740be
commit 423525ead5
3 changed files with 238 additions and 93 deletions

View File

@@ -25,7 +25,7 @@ from rich.progress import (
TextColumn,
)
from raw_data_preprocessor import RAW_DATA_PATH, load_raw_data
from src.config.config import global_config
from src.config.config import global_config, model_config
from src.llm_models.utils_model import LLMRequest
from dotenv import load_dotenv
@@ -96,11 +96,11 @@ open_ie_doc_lock = Lock()
shutdown_event = Event()
lpmm_entity_extract_llm = LLMRequest(
model=global_config.model.lpmm_entity_extract,
model_set=model_config.model_task_config.lpmm_entity_extract,
request_type="lpmm.entity_extract"
)
lpmm_rdf_build_llm = LLMRequest(
model=global_config.model.lpmm_rdf_build,
model_set=model_config.model_task_config.lpmm_rdf_build,
request_type="lpmm.rdf_build"
)
def process_single_text(pg_hash, raw_data):