From ab2b7ec9edcfbe3b2447d9a484deb1c0ac5bfba8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=A2=A8=E6=A2=93=E6=9F=92?= <1787882683@qq.com> Date: Sat, 7 Jun 2025 13:46:24 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E7=9F=A5=E8=AF=86=E5=BA=93=E5=B0=8F?= =?UTF-8?q?=E9=87=8D=E6=9E=84x2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- scripts/import_openie.py | 12 ++++++------ scripts/info_extraction.py | 10 +++++----- scripts/raw_data_preprocessor.py | 2 +- src/chat/knowledge/embedding_store.py | 4 ++-- src/chat/knowledge/kg_manager.py | 4 ++-- src/chat/knowledge/knowledge_lib.py | 14 +++++++------- src/chat/knowledge/lpmmconfig.py | 4 ++-- src/chat/knowledge/mem_active_manager.py | 2 +- src/chat/knowledge/qa_manager.py | 2 +- 9 files changed, 27 insertions(+), 27 deletions(-) diff --git a/scripts/import_openie.py b/scripts/import_openie.py index 90579bcef..66faaaf11 100644 --- a/scripts/import_openie.py +++ b/scripts/import_openie.py @@ -10,13 +10,13 @@ from time import sleep sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from src.chat.knowledge.src.lpmmconfig import PG_NAMESPACE, global_config -from src.chat.knowledge.src.embedding_store import EmbeddingManager -from src.chat.knowledge.src.llm_client import LLMClient -from src.chat.knowledge.src.open_ie import OpenIE -from src.chat.knowledge.src.kg_manager import KGManager +from src.chat.knowledge.lpmmconfig import PG_NAMESPACE, global_config +from src.chat.knowledge.embedding_store import EmbeddingManager +from src.chat.knowledge.llm_client import LLMClient +from src.chat.knowledge.open_ie import OpenIE +from src.chat.knowledge.kg_manager import KGManager from src.common.logger import get_module_logger -from src.chat.knowledge.src.utils.hash import get_sha256 +from src.chat.knowledge.utils.hash import get_sha256 # 添加项目根目录到 sys.path diff --git a/scripts/info_extraction.py b/scripts/info_extraction.py index 29e327300..2b18f8e37 100644 --- a/scripts/info_extraction.py +++ b/scripts/info_extraction.py @@ -13,11 +13,11 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from rich.progress import Progress # 替换为 rich 进度条 from src.common.logger import get_module_logger -from src.chat.knowledge.src.lpmmconfig import global_config -from src.chat.knowledge.src.ie_process import info_extract_from_str -from src.chat.knowledge.src.llm_client import LLMClient -from src.chat.knowledge.src.open_ie import OpenIE -from src.chat.knowledge.src.raw_processing import load_raw_data +from src.chat.knowledge.lpmmconfig import global_config +from src.chat.knowledge.ie_process import info_extract_from_str +from src.chat.knowledge.llm_client import LLMClient +from src.chat.knowledge.open_ie import OpenIE +from src.chat.knowledge.raw_processing import load_raw_data from rich.progress import ( BarColumn, TimeElapsedColumn, diff --git a/scripts/raw_data_preprocessor.py b/scripts/raw_data_preprocessor.py index 5ac3dd67c..35fb21c9d 100644 --- a/scripts/raw_data_preprocessor.py +++ b/scripts/raw_data_preprocessor.py @@ -6,7 +6,7 @@ import datetime # 新增导入 sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from src.common.logger_manager import get_logger -from src.chat.knowledge.src.lpmmconfig import global_config +from src.chat.knowledge.lpmmconfig import global_config logger = get_logger("lpmm") ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) diff --git a/src/chat/knowledge/embedding_store.py b/src/chat/knowledge/embedding_store.py index b4d1f9d0f..90987576b 100644 --- a/src/chat/knowledge/embedding_store.py +++ b/src/chat/knowledge/embedding_store.py @@ -12,7 +12,7 @@ import faiss from .llm_client import LLMClient from .lpmmconfig import ENT_NAMESPACE, PG_NAMESPACE, REL_NAMESPACE, global_config -from src.chat.knowledge.utils import get_sha256 +from .utils.hash import get_sha256 from .global_logger import logger from rich.traceback import install from rich.progress import ( @@ -27,7 +27,7 @@ from rich.progress import ( ) install(extra_lines=3) -ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) EMBEDDING_DATA_DIR = ( os.path.join(ROOT_PATH, "data", "embedding") if global_config["persistence"]["embedding_data_dir"] is None diff --git a/src/chat/knowledge/kg_manager.py b/src/chat/knowledge/kg_manager.py index f9d600f89..1ff651b5e 100644 --- a/src/chat/knowledge/kg_manager.py +++ b/src/chat/knowledge/kg_manager.py @@ -18,7 +18,7 @@ from rich.progress import ( from quick_algo import di_graph, pagerank -from src.chat.knowledge.utils import get_sha256 +from .utils.hash import get_sha256 from .embedding_store import EmbeddingManager, EmbeddingStoreItem from .lpmmconfig import ( ENT_NAMESPACE, @@ -31,7 +31,7 @@ from .lpmmconfig import ( from .global_logger import logger -ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) KG_DIR = ( os.path.join(ROOT_PATH, "data/rag") if global_config["persistence"]["rag_data_dir"] is None diff --git a/src/chat/knowledge/knowledge_lib.py b/src/chat/knowledge/knowledge_lib.py index 14340bb71..6a4fcd4ea 100644 --- a/src/chat/knowledge/knowledge_lib.py +++ b/src/chat/knowledge/knowledge_lib.py @@ -1,10 +1,10 @@ -from .src.lpmmconfig import PG_NAMESPACE, global_config -from .src.embedding_store import EmbeddingManager -from .src.llm_client import LLMClient -from .src.mem_active_manager import MemoryActiveManager -from .src.qa_manager import QAManager -from .src.kg_manager import KGManager -from .src.global_logger import logger +from src.chat.knowledge.lpmmconfig import PG_NAMESPACE, global_config +from src.chat.knowledge.embedding_store import EmbeddingManager +from src.chat.knowledge.llm_client import LLMClient +from src.chat.knowledge.mem_active_manager import MemoryActiveManager +from src.chat.knowledge.qa_manager import QAManager +from src.chat.knowledge.kg_manager import KGManager +from src.chat.knowledge.global_logger import logger # try: # import quick_algo # except ImportError: diff --git a/src/chat/knowledge/lpmmconfig.py b/src/chat/knowledge/lpmmconfig.py index 387a7b291..6cb91db25 100644 --- a/src/chat/knowledge/lpmmconfig.py +++ b/src/chat/knowledge/lpmmconfig.py @@ -45,7 +45,7 @@ def _load_config(config, config_file_path): if "llm_providers" in file_config: for provider in file_config["llm_providers"]: if provider["name"] not in config["llm_providers"]: - config["llm_providers"][provider["name"]] = dict() + config["llm_providers"][provider["name"]] = {} config["llm_providers"][provider["name"]]["base_url"] = provider["base_url"] config["llm_providers"][provider["name"]]["api_key"] = provider["api_key"] @@ -135,6 +135,6 @@ global_config = dict( # _load_config(global_config, parser.parse_args().config_path) # file_path = os.path.abspath(__file__) # dir_path = os.path.dirname(file_path) -ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) +ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..")) config_path = os.path.join(ROOT_PATH, "config", "lpmm_config.toml") _load_config(global_config, config_path) diff --git a/src/chat/knowledge/mem_active_manager.py b/src/chat/knowledge/mem_active_manager.py index 9c6e65e9a..3998c0664 100644 --- a/src/chat/knowledge/mem_active_manager.py +++ b/src/chat/knowledge/mem_active_manager.py @@ -1,7 +1,7 @@ from .lpmmconfig import global_config from .embedding_store import EmbeddingManager from .llm_client import LLMClient -from src.chat.knowledge.utils import dyn_select_top_k +from .utils.dyn_topk import dyn_select_top_k class MemoryActiveManager: diff --git a/src/chat/knowledge/qa_manager.py b/src/chat/knowledge/qa_manager.py index e76f29d7b..b6bbd1207 100644 --- a/src/chat/knowledge/qa_manager.py +++ b/src/chat/knowledge/qa_manager.py @@ -8,7 +8,7 @@ from .embedding_store import EmbeddingManager from .llm_client import LLMClient from .kg_manager import KGManager from .lpmmconfig import global_config -from src.chat.knowledge.utils import dyn_select_top_k +from .utils.dyn_topk import dyn_select_top_k MAX_KNOWLEDGE_LENGTH = 10000 # 最大知识长度