gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 知识库小重构x2

Browse Source
This commit is contained in:
墨梓柒
2025-06-07 13:46:24 +08:00
parent 0038d9ff60
commit ab2b7ec9ed
9 changed files with 27 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
scripts/import_openie.py
View File

@@ -10,13 +10,13 @@ from time import sleep
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from src.chat.knowledge.src.lpmmconfig import PG_NAMESPACE, global_config from src.chat.knowledge.lpmmconfig import PG_NAMESPACE, global_config
from src.chat.knowledge.src.embedding_store import EmbeddingManager from src.chat.knowledge.embedding_store import EmbeddingManager
from src.chat.knowledge.src.llm_client import LLMClient from src.chat.knowledge.llm_client import LLMClient
from src.chat.knowledge.src.open_ie import OpenIE from src.chat.knowledge.open_ie import OpenIE
from src.chat.knowledge.src.kg_manager import KGManager from src.chat.knowledge.kg_manager import KGManager
from src.common.logger import get_module_logger from src.common.logger import get_module_logger
from src.chat.knowledge.src.utils.hash import get_sha256 from src.chat.knowledge.utils.hash import get_sha256
# 添加项目根目录到 sys.path # 添加项目根目录到 sys.path

10
scripts/info_extraction.py
View File

@@ -13,11 +13,11 @@ sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from rich.progress import Progress # 替换为 rich 进度条 from rich.progress import Progress # 替换为 rich 进度条
from src.common.logger import get_module_logger from src.common.logger import get_module_logger
from src.chat.knowledge.src.lpmmconfig import global_config from src.chat.knowledge.lpmmconfig import global_config
from src.chat.knowledge.src.ie_process import info_extract_from_str from src.chat.knowledge.ie_process import info_extract_from_str
from src.chat.knowledge.src.llm_client import LLMClient from src.chat.knowledge.llm_client import LLMClient
from src.chat.knowledge.src.open_ie import OpenIE from src.chat.knowledge.open_ie import OpenIE
from src.chat.knowledge.src.raw_processing import load_raw_data from src.chat.knowledge.raw_processing import load_raw_data
from rich.progress import ( from rich.progress import (
BarColumn, BarColumn,
TimeElapsedColumn, TimeElapsedColumn,

2
scripts/raw_data_preprocessor.py
View File

@@ -6,7 +6,7 @@ import datetime # 新增导入
sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from src.common.logger_manager import get_logger from src.common.logger_manager import get_logger
from src.chat.knowledge.src.lpmmconfig import global_config from src.chat.knowledge.lpmmconfig import global_config
logger = get_logger("lpmm") logger = get_logger("lpmm")
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))

4
src/chat/knowledge/embedding_store.py
View File

@@ -12,7 +12,7 @@ import faiss
from .llm_client import LLMClient from .llm_client import LLMClient
from .lpmmconfig import ENT_NAMESPACE, PG_NAMESPACE, REL_NAMESPACE, global_config from .lpmmconfig import ENT_NAMESPACE, PG_NAMESPACE, REL_NAMESPACE, global_config
from src.chat.knowledge.utils import get_sha256 from .utils.hash import get_sha256
from .global_logger import logger from .global_logger import logger
from rich.traceback import install from rich.traceback import install
from rich.progress import ( from rich.progress import (
@@ -27,7 +27,7 @@ from rich.progress import (
) )
install(extra_lines=3) install(extra_lines=3)
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
EMBEDDING_DATA_DIR = ( EMBEDDING_DATA_DIR = (
os.path.join(ROOT_PATH, "data", "embedding") os.path.join(ROOT_PATH, "data", "embedding")
if global_config["persistence"]["embedding_data_dir"] is None if global_config["persistence"]["embedding_data_dir"] is None

4
src/chat/knowledge/kg_manager.py
View File

@@ -18,7 +18,7 @@ from rich.progress import (
from quick_algo import di_graph, pagerank from quick_algo import di_graph, pagerank
from src.chat.knowledge.utils import get_sha256 from .utils.hash import get_sha256
from .embedding_store import EmbeddingManager, EmbeddingStoreItem from .embedding_store import EmbeddingManager, EmbeddingStoreItem
from .lpmmconfig import ( from .lpmmconfig import (
ENT_NAMESPACE, ENT_NAMESPACE,
@@ -31,7 +31,7 @@ from .lpmmconfig import (
from .global_logger import logger from .global_logger import logger
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
KG_DIR = ( KG_DIR = (
os.path.join(ROOT_PATH, "data/rag") os.path.join(ROOT_PATH, "data/rag")
if global_config["persistence"]["rag_data_dir"] is None if global_config["persistence"]["rag_data_dir"] is None

14
src/chat/knowledge/knowledge_lib.py
View File

@@ -1,10 +1,10 @@
from .src.lpmmconfig import PG_NAMESPACE, global_config from src.chat.knowledge.lpmmconfig import PG_NAMESPACE, global_config
from .src.embedding_store import EmbeddingManager from src.chat.knowledge.embedding_store import EmbeddingManager
from .src.llm_client import LLMClient from src.chat.knowledge.llm_client import LLMClient
from .src.mem_active_manager import MemoryActiveManager from src.chat.knowledge.mem_active_manager import MemoryActiveManager
from .src.qa_manager import QAManager from src.chat.knowledge.qa_manager import QAManager
from .src.kg_manager import KGManager from src.chat.knowledge.kg_manager import KGManager
from .src.global_logger import logger from src.chat.knowledge.global_logger import logger
# try: # try:
# import quick_algo # import quick_algo
# except ImportError: # except ImportError:

4
src/chat/knowledge/lpmmconfig.py
View File

@@ -45,7 +45,7 @@ def _load_config(config, config_file_path):
if "llm_providers" in file_config: if "llm_providers" in file_config:
for provider in file_config["llm_providers"]: for provider in file_config["llm_providers"]:
if provider["name"] not in config["llm_providers"]: if provider["name"] not in config["llm_providers"]:
config["llm_providers"][provider["name"]] = dict() config["llm_providers"][provider["name"]] = {}
config["llm_providers"][provider["name"]]["base_url"] = provider["base_url"] config["llm_providers"][provider["name"]]["base_url"] = provider["base_url"]
config["llm_providers"][provider["name"]]["api_key"] = provider["api_key"] config["llm_providers"][provider["name"]]["api_key"] = provider["api_key"]
@@ -135,6 +135,6 @@ global_config = dict(
# _load_config(global_config, parser.parse_args().config_path) # _load_config(global_config, parser.parse_args().config_path)
# file_path = os.path.abspath(__file__) # file_path = os.path.abspath(__file__)
# dir_path = os.path.dirname(file_path) # dir_path = os.path.dirname(file_path)
ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", "..", "..")) ROOT_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..", ".."))
config_path = os.path.join(ROOT_PATH, "config", "lpmm_config.toml") config_path = os.path.join(ROOT_PATH, "config", "lpmm_config.toml")
_load_config(global_config, config_path) _load_config(global_config, config_path)

2
src/chat/knowledge/mem_active_manager.py
View File

@@ -1,7 +1,7 @@
from .lpmmconfig import global_config from .lpmmconfig import global_config
from .embedding_store import EmbeddingManager from .embedding_store import EmbeddingManager
from .llm_client import LLMClient from .llm_client import LLMClient
from src.chat.knowledge.utils import dyn_select_top_k from .utils.dyn_topk import dyn_select_top_k
class MemoryActiveManager: class MemoryActiveManager:

2
src/chat/knowledge/qa_manager.py
View File

@@ -8,7 +8,7 @@ from .embedding_store import EmbeddingManager
from .llm_client import LLMClient from .llm_client import LLMClient
from .kg_manager import KGManager from .kg_manager import KGManager
from .lpmmconfig import global_config from .lpmmconfig import global_config
from src.chat.knowledge.utils import dyn_select_top_k from .utils.dyn_topk import dyn_select_top_k
MAX_KNOWLEDGE_LENGTH = 10000 # 最大知识长度 MAX_KNOWLEDGE_LENGTH = 10000 # 最大知识长度