孩子们,知识库的高并发太恐怖了
This commit is contained in:
@@ -204,7 +204,7 @@ def extract_information(paragraphs_dict, model_set):
|
||||
|
||||
failed_hashes, open_ie_docs = [], []
|
||||
|
||||
with ThreadPoolExecutor(max_workers=5) as executor:
|
||||
with ThreadPoolExecutor(max_workers=3) as executor:
|
||||
f_to_hash = {
|
||||
executor.submit(extract_info_sync, p_hash, p, model_set): p_hash
|
||||
for p_hash, p in paragraphs_dict.items()
|
||||
|
||||
@@ -30,8 +30,8 @@ from .utils.hash import get_sha256
|
||||
install(extra_lines=3)
|
||||
|
||||
# 多线程embedding配置常量
|
||||
DEFAULT_MAX_WORKERS = 10 # 默认最大线程数
|
||||
DEFAULT_CHUNK_SIZE = 10 # 默认每个线程处理的数据块大小
|
||||
DEFAULT_MAX_WORKERS = 3 # 默认最大线程数
|
||||
DEFAULT_CHUNK_SIZE = 5 # 默认每个线程处理的数据块大小
|
||||
MIN_CHUNK_SIZE = 1 # 最小分块大小
|
||||
MAX_CHUNK_SIZE = 50 # 最大分块大小
|
||||
MIN_WORKERS = 1 # 最小线程数
|
||||
|
||||
Reference in New Issue
Block a user