refactor(storage): 迁移记忆系统架构至现代化Vector DB方案
重构记忆存储层,采用ChromaDB作为核心向量数据库引擎,提升系统可扩展性和查询性能。通过引入VectorMemoryStorage替代原有的UnifiedMemoryStorage实现,实现了更高效的向量索引和检索机制。 主要变更包括: - 架构层面:完全重构记忆存储抽象层,移除577行旧存储实现代码 - 配置系统:新增41项Vector DB专用配置参数,支持细粒度性能调优 - 查询优化:增强ChromaDB查询条件处理器,添加智能回退机制 - 系统集成:更新记忆系统初始化流程,适配新的存储接口 - 类型安全:修复异步调用类型不匹配问题 BREAKING CHANGE: 记忆存储API接口发生重大变更,UnifiedMemoryStorage相关类已废弃
This commit is contained in:
@@ -98,13 +98,79 @@ class ChromaDBImpl(VectorDBBase):
|
||||
"n_results": n_results,
|
||||
**kwargs,
|
||||
}
|
||||
|
||||
# 修复ChromaDB的where条件格式
|
||||
if where:
|
||||
query_params["where"] = where
|
||||
processed_where = self._process_where_condition(where)
|
||||
if processed_where:
|
||||
query_params["where"] = processed_where
|
||||
|
||||
return collection.query(**query_params)
|
||||
except Exception as e:
|
||||
logger.error(f"查询集合 '{collection_name}' 失败: {e}")
|
||||
# 如果查询失败,尝试不使用where条件重新查询
|
||||
try:
|
||||
fallback_params = {
|
||||
"query_embeddings": query_embeddings,
|
||||
"n_results": n_results,
|
||||
}
|
||||
logger.warning(f"使用回退查询模式(无where条件)")
|
||||
return collection.query(**fallback_params)
|
||||
except Exception as fallback_e:
|
||||
logger.error(f"回退查询也失败: {fallback_e}")
|
||||
return {}
|
||||
|
||||
def _process_where_condition(self, where: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
||||
"""
|
||||
处理where条件,转换为ChromaDB支持的格式
|
||||
ChromaDB支持的格式:
|
||||
- 简单条件: {"field": "value"}
|
||||
- 操作符条件: {"field": {"$op": "value"}}
|
||||
- AND条件: {"$and": [condition1, condition2]}
|
||||
- OR条件: {"$or": [condition1, condition2]}
|
||||
"""
|
||||
if not where:
|
||||
return None
|
||||
|
||||
try:
|
||||
# 如果只有一个字段,直接返回
|
||||
if len(where) == 1:
|
||||
key, value = next(iter(where.items()))
|
||||
|
||||
# 处理列表值(如memory_types)
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1:
|
||||
return {key: value[0]}
|
||||
else:
|
||||
# 多个值使用 $in 操作符
|
||||
return {key: {"$in": value}}
|
||||
else:
|
||||
return {key: value}
|
||||
|
||||
# 多个字段使用 $and 操作符
|
||||
conditions = []
|
||||
for key, value in where.items():
|
||||
if isinstance(value, list):
|
||||
if len(value) == 1:
|
||||
conditions.append({key: value[0]})
|
||||
else:
|
||||
conditions.append({key: {"$in": value}})
|
||||
else:
|
||||
conditions.append({key: value})
|
||||
|
||||
return {"$and": conditions}
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"处理where条件失败: {e}, 使用简化条件")
|
||||
# 回退到只使用第一个条件
|
||||
if where:
|
||||
key, value = next(iter(where.items()))
|
||||
if isinstance(value, list) and value:
|
||||
return {key: value[0]}
|
||||
elif not isinstance(value, list):
|
||||
return {key: value}
|
||||
return None
|
||||
|
||||
def get(
|
||||
self,
|
||||
collection_name: str,
|
||||
@@ -119,16 +185,33 @@ class ChromaDBImpl(VectorDBBase):
|
||||
collection = self.get_or_create_collection(collection_name)
|
||||
if collection:
|
||||
try:
|
||||
# 处理where条件
|
||||
processed_where = None
|
||||
if where:
|
||||
processed_where = self._process_where_condition(where)
|
||||
|
||||
return collection.get(
|
||||
ids=ids,
|
||||
where=where,
|
||||
where=processed_where,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
where_document=where_document,
|
||||
include=include,
|
||||
include=include or ["documents", "metadatas", "embeddings"],
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"从集合 '{collection_name}' 获取数据失败: {e}")
|
||||
# 如果获取失败,尝试不使用where条件重新获取
|
||||
try:
|
||||
logger.warning(f"使用回退获取模式(无where条件)")
|
||||
return collection.get(
|
||||
ids=ids,
|
||||
limit=limit,
|
||||
offset=offset,
|
||||
where_document=where_document,
|
||||
include=include or ["documents", "metadatas", "embeddings"],
|
||||
)
|
||||
except Exception as fallback_e:
|
||||
logger.error(f"回退获取也失败: {fallback_e}")
|
||||
return {}
|
||||
|
||||
def delete(
|
||||
|
||||
Reference in New Issue
Block a user