From 584b5524152c0de8fc8e24cce8199b96ed3109d0 Mon Sep 17 00:00:00 2001 From: minecraft1024a Date: Mon, 18 Aug 2025 18:08:14 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(cache):=20=E5=A2=9E=E5=BC=BA=E5=B5=8C?= =?UTF-8?q?=E5=85=A5=E5=90=91=E9=87=8F=E5=A4=84=E7=90=86=E7=9A=84=E5=81=A5?= =?UTF-8?q?=E5=A3=AE=E6=80=A7=E5=92=8C=E9=AA=8C=E8=AF=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 `_validate_embedding` 方法，用于在存入缓存前对嵌入向量进行严格的格式检查、维度验证和数值有效性校验。 - 在缓存查询 (`get`) 和写入 (`set`) 流程中，集成此验证逻辑，确保只有合规的向量才能被处理和存储。 - 增加了在L1和L2向量索引操作中的异常捕获，防止因向量处理失败导致缓存功能中断，提升了系统的整体稳定性。 --- src/common/cache_manager.py | 74 ++++++++++++++++++++++++++++++------- 1 file changed, 61 insertions(+), 13 deletions(-) diff --git a/src/common/cache_manager.py b/src/common/cache_manager.py index 28fcd0d87..9b24a7377 100644 --- a/src/common/cache_manager.py +++ b/src/common/cache_manager.py @@ -65,6 +65,43 @@ class CacheManager: """) conn.commit() + def _validate_embedding(self, embedding_result: Any) -> Optional[np.ndarray]: + """ + 验证和标准化嵌入向量格式 + """ + try: + if embedding_result is None: + return None + + # 确保embedding_result是一维数组或列表 + if isinstance(embedding_result, (list, tuple, np.ndarray)): + # 转换为numpy数组进行处理 + embedding_array = np.array(embedding_result) + + # 如果是多维数组，展平它 + if embedding_array.ndim > 1: + embedding_array = embedding_array.flatten() + + # 检查维度是否符合预期 + expected_dim = global_config.lpmm_knowledge.embedding_dimension + if embedding_array.shape[0] != expected_dim: + logger.warning(f"嵌入向量维度不匹配: 期望 {expected_dim}, 实际 {embedding_array.shape[0]}") + return None + + # 检查是否包含有效的数值 + if np.isnan(embedding_array).any() or np.isinf(embedding_array).any(): + logger.warning("嵌入向量包含无效的数值 (NaN 或 Inf)") + return None + + return embedding_array.astype('float32') + else: + logger.warning(f"嵌入结果格式不支持: {type(embedding_result)}") + return None + + except Exception as e: + logger.error(f"验证嵌入向量时发生错误: {e}") + return None + def _generate_key(self, tool_name: str, function_args: Dict[str, Any], tool_class: Any) -> str: """生成确定性的缓存键，包含代码哈希以实现自动失效。""" try: @@ -102,7 +139,9 @@ class CacheManager: if semantic_query and self.embedding_model: embedding_result = await self.embedding_model.get_embedding(semantic_query) if embedding_result: - query_embedding = np.array([embedding_result], dtype='float32') + validated_embedding = self._validate_embedding(embedding_result) + if validated_embedding is not None: + query_embedding = np.array([validated_embedding], dtype='float32') # 步骤 2a: L1 语义缓存 (FAISS) if query_embedding is not None and self.l1_vector_index.ntotal > 0: @@ -153,10 +192,13 @@ class CacheManager: # 回填 L1 self.l1_kv_cache[key] = {"data": data, "expires_at": expires_at} if query_embedding is not None: - new_id = self.l1_vector_index.ntotal - faiss.normalize_L2(query_embedding) - self.l1_vector_index.add(x=query_embedding) - self.l1_vector_id_to_key[new_id] = key + try: + new_id = self.l1_vector_index.ntotal + faiss.normalize_L2(query_embedding) + self.l1_vector_index.add(x=query_embedding) + self.l1_vector_id_to_key[new_id] = key + except Exception as e: + logger.error(f"回填L1向量索引时发生错误: {e}") return data logger.debug(f"缓存未命中: {key}") @@ -186,14 +228,20 @@ class CacheManager: if semantic_query and self.embedding_model: embedding_result = await self.embedding_model.get_embedding(semantic_query) if embedding_result: - embedding = np.array([embedding_result], dtype='float32') - # 写入 L1 Vector - new_id = self.l1_vector_index.ntotal - faiss.normalize_L2(embedding) - self.l1_vector_index.add(x=embedding) - self.l1_vector_id_to_key[new_id] = key - # 写入 L2 Vector - self.chroma_collection.add(embeddings=embedding.tolist(), ids=[key]) + validated_embedding = self._validate_embedding(embedding_result) + if validated_embedding is not None: + try: + embedding = np.array([validated_embedding], dtype='float32') + # 写入 L1 Vector + new_id = self.l1_vector_index.ntotal + faiss.normalize_L2(embedding) + self.l1_vector_index.add(x=embedding) + self.l1_vector_id_to_key[new_id] = key + # 写入 L2 Vector + self.chroma_collection.add(embeddings=embedding.tolist(), ids=[key]) + except Exception as e: + logger.error(f"写入语义缓存时发生错误: {e}") + # 继续执行，不影响主要缓存功能 logger.info(f"已缓存条目: {key}, TTL: {ttl}s") From 483c470acff5a0920cb6e4c460139af09508102e Mon Sep 17 00:00:00 2001 From: minecraft1024a Date: Mon, 18 Aug 2025 18:29:42 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E5=99=A8=E4=B8=AD=E7=9A=84=E5=BC=82=E5=B8=B8?= =?UTF-8?q?=E5=A4=84=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 将捕获的异常类型从 `TypeError` 和 `OSError` 修改为 `Exception`，以涵盖更多潜在错误。 - 增强日志记录，提供更清晰的类名和简化的错误信息，便于调试和问题追踪。 --- src/common/cache_manager.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/common/cache_manager.py b/src/common/cache_manager.py index 9b24a7377..455b53aa6 100644 --- a/src/common/cache_manager.py +++ b/src/common/cache_manager.py @@ -107,9 +107,13 @@ class CacheManager: try: source_code = inspect.getsource(tool_class) code_hash = hashlib.md5(source_code.encode()).hexdigest() - except (TypeError, OSError) as e: + except Exception as e: code_hash = "unknown" - logger.warning(f"无法获取 {tool_class.__name__} 的源代码，代码哈希将为 'unknown'。错误: {e}") + # 获取更清晰的类名 + class_name = getattr(tool_class, '__name__', str(tool_class)) + # 简化错误信息 + error_msg = str(e).replace(str(tool_class), class_name) + logger.warning(f"无法获取 {class_name} 的源代码，代码哈希将为 'unknown'。原因: {error_msg}") try: sorted_args = json.dumps(function_args, sort_keys=True) except TypeError: