gardel/Mofox-Core
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(api): 修复记忆可视化中重复的边

Browse Source 
在从文件或内存管理器加载图数据时，由于遍历所有记忆（memory）并将其包含的边（edge）添加到列表中，导致了边的重复。当多个记忆共享同一条边时，这条边会被多次添加到最终的边列表中，造成前端可视化图中出现冗余的连接。

此提交通过引入一个集合（set）或字典（dict）来跟踪已经处理过的边的 ID，确保每条边只被添加一次，从而解决了重复边的问题。
This commit is contained in:
minecraft1024a
2025-11-07 22:03:09 +08:00
parent b0ee26652e
commit f8e42f3348
2 changed files with 283 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

209
src/api/memory_visualizer_router.py
View File

@@ -61,15 +61,27 @@ def find_available_data_files() -> List[Path]:
return sorted(files, key=lambda f: f.stat().st_mtime, reverse=True)
def load_graph_data_from_file(file_path: Optional[Path] = None) -> Dict[str, Any]:
"""从磁盘加载图数据"""
def load_graph_data_from_file(
file_path: Optional[Path] = None,
nodes_page: Optional[int] = None,
nodes_per_page: Optional[int] = None,
edges_page: Optional[int] = None,
edges_per_page: Optional[int] = None,
) -> Dict[str, Any]:
"""
从磁盘加载图数据, 支持分页。
如果不提供分页参数, 则加载并缓存所有数据。
"""
global graph_data_cache, current_data_file
# 如果是请求分页数据, 则不使用缓存的全量数据
is_paged_request = nodes_page is not None or edges_page is not None
if file_path and file_path != current_data_file:
graph_data_cache = None
current_data_file = file_path
if graph_data_cache:
if graph_data_cache and not is_paged_request:
return graph_data_cache
try:
@@ -84,53 +96,78 @@ def load_graph_data_from_file(file_path: Optional[Path] = None) -> Dict[str, Any
if not graph_file.exists():
return {"error": f"文件不存在: {graph_file}", "nodes": [], "edges": [], "stats": {}}
with open(graph_file, "r", encoding="utf-8") as f:
data = orjson.loads(f.read())
# 只有在没有缓存时才从磁盘读取和处理文件
if not graph_data_cache:
with open(graph_file, "r", encoding="utf-8") as f:
data = orjson.loads(f.read())
nodes = data.get("nodes", [])
edges = data.get("edges", [])
metadata = data.get("metadata", {})
nodes = data.get("nodes", [])
edges = data.get("edges", [])
metadata = data.get("metadata", {})
nodes_dict = {
node["id"]: {
**node,
"label": node.get("content", ""),
"group": node.get("node_type", ""),
"title": f"{node.get('node_type', '')}: {node.get('content', '')}",
nodes_dict = {
node["id"]: {
**node,
"label": node.get("content", ""),
"group": node.get("node_type", ""),
"title": f"{node.get('node_type', '')}: {node.get('content', '')}",
}
for node in nodes
if node.get("id")
}
for node in nodes
if node.get("id")
}
edges_list = [
{
**edge,
"from": edge.get("source", edge.get("source_id")),
"to": edge.get("target", edge.get("target_id")),
"label": edge.get("relation", ""),
"arrows": "to",
edges_list = []
seen_edge_ids = set()
for edge in edges:
edge_id = edge.get("id")
if edge_id and edge_id not in seen_edge_ids:
edges_list.append(
{
**edge,
"from": edge.get("source", edge.get("source_id")),
"to": edge.get("target", edge.get("target_id")),
"label": edge.get("relation", ""),
"arrows": "to",
}
)
seen_edge_ids.add(edge_id)
stats = metadata.get("statistics", {})
total_memories = stats.get("total_memories", 0)
graph_data_cache = {
"nodes": list(nodes_dict.values()),
"edges": edges_list,
"memories": [], # TODO: 未来也可以考虑分页加载记忆
"stats": {
"total_nodes": len(nodes_dict),
"total_edges": len(edges_list),
"total_memories": total_memories,
},
"current_file": str(graph_file),
"file_size": graph_file.stat().st_size,
"file_modified": datetime.fromtimestamp(graph_file.stat().st_mtime).isoformat(),
}
for edge in edges
]
stats = metadata.get("statistics", {})
total_memories = stats.get("total_memories", 0)
# 如果是分页请求, 则从缓存中切片数据
if is_paged_request:
paged_data = graph_data_cache.copy() # 浅拷贝一份, 避免修改缓存
# 分页节点
if nodes_page is not None and nodes_per_page is not None:
node_start = (nodes_page - 1) * nodes_per_page
node_end = node_start + nodes_per_page
paged_data["nodes"] = graph_data_cache["nodes"][node_start:node_end]
# 分页边
if edges_page is not None and edges_per_page is not None:
edge_start = (edges_page - 1) * edges_per_page
edge_end = edge_start + edges_per_page
paged_data["edges"] = graph_data_cache["edges"][edge_start:edge_end]
return paged_data
graph_data_cache = {
"nodes": list(nodes_dict.values()),
"edges": edges_list,
"memories": [],
"stats": {
"total_nodes": len(nodes_dict),
"total_edges": len(edges_list),
"total_memories": total_memories,
},
"current_file": str(graph_file),
"file_size": graph_file.stat().st_size,
"file_modified": datetime.fromtimestamp(graph_file.stat().st_mtime).isoformat(),
}
return graph_data_cache
except Exception as e:
import traceback
@@ -151,7 +188,7 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
all_memories = memory_manager.graph_store.get_all_memories()
nodes_dict = {}
edges_list = []
edges_dict = {}
memory_info = []
for memory in all_memories:
@@ -173,8 +210,8 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
"title": f"{node.node_type.value}: {node.content}",
}
for edge in memory.edges:
edges_list.append( # noqa: PERF401
{
if edge.id not in edges_dict:
edges_dict[edge.id] = {
"id": edge.id,
"from": edge.source_id,
"to": edge.target_id,
@@ -182,7 +219,8 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
"arrows": "to",
"memory_id": memory.id,
}
)
edges_list = list(edges_dict.values())
stats = memory_manager.get_statistics()
return {
@@ -197,28 +235,67 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
"current_file": "memory_manager (实时数据)",
}
@router.get("/api/graph/paged")
async def get_paged_graph(
nodes_page: int = 1, nodes_per_page: int = 100, edges_page: int = 1, edges_per_page: int = 200
):
"""获取分页的记忆图数据"""
try:
# 确保全量数据已加载到缓存
full_data = load_graph_data_from_file()
if "error" in full_data:
raise HTTPException(status_code=404, detail=full_data["error"])
# 从缓存中获取全量数据
all_nodes = full_data.get("nodes", [])
all_edges = full_data.get("edges", [])
total_nodes = len(all_nodes)
total_edges = len(all_edges)
# 计算节点分页
node_start = (nodes_page - 1) * nodes_per_page
node_end = node_start + nodes_per_page
paginated_nodes = all_nodes[node_start:node_end]
# 计算边分页
edge_start = (edges_page - 1) * edges_per_page
edge_end = edge_start + edges_per_page
paginated_edges = all_edges[edge_start:edge_end]
return JSONResponse(
content={
"success": True,
"data": {
"nodes": paginated_nodes,
"edges": paginated_edges,
"pagination": {
"nodes": {
"page": nodes_page,
"per_page": nodes_per_page,
"total": total_nodes,
"total_pages": (total_nodes + nodes_per_page - 1) // nodes_per_page,
},
"edges": {
"page": edges_page,
"per_page": edges_per_page,
"total": total_edges,
"total_pages": (total_edges + edges_per_page - 1) // edges_per_page,
},
},
},
}
)
except Exception as e:
return JSONResponse(content={"success": False, "error": str(e)}, status_code=500)
@router.get("/api/graph/full")
async def get_full_graph():
"""获取完整记忆图数据"""
try:
from src.memory_graph.manager_singleton import get_memory_manager
memory_manager = get_memory_manager()
data = {}
if memory_manager and memory_manager._initialized:
data = _format_graph_data_from_manager(memory_manager)
else:
# 如果内存管理器不可用，则从文件加载
data = load_graph_data_from_file()
return JSONResponse(content={"success": True, "data": data})
except Exception as e:
import traceback
traceback.print_exc()
return JSONResponse(content={"success": False, "error": str(e)}, status_code=500)
async def get_full_graph_deprecated():
"""
(已废弃) 获取完整记忆图数据。
此接口现在只返回第一页的数据, 请使用 /api/graph/paged 进行分页获取。
"""
return await get_paged_graph(nodes_page=1, nodes_per_page=100, edges_page=1, edges_per_page=200)
@router.get("/api/files")

180
src/api/templates/visualizer.html
View File

@@ -533,11 +533,18 @@
let network = null;
let availableFiles = [];
let graphData = {
nodes: [],
edges: [],
memories: []
nodes: new vis.DataSet([])
,
edges: new vis.DataSet([])
};
let originalData = null;
let originalData = null; // 用于过滤器
// 分页状态
let pagination = {
nodes: { page: 1, per_page: 200, total_pages: 1, total: 0 },
edges: { page: 1, per_page: 500, total_pages: 1, total: 0 }
};
let isLoading = false;
// 节点颜色配置
const nodeColors = {
@@ -653,35 +660,93 @@
});
}
// 加载图形数据
// 重置并加载第一页数据
async function loadGraph() {
if (isLoading) return;
console.log('开始加载初始图数据...');
// 重置状态
graphData.nodes.clear();
graphData.edges.clear();
pagination.nodes.page = 1;
pagination.edges.page = 1;
try {
document.getElementById('loading').style.display = 'block';
const response = await fetch('/visualizer/api/graph/full');
const result = await response.json();
if (result.success) {
originalData = result.data;
updateGraph(result.data);
updateStats(result.data.stats);
// 先获取一次完整的统计信息
const statsResponse = await fetch('/visualizer/api/stats');
const statsResult = await statsResponse.json();
if(statsResult.success) {
updateStats(statsResult.data);
pagination.nodes.total = statsResult.data.total_nodes;
pagination.edges.total = statsResult.data.total_edges;
pagination.nodes.total_pages = Math.ceil(statsResult.data.total_nodes / pagination.nodes.per_page);
pagination.edges.total_pages = Math.ceil(statsResult.data.total_edges / pagination.edges.per_page);
} else {
alert('加载失败: ' + result.error);
throw new Error('获取统计信息失败: ' + statsResult.error);
}
// 加载第一页
await loadMoreData();
} catch (error) {
console.error('加载图形失败:', error);
alert('加载失败: ' + error.message);
} finally {
document.getElementById('loading').style.display = 'none';
console.error('初始加载失败:', error);
alert('初始加载失败: ' + error.message);
}
}
// 更新图形显示
// 加载更多数据（分页核心）
async function loadMoreData() {
if (isLoading) return;
const canLoadNodes = pagination.nodes.page <= pagination.nodes.total_pages;
const canLoadEdges = pagination.edges.page <= pagination.edges.total_pages;
if (!canLoadNodes && !canLoadEdges) {
console.log('所有数据已加载完毕');
return;
}
isLoading = true;
document.getElementById('loading').style.display = 'block';
try {
const url = `/visualizer/api/graph/paged?nodes_page=${pagination.nodes.page}&nodes_per_page=${pagination.nodes.per_page}&edges_page=${pagination.edges.page}&edges_per_page=${pagination.edges.per_page}`;
console.log(`正在请求: ${url}`);
const response = await fetch(url);
const result = await response.json();
if (result.success) {
console.log(`成功获取 ${result.data.nodes.length} 个节点, ${result.data.edges.length} 个边`);
updateGraph(result.data); // 追加数据
// 更新分页信息
if (result.data.pagination) {
pagination.nodes.page++;
pagination.edges.page++;
}
} else {
throw new Error('加载分页数据失败: ' + result.error);
}
} catch (error) {
console.error('加载更多数据失败:', error);
alert('加载失败: ' + error.message);
} finally {
isLoading = false;
document.getElementById('loading').style.display = 'none';
}
}
// 更新图形显示（追加数据）
function updateGraph(data) {
graphData = data;
// originalData 用于过滤器, 这里只追加, 不完全覆盖
if (!originalData) {
originalData = { nodes: [], edges: [] };
}
originalData.nodes.push(...data.nodes);
originalData.edges.push(...data.edges);
// 处理节点数据
const nodes = data.nodes.map(node => ({
const newNodes = data.nodes.map(node => ({
id: node.id,
label: node.label,
title: node.title,
@@ -691,25 +756,31 @@
}));
// 处理边数据
const edges = data.edges.map(edge => ({
const newEdges = data.edges.map(edge => ({
id: edge.id,
from: edge.from,
to: edge.to,
label: edge.label,
title: edge.title,
width: edge.importance * 3 + 1
width: (edge.importance || 0.5) * 2 + 1
}));
// 更新网络
network.setData({
nodes: new vis.DataSet(nodes),
edges: new vis.DataSet(edges)
});
// 注意：setData 会自动触发物理引擎重新布局
// stabilizationIterationsDone 事件监听器会自动停止物理引擎
// 追加数据到 DataSet
if (newNodes.length > 0) {
graphData.nodes.add(newNodes);
}
if (newEdges.length > 0) {
graphData.edges.add(newEdges);
}
// 第一次加载时设置数据
if (pagination.nodes.page === 2) { // 意味着第一页刚加载完
network.setData({
nodes: graphData.nodes,
edges: graphData.edges
});
}
}
// 更新统计信息
function updateStats(stats) {
document.getElementById('statNodes').textContent = stats.total_nodes;
@@ -1163,13 +1234,42 @@
closeFileSelector();
}
}
// 页面加载完成后初始化
window.addEventListener('load', function() {
initNetwork();
loadGraph(); // 加载初始数据
loadFileList();
// 页面加载完成后初始化
window.addEventListener('load', function() {
initNetwork();
loadGraph();
loadFileList();
});
// 添加滚动加载监听器
const graphContainer = document.getElementById('memory-graph');
graphContainer.addEventListener('mousewheel', async (event) => {
if(network) {
const canvasHeight = network.canvas.body.height;
const viewPosition = network.getViewPosition();
const scale = network.getScale();
const viewHeight = canvasHeight / scale;
// 简单的滚动到底部检测（可能需要根据实际情况微调）
if (event.deltaY > 0 && !isLoading) {
const isAtBottom = viewPosition.y > (canvasHeight/2 - viewHeight/2) * 0.8;
if (isAtBottom) {
console.log("滚动到底部，加载更多数据...");
await loadMoreData();
}
}
}
});
// 添加一个按钮用于手动加载
const loadMoreBtn = document.createElement('button');
loadMoreBtn.textContent = '加载更多';
loadMoreBtn.className = 'btn';
loadMoreBtn.style.position = 'absolute';
loadMoreBtn.style.bottom = '20px';
loadMoreBtn.style.right = '20px';
loadMoreBtn.style.zIndex = '10';
loadMoreBtn.onclick = loadMoreData;
document.querySelector('.graph-container').appendChild(loadMoreBtn);
});
</script>
</body>
</html>