From 679c7875516e331d59e1bf48c79ad8ee2ac2b858 Mon Sep 17 00:00:00 2001
From: minecraft1024a <wwwww95915@qq.com>
Date: Fri, 7 Nov 2025 22:03:09 +0800
Subject: [PATCH] =?UTF-8?q?fix(api):=20=E4=BF=AE=E5=A4=8D=E8=AE=B0?=
 =?UTF-8?q?=E5=BF=86=E5=8F=AF=E8=A7=86=E5=8C=96=E4=B8=AD=E9=87=8D=E5=A4=8D?=
 =?UTF-8?q?=E7=9A=84=E8=BE=B9?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

在从文件或内存管理器加载图数据时，由于遍历所有记忆（memory）并将其包含的边（edge）添加到列表中，导致了边的重复。当多个记忆共享同一条边时，这条边会被多次添加到最终的边列表中，造成前端可视化图中出现冗余的连接。

此提交通过引入一个集合（set）或字典（dict）来跟踪已经处理过的边的 ID，确保每条边只被添加一次，从而解决了重复边的问题。
---
 src/api/memory_visualizer_router.py | 209 +++++++++++++++++++---------
 src/api/templates/visualizer.html   | 180 ++++++++++++++++++------
 2 files changed, 283 insertions(+), 106 deletions(-)

diff --git a/src/api/memory_visualizer_router.py b/src/api/memory_visualizer_router.py
index b35c1c074..1f0eb27ee 100644
--- a/src/api/memory_visualizer_router.py
+++ b/src/api/memory_visualizer_router.py
@@ -61,15 +61,27 @@ def find_available_data_files() -> List[Path]:
     return sorted(files, key=lambda f: f.stat().st_mtime, reverse=True)
 
 
-def load_graph_data_from_file(file_path: Optional[Path] = None) -> Dict[str, Any]:
-    """从磁盘加载图数据"""
+def load_graph_data_from_file(
+    file_path: Optional[Path] = None,
+    nodes_page: Optional[int] = None,
+    nodes_per_page: Optional[int] = None,
+    edges_page: Optional[int] = None,
+    edges_per_page: Optional[int] = None,
+) -> Dict[str, Any]:
+    """
+    从磁盘加载图数据, 支持分页。
+    如果不提供分页参数, 则加载并缓存所有数据。
+    """
     global graph_data_cache, current_data_file
 
+    # 如果是请求分页数据, 则不使用缓存的全量数据
+    is_paged_request = nodes_page is not None or edges_page is not None
+
     if file_path and file_path != current_data_file:
         graph_data_cache = None
         current_data_file = file_path
 
-    if graph_data_cache:
+    if graph_data_cache and not is_paged_request:
         return graph_data_cache
 
     try:
@@ -84,53 +96,78 @@ def load_graph_data_from_file(file_path: Optional[Path] = None) -> Dict[str, Any
         if not graph_file.exists():
             return {"error": f"文件不存在: {graph_file}", "nodes": [], "edges": [], "stats": {}}
 
-        with open(graph_file, "r", encoding="utf-8") as f:
-            data = orjson.loads(f.read())
+        # 只有在没有缓存时才从磁盘读取和处理文件
+        if not graph_data_cache:
+            with open(graph_file, "r", encoding="utf-8") as f:
+                data = orjson.loads(f.read())
 
-        nodes = data.get("nodes", [])
-        edges = data.get("edges", [])
-        metadata = data.get("metadata", {})
+            nodes = data.get("nodes", [])
+            edges = data.get("edges", [])
+            metadata = data.get("metadata", {})
 
-        nodes_dict = {
-            node["id"]: {
-                **node,
-                "label": node.get("content", ""),
-                "group": node.get("node_type", ""),
-                "title": f"{node.get('node_type', '')}: {node.get('content', '')}",
+            nodes_dict = {
+                node["id"]: {
+                    **node,
+                    "label": node.get("content", ""),
+                    "group": node.get("node_type", ""),
+                    "title": f"{node.get('node_type', '')}: {node.get('content', '')}",
+                }
+                for node in nodes
+                if node.get("id")
             }
-            for node in nodes
-            if node.get("id")
-        }
 
-        edges_list = [
-            {
-                **edge,
-                "from": edge.get("source", edge.get("source_id")),
-                "to": edge.get("target", edge.get("target_id")),
-                "label": edge.get("relation", ""),
-                "arrows": "to",
+            edges_list = []
+            seen_edge_ids = set()
+            for edge in edges:
+                edge_id = edge.get("id")
+                if edge_id and edge_id not in seen_edge_ids:
+                    edges_list.append(
+                        {
+                            **edge,
+                            "from": edge.get("source", edge.get("source_id")),
+                            "to": edge.get("target", edge.get("target_id")),
+                            "label": edge.get("relation", ""),
+                            "arrows": "to",
+                        }
+                    )
+                    seen_edge_ids.add(edge_id)
+
+            stats = metadata.get("statistics", {})
+            total_memories = stats.get("total_memories", 0)
+
+            graph_data_cache = {
+                "nodes": list(nodes_dict.values()),
+                "edges": edges_list,
+                "memories": [], # TODO: 未来也可以考虑分页加载记忆
+                "stats": {
+                    "total_nodes": len(nodes_dict),
+                    "total_edges": len(edges_list),
+                    "total_memories": total_memories,
+                },
+                "current_file": str(graph_file),
+                "file_size": graph_file.stat().st_size,
+                "file_modified": datetime.fromtimestamp(graph_file.stat().st_mtime).isoformat(),
             }
-            for edge in edges
-        ]
 
-        stats = metadata.get("statistics", {})
-        total_memories = stats.get("total_memories", 0)
+        # 如果是分页请求, 则从缓存中切片数据
+        if is_paged_request:
+            paged_data = graph_data_cache.copy() # 浅拷贝一份, 避免修改缓存
+            
+            # 分页节点
+            if nodes_page is not None and nodes_per_page is not None:
+                node_start = (nodes_page - 1) * nodes_per_page
+                node_end = node_start + nodes_per_page
+                paged_data["nodes"] = graph_data_cache["nodes"][node_start:node_end]
+            
+            # 分页边
+            if edges_page is not None and edges_per_page is not None:
+                edge_start = (edges_page - 1) * edges_per_page
+                edge_end = edge_start + edges_per_page
+                paged_data["edges"] = graph_data_cache["edges"][edge_start:edge_end]
+                
+            return paged_data
 
-        graph_data_cache = {
-            "nodes": list(nodes_dict.values()),
-            "edges": edges_list,
-            "memories": [],
-            "stats": {
-                "total_nodes": len(nodes_dict),
-                "total_edges": len(edges_list),
-                "total_memories": total_memories,
-            },
-            "current_file": str(graph_file),
-            "file_size": graph_file.stat().st_size,
-            "file_modified": datetime.fromtimestamp(graph_file.stat().st_mtime).isoformat(),
-        }
         return graph_data_cache
-
     except Exception as e:
         import traceback
 
@@ -151,7 +188,7 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
 
     all_memories = memory_manager.graph_store.get_all_memories()
     nodes_dict = {}
-    edges_list = []
+    edges_dict = {}
     memory_info = []
 
     for memory in all_memories:
@@ -173,8 +210,8 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
                     "title": f"{node.node_type.value}: {node.content}",
                 }
         for edge in memory.edges:
-            edges_list.append(  # noqa: PERF401
-                {
+            if edge.id not in edges_dict:
+                edges_dict[edge.id] = {
                     "id": edge.id,
                     "from": edge.source_id,
                     "to": edge.target_id,
@@ -182,7 +219,8 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
                     "arrows": "to",
                     "memory_id": memory.id,
                 }
-            )
+    
+    edges_list = list(edges_dict.values())
 
     stats = memory_manager.get_statistics()
     return {
@@ -197,28 +235,67 @@ def _format_graph_data_from_manager(memory_manager) -> Dict[str, Any]:
         "current_file": "memory_manager (实时数据)",
     }
 
+@router.get("/api/graph/paged")
+async def get_paged_graph(
+    nodes_page: int = 1, nodes_per_page: int = 100, edges_page: int = 1, edges_per_page: int = 200
+):
+    """获取分页的记忆图数据"""
+    try:
+        # 确保全量数据已加载到缓存
+        full_data = load_graph_data_from_file()
+        if "error" in full_data:
+             raise HTTPException(status_code=404, detail=full_data["error"])
+
+        # 从缓存中获取全量数据
+        all_nodes = full_data.get("nodes", [])
+        all_edges = full_data.get("edges", [])
+        total_nodes = len(all_nodes)
+        total_edges = len(all_edges)
+        
+        # 计算节点分页
+        node_start = (nodes_page - 1) * nodes_per_page
+        node_end = node_start + nodes_per_page
+        paginated_nodes = all_nodes[node_start:node_end]
+
+        # 计算边分页
+        edge_start = (edges_page - 1) * edges_per_page
+        edge_end = edge_start + edges_per_page
+        paginated_edges = all_edges[edge_start:edge_end]
+
+        return JSONResponse(
+            content={
+                "success": True,
+                "data": {
+                    "nodes": paginated_nodes,
+                    "edges": paginated_edges,
+                    "pagination": {
+                        "nodes": {
+                            "page": nodes_page,
+                            "per_page": nodes_per_page,
+                            "total": total_nodes,
+                            "total_pages": (total_nodes + nodes_per_page - 1) // nodes_per_page,
+                        },
+                        "edges": {
+                            "page": edges_page,
+                            "per_page": edges_per_page,
+                            "total": total_edges,
+                            "total_pages": (total_edges + edges_per_page - 1) // edges_per_page,
+                        },
+                    },
+                },
+            }
+        )
+    except Exception as e:
+        return JSONResponse(content={"success": False, "error": str(e)}, status_code=500)
+
 
 @router.get("/api/graph/full")
-async def get_full_graph():
-    """获取完整记忆图数据"""
-    try:
-        from src.memory_graph.manager_singleton import get_memory_manager
-
-        memory_manager = get_memory_manager()
-
-        data = {}
-        if memory_manager and memory_manager._initialized:
-            data = _format_graph_data_from_manager(memory_manager)
-        else:
-            # 如果内存管理器不可用，则从文件加载
-            data = load_graph_data_from_file()
-
-        return JSONResponse(content={"success": True, "data": data})
-    except Exception as e:
-        import traceback
-
-        traceback.print_exc()
-        return JSONResponse(content={"success": False, "error": str(e)}, status_code=500)
+async def get_full_graph_deprecated():
+    """
+    (已废弃) 获取完整记忆图数据。
+    此接口现在只返回第一页的数据, 请使用 /api/graph/paged 进行分页获取。
+    """
+    return await get_paged_graph(nodes_page=1, nodes_per_page=100, edges_page=1, edges_per_page=200)
 
 
 @router.get("/api/files")
diff --git a/src/api/templates/visualizer.html b/src/api/templates/visualizer.html
index 47c105863..9c44a420f 100644
--- a/src/api/templates/visualizer.html
+++ b/src/api/templates/visualizer.html
@@ -533,11 +533,18 @@
         let network = null;
         let availableFiles = [];
         let graphData = {
-            nodes: [],
-            edges: [],
-            memories: []
+            nodes: new vis.DataSet([])
+,
+            edges: new vis.DataSet([])
         };
-        let originalData = null;
+        let originalData = null; // 用于过滤器
+
+        // 分页状态
+        let pagination = {
+            nodes: { page: 1, per_page: 200, total_pages: 1, total: 0 },
+            edges: { page: 1, per_page: 500, total_pages: 1, total: 0 }
+        };
+        let isLoading = false;
 
         // 节点颜色配置
         const nodeColors = {
@@ -653,35 +660,93 @@
             });
         }
 
-        // 加载图形数据
+        // 重置并加载第一页数据
         async function loadGraph() {
+            if (isLoading) return;
+            console.log('开始加载初始图数据...');
+            
+            // 重置状态
+            graphData.nodes.clear();
+            graphData.edges.clear();
+            pagination.nodes.page = 1;
+            pagination.edges.page = 1;
+            
             try {
-                document.getElementById('loading').style.display = 'block';
-                
-                const response = await fetch('/visualizer/api/graph/full');
-                const result = await response.json();
-
-                if (result.success) {
-                    originalData = result.data;
-                    updateGraph(result.data);
-                    updateStats(result.data.stats);
+                // 先获取一次完整的统计信息
+                const statsResponse = await fetch('/visualizer/api/stats');
+                const statsResult = await statsResponse.json();
+                if(statsResult.success) {
+                    updateStats(statsResult.data);
+                    pagination.nodes.total = statsResult.data.total_nodes;
+                    pagination.edges.total = statsResult.data.total_edges;
+                    pagination.nodes.total_pages = Math.ceil(statsResult.data.total_nodes / pagination.nodes.per_page);
+                    pagination.edges.total_pages = Math.ceil(statsResult.data.total_edges / pagination.edges.per_page);
                 } else {
-                    alert('加载失败: ' + result.error);
+                     throw new Error('获取统计信息失败: ' + statsResult.error);
                 }
+
+                // 加载第一页
+                await loadMoreData();
+
             } catch (error) {
-                console.error('加载图形失败:', error);
-                alert('加载失败: ' + error.message);
-            } finally {
-                document.getElementById('loading').style.display = 'none';
+                console.error('初始加载失败:', error);
+                alert('初始加载失败: ' + error.message);
             }
         }
 
-        // 更新图形显示
+        // 加载更多数据（分页核心）
+        async function loadMoreData() {
+            if (isLoading) return;
+            
+            const canLoadNodes = pagination.nodes.page <= pagination.nodes.total_pages;
+            const canLoadEdges = pagination.edges.page <= pagination.edges.total_pages;
+
+            if (!canLoadNodes && !canLoadEdges) {
+                console.log('所有数据已加载完毕');
+                return;
+            }
+
+            isLoading = true;
+            document.getElementById('loading').style.display = 'block';
+
+            try {
+                const url = `/visualizer/api/graph/paged?nodes_page=${pagination.nodes.page}&nodes_per_page=${pagination.nodes.per_page}&edges_page=${pagination.edges.page}&edges_per_page=${pagination.edges.per_page}`;
+                console.log(`正在请求: ${url}`);
+                const response = await fetch(url);
+                const result = await response.json();
+
+                if (result.success) {
+                    console.log(`成功获取 ${result.data.nodes.length} 个节点, ${result.data.edges.length} 个边`);
+                    updateGraph(result.data); // 追加数据
+                    
+                    // 更新分页信息
+                    if (result.data.pagination) {
+                        pagination.nodes.page++;
+                        pagination.edges.page++;
+                    }
+                } else {
+                    throw new Error('加载分页数据失败: ' + result.error);
+                }
+            } catch (error) {
+                console.error('加载更多数据失败:', error);
+                alert('加载失败: ' + error.message);
+            } finally {
+                isLoading = false;
+                document.getElementById('loading').style.display = 'none';
+            }
+        }
+        // 更新图形显示（追加数据）
         function updateGraph(data) {
-            graphData = data;
+            // originalData 用于过滤器, 这里只追加, 不完全覆盖
+            if (!originalData) {
+                originalData = { nodes: [], edges: [] };
+            }
+            originalData.nodes.push(...data.nodes);
+            originalData.edges.push(...data.edges);
+
 
             // 处理节点数据
-            const nodes = data.nodes.map(node => ({
+            const newNodes = data.nodes.map(node => ({
                 id: node.id,
                 label: node.label,
                 title: node.title,
@@ -691,25 +756,31 @@
             }));
 
             // 处理边数据
-            const edges = data.edges.map(edge => ({
+            const newEdges = data.edges.map(edge => ({
                 id: edge.id,
                 from: edge.from,
                 to: edge.to,
                 label: edge.label,
                 title: edge.title,
-                width: edge.importance * 3 + 1
+                width: (edge.importance || 0.5) * 2 + 1
             }));
-
-            // 更新网络
-            network.setData({
-                nodes: new vis.DataSet(nodes),
-                edges: new vis.DataSet(edges)
-            });
-
-            // 注意：setData 会自动触发物理引擎重新布局
-            // stabilizationIterationsDone 事件监听器会自动停止物理引擎
+            
+            // 追加数据到 DataSet
+            if (newNodes.length > 0) {
+                graphData.nodes.add(newNodes);
+            }
+            if (newEdges.length > 0) {
+                graphData.edges.add(newEdges);
+            }
+            
+            // 第一次加载时设置数据
+            if (pagination.nodes.page === 2) { // 意味着第一页刚加载完
+                 network.setData({
+                    nodes: graphData.nodes,
+                    edges: graphData.edges
+                });
+            }
         }
-
         // 更新统计信息
         function updateStats(stats) {
             document.getElementById('statNodes').textContent = stats.total_nodes;
@@ -1163,13 +1234,42 @@
                 closeFileSelector();
             }
         }
+// 页面加载完成后初始化
+window.addEventListener('load', function() {
+    initNetwork();
+    loadGraph(); // 加载初始数据
+    loadFileList();
 
-        // 页面加载完成后初始化
-        window.addEventListener('load', function() {
-            initNetwork();
-            loadGraph();
-            loadFileList();
-        });
+    // 添加滚动加载监听器
+    const graphContainer = document.getElementById('memory-graph');
+    graphContainer.addEventListener('mousewheel', async (event) => {
+        if(network) {
+            const canvasHeight = network.canvas.body.height;
+            const viewPosition = network.getViewPosition();
+            const scale = network.getScale();
+            const viewHeight = canvasHeight / scale;
+
+            // 简单的滚动到底部检测（可能需要根据实际情况微调）
+            if (event.deltaY > 0 && !isLoading) {
+                const isAtBottom = viewPosition.y > (canvasHeight/2 - viewHeight/2) * 0.8;
+                if (isAtBottom) {
+                    console.log("滚动到底部，加载更多数据...");
+                    await loadMoreData();
+                }
+            }
+        }
+    });
+     // 添加一个按钮用于手动加载
+    const loadMoreBtn = document.createElement('button');
+    loadMoreBtn.textContent = '加载更多';
+    loadMoreBtn.className = 'btn';
+    loadMoreBtn.style.position = 'absolute';
+    loadMoreBtn.style.bottom = '20px';
+    loadMoreBtn.style.right = '20px';
+    loadMoreBtn.style.zIndex = '10';
+    loadMoreBtn.onclick = loadMoreData;
+    document.querySelector('.graph-container').appendChild(loadMoreBtn);
+});
     </script>
 </body>
 </html>