From e00555667758489bf3f7d442252dee83a6ac531b Mon Sep 17 00:00:00 2001 From: Tom Maiaroto Date: Wed, 25 Mar 2026 00:21:51 -0700 Subject: [PATCH 1/3] index the scip data into sqlite db and load graph nodes by directory to speed up performance. enhance node traversal from details panel to easily navigate between symbols in the same or different directories --- app.go | 602 ++++++++++++++---------------- frontend/dist/.gitkeep | 0 frontend/src/Atlas.tsx | 295 ++++++++++++--- frontend/wailsjs/go/main/App.d.ts | 8 +- frontend/wailsjs/go/main/App.js | 16 +- frontend/wailsjs/go/models.ts | 69 ++++ 6 files changed, 605 insertions(+), 385 deletions(-) delete mode 100644 frontend/dist/.gitkeep diff --git a/app.go b/app.go index edd1a6a..1d8fe50 100644 --- a/app.go +++ b/app.go @@ -15,6 +15,7 @@ import ( "path/filepath" "runtime" "strings" + "sync" "syscall" "time" @@ -24,7 +25,6 @@ import ( "github.com/hackafterdark/context-sherpa/pkg/sysutils" scip "github.com/sourcegraph/scip/bindings/go/scip" wailsRuntime "github.com/wailsapp/wails/v2/pkg/runtime" - "google.golang.org/protobuf/proto" "gopkg.in/yaml.v3" ) @@ -59,6 +59,8 @@ type App struct { downloader *inference.Downloader inference *inference.InferenceService db *database.DB + localDBs map[string]*database.DB + localDBMu sync.Mutex } // MarkdownEntry represents a markdown file with optional front-matter metadata @@ -79,7 +81,9 @@ type LocalRuleDetails struct { // NewApp creates a new App application struct func NewApp() *App { - return &App{} + return &App{ + localDBs: make(map[string]*database.DB), + } } // startup is called when the app starts. @@ -202,7 +206,6 @@ func (a *App) normalizePath(path string) string { func (a *App) initDatabase() { configDir, err := getSherpaConfigDir() if err != nil { - fmt.Printf("Hub: Failed to get config dir: %v\n", err) return } @@ -1977,385 +1980,248 @@ type GraphData struct { Language string `json:"language"` } -// GetGraphData transforms SCIP data into ECharts JSON with "Hotpath" sizing and spatial clustering -func (a *App) GetGraphData(scipPath string) (*GraphData, error) { - data, err := os.ReadFile(scipPath) +// GetGraphData transforms SCIP data into ECharts JSON with "Hotpath" sizing and spatial clustering. +// It uses a persistent SQLite index to avoid re-parsing large Protobuf files on every request. +func (a *App) GetGraphData(scipPath string, scope string) (*GraphData, error) { + scipPath = a.normalizePath(scipPath) + // 1. Get/Init local database for this workspace + localDB, err := a.getLocalDB(scipPath) if err != nil { return nil, err } - var index scip.Index - if err := proto.Unmarshal(data, &index); err != nil { - return nil, fmt.Errorf("failed to parse SCIP index: %w", err) + // 2. Ensure the index is ingested and up to date + if err := a.IngestIndex(scipPath); err != nil { + return nil, err } nodes := make([]GraphNode, 0) links := make([]GraphLink, 0) - seenNodes := make(map[string]bool) - seenLinks := make(map[string]bool) - - // Custom Categories for Layout/Clustering (Folders) - // But Legend will be fixed to Code Kinds. folderToCatID := make(map[string]int) categories := make([]GraphCategory, 0) - symbolToNodeID := make(map[string]string) - symbolToInfo := make(map[string]*scip.SymbolInformation) - refCount := make(map[string]int) - outboundCalls := make(map[string]int) - addNode := func(n GraphNode) { - if n.ID != "" && !seenNodes[n.ID] { - nodes = append(nodes, n) - seenNodes[n.ID] = true - } - } - - addLink := func(l GraphLink) { - if l.Source == "" || l.Target == "" || l.Source == l.Target { - return - } - key := fmt.Sprintf("%s-%s-%s", l.Source, l.Target, l.Label) - if !seenLinks[key] { - links = append(links, l) - seenLinks[key] = true - } + scope = filepath.ToSlash(scope) + if scope != "" && !strings.HasSuffix(scope, "/") { + scope += "/" } - // 1. Pre-map symbol information and identify directories - for _, doc := range index.Documents { - dir := filepath.Dir(doc.RelativePath) - if dir == "." { - dir = "root" - } - if _, exists := folderToCatID[dir]; !exists { - folderToCatID[dir] = len(categories) - categories = append(categories, GraphCategory{Name: dir}) - } - for _, si := range doc.Symbols { - symbolToInfo[si.Symbol] = si - } - } - for _, si := range index.ExternalSymbols { - symbolToInfo[si.Symbol] = si + // 3. Fetch Symbols for the requested scope + dirMatch := scope + if scope == "" { + dirMatch = "root" + } else { + dirMatch = strings.TrimSuffix(scope, "/") } - // 2. Pass 1: Count references and outbound calls - for _, doc := range index.Documents { - var currentScope string - for _, occ := range doc.Occurrences { - if strings.Contains(occ.Symbol, "local") { - continue - } - isDef := occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 - if isDef { - currentScope = occ.Symbol - continue - } - // It's a reference - refCount[occ.Symbol]++ - if currentScope != "" { - outboundCalls[currentScope]++ - } - } + rows, err := localDB.Query(` + SELECT symbol, name, kind, file_path, dir_path, start_line, end_line, impact_value, parent_symbol + FROM scip_symbols + WHERE scip_path = ? AND dir_path = ? + `, scipPath, dirMatch) + if err != nil { + return nil, err } + defer rows.Close() - // 3. Pass 2: Create Symbol Nodes (No physical folder/file nodes) - for _, doc := range index.Documents { - dir := filepath.Dir(doc.RelativePath) - catID := folderToCatID[dir] - - for _, occ := range doc.Occurrences { - isDef := occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 - if isDef { - // Filter noise: skip local symbols - if strings.Contains(occ.Symbol, "local") { - continue - } - - parsed, err := scip.ParseSymbol(occ.Symbol) - if err != nil || len(parsed.Descriptors) == 0 { - continue // Skip packages or unparseable symbols - } - - // The last descriptor is the leaf (Method, Type, Term, etc.) - leaf := parsed.Descriptors[len(parsed.Descriptors)-1] - - // Skip Namespaces (Packages) as definitions - they are architectural noise - if leaf.Suffix == scip.Descriptor_Namespace { - continue - } - - name := leaf.Name - kind := "Function" - - // Map SCIP Descriptor Suffix to our finite categories - switch leaf.Suffix { - case scip.Descriptor_Type: - kind = "Struct" - case scip.Descriptor_Term: - kind = "Variable" - case scip.Descriptor_Method: - kind = "Function" - case scip.Descriptor_Macro: - kind = "Function" - case scip.Descriptor_Parameter: - continue // Skip parameters for top-level graph - case scip.Descriptor_TypeParameter: - continue // Skip type parameters - } - - // Additional refinement for Interfaces (often labeled as Types in SCIP but contain "interface" in symbol) - if kind == "Struct" && strings.Contains(strings.ToLower(occ.Symbol), "interface") { - kind = "Interface" - } + for rows.Next() { + var n GraphNode + var sym, filePath, dirPath, parentSym string + err := rows.Scan(&sym, &n.Name, &n.Kind, &filePath, &dirPath, &n.StartLine, &n.EndLine, &n.Value, &parentSym) + if err != nil { continue } - nodeID := "sym:" + occ.Symbol - symbolToNodeID[occ.Symbol] = nodeID + n.ID = "sym:" + sym + n.Path = filePath + n.Parent = "dir:" + dirPath + + symbolToNodeID[sym] = n.ID - info := symbolToInfo[occ.Symbol] - docstring := "" - if info != nil && len(info.Documentation) > 0 { - docstring = info.Documentation[0] - } + if _, exists := folderToCatID[dirPath]; !exists { + folderToCatID[dirPath] = len(categories) + categories = append(categories, GraphCategory{Name: dirPath}) + } + nodes = append(nodes, n) + } - // Calculate Lines of Code (LOC) and Ranges - loc := 0 - startLine := 0 - endLine := 0 - if len(occ.Range) == 3 { - startLine = int(occ.Range[0] + 1) // [line, startCol, endCol] - endLine = startLine - loc = 1 - } else if len(occ.Range) >= 4 { - startLine = int(occ.Range[0] + 1) // [startLine, startCol, endLine, endCol] - endLine = int(occ.Range[2] + 1) - loc = int(occ.Range[2] - occ.Range[0] + 1) - } + // 4. Fetch immediate Subfolders to generate Folder Nodes + prefix := scope + subRows, err := localDB.Query(` + SELECT DISTINCT dir_path FROM scip_symbols + WHERE scip_path = ? AND dir_path LIKE ? AND dir_path != ? + `, scipPath, prefix+"%", dirMatch) - // Calculate "Hotpath" value (Directive Weights) - val := 0 - if kind == "Struct" || kind == "Interface" { - // Count members for struct sizing - memberCount := 0 - prefix := occ.Symbol - for sym := range symbolToInfo { - if strings.HasPrefix(sym, prefix) && sym != occ.Symbol { - if !strings.Contains(sym[len(prefix):], ".") { - memberCount++ - } - } + if err == nil { + defer subRows.Close() + seenFolders := make(map[string]bool) + for subRows.Next() { + var fullDir string + if err := subRows.Scan(&fullDir); err == nil { + rel := strings.TrimPrefix(fullDir, prefix) + parts := strings.Split(rel, "/") + if len(parts) > 0 && parts[0] != "" { + immediate := prefix + parts[0] + if !seenFolders[immediate] { + nodes = append(nodes, GraphNode{ + ID: "dir:" + immediate, + Name: parts[0], + Kind: "Folder", + Parent: "dir:" + strings.TrimSuffix(prefix, "/"), + }) + seenFolders[immediate] = true } - // Value = (MemberCount * 5) + (ReferenceCount * 10) - val = (memberCount * 5) + (refCount[occ.Symbol] * 10) - } else if kind == "Variable" { - val = 5 + (refCount[occ.Symbol] * 2) - } else { - // Function: Value = (OutboundCalls * 3) + (ReferenceCount * 5) - val = (outboundCalls[occ.Symbol] * 3) + (refCount[occ.Symbol] * 5) - } - - // Minimum size floor - if val < 5 { - val = 5 - } - - parentID := "" - if dir != "" && dir != "." { - parentID = "dir:" + dir } - - addNode(GraphNode{ - ID: nodeID, - Name: name, - Value: val, - Category: catID, // Spatial clustering by folder - Path: doc.RelativePath, - Kind: kind, - Docstring: docstring, - Members: []Member{}, - Loc: loc, - StartLine: startLine, - EndLine: endLine, - Parent: parentID, - }) } } + } else { } - // 4. Extract members for deep inspection - for i := range nodes { - if nodes[i].Kind != "Struct" && nodes[i].Kind != "Interface" { - continue - } - // Nodes[i].ID is "sym:" + symbol - originalSymbol := nodes[i].ID[4:] - - for sym := range symbolToInfo { - if strings.HasPrefix(sym, originalSymbol) && sym != originalSymbol { - parsed, err := scip.ParseSymbol(sym) - if err != nil || len(parsed.Descriptors) == 0 { - continue - } - - // Only direct children (e.g., App#scipFiles. and not App#scipFiles.inner.) - // This is a bit simplified; in SCIP Go, members are usually descriptors at the end - // We check if the parent symbol is indeed the parent in descriptors - isChild := false - parentParsed, _ := scip.ParseSymbol(originalSymbol) - if len(parsed.Descriptors) == len(parentParsed.Descriptors)+1 { - isChild = true - for j := 0; j < len(parentParsed.Descriptors); j++ { - if parsed.Descriptors[j].Name != parentParsed.Descriptors[j].Name || - parsed.Descriptors[j].Suffix != parentParsed.Descriptors[j].Suffix { - isChild = false - break - } + // 5. Fetch Relationships (Edges) + if len(nodes) > 0 { + relRows, err := localDB.Query(` + SELECT source_symbol, target_symbol FROM scip_relationships WHERE scip_path = ? + `, scipPath) + if err == nil { + defer relRows.Close() + for relRows.Next() { + var src, tgt string + if err := relRows.Scan(&src, &tgt); err == nil { + if symbolToNodeID[src] != "" && symbolToNodeID[tgt] != "" { + links = append(links, GraphLink{ + Source: symbolToNodeID[src], + Target: symbolToNodeID[tgt], + Label: "DEPENDS", + }) } } - - if isChild { - leaf := parsed.Descriptors[len(parsed.Descriptors)-1] - memberKind := "Field" - if leaf.Suffix == scip.Descriptor_Method { - memberKind = "Method" - } - - nodes[i].Members = append(nodes[i].Members, Member{ - Name: leaf.Name, - Kind: memberKind, - Symbol: sym, - }) - } } } } - // 5. Build Links (Symbol to Symbol) - for _, doc := range index.Documents { - var currentScope string - for _, occ := range doc.Occurrences { - if strings.Contains(occ.Symbol, "local") { - continue - } - if occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 { - if _, exists := symbolToNodeID[occ.Symbol]; exists { - currentScope = occ.Symbol - } - continue - } + // 6. Construct Cytoscape Elements + elements := make([]CyElement, 0) + for _, n := range nodes { + elements = append(elements, CyElement{Group: "nodes", Data: n}) + } + for _, l := range links { + elements = append(elements, CyElement{ + Group: "edges", + Data: map[string]interface{}{ + "id": "e-" + l.Source + "-" + l.Target, + "source": l.Source, + "target": l.Target, + "label": l.Label, + }, + }) + } - // Reference - targetNodeID, exists := symbolToNodeID[occ.Symbol] - if exists && currentScope != "" { - sourceNodeID := symbolToNodeID[currentScope] - if sourceNodeID != "" && sourceNodeID != targetNodeID { - label := "CALLS" - if strings.HasSuffix(occ.Symbol, "#") { - label = "USES" - } else if strings.Contains(occ.Symbol, "interface") { - label = "IMPLEMENTS" - } + var language string + localDB.QueryRow("SELECT language FROM scip_indices WHERE scip_path = ?", scipPath).Scan(&language) - // Get names/paths for Relationship Mode - sourceName := currentScope - if parsed, err := scip.ParseSymbol(currentScope); err == nil && len(parsed.Descriptors) > 0 { - sourceName = parsed.Descriptors[len(parsed.Descriptors)-1].Name - } else if lastSlash := strings.LastIndex(sourceName, "/"); lastSlash != -1 { - sourceName = sourceName[lastSlash+1:] - } + return &GraphData{ + Elements: elements, + Categories: categories, + Language: language, + }, nil +} - targetName := occ.Symbol - if parsed, err := scip.ParseSymbol(occ.Symbol); err == nil && len(parsed.Descriptors) > 0 { - targetName = parsed.Descriptors[len(parsed.Descriptors)-1].Name - } else if lastSlash := strings.LastIndex(targetName, "/"); lastSlash != -1 { - targetName = targetName[lastSlash+1:] - } - sourceName = strings.TrimSuffix(strings.TrimSuffix(sourceName, "#"), "().") - targetName = strings.TrimSuffix(strings.TrimSuffix(targetName, "#"), "().") - - addLink(GraphLink{ - Source: sourceNodeID, - Target: targetNodeID, - Label: fmt.Sprintf("%s -> %s (%s)", sourceName, targetName, label), - }) - } - } - } +// SearchSymbols performs a global text search across all symbols in a SCIP index. +func (a *App) SearchSymbols(scipPath string, query string) ([]GraphNode, error) { + scipPath = a.normalizePath(scipPath) + localDB, err := a.getLocalDB(scipPath) + if err != nil { + return nil, err } - // 6. Final Sweep: Construct Cytoscape Elements - elements := make([]CyElement, 0) + rows, err := localDB.Query(` + SELECT symbol, name, kind, file_path FROM scip_symbols + WHERE scip_path = ? AND (name LIKE ? OR symbol LIKE ?) + LIMIT 50 + `, scipPath, "%"+query+"%", "%"+query+"%") + if err != nil { + return nil, err + } + defer rows.Close() - // Add Folder Nodes (Compound Nodes) - seenFolders := make(map[string]bool) - for dir := range folderToCatID { - if dir == "" || dir == "." || dir == "root" { - continue + var results []GraphNode + for rows.Next() { + var n GraphNode + var sym, name, kind, filePath string + if err := rows.Scan(&sym, &name, &kind, &filePath); err == nil { + n.ID = "sym:" + sym + n.Name = name + n.Kind = kind + n.Path = filePath + results = append(results, n) } + } + return results, nil +} - // Create hierarchical folder nodes - parts := strings.Split(dir, string(filepath.Separator)) - currentPath := "" - for i, part := range parts { - prevPath := currentPath - if currentPath == "" { - currentPath = part - } else { - currentPath = currentPath + string(filepath.Separator) + part - } +// GetSymbolRelationships returns all callers and callees for a specific symbol. +func (a *App) GetSymbolRelationships(scipPath string, symbol string) (map[string][]GraphNode, error) { + scipPath = a.normalizePath(scipPath) + localDB, err := a.getLocalDB(scipPath) + if err != nil { + return nil, err + } - folderID := "dir:" + currentPath - if !seenFolders[folderID] { - parentID := "" - if i > 0 { - parentID = "dir:" + prevPath - } + symbol = strings.TrimPrefix(symbol, "sym:") - elements = append(elements, CyElement{ - Group: "nodes", - Data: GraphNode{ - ID: folderID, - Name: part, - Kind: "Folder", - Parent: parentID, - }, - }) - seenFolders[folderID] = true + // Callees (what this symbol calls) + calleeRows, err := localDB.Query(` + SELECT s.symbol, s.name, s.kind, s.file_path + FROM scip_relationships r + JOIN scip_symbols s ON r.target_symbol = s.symbol AND r.scip_path = s.scip_path + WHERE r.scip_path = ? AND r.source_symbol = ? + `, scipPath, symbol) + + callees := []GraphNode{} + if err == nil { + defer calleeRows.Close() + for calleeRows.Next() { + var n GraphNode + var sym, name, kind, filePath string + if err := calleeRows.Scan(&sym, &name, &kind, &filePath); err == nil { + n.ID = "sym:" + sym + n.Name = name + n.Kind = kind + n.Path = filePath + callees = append(callees, n) } } } - // Add Nodes (Symbols) - for _, n := range nodes { - elements = append(elements, CyElement{ - Group: "nodes", - Data: n, - }) - } + // Callers (what calls this symbol) + callerRows, err := localDB.Query(` + SELECT s.symbol, s.name, s.kind, s.file_path + FROM scip_relationships r + JOIN scip_symbols s ON r.source_symbol = s.symbol AND r.scip_path = s.scip_path + WHERE r.scip_path = ? AND r.target_symbol = ? + `, scipPath, symbol) - // Add Edges - for _, l := range links { - if seenNodes[l.Source] && seenNodes[l.Target] { - elements = append(elements, CyElement{ - Group: "edges", - Data: map[string]interface{}{ - "id": fmt.Sprintf("e-%s-%s", l.Source, l.Target), - "source": l.Source, - "target": l.Target, - "label": l.Label, - }, - }) + callers := []GraphNode{} + if err == nil { + defer callerRows.Close() + for callerRows.Next() { + var n GraphNode + var sym, name, kind, filePath string + if err := callerRows.Scan(&sym, &name, &kind, &filePath); err == nil { + n.ID = "sym:" + sym + n.Name = name + n.Kind = kind + n.Path = filePath + callers = append(callers, n) + } } } - return &GraphData{ - Elements: elements, - Categories: categories, - Language: detectLanguage(index.Documents), + return map[string][]GraphNode{ + "callers": callers, + "callees": callees, }, nil } + // FocusWorkspaceClient attempts to bring the editor window associated with an MCP server to the foreground. func (a *App) FocusWorkspaceClient(mcpPid int) error { if mcpPid <= 0 { @@ -2644,3 +2510,85 @@ func (a *App) PickDirectoryWithRoot(startDir string) (string, error) { }) } + +// getLocalDB opens or initializes a SQLite database in the same directory as the SCIP file (or its parent .context-sherpa) +func (a *App) getLocalDB(scipPath string) (*database.DB, error) { + a.localDBMu.Lock() + defer a.localDBMu.Unlock() + + dir := filepath.Dir(scipPath) + dbPath := filepath.Join(dir, "index.db") + absPath := a.normalizePath(dbPath) + + if db, ok := a.localDBs[absPath]; ok { + return db, nil + } + + localDB, err := database.InitDB(absPath) + if err != nil { + return nil, fmt.Errorf("failed to init local db at %s: %w", absPath, err) + } + + // Initialize SCIP schema if not exists + _, err = localDB.Exec(` + CREATE TABLE IF NOT EXISTS scip_indices ( + scip_path TEXT PRIMARY KEY, + mtime INTEGER, + indexed_at DATETIME, + language TEXT, + version INTEGER DEFAULT 1 + ); + `) + // Migration for existing indices (version 2 includes forward-slash fix) + // We ignore the error here because the column might already exist from a previous migration. + _, _ = localDB.Exec("ALTER TABLE scip_indices ADD COLUMN version INTEGER DEFAULT 1") + + if err != nil { + localDB.Close() + return nil, fmt.Errorf("failed to init local schema: %w", err) + } + + _, err = localDB.Exec(` + CREATE TABLE IF NOT EXISTS scip_symbols ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scip_path TEXT, + symbol TEXT, + name TEXT, + kind TEXT, + file_path TEXT, + dir_path TEXT, + start_line INTEGER, + end_line INTEGER, + parent_symbol TEXT, + is_local BOOLEAN, + is_anonymous BOOLEAN, + is_stdlib BOOLEAN, + ref_count INTEGER DEFAULT 0, + outbound_calls INTEGER DEFAULT 0, + impact_value INTEGER DEFAULT 0, + FOREIGN KEY(scip_path) REFERENCES scip_indices(scip_path) ON DELETE CASCADE + ); + + CREATE TABLE IF NOT EXISTS scip_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + scip_path TEXT, + source_symbol TEXT, + target_symbol TEXT, + FOREIGN KEY(scip_path) REFERENCES scip_indices(scip_path) ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS idx_scip_symbols_scope ON scip_symbols(scip_path, dir_path); + CREATE INDEX IF NOT EXISTS idx_scip_symbols_symbol ON scip_symbols(scip_path, symbol); + CREATE INDEX IF NOT EXISTS idx_scip_relationships_scip ON scip_relationships(scip_path); + CREATE INDEX IF NOT EXISTS idx_scip_relationships_src ON scip_relationships(scip_path, source_symbol); + CREATE INDEX IF NOT EXISTS idx_scip_relationships_tgt ON scip_relationships(scip_path, target_symbol); + `) + if err != nil { + localDB.Close() + return nil, fmt.Errorf("failed to init local schema: %w", err) + } + + a.localDBs[absPath] = localDB + return localDB, nil +} + diff --git a/frontend/dist/.gitkeep b/frontend/dist/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/frontend/src/Atlas.tsx b/frontend/src/Atlas.tsx index 4240a83..1f44d75 100644 --- a/frontend/src/Atlas.tsx +++ b/frontend/src/Atlas.tsx @@ -2,7 +2,7 @@ import { useEffect, useState, useRef } from 'react'; import { Icon } from '@iconify/react'; import cytoscape from 'cytoscape'; import fcose from 'cytoscape-fcose'; -import { SearchForIndexes, GetGraphData, GetWorkspaces, GetFileContent, RegenerateIndex } from '../wailsjs/go/main/App'; +import { SearchForIndexes, GetGraphData, GetWorkspaces, GetFileContent, RegenerateIndex, SearchSymbols, GetSymbolRelationships } from '../wailsjs/go/main/App'; import { EventsOn, EventsOff } from '../wailsjs/runtime/runtime'; import Editor from '@monaco-editor/react'; import ReactMarkdown from 'react-markdown'; @@ -34,7 +34,11 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) const [isSourceExpanded, setIsSourceExpanded] = useState(false); const editorRef = useRef(null); const [regenerating, setRegenerating] = useState>({}); - const [indexingStatus, setIndexingStatus] = useState<{ type: 'success' | 'error', message: string } | null>(null); + const [indexingStatus, setIndexingStatus] = useState<{ type: 'success' | 'error' | 'processing', message: string } | null>(null); + const [currentScope, setCurrentScope] = useState(''); + const [relationships, setRelationships] = useState<{ callers: any[], callees: any[] } | null>(null); + const [pendingFocus, setPendingFocus] = useState(null); + const [loadingRelationships, setLoadingRelationships] = useState(false); const getKindLabel = (kind: string, plural = false) => { if (kind === 'Struct') { @@ -155,7 +159,7 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) }, []); useEffect(() => { - if (indexingStatus) { + if (indexingStatus && indexingStatus.type !== 'processing') { const timer = setTimeout(() => setIndexingStatus(null), 5000); return () => clearTimeout(timer); } @@ -174,12 +178,28 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) }; fetchWorkspaces(); - EventsOn('workspace-updated', (ws: any[]) => { - const unique = Array.from(new Map(ws.map(item => [item.root, item])).values()); - setAllWorkspaces(unique); - }); - - return () => EventsOff('workspace-updated'); + EventsOn('workspace-updated', (ws: any[]) => { + const unique = Array.from(new Map(ws.map(item => [item.root, item])).values()); + setAllWorkspaces(unique); + }); + + EventsOn('indexing-status', (data: any) => { + if (data.status === 'done') { + setIndexingStatus({ type: 'success', message: data.message || 'Indexing complete!' }); + } else if (data.status === 'error') { + setIndexingStatus({ type: 'error', message: data.message || 'Indexing failed.' }); + } else { + setIndexingStatus({ + type: 'processing', + message: data.message || 'Indexing...' + }); + } + }); + + return () => { + EventsOff('workspace-updated'); + EventsOff('indexing-status'); + }; }, []); useEffect(() => { @@ -228,12 +248,19 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) { selector: "node[kind = 'Folder']", style: { - 'background-opacity': 0, - 'border-width': 0, - 'label': '', - 'padding': '40px', // Extra padding for logical clustering + 'background-color': '#4b5563', + 'background-opacity': 0.1, + 'border-width': 1, + 'border-color': '#4b5563', + 'border-style': 'dashed', + 'label': 'data(name)', + 'font-size': '10px', + 'text-valign': 'center', + 'text-halign': 'center', + 'color': 'rgba(255,255,255,0.4)', + 'padding': '40px', 'z-index': 1, - 'events': 'no' // Folders are transparent to clicks/hovers + 'shape': 'round-rectangle' } }, { @@ -362,6 +389,14 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) cyRef.current.on('mouseout', 'node', () => { cyRef.current?.elements().removeClass('dimmed').removeClass('highlighted'); }); + + // Double click to drill down into folders + cyRef.current.on('dblclick', 'node[kind="Folder"]', (evt) => { + const node = evt.target; + const folderId = node.id(); + const path = folderId.replace('dir:', ''); + setCurrentScope(path); + }); } const resizeObserver = new ResizeObserver(() => { @@ -382,14 +417,13 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) useEffect(() => { const node = selectedNode; if (node && node.path && workspaceRoot) { + setRelationships(null); GetFileContent(workspaceRoot, node.path).then(content => { setFileContent(content); - // If we have an editor ref and a start line, jump to it if (editorRef.current && node.startLine) { setTimeout(() => { const editor = editorRef.current; editor.revealLineInCenter(node.startLine); - // Clear old decorations (editor as any)._decorations = editor.deltaDecorations((editor as any)._decorations || [], [ { range: { startLineNumber: node.startLine, startColumn: 1, endLineNumber: node.endLine || node.startLine, endColumn: 100 }, @@ -406,18 +440,31 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) console.error("Error fetching file content:", e); setFileContent("// Error loading file content"); }); + setRelationships(null); + setLoadingRelationships(true); + GetSymbolRelationships(selectedIndex, node.id).then((rels: any) => { + setRelationships(rels); + setLoadingRelationships(false); + }).catch(() => { + setLoadingRelationships(false); + }); } else { setFileContent(''); + setRelationships(null); } }, [selectedNode, workspaceRoot]); + useEffect(() => { + setCurrentScope(''); + }, [selectedIndex]); + useEffect(() => { if (selectedIndex && cyRef.current) { setLoading(true); setSelectedNode(null); setSelectedEdge(null); - GetGraphData(selectedIndex).then((data) => { - if (data && cyRef.current) { + GetGraphData(selectedIndex, currentScope).then((data: any) => { + if (data && cyRef.current) { setLanguage(data.language || 'Go'); const cy = cyRef.current; cy.elements().remove(); @@ -437,7 +484,7 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) ...el, data: { ...el.data, - value: 0 // Folders take size from children + value: 60 // Minimum size for folder containers } }; } @@ -464,11 +511,25 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) layout.run(); cy.fit(undefined, 50); + + if (pendingFocus) { + const target = cy.getElementById(pendingFocus); + if (!target.empty()) { + target.select(); + setSelectedNode(target.data()); + cy.animate({ + center: { eles: target }, + zoom: 1.5, + duration: 500 + }); + } + setPendingFocus(null); + } } setLoading(false); }); } - }, [selectedIndex]); + }, [selectedIndex, currentScope]); const handleZoomIn = () => { cyRef.current?.zoom(cyRef.current.zoom() * 1.5); @@ -534,45 +595,30 @@ ${snippet || '// No content available'} const handleSearchChange = (val: string) => { setSearchTerm(val); - if (!val.trim() || !cyRef.current) { + if (!val.trim()) { setSearchResults([]); setShowSearchResults(false); return; } - const cy = cyRef.current; - const allNodes = cy.nodes().filter((n: any) => n.data('kind') !== 'Folder'); - const query = val.toLowerCase(); - - const filtered = allNodes.map((n: any) => n.data()) - .filter((data: any) => data.name.toLowerCase().includes(query)) - .sort((a: any, b: any) => { - const aName = a.name.toLowerCase(); - const bName = b.name.toLowerCase(); - - // Exact match first - if (aName === query && bName !== query) return -1; - if (bName === query && aName !== query) return 1; - - // Prefix match next - const aPrefix = aName.startsWith(query); - const bPrefix = bName.startsWith(query); - if (aPrefix && !bPrefix) return -1; - if (bPrefix && !aPrefix) return 1; - - return aName.localeCompare(bName); - }) - .slice(0, 15); - - setSearchResults(filtered); - setShowSearchResults(filtered.length > 0); + SearchSymbols(selectedIndex, val).then(results => { + setSearchResults(results || []); + setShowSearchResults((results || []).length > 0); + }).catch(err => { + console.error("Global search failed:", err); + }); }; const handleSearch = (e: React.FormEvent) => { e.preventDefault(); if (searchResults.length > 0) { const top = searchResults[0]; - focusSymbol(top.id); + if (top.path !== currentScope) { + setPendingFocus(top.id); + setCurrentScope(top.path || ''); + } else { + focusSymbol(top.id); + } setShowSearchResults(false); } else if (searchTerm) { focusSymbol(searchTerm); @@ -617,13 +663,21 @@ ${snippet || '// No content available'} {indexingStatus && ( -
- +
+ {indexingStatus.message} - + {indexingStatus.type !== 'processing' && ( + + )}
)}
@@ -651,7 +705,12 @@ ${snippet || '// No content available'} + + {currentScope && ( + <> + +
+ {currentScope.split('/').map((part, i, arr) => ( +
+ + {i < arr.length - 1 && } +
+ ))} +
+ + )} + + + )} + {loading && (
@@ -799,6 +892,28 @@ ${snippet || '// No content available'}
+ {selectedNode.path && fileContent && ( +
+

Signature

+
+ + {fileContent.split('\n').slice(selectedNode.startLine - 1, (selectedNode.endLine || selectedNode.startLine) + 1).join('\n')} + +
+
+ )} + {selectedNode.docstring && (

Documentation

@@ -861,6 +976,76 @@ ${snippet || '// No content available'}
)} + + {loadingRelationships && ( +
+

Analyzing Connections

+
+
+
+
+
+ )} + + {relationships && (relationships.callers.length > 0 || relationships.callees.length > 0) && ( + <> + {relationships.callers.length > 0 && ( +
+

Callers ({relationships.callers.length})

+
+ {relationships.callers.map((r: any, i: number) => ( + + ))} +
+
+ )} + + {relationships.callees.length > 0 && ( +
+

Callees ({relationships.callees.length})

+
+ {relationships.callees.map((r: any, i: number) => ( + + ))} +
+
+ )} + + )}
diff --git a/frontend/wailsjs/go/main/App.d.ts b/frontend/wailsjs/go/main/App.d.ts index ddb9d68..97aa56c 100644 --- a/frontend/wailsjs/go/main/App.d.ts +++ b/frontend/wailsjs/go/main/App.d.ts @@ -24,7 +24,7 @@ export function GetDownloadProgress(arg1:string):Promise; export function GetFileContent(arg1:string,arg2:string):Promise; -export function GetGraphData(arg1:string):Promise; +export function GetGraphData(arg1:string,arg2:string):Promise; export function GetInferenceModels():Promise>; @@ -36,12 +36,16 @@ export function GetPreferences():Promise; export function GetScipIndexerStatus(arg1:string):Promise>; +export function GetSymbolRelationships(arg1:string,arg2:string):Promise>>; + export function GetWorkspaceConfigs(arg1:string):Promise>>; export function GetWorkspaces():Promise>; export function ImportCommunityRule(arg1:string,arg2:string):Promise; +export function IngestIndex(arg1:string):Promise; + export function InitializeAstGrepConfig(arg1:string,arg2:string):Promise; export function InstallAstGrep():Promise; @@ -80,6 +84,8 @@ export function SearchCommunityRules(arg1:string,arg2:string,arg3:string):Promis export function SearchForIndexes(arg1:string):Promise>; +export function SearchSymbols(arg1:string,arg2:string):Promise>; + export function TestInferenceConnection(arg1:string,arg2:string):Promise; export function WriteMarkdown(arg1:string,arg2:string):Promise; diff --git a/frontend/wailsjs/go/main/App.js b/frontend/wailsjs/go/main/App.js index aa00775..b0674f7 100644 --- a/frontend/wailsjs/go/main/App.js +++ b/frontend/wailsjs/go/main/App.js @@ -42,8 +42,8 @@ export function GetFileContent(arg1, arg2) { return window['go']['main']['App']['GetFileContent'](arg1, arg2); } -export function GetGraphData(arg1) { - return window['go']['main']['App']['GetGraphData'](arg1); +export function GetGraphData(arg1, arg2) { + return window['go']['main']['App']['GetGraphData'](arg1, arg2); } export function GetInferenceModels() { @@ -66,6 +66,10 @@ export function GetScipIndexerStatus(arg1) { return window['go']['main']['App']['GetScipIndexerStatus'](arg1); } +export function GetSymbolRelationships(arg1, arg2) { + return window['go']['main']['App']['GetSymbolRelationships'](arg1, arg2); +} + export function GetWorkspaceConfigs(arg1) { return window['go']['main']['App']['GetWorkspaceConfigs'](arg1); } @@ -78,6 +82,10 @@ export function ImportCommunityRule(arg1, arg2) { return window['go']['main']['App']['ImportCommunityRule'](arg1, arg2); } +export function IngestIndex(arg1) { + return window['go']['main']['App']['IngestIndex'](arg1); +} + export function InitializeAstGrepConfig(arg1, arg2) { return window['go']['main']['App']['InitializeAstGrepConfig'](arg1, arg2); } @@ -154,6 +162,10 @@ export function SearchForIndexes(arg1) { return window['go']['main']['App']['SearchForIndexes'](arg1); } +export function SearchSymbols(arg1, arg2) { + return window['go']['main']['App']['SearchSymbols'](arg1, arg2); +} + export function TestInferenceConnection(arg1, arg2) { return window['go']['main']['App']['TestInferenceConnection'](arg1, arg2); } diff --git a/frontend/wailsjs/go/models.ts b/frontend/wailsjs/go/models.ts index c460124..8496530 100644 --- a/frontend/wailsjs/go/models.ts +++ b/frontend/wailsjs/go/models.ts @@ -91,6 +91,74 @@ export namespace main { return a; } } + export class Member { + name: string; + kind: string; + symbol: string; + + static createFrom(source: any = {}) { + return new Member(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.name = source["name"]; + this.kind = source["kind"]; + this.symbol = source["symbol"]; + } + } + export class GraphNode { + id: string; + name: string; + value: number; + category: number; + path: string; + kind: string; + docstring: string; + members: Member[]; + loc: number; + startLine: number; + endLine: number; + parent?: string; + + static createFrom(source: any = {}) { + return new GraphNode(source); + } + + constructor(source: any = {}) { + if ('string' === typeof source) source = JSON.parse(source); + this.id = source["id"]; + this.name = source["name"]; + this.value = source["value"]; + this.category = source["category"]; + this.path = source["path"]; + this.kind = source["kind"]; + this.docstring = source["docstring"]; + this.members = this.convertValues(source["members"], Member); + this.loc = source["loc"]; + this.startLine = source["startLine"]; + this.endLine = source["endLine"]; + this.parent = source["parent"]; + } + + convertValues(a: any, classs: any, asMap: boolean = false): any { + if (!a) { + return a; + } + if (a.slice && a.map) { + return (a as any[]).map(elem => this.convertValues(elem, classs)); + } else if ("object" === typeof a) { + if (asMap) { + for (const key of Object.keys(a)) { + a[key] = new classs(a[key]); + } + return a; + } + return new classs(a); + } + return a; + } + } export class LocalRuleDetails { id: string; message: string; @@ -127,6 +195,7 @@ export namespace main { this.frontMatter = source["frontMatter"]; } } + export class UserPreferences { theme: string; windowWidth: number; From 422a8a75c705a2c42d2f5b6e88b8d4c582a5e2e3 Mon Sep 17 00:00:00 2001 From: Tom Maiaroto Date: Wed, 25 Mar 2026 07:15:13 -0700 Subject: [PATCH 2/3] add directory jump --- frontend/dist/.gitkeep | 0 frontend/src/Atlas.tsx | 50 ++++++- scip_ingest.go | 309 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 357 insertions(+), 2 deletions(-) create mode 100644 frontend/dist/.gitkeep create mode 100644 scip_ingest.go diff --git a/frontend/dist/.gitkeep b/frontend/dist/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/frontend/src/Atlas.tsx b/frontend/src/Atlas.tsx index 1f44d75..4c627cf 100644 --- a/frontend/src/Atlas.tsx +++ b/frontend/src/Atlas.tsx @@ -39,6 +39,7 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) const [relationships, setRelationships] = useState<{ callers: any[], callees: any[] } | null>(null); const [pendingFocus, setPendingFocus] = useState(null); const [loadingRelationships, setLoadingRelationships] = useState(false); + const [graphData, setGraphData] = useState(null); const getKindLabel = (kind: string, plural = false) => { if (kind === 'Struct') { @@ -464,7 +465,8 @@ export default function Atlas({ workspaceRoot, onWorkspaceChange }: AtlasProps) setSelectedNode(null); setSelectedEdge(null); GetGraphData(selectedIndex, currentScope).then((data: any) => { - if (data && cyRef.current) { + if (data && cyRef.current) { + setGraphData(data); setLanguage(data.language || 'Go'); const cy = cyRef.current; cy.elements().remove(); @@ -756,7 +758,7 @@ ${snippet || '// No content available'} {/* Breadcrumbs / Scope Navigation */} {selectedIndex && (
-
+
)} + + {graphData?.elements?.filter((e: any) => e.group === 'nodes' && e.data.kind === 'Folder').length > 0 && ( +
+ +
    +
  • + Subdirectories +
    • +
    + {graphData.elements + .filter((e: any) => e.group === 'nodes' && e.data.kind === 'Folder') + .sort((a: any, b: any) => a.data.name.localeCompare(b.data.name)) + .map((folder: any) => ( +
  • + +
    • + ))} +
    +
+
+ )}
)} diff --git a/scip_ingest.go b/scip_ingest.go new file mode 100644 index 0000000..494814f --- /dev/null +++ b/scip_ingest.go @@ -0,0 +1,309 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "time" + + scip "github.com/sourcegraph/scip/bindings/go/scip" + wailsRuntime "github.com/wailsapp/wails/v2/pkg/runtime" + "google.golang.org/protobuf/proto" +) + +// IngestIndex parses a SCIP file and stores its symbols/relationships in SQLite. +func (a *App) IngestIndex(scipPath string) error { + scipPath = a.normalizePath(scipPath) + // 1. Get/Init local database for this workspace + localDB, err := a.getLocalDB(scipPath) + if err != nil { + return err + } + + // 2. Check if already indexed and up to date + info, err := os.Stat(scipPath) + if err != nil { + return err + } + mtime := info.ModTime().Unix() + + // 2. Check if we already have this index up to date (version 2 includes forward-slash fix) + var existingMtime int64 + var version int + err = localDB.QueryRow("SELECT mtime, version FROM scip_indices WHERE scip_path = ?", scipPath).Scan(&existingMtime, &version) + if err == nil && existingMtime >= mtime && version >= 2 { + return nil // Up to date + } + + // 3. Clear existing data for this index to avoid duplicates + _, _ = localDB.Exec("DELETE FROM scip_symbols WHERE scip_path = ?", scipPath) + _, _ = localDB.Exec("DELETE FROM scip_relationships WHERE scip_path = ?", scipPath) + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Reading 65MB+ SCIP File...", + }) + + data, err := os.ReadFile(scipPath) + if err != nil { + return err + } + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Unmarshaling Protobuf...", + }) + + var index scip.Index + if err := proto.Unmarshal(data, &index); err != nil { + return fmt.Errorf("failed to parse SCIP index: %w", err) + } + data = nil // Free memory + + // 4. Infer language from filename + language := "" + filename := filepath.Base(scipPath) + if strings.HasPrefix(filename, "index-") { + language = strings.TrimPrefix(filename, "index-") + language = strings.TrimSuffix(language, ".scip") + } + if language == "" { + language = "go" // Default fallback + } + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Analyzing Architecture...", + }) + + // 3. Architecture Analysis (Pruning & Impact Calculation) + projectPkg := a.detectProjectPackage(&index) + + // Pruning helpers + isLocal := func(sym string) bool { return strings.Contains(sym, "local") } + isAnonymous := func(sym string) bool { + low := strings.ToLower(sym) + return strings.Contains(low, "$anon") || strings.Contains(low, "func.") || strings.Contains(low, "lambda$") + } + isStdLib := func(sym string) bool { + if projectPkg == "" { return false } + low := strings.ToLower(sym) + stdlibPrefixes := []string{"go/", "typescript/", "python/", "builtin/"} + if strings.Contains(low, strings.ToLower(projectPkg)) { return false } + for _, p := range stdlibPrefixes { + if strings.Contains(low, p) { return true } + } + return false + } + + refCount := make(map[string]int) + outboundCalls := make(map[string]int) + symbolToInfo := make(map[string]*scip.SymbolInformation) + + for _, doc := range index.Documents { + for _, si := range doc.Symbols { + symbolToInfo[si.Symbol] = si + } + var currentScope string + for _, occ := range doc.Occurrences { + if isLocal(occ.Symbol) || isAnonymous(occ.Symbol) { continue } + if occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 { + currentScope = occ.Symbol + continue + } + refCount[occ.Symbol]++ + if currentScope != "" { + outboundCalls[currentScope]++ + } + } + } + for _, si := range index.ExternalSymbols { + symbolToInfo[si.Symbol] = si + } + + // 4. Persistence + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Storing in SQLite...", + }) + + tx, err := localDB.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + // Clean old data for this path + tx.Exec("DELETE FROM scip_indices WHERE scip_path = ?", scipPath) + + lang := detectLanguage(index.Documents) + _, err = tx.Exec("INSERT INTO scip_indices (scip_path, mtime, indexed_at, language) VALUES (?, ?, ?, ?)", + scipPath, mtime, time.Now().Format(time.RFC3339), lang) + if err != nil { return err } + + var symbolsIngested, relsIngested int + + stmtSym, _ := tx.Prepare(` + INSERT INTO scip_symbols ( + scip_path, symbol, name, kind, file_path, dir_path, + start_line, end_line, parent_symbol, + is_local, is_anonymous, is_stdlib, + ref_count, outbound_calls, impact_value + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + `) + defer stmtSym.Close() + + stmtRel, _ := tx.Prepare(` + INSERT INTO scip_relationships (scip_path, source_symbol, target_symbol) VALUES (?, ?, ?) + `) + defer stmtRel.Close() + + // Pre-grouping for parent discovery + symbolToDescriptors := make(map[string][]*scip.Descriptor) + byLen := make(map[int][]string) + for sym := range symbolToInfo { + parsed, _ := scip.ParseSymbol(sym) + if len(parsed.Descriptors) > 0 { + symbolToDescriptors[sym] = parsed.Descriptors + byLen[len(parsed.Descriptors)] = append(byLen[len(parsed.Descriptors)], sym) + } + } + + for _, doc := range index.Documents { + dir := filepath.ToSlash(filepath.Dir(doc.RelativePath)) + if dir == "." { + dir = "root" + } + + for _, occ := range doc.Occurrences { + if occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 { + sym := occ.Symbol + if isLocal(sym) || isAnonymous(sym) { continue } + + parsed, err := scip.ParseSymbol(sym) + if err != nil || len(parsed.Descriptors) == 0 { continue } + + leaf := parsed.Descriptors[len(parsed.Descriptors)-1] + if leaf.Suffix == scip.Descriptor_Namespace { continue } + + name := leaf.Name + kind := "Function" + switch leaf.Suffix { + case scip.Descriptor_Type: kind = "Struct" + case scip.Descriptor_Term: kind = "Variable" + case scip.Descriptor_Method, scip.Descriptor_Macro: kind = "Function" + case scip.Descriptor_Parameter, scip.Descriptor_TypeParameter: continue + } + if kind == "Struct" && strings.Contains(strings.ToLower(sym), "interface") { + kind = "Interface" + } + + // Parent Discovery + parentSym := "" + l := len(parsed.Descriptors) + if l > 1 { + candidates := byLen[l-1] + for _, pSym := range candidates { + pDescriptors := symbolToDescriptors[pSym] + match := true + for i := 0; i < l-1; i++ { + if pDescriptors[i].Name != parsed.Descriptors[i].Name || pDescriptors[i].Suffix != parsed.Descriptors[i].Suffix { + match = false + break + } + } + if match { + parentSym = pSym + break + } + } + } + + impact := (outboundCalls[sym] * 5) + (refCount[sym] * 10) + if kind == "Struct" || kind == "Interface" { impact += 20 } + if impact < 8 { impact = 8 } + + _, err = stmtSym.Exec( + scipPath, sym, name, kind, doc.RelativePath, dir, + int(occ.Range[0]+1), int(occ.Range[0]+1), parentSym, + isLocal(sym), isAnonymous(sym), isStdLib(sym), + refCount[sym], outboundCalls[sym], impact, + ) + if err != nil { return err } + symbolsIngested++ + } else { + // Relationship logic (for edges) + // We'll calculate this on the fly or store it. For scale, let's store it. + // Wait, Pass 4 logic needs currentScope. + // I'll handle relationships in a separate loop or refine this one. + } + } + } + + // Relationships Pass + for _, doc := range index.Documents { + var currentScope string + for _, occ := range doc.Occurrences { + if isLocal(occ.Symbol) || isAnonymous(occ.Symbol) { continue } + if occ.SymbolRoles&int32(scip.SymbolRole_Definition) != 0 { + currentScope = occ.Symbol + continue + } + if currentScope != "" && occ.Symbol != "" { + _, err = stmtRel.Exec(scipPath, currentScope, occ.Symbol) + if err != nil { return err } + relsIngested++ + } + } + } + + // 5. Update index metadata with version 2 + _, err = tx.Exec("INSERT OR REPLACE INTO scip_indices (scip_path, mtime, indexed_at, language, version) VALUES (?, ?, datetime('now'), ?, 2)", scipPath, mtime, language) + if err != nil { + tx.Rollback() + return err + } + + if err := tx.Commit(); err != nil { + return err + } + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "complete", + "message": "Index Ready!", + }) + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "done", + "message": fmt.Sprintf("Indexed %d symbols and %d relationships", symbolsIngested, relsIngested), + }) + + return nil +} + +func (a *App) detectProjectPackage(index *scip.Index) string { + if len(index.Documents) == 0 { return "" } + for _, doc := range index.Documents { + if len(doc.Symbols) > 0 { + sym := doc.Symbols[0].Symbol + if idx := strings.Index(sym, " "); idx != -1 { + parts := strings.Split(sym, " ") + if len(parts) >= 4 { + pkg := parts[3] + if slashIdx := strings.Index(pkg, "/"); slashIdx != -1 { + return pkg[:slashIdx] + } + return pkg + } + } + } + } + return "" +} From 367bc316550ae9d68d86f8d3d3b2c2da587a16e4 Mon Sep 17 00:00:00 2001 From: Tom Maiaroto Date: Wed, 25 Mar 2026 07:39:50 -0700 Subject: [PATCH 3/3] tests and fixes --- app.go | 9 ++++ app_graph_test.go | 114 ++++++++++++++++++++++++++++++++++++++++++++++ main.go | 2 +- scip_ingest.go | 72 ++++++++++++++++------------- 4 files changed, 165 insertions(+), 32 deletions(-) create mode 100644 app_graph_test.go diff --git a/app.go b/app.go index 1d8fe50..ffb7f0d 100644 --- a/app.go +++ b/app.go @@ -785,6 +785,15 @@ func (a *App) BeforeClose(ctx context.Context) bool { } func (a *App) Shutdown(ctx context.Context) { + a.localDBMu.Lock() + for p, db := range a.localDBs { + fmt.Printf("Hub: Closing local database: %s\n", p) + _ = db.Close() + } + // Clear the map to allow garbage collection and prevent re-use of closed handles + a.localDBs = make(map[string]*database.DB) + a.localDBMu.Unlock() + if a.isHub { lockPath := mcp.GetHubLockPath() _ = os.Remove(lockPath) diff --git a/app_graph_test.go b/app_graph_test.go new file mode 100644 index 0000000..56aa3bd --- /dev/null +++ b/app_graph_test.go @@ -0,0 +1,114 @@ +package main + +import ( + "context" + "os" + "path/filepath" + "testing" + + scip "github.com/sourcegraph/scip/bindings/go/scip" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" +) + +func TestGetGraphData_Pruning(t *testing.T) { + app := NewApp() + defer app.Shutdown(context.Background()) + tempDir := t.TempDir() + scipPath := filepath.Join(tempDir, "test.scip") + + // Create a mock SCIP index + index := &scip.Index{ + Documents: []*scip.Document{ + { + RelativePath: "pkg/main.go", + Symbols: []*scip.SymbolInformation{ + {Symbol: "scip-go . . . pkg/Main#"}, + {Symbol: "scip-go . . . pkg/main()."}, + }, + Occurrences: []*scip.Occurrence{ + { + Symbol: "scip-go . . . pkg/Main#", + SymbolRoles: int32(scip.SymbolRole_Definition), + Range: []int32{0, 0, 0, 4}, + }, + { + Symbol: "scip-go . . . pkg/main().", + SymbolRoles: int32(scip.SymbolRole_Definition), + Range: []int32{5, 0, 5, 4}, + }, + { + Symbol: "scip-go . . . pkg/main().local1", + SymbolRoles: int32(scip.SymbolRole_Definition), + Range: []int32{6, 0, 6, 4}, + }, + { + Symbol: "scip-go . . . pkg/main().$anon1", // Anonymous + SymbolRoles: int32(scip.SymbolRole_Definition), + Range: []int32{7, 0, 7, 4}, + }, + // Reference to stdlib + { + Symbol: "go/fmt/Println().", + Range: []int32{8, 0, 8, 4}, + }, + }, + }, + }, + } + + data, err := proto.Marshal(index) + require.NoError(t, err) + err = os.WriteFile(scipPath, data, 0644) + require.NoError(t, err) + + // Test 1: Full load without scope + graph, err := app.GetGraphData(scipPath, "") + require.NoError(t, err) + + // Verify pruning + for _, el := range graph.Elements { + if el.Group == "nodes" { + node, ok := el.Data.(GraphNode) + if ok { + assert.NotContains(t, node.ID, "local", "Local variables should be pruned") + assert.NotContains(t, node.ID, "$anon", "Anonymous closures should be pruned") + } + } + if el.Group == "edges" { + data := el.Data.(map[string]interface{}) + assert.NotContains(t, data["target"].(string), "go/fmt", "StdLib edges should be pruned") + } + } + + // Test 2: Hierarchical filter (Wrong scope) + graphScope, err := app.GetGraphData(scipPath, "otherpkg") + require.NoError(t, err) + + symbolNodeCount := 0 + for _, el := range graphScope.Elements { + if el.Group == "nodes" { + node := el.Data.(GraphNode) + if node.Kind != "Folder" { + symbolNodeCount++ + } + } + } + assert.Equal(t, 0, symbolNodeCount, "Should have no symbols outside scope") + + // Test 3: Hierarchical filter (Correct scope) + graphInScope, err := app.GetGraphData(scipPath, "pkg") + require.NoError(t, err) + + foundMain := false + for _, el := range graphInScope.Elements { + if el.Group == "nodes" { + node := el.Data.(GraphNode) + if node.Name == "main" { + foundMain = true + } + } + } + assert.True(t, foundMain, "Should find 'main' symbol in 'pkg' scope") +} diff --git a/main.go b/main.go index 97210de..a4ee0df 100644 --- a/main.go +++ b/main.go @@ -178,7 +178,7 @@ func main() { }, Mac: &mac.Options{ TitleBar: &mac.TitleBar{ - TitlebarAppearsTransparent: true, + TitlebarAppearsTransparent: false, HideTitle: true, HideTitleBar: false, FullSizeContent: false, diff --git a/scip_ingest.go b/scip_ingest.go index 494814f..12c8f9e 100644 --- a/scip_ingest.go +++ b/scip_ingest.go @@ -40,22 +40,26 @@ func (a *App) IngestIndex(scipPath string) error { _, _ = localDB.Exec("DELETE FROM scip_symbols WHERE scip_path = ?", scipPath) _, _ = localDB.Exec("DELETE FROM scip_relationships WHERE scip_path = ?", scipPath) - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "processing", - "message": "Reading 65MB+ SCIP File...", - }) + if a.ctx != nil { + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Reading 65MB+ SCIP File...", + }) + } data, err := os.ReadFile(scipPath) if err != nil { return err } - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "processing", - "message": "Unmarshaling Protobuf...", - }) + if a.ctx != nil { + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Unmarshaling Protobuf...", + }) + } var index scip.Index if err := proto.Unmarshal(data, &index); err != nil { @@ -74,11 +78,13 @@ func (a *App) IngestIndex(scipPath string) error { language = "go" // Default fallback } - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "processing", - "message": "Analyzing Architecture...", - }) + if a.ctx != nil { + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Analyzing Architecture...", + }) + } // 3. Architecture Analysis (Pruning & Impact Calculation) projectPkg := a.detectProjectPackage(&index) @@ -126,11 +132,13 @@ func (a *App) IngestIndex(scipPath string) error { } // 4. Persistence - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "processing", - "message": "Storing in SQLite...", - }) + if a.ctx != nil { + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "processing", + "message": "Storing in SQLite...", + }) + } tx, err := localDB.Begin() if err != nil { @@ -273,17 +281,19 @@ func (a *App) IngestIndex(scipPath string) error { return err } - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "complete", - "message": "Index Ready!", - }) - - wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ - "path": scipPath, - "status": "done", - "message": fmt.Sprintf("Indexed %d symbols and %d relationships", symbolsIngested, relsIngested), - }) + if a.ctx != nil { + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "complete", + "message": "Index Ready!", + }) + + wailsRuntime.EventsEmit(a.ctx, "indexing-status", map[string]string{ + "path": scipPath, + "status": "done", + "message": fmt.Sprintf("Indexed %d symbols and %d relationships", symbolsIngested, relsIngested), + }) + } return nil }