From b887331a0c196f52ccf91452130ea1e506748d4b Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Tue, 14 Jul 2026 11:34:19 +1000 Subject: [PATCH 01/26] Add built-in MCP server (dexter mcp) Expose the index to AI agents over the Model Context Protocol, modeled on gopls mcp. Nine tools, addressed by module/function name rather than file positions because Elixir modules are not tied to files: - dexter_workspace, dexter_search, dexter_definition, dexter_references, dexter_module_api, dexter_file_outline, dexter_implementations, dexter_call_hierarchy, dexter_reindex Transports: stdio (dexter mcp), streamable HTTP (dexter mcp --listen), and attached mode on a running LSP session (dexter lsp --mcp-listen) sharing open buffers and caches. dexter mcp --instructions prints an agent-facing usage guide. Reuses the LSP server internals: reindexing via the extracted Server.ReindexWorkspace (backgroundReindex body, now also callable blocking), reference collection via Server.CollectReferences, and doc extraction via the tokenizer. No index schema or parser changes. Uses the official github.com/modelcontextprotocol/go-sdk. --- CHANGELOG.md | 6 + README.md | 28 +++ cmd/main.go | 133 +++++++++++++- docs/architecture.md | 1 + go.mod | 8 +- go.sum | 26 ++- integration_test.go | 174 ++++++++++++++++++ internal/lsp/api.go | 114 ++++++++++++ internal/lsp/hover.go | 2 +- internal/lsp/server.go | 207 +++++++++++---------- internal/mcp/call_hierarchy.go | 89 +++++++++ internal/mcp/definition.go | 115 ++++++++++++ internal/mcp/file_outline.go | 109 ++++++++++++ internal/mcp/implementations.go | 120 +++++++++++++ internal/mcp/instructions.md | 27 +++ internal/mcp/mcp.go | 160 +++++++++++++++++ internal/mcp/mcp_test.go | 184 +++++++++++++++++++ internal/mcp/module_api.go | 194 ++++++++++++++++++++ internal/mcp/references.go | 67 +++++++ internal/mcp/reindex.go | 24 +++ internal/mcp/search.go | 39 ++++ internal/mcp/serve.go | 20 +++ internal/mcp/tools_test.go | 307 ++++++++++++++++++++++++++++++++ internal/mcp/workspace.go | 82 +++++++++ internal/store/store.go | 48 +++++ internal/store/store_test.go | 86 +++++++++ 26 files changed, 2246 insertions(+), 124 deletions(-) create mode 100644 internal/lsp/api.go create mode 100644 internal/mcp/call_hierarchy.go create mode 100644 internal/mcp/definition.go create mode 100644 internal/mcp/file_outline.go create mode 100644 internal/mcp/implementations.go create mode 100644 internal/mcp/instructions.md create mode 100644 internal/mcp/mcp.go create mode 100644 internal/mcp/mcp_test.go create mode 100644 internal/mcp/module_api.go create mode 100644 internal/mcp/references.go create mode 100644 internal/mcp/reindex.go create mode 100644 internal/mcp/search.go create mode 100644 internal/mcp/serve.go create mode 100644 internal/mcp/tools_test.go create mode 100644 internal/mcp/workspace.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fe4c935..eaae6b2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [Unreleased] + +### Added + +- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Nine tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide + ## [0.7.1] - 2026-06-12 ### Added diff --git a/README.md b/README.md index c3f5726..b2e835b 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ A fast, full-featured Elixir LSP optimized for large Elixir codebases. - [Look up definitions](#look-up-definitions) - [Find references](#find-references) - [Reindexing files manually](#reindexing-files-manually) +- [MCP server](#mcp-server) - [Hover documentation](#hover-documentation) - [Cursor-position-aware resolution](#cursor-position-aware-resolution) - [Rename](#rename) @@ -453,6 +454,33 @@ When running as an LSP server, dexter automatically: - Runs an incremental reindex on startup - Watches `.git/HEAD` for branch switches and reindexes when detected +## MCP server + +Dexter includes a built-in [Model Context Protocol](https://modelcontextprotocol.io) server, modeled on `gopls mcp`, so AI agents can navigate Elixir codebases through the index instead of grep. Tools cover symbol search, definitions with docs and specs, references, module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. + +Register it with your MCP client. For Claude Code: + +```sh +claude mcp add dexter -- dexter mcp +``` + +Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use, keeps the index fresh across git branch switches, and exposes a `dexter_reindex` tool for agents to call after editing files. + +Useful variants: + +```sh +# Serve over streamable HTTP instead of stdio +dexter mcp --listen localhost:8092 + +# Print the agent-facing usage guide (save as context for clients that want it) +dexter mcp --instructions + +# Expose MCP from a running LSP session (shares open buffers and caches) +dexter lsp --mcp-listen=localhost:8092 +``` + +The MCP server and an editor LSP can run side by side: both read the same `.dexter/dexter.db` index. + ## Hover documentation Dexter serves hover docs (`textDocument/hover`) for functions, modules, and types. When you hover over a symbol, it looks up the definition in the index and reads the `@doc`, `@moduledoc`, `@typedoc`, or `@spec` annotations from the source file. diff --git a/cmd/main.go b/cmd/main.go index 2434880..3252d15 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -1,17 +1,23 @@ package main import ( + "context" "fmt" "io/fs" "log" + "net" + "net/http" "os" + "os/signal" "path/filepath" "runtime" "sync" "sync/atomic" + "syscall" "time" dexter_lsp "github.com/remoteoss/dexter/internal/lsp" + dexter_mcp "github.com/remoteoss/dexter/internal/mcp" "github.com/remoteoss/dexter/internal/parser" "github.com/remoteoss/dexter/internal/stdlib" "github.com/remoteoss/dexter/internal/store" @@ -100,6 +106,7 @@ func main() { }, } + var lspMCPListen string lspCmd := &cobra.Command{ Use: "lsp [path]", Short: "Start the LSP server (stdio)", @@ -109,10 +116,33 @@ func main() { if err != nil { return err } - cmdLSP(projectRoot) + cmdLSP(projectRoot, lspMCPListen) return nil }, } + lspCmd.Flags().StringVar(&lspMCPListen, "mcp-listen", "", "Also serve MCP over streamable HTTP on this address, sharing the LSP session") + + var mcpListen string + var mcpInstructions bool + mcpCmd := &cobra.Command{ + Use: "mcp [path]", + Short: "Start the MCP server (stdio)", + Args: cobra.MaximumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + if mcpInstructions { + fmt.Print(dexter_mcp.Instructions) + return nil + } + projectRoot, err := resolvePath(args, 0) + if err != nil { + return err + } + cmdMCP(projectRoot, mcpListen) + return nil + }, + } + mcpCmd.Flags().StringVar(&mcpListen, "listen", "", "Serve MCP over streamable HTTP on this address instead of stdio") + mcpCmd.Flags().BoolVar(&mcpInstructions, "instructions", false, "Print the MCP instructions file and exit") versionCmd := &cobra.Command{ Use: "version", @@ -122,7 +152,7 @@ func main() { }, } - rootCmd.AddCommand(initCmd, reindexCmd, lookupCmd, referencesCmd, lspCmd, versionCmd) + rootCmd.AddCommand(initCmd, reindexCmd, lookupCmd, referencesCmd, lspCmd, mcpCmd, versionCmd) if err := rootCmd.Execute(); err != nil { os.Exit(1) @@ -498,9 +528,57 @@ func cmdReferences(projectRoot string, module string, function string) { } } -func cmdLSP(projectRoot string) { +func cmdLSP(projectRoot string, mcpListen string) { projectRoot = findProjectRoot(projectRoot) + s := openStoreForServer(projectRoot) + defer func() { + if err := s.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to close store: %v\n", err) + } + }() + + log.SetOutput(os.Stderr) + log.Printf("Dexter LSP v%s starting (root: %s)", version.Version, projectRoot) + + server := dexter_lsp.NewServer(s, projectRoot) + + // Attached MCP mode: serve MCP over HTTP from the same process, sharing + // the live LSP server so tools see open editor buffers and warm caches. + // The LSP connection's lifetime is authoritative: when it ends, the MCP + // listener goes with it. + var httpSrv *http.Server + if mcpListen != "" { + ln, err := net.Listen("tcp", mcpListen) + if err != nil { + fatal(err) + } + log.Printf("MCP server listening on %s", ln.Addr()) + h := dexter_mcp.NewHandler(dexter_mcp.Config{LSP: server, Store: s, ProjectRoot: projectRoot}) + httpSrv = &http.Server{Handler: dexter_mcp.HTTPHandler(h)} + go func() { + if err := httpSrv.Serve(ln); err != nil && err != http.ErrServerClosed { + log.Printf("MCP server error: %v", err) + } + }() + } + + serveErr := dexter_lsp.Serve(server, os.Stdin, os.Stdout) + + if httpSrv != nil { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = httpSrv.Shutdown(shutdownCtx) + } + if serveErr != nil { + fatal(serveErr) + } +} + +// openStoreForServer opens the index with the recovery behavior long-running +// servers need: a corrupted database or an index version mismatch triggers a +// full rebuild instead of an error. +func openStoreForServer(projectRoot string) *store.Store { const maxOpenAttempts = 3 var s *store.Store for attempt := 1; ; attempt++ { @@ -532,16 +610,59 @@ func cmdLSP(projectRoot string) { fatal(openErr) } } + return s +} + +// cmdMCP starts the headless MCP server. Logs go to stderr; stdout belongs to +// the MCP stdio transport. +func cmdMCP(projectRoot string, listen string) { + projectRoot = findProjectRoot(projectRoot) + + log.SetOutput(os.Stderr) + s := openStoreForServer(projectRoot) defer func() { if err := s.Close(); err != nil { fmt.Fprintf(os.Stderr, "Warning: failed to close store: %v\n", err) } }() - log.SetOutput(os.Stderr) - log.Printf("Dexter LSP v%s starting (root: %s)", version.Version, projectRoot) + server := dexter_lsp.NewServer(s, projectRoot) + if root, ok := stdlib.Resolve(s, "", projectRoot); ok { + server.SetStdlibRoot(root) + } + + // Serve only once the index reflects the current tree: an empty index is + // built from scratch, an existing one gets a fast incremental update. + server.ReindexWorkspace() + server.WatchGitHead() + + h := dexter_mcp.NewHandler(dexter_mcp.Config{LSP: server, Store: s, ProjectRoot: projectRoot}) + + log.Printf("Dexter MCP v%s starting (root: %s)", version.Version, projectRoot) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + if listen != "" { + ln, err := net.Listen("tcp", listen) + if err != nil { + fatal(err) + } + log.Printf("MCP server listening on %s", ln.Addr()) + httpSrv := &http.Server{Handler: dexter_mcp.HTTPHandler(h)} + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = httpSrv.Shutdown(shutdownCtx) + }() + if err := httpSrv.Serve(ln); err != nil && err != http.ErrServerClosed { + fatal(err) + } + return + } - if err := dexter_lsp.Serve(os.Stdin, os.Stdout, s, projectRoot); err != nil { + if err := dexter_mcp.RunStdio(ctx, h); err != nil && ctx.Err() == nil { fatal(err) } } diff --git a/docs/architecture.md b/docs/architecture.md index 00e2610..5484102 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -9,6 +9,7 @@ Dexter is a fast Elixir LSP server. It indexes module and function definitions f - `internal/store/` — SQLite layer. Tables: `files` (path + mtime), `definitions` (module, function, kind, line, file_path, delegate_to, delegate_as), `refs` (module, function, line, file_path, kind). - `internal/lsp/` — LSP server. `server.go` handles all LSP methods. `elixir.go` contains pure functions for cursor expression extraction, alias/import/use extraction (tokenizer-based), and use-chain parsing. `rename.go` has rename helpers. `hover.go` has hover formatting. `documents.go` is an in-memory open-buffer store. - `internal/treesitter/` — Tree-sitter integration for scope-aware variable rename and go-to-references. +- `internal/mcp/` — Model Context Protocol server (`dexter mcp`). One file per tool, gopls-style; tools are name-based (module/function, not file+position) and call the store plus the exported facade in `internal/lsp/api.go`. ## LSP feature map diff --git a/go.mod b/go.mod index c2da4ee..94ffde1 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.26.1 require ( github.com/mattn/go-sqlite3 v1.14.38 + github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/spf13/cobra v1.10.2 github.com/tree-sitter/go-tree-sitter v0.25.0 github.com/tree-sitter/tree-sitter-elixir v0.3.5 @@ -16,13 +17,16 @@ require ( replace github.com/tree-sitter/tree-sitter-elixir => github.com/elixir-lang/tree-sitter-elixir v0.3.5 require ( + github.com/google/jsonschema-go v0.4.3 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mattn/go-pointer v0.0.1 // indirect github.com/segmentio/asm v1.1.3 // indirect - github.com/segmentio/encoding v0.3.4 // indirect + github.com/segmentio/encoding v0.5.4 // indirect github.com/spf13/pflag v1.0.9 // indirect + github.com/yosida95/uritemplate/v3 v3.0.2 // indirect go.lsp.dev/pkg v0.0.0-20210717090340-384b27a52fb2 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect - golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 // indirect + golang.org/x/oauth2 v0.35.0 // indirect + golang.org/x/sys v0.41.0 // indirect ) diff --git a/go.sum b/go.sum index a8ff704..6a99e0e 100644 --- a/go.sum +++ b/go.sum @@ -6,9 +6,13 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elixir-lang/tree-sitter-elixir v0.3.5 h1:Ir60dE/aHPt80uil58ukW1CTC+15l4jHax/iHBsW9HI= github.com/elixir-lang/tree-sitter-elixir v0.3.5/go.mod h1:wNBVf64kzvhSbZ8ojVtBF1jRiqGY0lsuK5Kx/60s6Z0= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= -github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/jsonschema-go v0.4.3 h1:/DBOLZTfDow7pe2GmaJNhltueGTtDKICi8V8p+DQPd0= +github.com/google/jsonschema-go v0.4.3/go.mod h1:r5quNTdLOYEz95Ru18zA0ydNbBuYoo9tgaYcxEYhJVE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -18,6 +22,8 @@ github.com/mattn/go-pointer v0.0.1 h1:n+XhsuGeVO6MEAp7xyEukFINEa+Quek5psIR/ylA6o github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-sqlite3 v1.14.38 h1:tDUzL85kMvOrvpCt8P64SbGgVFtJB11GPi2AdmITgb4= github.com/mattn/go-sqlite3 v1.14.38/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= +github.com/modelcontextprotocol/go-sdk v1.6.1 h1:0zOSupjKUxPKSocPT1Wtago+mUHU2/uZ4xSOY0FGReU= +github.com/modelcontextprotocol/go-sdk v1.6.1/go.mod h1:kzm3kzFL1/+AziGOE0nUs3gvPoNxMCvkxokMkuFapXQ= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= @@ -25,8 +31,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/asm v1.1.3 h1:WM03sfUOENvvKexOLp+pCqgb/WDjsi7EK8gIsICtzhc= github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQy1EJg= -github.com/segmentio/encoding v0.3.4 h1:WM4IBnxH8B9TakiM2QD5LyNl9JSndh88QbHqVC+Pauc= -github.com/segmentio/encoding v0.3.4/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= +github.com/segmentio/encoding v0.5.4 h1:OW1VRern8Nw6ITAtwSZ7Idrl3MXCFwXHPgqESYfvNt0= +github.com/segmentio/encoding v0.5.4/go.mod h1:HS1ZKa3kSN32ZHVZ7ZLPLXWvOVIiZtyJnO1gPH1sKt0= github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= @@ -62,6 +68,8 @@ github.com/tree-sitter/tree-sitter-ruby v0.23.1 h1:T/NKHUA+iVbHM440hFx+lzVOzS4dV github.com/tree-sitter/tree-sitter-ruby v0.23.1/go.mod h1:kUS4kCCQloFcdX6sdpr8p6r2rogbM6ZjTox5ZOQy8cA= github.com/tree-sitter/tree-sitter-rust v0.23.2 h1:6AtoooCW5GqNrRpfnvl0iUhxTAZEovEmLKDbyHlfw90= github.com/tree-sitter/tree-sitter-rust v0.23.2/go.mod h1:hfeGWic9BAfgTrc7Xf6FaOAguCFJRo3RBbs7QJ6D7MI= +github.com/yosida95/uritemplate/v3 v3.0.2 h1:Ed3Oyj9yrmi9087+NczuL5BwkIc4wvTb5zIM+UJPGz4= +github.com/yosida95/uritemplate/v3 v3.0.2/go.mod h1:ILOh0sOhIJR3+L/8afwt/kE++YT040gmv5BQTMR2HP4= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.lsp.dev/jsonrpc2 v0.10.0 h1:Pr/YcXJoEOTMc/b6OTmcR1DPJ3mSWl/SWiU1Cct6VmI= go.lsp.dev/jsonrpc2 v0.10.0/go.mod h1:fmEzIdXPi/rf6d4uFcayi8HpFP1nBF99ERP1htC72Ac= @@ -90,6 +98,8 @@ golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/oauth2 v0.35.0 h1:Mv2mzuHuZuY2+bkyWXIHMfhNdJAdwW3FuWeCPYN5GVQ= +golang.org/x/oauth2 v0.35.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -97,9 +107,8 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211110154304-99a53858aa08/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= -golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= +golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -107,10 +116,11 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/integration_test.go b/integration_test.go index 1f43e58..a3328a0 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,11 +1,16 @@ package main import ( + "bufio" + "context" "os" "os/exec" "path/filepath" "strings" "testing" + "time" + + sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" "github.com/remoteoss/dexter/internal/store" ) @@ -564,3 +569,172 @@ func TestIntegration_LegacyMigration(t *testing.T) { t.Errorf("expected lookup to work after migration, got: %s", out) } } + +// mcpConnect spawns `dexter mcp ` over stdio and returns a connected +// MCP client session. +func mcpConnect(t *testing.T, binary, root string) *sdkmcp.ClientSession { + t.Helper() + cmd := exec.Command(binary, "mcp", root) + cmd.Dir = root + cmd.Stderr = os.Stderr + client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "integration-test", Version: "0.0.1"}, nil) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + t.Cleanup(cancel) + session, err := client.Connect(ctx, &sdkmcp.CommandTransport{Command: cmd}, nil) + if err != nil { + t.Fatalf("connecting to dexter mcp: %v", err) + } + t.Cleanup(func() { _ = session.Close() }) + return session +} + +func mcpToolText(t *testing.T, res *sdkmcp.CallToolResult) string { + t.Helper() + var b strings.Builder + for _, c := range res.Content { + if tc, ok := c.(*sdkmcp.TextContent); ok { + b.WriteString(tc.Text) + } + } + return b.String() +} + +func TestIntegration_MCPStdio(t *testing.T) { + binary := buildDexter(t) + root := scaffoldProject(t) + runDexter(t, binary, root, "init", root) + + session := mcpConnect(t, binary, root) + ctx := context.Background() + + tools, err := session.ListTools(ctx, nil) + if err != nil { + t.Fatal(err) + } + names := map[string]bool{} + for _, tool := range tools.Tools { + names[tool.Name] = true + } + for _, want := range []string{"dexter_workspace", "dexter_search", "dexter_definition", "dexter_references", "dexter_module_api", "dexter_file_outline", "dexter_implementations", "dexter_call_hierarchy", "dexter_reindex"} { + if !names[want] { + t.Errorf("tool %s not advertised; got %v", want, names) + } + } + + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "dexter_workspace"}) + if err != nil { + t.Fatal(err) + } + if res.IsError { + t.Fatalf("dexter_workspace errored: %s", mcpToolText(t, res)) + } + out := mcpToolText(t, res) + for _, want := range []string{"Project root:", "mix.exs", "definitions"} { + if !strings.Contains(out, want) { + t.Errorf("workspace output missing %q:\n%s", want, out) + } + } + + res, err = session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "dexter_definition", Arguments: map[string]any{"module": "MyApp.Repo", "function": "get"}}) + if err != nil { + t.Fatal(err) + } + out = mcpToolText(t, res) + if !strings.Contains(out, "lib/my_app/repo.ex") { + t.Errorf("definition output missing location:\n%s", out) + } +} + +func TestIntegration_MCPStdio_EmptyIndexBuildsOnStartup(t *testing.T) { + binary := buildDexter(t) + root := scaffoldProject(t) + // No `dexter init`: the MCP server must build the index before serving. + + session := mcpConnect(t, binary, root) + res, err := session.CallTool(context.Background(), &sdkmcp.CallToolParams{Name: "dexter_search", Arguments: map[string]any{"query": "process_event"}}) + if err != nil { + t.Fatal(err) + } + out := mcpToolText(t, res) + if !strings.Contains(out, "MyApp.Handlers.Webhooks.process_event") { + t.Errorf("search after auto-index missing symbol:\n%s", out) + } +} + +func TestIntegration_MCPInstructions(t *testing.T) { + binary := buildDexter(t) + out := runDexter(t, binary, t.TempDir(), "mcp", "--instructions") + for _, want := range []string{"dexter_workspace", "dexter_reindex"} { + if !strings.Contains(out, want) { + t.Errorf("instructions missing %q", want) + } + } +} + +// TestIntegration_LSPWithMCPListen starts `dexter lsp --mcp-listen=localhost:0` +// (LSP on stdio, MCP over streamable HTTP from the same process) and calls an +// MCP tool while the LSP is running. +func TestIntegration_LSPWithMCPListen(t *testing.T) { + binary := buildDexter(t) + root := scaffoldProject(t) + runDexter(t, binary, root, "init", root) + + cmd := exec.Command(binary, "lsp", "--mcp-listen=localhost:0", root) + cmd.Dir = root + stdin, err := cmd.StdinPipe() // held open: the LSP session's lifetime + if err != nil { + t.Fatal(err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = stdin.Close() + _ = cmd.Process.Kill() + _, _ = cmd.Process.Wait() + }) + + // Parse the bound address from stderr. + addrCh := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + line := scanner.Text() + if i := strings.Index(line, "MCP server listening on "); i >= 0 { + addrCh <- strings.TrimSpace(line[i+len("MCP server listening on "):]) + break + } + } + // Keep draining so the child never blocks on a full stderr pipe. + for scanner.Scan() { + } + }() + var addr string + select { + case addr = <-addrCh: + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for MCP listen address on stderr") + } + + client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "integration-test", Version: "0.0.1"}, nil) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + session, err := client.Connect(ctx, &sdkmcp.StreamableClientTransport{Endpoint: "http://" + addr}, nil) + if err != nil { + t.Fatalf("connecting to attached MCP server: %v", err) + } + defer func() { _ = session.Close() }() + + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "dexter_definition", Arguments: map[string]any{"module": "MyApp.Repo", "function": "get"}}) + if err != nil { + t.Fatal(err) + } + out := mcpToolText(t, res) + if !strings.Contains(out, "lib/my_app/repo.ex") { + t.Errorf("attached-mode definition output missing location:\n%s", out) + } +} diff --git a/internal/lsp/api.go b/internal/lsp/api.go new file mode 100644 index 0000000..433f199 --- /dev/null +++ b/internal/lsp/api.go @@ -0,0 +1,114 @@ +package lsp + +import ( + "context" + "io" + "sort" + "strings" + + "go.lsp.dev/jsonrpc2" + "go.lsp.dev/protocol" + "go.uber.org/zap" + + "github.com/remoteoss/dexter/internal/store" +) + +// This file is the exported, name-based surface of the LSP server used by +// callers outside the LSP session (the MCP server and the CLI). Everything +// here delegates to the same internals the LSP handlers use, so results are +// identical regardless of which front end asked. + +// Serve runs the Server over the given reader/writer (typically +// stdin/stdout). It blocks until the connection closes. +func Serve(server *Server, in io.Reader, out io.Writer) error { + logger, _ := zap.NewProduction() + stream := jsonrpc2.NewStream(stdinoutCloser{in, out}) + conn := jsonrpc2.NewConn(stream) + server.client = protocol.ClientDispatcher(conn, logger) + server.conn = conn + + handler := protocol.ServerHandler(server, nil) + ctx := context.Background() + + conn.Go(ctx, handler) + <-conn.Done() + return conn.Err() +} + +// SetStdlibRoot records the Elixir stdlib directory so lookups can classify +// stdlib symbols. The LSP session sets this during Initialize; headless +// callers (MCP) set it explicitly after resolving the stdlib themselves. +func (s *Server) SetStdlibRoot(root string) { + s.stdlibRoot = root +} + +// StdlibRoot returns the Elixir stdlib directory, or "" if not detected. In +// attached MCP mode this is set by Initialize after the Handler is built, so +// callers must read it per request rather than caching it. +func (s *Server) StdlibRoot() string { + return s.stdlibRoot +} + +// CollectReferences gathers references to module (or module.function) across +// the workspace, name-based. It mirrors the collection performed by the LSP +// References handler: direct refs, transitive refs through static __using__ +// import chains, bare intra-module calls in definition files, and refs to +// defdelegate facades that target the function. Results are deduplicated by +// file+line, stdlib-filtered, and sorted by file then line. +func (s *Server) CollectReferences(module, function string) []store.ReferenceResult { + refResults, err := s.store.LookupReferences(module, function) + if err != nil { + return nil + } + + if function != "" { + // Transitive refs via static __using__ import chains. Call sites of + // use-injected functions are attributed to the injecting module in the + // store, so we look up refs under each injector too. + for _, mod := range s.findModulesWhoseUsingImports(module) { + if transitive, err := s.store.LookupReferences(mod, function); err == nil { + refResults = append(refResults, transitive...) + } + } + + // Bare intra-module calls in definition files are not indexed. + refResults = append(refResults, s.findBareCallRefs(module, function)...) + + // Follow defdelegate in reverse: calls to facades that delegate here. + if s.followDelegates { + if delegates, err := s.store.LookupDelegatesTo(module, function); err == nil { + for _, del := range delegates { + if delegateRefs, err := s.store.LookupReferences(del.Module, del.Function); err == nil { + refResults = append(refResults, delegateRefs...) + } + refResults = append(refResults, s.findBareCallRefs(del.Module, del.Function)...) + } + } + } + } + + type refKey struct { + filePath string + line int + } + seen := make(map[refKey]struct{}, len(refResults)) + var out []store.ReferenceResult + for _, r := range refResults { + if s.stdlibRoot != "" && strings.HasPrefix(r.FilePath, s.stdlibRoot) { + continue + } + k := refKey{r.FilePath, r.Line} + if _, ok := seen[k]; ok { + continue + } + seen[k] = struct{}{} + out = append(out, r) + } + sort.Slice(out, func(i, j int) bool { + if out[i].FilePath != out[j].FilePath { + return out[i].FilePath < out[j].FilePath + } + return out[i].Line < out[j].Line + }) + return out +} diff --git a/internal/lsp/hover.go b/internal/lsp/hover.go index 104d737..e4aebc4 100644 --- a/internal/lsp/hover.go +++ b/internal/lsp/hover.go @@ -9,7 +9,7 @@ import ( ) func (s *Server) hoverFromFile(function string, result store.LookupResult) (*protocol.Hover, error) { - text, _, ok := s.readFileText(result.FilePath) + text, _, ok := s.ReadFileText(result.FilePath) if !ok { return nil, nil } diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 93603cb..5ee9966 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -22,7 +22,6 @@ import ( "go.lsp.dev/jsonrpc2" "go.lsp.dev/protocol" "go.lsp.dev/uri" - "go.uber.org/zap" "github.com/remoteoss/dexter/internal/parser" "github.com/remoteoss/dexter/internal/stdlib" @@ -136,24 +135,6 @@ type stdinoutCloser struct { func (s stdinoutCloser) Close() error { return nil } -// Serve starts the LSP server on the given reader/writer (typically stdin/stdout). -func Serve(in io.Reader, out io.Writer, s *store.Store, projectRoot string) error { - server := NewServer(s, projectRoot) - - logger, _ := zap.NewProduction() - stream := jsonrpc2.NewStream(stdinoutCloser{in, out}) - conn := jsonrpc2.NewConn(stream) - server.client = protocol.ClientDispatcher(conn, logger) - server.conn = conn - - handler := protocol.ServerHandler(server, nil) - ctx := context.Background() - - conn.Go(ctx, handler) - <-conn.Done() - return conn.Err() -} - // backgroundReindex runs in the background. If the index is empty it does a // full init, otherwise it does an incremental mtime-based update. func (s *Server) backgroundReindex() { @@ -164,98 +145,110 @@ func (s *Server) backgroundReindex() { return } defer s.reindexing.Unlock() + s.reindexWorkspace() + }() +} - start := time.Now() - reindexed := 0 - isEmpty := s.store.IsEmpty() - - if isEmpty { - log.Printf("No index found, building from scratch...") - if s.client != nil { - if err := s.client.ShowMessage(context.Background(), &protocol.ShowMessageParams{ - Type: protocol.MessageTypeInfo, - Message: "Dexter: building index for the first time, go-to-definition will be available shortly...", - }); err != nil { - log.Printf("ShowMessage: %v", err) - } +// ReindexWorkspace runs the same full-or-incremental reindex as +// backgroundReindex, but blocking, and reports how many files were updated. +func (s *Server) ReindexWorkspace() (int, time.Duration) { + s.reindexing.Lock() + defer s.reindexing.Unlock() + return s.reindexWorkspace() +} + +func (s *Server) reindexWorkspace() (int, time.Duration) { + start := time.Now() + reindexed := 0 + isEmpty := s.store.IsEmpty() + + if isEmpty { + log.Printf("No index found, building from scratch...") + if s.client != nil { + if err := s.client.ShowMessage(context.Background(), &protocol.ShowMessageParams{ + Type: protocol.MessageTypeInfo, + Message: "Dexter: building index for the first time, go-to-definition will be available shortly...", + }); err != nil { + log.Printf("ShowMessage: %v", err) } } + } - seen := make(map[string]struct{}) - walkAndIndex := func(root string, indexRefs bool) { - _ = parser.WalkElixirFiles(root, func(path string, d fs.DirEntry) error { - seen[path] = struct{}{} - - if !isEmpty { - info, err := d.Info() - if err != nil { - return nil - } - storedMtime, found := s.store.GetFileMtime(path) - currentMtime := info.ModTime().UnixNano() - if found && storedMtime == currentMtime { - return nil - } - } + seen := make(map[string]struct{}) + walkAndIndex := func(root string, indexRefs bool) { + _ = parser.WalkElixirFiles(root, func(path string, d fs.DirEntry) error { + seen[path] = struct{}{} - defs, refs, err := parser.ParseFile(path) + if !isEmpty { + info, err := d.Info() if err != nil { return nil } - if !indexRefs { - refs = nil - } - if err := s.store.IndexFileWithRefs(path, defs, refs); err != nil { - log.Printf("Warning: reindex %s: %v", path, err) + storedMtime, found := s.store.GetFileMtime(path) + currentMtime := info.ModTime().UnixNano() + if found && storedMtime == currentMtime { + return nil } - reindexed++ + } + + defs, refs, err := parser.ParseFile(path) + if err != nil { return nil - }) - } + } + if !indexRefs { + refs = nil + } + if err := s.store.IndexFileWithRefs(path, defs, refs); err != nil { + log.Printf("Warning: reindex %s: %v", path, err) + } + reindexed++ + return nil + }) + } - // Index stdlib first (definitions only). - if s.stdlibRoot != "" { - walkAndIndex(s.stdlibRoot, false) - } + // Index stdlib first (definitions only). + if s.stdlibRoot != "" { + walkAndIndex(s.stdlibRoot, false) + } - walkAndIndex(s.projectRoot, true) + walkAndIndex(s.projectRoot, true) - // Prune store entries for files no longer on disk - if storedPaths, err := s.store.ListFilePaths(); err == nil { - var toRemove []string - for _, storedPath := range storedPaths { - if _, ok := seen[storedPath]; !ok { - toRemove = append(toRemove, storedPath) - } - } - if len(toRemove) > 0 { - _ = s.store.RemoveFiles(toRemove) + // Prune store entries for files no longer on disk + if storedPaths, err := s.store.ListFilePaths(); err == nil { + var toRemove []string + for _, storedPath := range storedPaths { + if _, ok := seen[storedPath]; !ok { + toRemove = append(toRemove, storedPath) } } - - // Collapse the WAL back to disk now that the (potentially large) reindex - // is complete, so the -wal file does not stay parked at its high-water - // mark for the lifetime of the LSP process. - if err := s.store.Checkpoint(); err != nil { - log.Printf("Warning: WAL checkpoint after reindex: %v", err) + if len(toRemove) > 0 { + _ = s.store.RemoveFiles(toRemove) } + } - elapsed := time.Since(start).Round(time.Millisecond) - log.Printf("Background reindex: %d files updated (%s)", reindexed, elapsed) + // Collapse the WAL back to disk now that the (potentially large) reindex + // is complete, so the -wal file does not stay parked at its high-water + // mark for the lifetime of the LSP process. + if err := s.store.Checkpoint(); err != nil { + log.Printf("Warning: WAL checkpoint after reindex: %v", err) + } - if isEmpty && s.client != nil { - if err := s.client.ShowMessage(context.Background(), &protocol.ShowMessageParams{ - Type: protocol.MessageTypeInfo, - Message: fmt.Sprintf("Dexter: index built (%d files in %s)", reindexed, elapsed), - }); err != nil { - log.Printf("ShowMessage: %v", err) - } + elapsed := time.Since(start).Round(time.Millisecond) + log.Printf("Background reindex: %d files updated (%s)", reindexed, elapsed) + + if isEmpty && s.client != nil { + if err := s.client.ShowMessage(context.Background(), &protocol.ShowMessageParams{ + Type: protocol.MessageTypeInfo, + Message: fmt.Sprintf("Dexter: index built (%d files in %s)", reindexed, elapsed), + }); err != nil { + log.Printf("ShowMessage: %v", err) } - }() + } + return reindexed, elapsed } -// watchGitHead polls .git/HEAD mtime and triggers reindex on branch switches. -func (s *Server) watchGitHead() { +// WatchGitHead polls .git/HEAD mtime and triggers reindex on branch switches. +func (s *Server) WatchGitHead() { go func() { headPath := filepath.Join(s.projectRoot, ".git", "HEAD") var lastMtime int64 @@ -397,7 +390,7 @@ func (s *Server) Initialize(ctx context.Context, params *protocol.InitializePara if !s.initialized { s.initialized = true s.backgroundReindex() - s.watchGitHead() + s.WatchGitHead() } if params.Capabilities.Window != nil && params.Capabilities.Window.ShowDocument != nil { @@ -1870,7 +1863,7 @@ func (s *Server) lookupThroughUseOf(fullModule, functionName string) []store.Loo if err != nil || len(modResults) == 0 { return nil } - fileText, _, ok := s.readFileText(modResults[0].FilePath) + fileText, _, ok := s.ReadFileText(modResults[0].FilePath) if !ok { return nil } @@ -4274,7 +4267,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro specPrefix := "@spec " + functionName callbackPrefix := "@callback " + functionName for filePath := range defFilePaths { - fileText, _, ok := s.readFileText(filePath) + fileText, _, ok := s.ReadFileText(filePath) if !ok { continue } @@ -4308,7 +4301,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro if r.Kind != "import" { continue } - lineText, ok := s.getFileLine(r.FilePath, r.Line) + lineText, ok := s.FileLine(r.FilePath, r.Line) if !ok { continue } @@ -4318,7 +4311,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro } } for filePath := range importFilePaths { - fileText, _, ok := s.readFileText(filePath) + fileText, _, ok := s.ReadFileText(filePath) if !ok { continue } @@ -4341,7 +4334,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro if s.isDepsFile(del.FilePath) { continue } - fileText, open, ok := s.readFileText(del.FilePath) + fileText, open, ok := s.ReadFileText(del.FilePath) if !ok { continue } @@ -4587,7 +4580,7 @@ func (mr *moduleRename) readFiles() map[string]moduleFileInfo { resultsCh := make(chan fileResult, len(mr.sitesByFile)) for fp := range mr.sitesByFile { go func() { - text, open, ok := mr.server.readFileText(fp) + text, open, ok := mr.server.ReadFileText(fp) if ok { resultsCh <- fileResult{fp, strings.Split(text, "\n"), open} } else { @@ -4896,7 +4889,7 @@ func (s *Server) buildTextEdits(sites []renameSite, oldToken, newToken string) * resultsCh := make(chan fileResult, len(sitesByFile)) for fp := range sitesByFile { go func() { - text, open, ok := s.readFileText(fp) + text, open, ok := s.ReadFileText(fp) if ok { resultsCh <- fileResult{fp, strings.Split(text, "\n"), open} } else { @@ -5075,11 +5068,11 @@ func isDepsFileUncached(filePath string) bool { } } -// readFileText returns the contents of filePath, preferring the in-memory +// ReadFileText returns the contents of filePath, preferring the in-memory // document store for editor-owned (didOpen) buffers. The second return // indicates whether the file is currently open in the editor — transient // entries loaded from disk via GetOrLoad are NOT reported as open. -func (s *Server) readFileText(filePath string) (text string, open bool, ok bool) { +func (s *Server) ReadFileText(filePath string) (text string, open bool, ok bool) { uri := string(uri.File(filePath)) if t, found := s.docs.GetIfOpen(uri); found { return t, true, true @@ -5090,12 +5083,12 @@ func (s *Server) readFileText(filePath string) (text string, open bool, ok bool) return "", false, false } -// getFileLine returns the text of line lineNum (1-based) from the file at +// FileLine returns the text of line lineNum (1-based) from the file at // filePath, preferring the in-memory document store for editor-owned // buffers. Transient entries loaded via GetOrLoad fall through to the // disk path. For closed files, only reads up to the target line instead // of the whole file. -func (s *Server) getFileLine(filePath string, lineNum int) (string, bool) { +func (s *Server) FileLine(filePath string, lineNum int) (string, bool) { // Editor-owned buffer: extract the single line from memory uri := string(uri.File(filePath)) if text, ok := s.docs.GetIfOpen(uri); ok { @@ -5135,7 +5128,7 @@ func (s *Server) findBareCallRefs(module, functionName string) []store.Reference } var refs []store.ReferenceResult for filePath := range defFilePaths { - fileText, _, ok := s.readFileText(filePath) + fileText, _, ok := s.ReadFileText(filePath) if !ok { continue } @@ -5227,7 +5220,7 @@ func (s *Server) SignatureHelp(ctx context.Context, params *protocol.SignatureHe } // Read the definition file, preferring the in-memory doc store - fileText, _, ok2 := s.readFileText(result.FilePath) + fileText, _, ok2 := s.ReadFileText(result.FilePath) if !ok2 { return nil, nil } @@ -5432,7 +5425,7 @@ func (s *Server) PrepareCallHierarchy(ctx context.Context, params *protocol.Call r := defResults[0] nameCol := 0 - if defLine, ok := s.getFileLine(r.FilePath, r.Line); ok { + if defLine, ok := s.FileLine(r.FilePath, r.Line); ok { if col := findTokenColumn(defLine, functionName); col >= 0 { nameCol = col } @@ -5514,7 +5507,7 @@ func (s *Server) IncomingCalls(ctx context.Context, params *protocol.CallHierarc } nameCol := 0 - if defLine, ok := s.getFileLine(r.FilePath, callerLine); ok { + if defLine, ok := s.FileLine(r.FilePath, callerLine); ok { if col := findTokenColumn(defLine, callerFunc); col >= 0 { nameCol = col } @@ -5604,7 +5597,7 @@ func (s *Server) OutgoingCalls(ctx context.Context, params *protocol.CallHierarc } nameCol := 0 - if defLine, ok := s.getFileLine(td.FilePath, td.Line); ok { + if defLine, ok := s.FileLine(td.FilePath, td.Line); ok { if col := findTokenColumn(defLine, key.function); col >= 0 { nameCol = col } diff --git a/internal/mcp/call_hierarchy.go b/internal/mcp/call_hierarchy.go new file mode 100644 index 0000000..741e9f9 --- /dev/null +++ b/internal/mcp/call_hierarchy.go @@ -0,0 +1,89 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + "go.lsp.dev/protocol" +) + +type CallHierarchyParams struct { + Module string `json:"module" jsonschema:"fully-qualified module owning the function"` + Function string `json:"function" jsonschema:"function name without arity"` + Direction string `json:"direction,omitempty" jsonschema:"'incoming' (callers), 'outgoing' (callees), or 'both' (default)"` +} + +const maxCallsPerDirection = 50 + +func (h *Handler) callHierarchyHandler(ctx context.Context, req *mcp.CallToolRequest, args CallHierarchyParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + function := strings.TrimSpace(args.Function) + if module == "" || function == "" { + return nil, nil, fmt.Errorf("module and function must not be empty") + } + direction := strings.ToLower(strings.TrimSpace(args.Direction)) + switch direction { + case "": + direction = "both" + case "incoming", "outgoing", "both": + default: + return nil, nil, fmt.Errorf("direction must be 'incoming', 'outgoing', or 'both', got %q", args.Direction) + } + + // The LSP call-hierarchy handlers are name-based: they only read the + // module/function pair from Item.Data, so a synthetic item works. + item := protocol.CallHierarchyItem{ + Data: map[string]interface{}{"module": module, "function": function}, + } + + var b strings.Builder + fmt.Fprintf(&b, "Call hierarchy for %s.%s:\n", module, function) + found := false + + if direction == "incoming" || direction == "both" { + calls, err := h.lsp.IncomingCalls(ctx, &protocol.CallHierarchyIncomingCallsParams{Item: item}) + if err != nil { + return nil, nil, fmt.Errorf("incoming calls: %w", err) + } + fmt.Fprintf(&b, "\nIncoming (callers): %d\n", len(calls)) + for i, c := range calls { + if i == maxCallsPerDirection { + fmt.Fprintf(&b, " ... and %d more\n", len(calls)-maxCallsPerDirection) + break + } + lines := make([]string, 0, len(c.FromRanges)) + for _, r := range c.FromRanges { + lines = append(lines, fmt.Sprintf("%d", r.Start.Line+1)) + } + fmt.Fprintf(&b, " ← %s (%s:%d) calls at line %s\n", c.From.Name, h.relPath(uriToPath(c.From.URI)), c.From.Range.Start.Line+1, strings.Join(lines, ", ")) + } + found = found || len(calls) > 0 + } + + if direction == "outgoing" || direction == "both" { + calls, err := h.lsp.OutgoingCalls(ctx, &protocol.CallHierarchyOutgoingCallsParams{Item: item}) + if err != nil { + return nil, nil, fmt.Errorf("outgoing calls: %w", err) + } + fmt.Fprintf(&b, "\nOutgoing (callees): %d\n", len(calls)) + for i, c := range calls { + if i == maxCallsPerDirection { + fmt.Fprintf(&b, " ... and %d more\n", len(calls)-maxCallsPerDirection) + break + } + fmt.Fprintf(&b, " → %s (%s:%d)\n", c.To.Name, h.relPath(uriToPath(c.To.URI)), c.To.Range.Start.Line+1) + } + found = found || len(calls) > 0 + } + + if !found { + fmt.Fprintf(&b, "\nNo calls found. Check the module/function names (dexter_search can help), or call dexter_reindex if files changed recently.\n") + } + return textResult(b.String()), nil, nil +} + +func uriToPath(u protocol.DocumentURI) string { + return u.Filename() +} diff --git a/internal/mcp/definition.go b/internal/mcp/definition.go new file mode 100644 index 0000000..2da320b --- /dev/null +++ b/internal/mcp/definition.go @@ -0,0 +1,115 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/lsp" + "github.com/remoteoss/dexter/internal/store" +) + +type DefinitionParams struct { + Module string `json:"module" jsonschema:"fully-qualified module name, e.g. MyApp.Accounts (aliases are not resolved)"` + Function string `json:"function,omitempty" jsonschema:"function/macro/type name without arity; omit to look up the module itself"` +} + +func (h *Handler) definitionHandler(ctx context.Context, req *mcp.CallToolRequest, args DefinitionParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + if module == "" { + return nil, nil, fmt.Errorf("module must not be empty") + } + function := strings.TrimSpace(args.Function) + + if function == "" { + return h.moduleDefinition(module) + } + + // Direct definitions first: they tell us whether this is a defdelegate facade. + direct, err := h.store.LookupFunction(module, function) + if err != nil { + return nil, nil, fmt.Errorf("looking up function: %w", err) + } + + var b strings.Builder + if len(direct) == 0 { + // No direct definition. The function may still resolve through a + // defdelegate chain recorded under a different arity/name form. + resolved, err := h.store.LookupFollowDelegate(module, function) + if err != nil { + return nil, nil, fmt.Errorf("looking up function: %w", err) + } + if len(resolved) == 0 { + return textResult(fmt.Sprintf("%s.%s is not in the index. It may be private to a use-chain (injected via __using__), dynamically generated by a macro, or misspelled. Try dexter_search or dexter_module_api %s.", module, function, module)), nil, nil + } + for _, r := range resolved { + h.writeDefinition(&b, module, function, r) + } + return textResult(b.String()), nil, nil + } + + for _, r := range direct { + h.writeDefinition(&b, module, function, r) + if r.Kind == "defdelegate" && r.DelegateTo != "" { + targetFn := function + if r.DelegateAs != "" { + targetFn = r.DelegateAs + } + targets, err := h.store.LookupFollowDelegate(module, function) + if err == nil && len(targets) > 0 { + fmt.Fprintf(&b, "\nDelegates to %s.%s:\n", r.DelegateTo, targetFn) + for _, t := range targets { + h.writeDefinition(&b, r.DelegateTo, targetFn, t) + } + } + } + } + return textResult(b.String()), nil, nil +} + +func (h *Handler) moduleDefinition(module string) (*mcp.CallToolResult, any, error) { + results, err := h.store.LookupModule(module) + if err != nil { + return nil, nil, fmt.Errorf("looking up module: %w", err) + } + if len(results) == 0 { + return textResult(fmt.Sprintf("Module %s is not in the index. Use dexter_search to find the right name, or dexter_reindex if it was just created.", module)), nil, nil + } + + var b strings.Builder + for _, r := range results { + fmt.Fprintf(&b, "%s %s - %s:%d\n", moduleKindLabel(r.Kind), module, h.relPath(r.FilePath), r.Line) + if r.Kind != "defimpl" { + if text, _, ok := h.lsp.ReadFileText(r.FilePath); ok { + if doc := lsp.NewTokenizedFile(text).ExtractModuledoc(r.Line - 1); doc != "" { + fmt.Fprintf(&b, "\n%s\n", strings.TrimRight(doc, "\n")) + } + } + } + } + return textResult(b.String()), nil, nil +} + +// writeDefinition renders one definition with location, @spec/@doc, and the +// definition head line. +func (h *Handler) writeDefinition(b *strings.Builder, module, function string, r store.LookupResult) { + fmt.Fprintf(b, "%s (%s) - %s:%d\n", symbolName(module, function, r.Arity), r.Kind, h.relPath(r.FilePath), r.Line) + + text, _, ok := h.lsp.ReadFileText(r.FilePath) + if !ok { + return + } + tf := lsp.NewTokenizedFile(text) + doc, spec := tf.ExtractDocAbove(r.Line - 1) + if spec != "" { + fmt.Fprintf(b, "%s\n", spec) + } + if head, ok := h.lsp.FileLine(r.FilePath, r.Line); ok { + fmt.Fprintf(b, "%s\n", strings.TrimRight(head, " \t")) + } + if doc != "" { + fmt.Fprintf(b, "\n%s\n", strings.TrimRight(doc, "\n")) + } +} diff --git a/internal/mcp/file_outline.go b/internal/mcp/file_outline.go new file mode 100644 index 0000000..e74390f --- /dev/null +++ b/internal/mcp/file_outline.go @@ -0,0 +1,109 @@ +package mcp + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/parser" +) + +type FileOutlineParams struct { + File string `json:"file" jsonschema:"path to a .ex/.exs file, absolute or relative to the project root"` +} + +func (h *Handler) fileOutlineHandler(ctx context.Context, req *mcp.CallToolRequest, args FileOutlineParams) (*mcp.CallToolResult, any, error) { + if strings.TrimSpace(args.File) == "" { + return nil, nil, fmt.Errorf("file must not be empty") + } + path := h.resolvePath(args.File) + if _, err := os.Stat(path); err != nil { + return textResult(fmt.Sprintf("File not found: %s", h.relPath(path))), nil, nil + } + + // Parse fresh from disk so the outline is correct even when the index is stale. + defs, _, err := parser.ParseFile(path) + if err != nil { + return nil, nil, fmt.Errorf("parsing %s: %w", h.relPath(path), err) + } + if len(defs) == 0 { + return textResult(fmt.Sprintf("%s defines no modules or functions.", h.relPath(path))), nil, nil + } + + // Split into module declarations (in line order) and their members. + type moduleEntry struct { + def parser.Definition + members []parser.Definition + } + var modules []*moduleEntry + byName := make(map[string]*moduleEntry) + var orphans []parser.Definition + + sorted := make([]parser.Definition, len(defs)) + copy(sorted, defs) + sort.SliceStable(sorted, func(i, j int) bool { return sorted[i].Line < sorted[j].Line }) + + for _, d := range sorted { + if d.Function == "" { + e := &moduleEntry{def: d} + modules = append(modules, e) + byName[d.Module] = e + } + } + for _, d := range sorted { + if d.Function == "" { + continue + } + if e, ok := byName[d.Module]; ok { + e.members = append(e.members, d) + } else { + orphans = append(orphans, d) + } + } + + var b strings.Builder + fmt.Fprintf(&b, "%s\n", h.relPath(path)) + for _, e := range modules { + fmt.Fprintf(&b, "\n%s %s (line %d)\n", moduleKindLabel(e.def.Kind), e.def.Module, e.def.Line) + for _, m := range e.members { + b.WriteString(" " + memberLine(m) + "\n") + } + } + for _, m := range orphans { + b.WriteString(memberLine(m) + "\n") + } + return textResult(b.String()), nil, nil +} + +func moduleKindLabel(kind string) string { + switch kind { + case "module": + return "defmodule" + default: // defprotocol, defimpl + return kind + } +} + +func memberLine(d parser.Definition) string { + label := d.Kind + switch d.Kind { + case "type", "opaque", "callback", "macrocallback": + label = "@" + d.Kind + } + line := fmt.Sprintf("%4d: %s %s/%d", d.Line, label, d.Function, d.Arity) + if d.Params != "" { + line += fmt.Sprintf(" (%s)", d.Params) + } + if d.DelegateTo != "" { + target := d.DelegateTo + if d.DelegateAs != "" { + target += "." + d.DelegateAs + } + line += " → " + target + } + return line +} diff --git a/internal/mcp/implementations.go b/internal/mcp/implementations.go new file mode 100644 index 0000000..6273eee --- /dev/null +++ b/internal/mcp/implementations.go @@ -0,0 +1,120 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type ImplementationsParams struct { + Module string `json:"module" jsonschema:"behaviour or protocol module, fully qualified"` + Function string `json:"function,omitempty" jsonschema:"callback name; when set, locate its definition in each implementor"` +} + +func (h *Handler) implementationsHandler(ctx context.Context, req *mcp.CallToolRequest, args ImplementationsParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + if module == "" { + return nil, nil, fmt.Errorf("module must not be empty") + } + + modResults, err := h.store.LookupModule(module) + if err != nil { + return nil, nil, fmt.Errorf("looking up module: %w", err) + } + + // Protocol: implementations are the defimpl rows indexed under the protocol name. + isProtocol := false + var impls, decls []int + for i, r := range modResults { + switch r.Kind { + case "defprotocol": + isProtocol = true + decls = append(decls, i) + case "defimpl": + impls = append(impls, i) + } + } + if isProtocol { + var b strings.Builder + fmt.Fprintf(&b, "%s is a protocol (defprotocol at %s:%d).\n", module, h.relPath(modResults[decls[0]].FilePath), modResults[decls[0]].Line) + if len(impls) == 0 { + fmt.Fprintf(&b, "No defimpl implementations found in the index.\n") + return textResult(b.String()), nil, nil + } + fmt.Fprintf(&b, "\nImplementations (%d):\n", len(impls)) + for _, i := range impls { + r := modResults[i] + fmt.Fprintf(&b, " %s:%d\n", h.relPath(r.FilePath), r.Line) + } + fmt.Fprintf(&b, "\nNote: the defimpl target type is on the cited line (defimpl %s, for: Type).\n", module) + return textResult(b.String()), nil, nil + } + + // Behaviour: modules that declare @behaviour or `use` this module. + implementors, err := h.store.LookupBehaviourImplementors(module) + if err != nil { + return nil, nil, fmt.Errorf("looking up implementors: %w", err) + } + if len(implementors) == 0 { + if len(modResults) == 0 { + return textResult(fmt.Sprintf("Module %s is not in the index. Use dexter_search to find the right name.", module)), nil, nil + } + return textResult(fmt.Sprintf("No modules declare @behaviour %s (or use it) in the index.", module)), nil, nil + } + + var b strings.Builder + + if args.Function != "" { + // Locate the callback's implementation in each implementor. + function := strings.TrimSpace(args.Function) + cbs, err := h.store.LookupCallbackDef(module, function) + if err != nil { + return nil, nil, fmt.Errorf("looking up callback: %w", err) + } + if len(cbs) == 0 { + return textResult(fmt.Sprintf("%s does not define a @callback named %s. List its callbacks with dexter_module_api.", module, function)), nil, nil + } + fmt.Fprintf(&b, "Implementations of callback %s.%s:\n", module, function) + arities := make(map[int]bool, len(cbs)) + for _, cb := range cbs { + arities[cb.Arity] = true + } + found := 0 + for _, impl := range implementors { + defs, err := h.store.LookupFunction(impl.Module, function) + if err != nil { + continue + } + for _, d := range defs { + if !arities[d.Arity] { + continue + } + fmt.Fprintf(&b, " %s - %s:%d\n", symbolName(impl.Module, function, d.Arity), h.relPath(d.FilePath), d.Line) + found++ + } + } + if found == 0 { + fmt.Fprintf(&b, " (none of the %d implementor(s) define %s; they may rely on a default implementation injected via use)\n", len(implementors), function) + } + return textResult(b.String()), nil, nil + } + + fmt.Fprintf(&b, "Modules implementing behaviour %s (%d):\n", module, len(implementors)) + const maxImpls = 50 + for i, impl := range implementors { + if i == maxImpls { + fmt.Fprintf(&b, " ... and %d more\n", len(implementors)-maxImpls) + break + } + fmt.Fprintf(&b, " %s - %s\n", impl.Module, h.relPath(impl.FilePath)) + } + if cbs, err := h.store.ListModuleCallbacks(module); err == nil && len(cbs) > 0 { + fmt.Fprintf(&b, "\nCallbacks defined by %s:\n", module) + for _, cb := range cbs { + fmt.Fprintf(&b, " @%s %s/%d\n", cb.Kind, cb.Function, cb.Arity) + } + } + return textResult(b.String()), nil, nil +} diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md new file mode 100644 index 0000000..a515005 --- /dev/null +++ b/internal/mcp/instructions.md @@ -0,0 +1,27 @@ +# Dexter: Elixir code intelligence + +Dexter indexes every module, function, and call site in this Elixir workspace +by parsing source directly (no compilation needed). Use these tools instead of +grep or reading whole files whenever you navigate or ask questions about +Elixir code: they resolve aliases, imports, defdelegate chains, use-chain +injection, and the Elixir stdlib, which text search cannot. + +Which tool for which question: + +- Locate a symbol by name fragment: `dexter_search` +- Where or what is Module.function: `dexter_definition` +- Understand a module before reading its source: `dexter_module_api` +- Who calls or uses something: `dexter_references` or `dexter_call_hierarchy` +- Implementations of a behaviour or protocol: `dexter_implementations` +- What a specific file defines: `dexter_file_outline` +- Project layout and index freshness: `dexter_workspace` + +After you create, edit, or delete Elixir files by any means, call +`dexter_reindex` (fast, incremental) so results stay accurate. Git branch +switches are picked up automatically. + +Elixir specifics: modules are not tied to files (use `dexter_file_outline` for +a file, `dexter_definition` for a module); pass fully-qualified module names, +not aliases; function names take no arity; functions defined inside a +`__using__` quote block may not be indexed, so an empty lookup can mean +macro-generated code. diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go new file mode 100644 index 0000000..f3bd22b --- /dev/null +++ b/internal/mcp/mcp.go @@ -0,0 +1,160 @@ +// Package mcp implements dexter's Model Context Protocol server. It exposes +// the index as a set of coarse, agent-oriented tools (modeled on gopls mcp), +// addressed by module/function name rather than file positions because Elixir +// modules are not tied to files. +package mcp + +import ( + _ "embed" + "fmt" + "path/filepath" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/lsp" + "github.com/remoteoss/dexter/internal/store" + "github.com/remoteoss/dexter/internal/version" +) + +// Instructions is the agent-facing usage guide, offered to MCP clients via the +// server's instructions field and printable with `dexter mcp --instructions`. +// +//go:embed instructions.md +var Instructions string + +// Handler carries the state shared by all tool handlers. In headless mode +// (`dexter mcp`) the lsp.Server is constructed without a client connection; in +// attached mode (`dexter lsp --mcp-listen`) it is the live LSP session, so +// tools see open editor buffers and warm caches. +type Handler struct { + lsp *lsp.Server + store *store.Store + projectRoot string +} + +type Config struct { + LSP *lsp.Server + Store *store.Store + ProjectRoot string +} + +func NewHandler(cfg Config) *Handler { + return &Handler{ + lsp: cfg.LSP, + store: cfg.Store, + projectRoot: cfg.ProjectRoot, + } +} + +// NewServer returns an MCP server with all dexter tools registered. +func NewServer(h *Handler) *mcp.Server { + srv := mcp.NewServer( + &mcp.Implementation{Name: "dexter", Title: "Dexter Elixir language tools", Version: version.Version}, + &mcp.ServerOptions{Instructions: Instructions}, + ) + + // The pointer hints distinguish explicit false from unset; clients must + // treat unset pessimistically (destructive, open world). + readOnly := &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: new(bool)} + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_workspace", + Annotations: readOnly, + Description: "Overview of the Elixir workspace: Mix projects, index size, stdlib status. Call once at the start of Elixir work.", + }, h.workspaceHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_search", + Annotations: readOnly, + Description: "Locate Elixir modules and functions by fuzzy name match. More precise than grep for finding symbols: results are exact definitions with file:line.", + }, h.searchHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_definition", + Annotations: readOnly, + Description: "Definition of an Elixir module or function by name: location, @doc/@spec, and source snippet, following defdelegate to the real implementation. Use instead of grep or reading files to answer where or what a symbol is.", + }, h.definitionHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_references", + Annotations: readOnly, + Description: "All call sites of an Elixir module or function, resolved through aliases, imports, and use-chain injection that grep cannot see. Use for any 'who calls or uses X' question.", + }, h.referencesHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_module_api", + Annotations: readOnly, + Description: "A module's public API in one call: moduledoc, functions with signatures and doc summaries, macros, delegates, types, callbacks, and submodules. Use before reading a module's source.", + }, h.moduleAPIHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_file_outline", + Annotations: readOnly, + Description: "Everything an Elixir file defines: modules, functions, macros, and types with line numbers. Use instead of reading a file to map its contents; one Elixir file can define many modules.", + }, h.fileOutlineHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_implementations", + Annotations: readOnly, + Description: "Implementations of an Elixir behaviour (@behaviour/use) or protocol (defimpl), optionally locating one callback in each implementor. Grep cannot resolve these relationships.", + }, h.implementationsHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_call_hierarchy", + Annotations: readOnly, + Description: "Incoming callers and outgoing callees of an Elixir function, with file:line locations. Use to trace execution paths without reading files.", + }, h.callHierarchyHandler) + + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_reindex", + Annotations: &mcp.ToolAnnotations{DestructiveHint: new(bool), IdempotentHint: true, OpenWorldHint: new(bool)}, + Description: "Update dexter's index after creating, editing, or deleting Elixir files so lookups stay accurate. Incremental and fast; the only tool that writes, and it writes only dexter's own index database.", + }, h.reindexHandler) + + return srv +} + +func textResult(text string) *mcp.CallToolResult { + return &mcp.CallToolResult{Content: []mcp.Content{&mcp.TextContent{Text: text}}} +} + +// relPath renders p relative to the project root when it is inside it. +func (h *Handler) relPath(p string) string { + if rel, err := filepath.Rel(h.projectRoot, p); err == nil && !strings.HasPrefix(rel, "..") { + return rel + } + return p +} + +// resolvePath interprets a user-supplied path against the project root. +func (h *Handler) resolvePath(p string) string { + if filepath.IsAbs(p) { + return p + } + return filepath.Join(h.projectRoot, p) +} + +// symbolName renders Module.function/arity (or just the module name). +func symbolName(module, function string, arity int) string { + if function == "" { + return module + } + return fmt.Sprintf("%s.%s/%d", module, function, arity) +} + +// firstDocLine returns the first non-empty line of a doc string, truncated. +func firstDocLine(doc string) string { + for _, line := range strings.Split(doc, "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + const max = 120 + if len(line) > max { + return line[:max-3] + "..." + } + return line + } + return "" +} diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go new file mode 100644 index 0000000..7a3180f --- /dev/null +++ b/internal/mcp/mcp_test.go @@ -0,0 +1,184 @@ +package mcp + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/lsp" + "github.com/remoteoss/dexter/internal/parser" + "github.com/remoteoss/dexter/internal/store" + "github.com/remoteoss/dexter/internal/version" +) + +// testEnv is a full in-memory MCP round trip: client session <-> server with +// all tools registered, backed by a real store in a temp dir. Going through +// the SDK session exercises schema inference and argument validation, not +// just the handler bodies. +type testEnv struct { + t *testing.T + store *store.Store + root string + session *mcp.ClientSession +} + +func setupTestEnv(t *testing.T) *testEnv { + t.Helper() + root := t.TempDir() + s, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + if err := s.SetIndexVersion(version.IndexVersion); err != nil { + t.Fatal(err) + } + + server := lsp.NewServer(s, root) + h := NewHandler(Config{LSP: server, Store: s, ProjectRoot: root}) + + ctx := context.Background() + serverTransport, clientTransport := mcp.NewInMemoryTransports() + serverSession, err := NewServer(h).Connect(ctx, serverTransport, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = serverSession.Close() }) + + client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "0.0.1"}, nil) + session, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = session.Close() }) + + return &testEnv{t: t, store: s, root: root, session: session} +} + +// indexFile writes an Elixir source file under the project root and indexes it. +func (e *testEnv) indexFile(relPath, content string) string { + e.t.Helper() + path := filepath.Join(e.root, relPath) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + e.t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + e.t.Fatal(err) + } + defs, refs, err := parser.ParseFile(path) + if err != nil { + e.t.Fatal(err) + } + if err := e.store.IndexFileWithRefs(path, defs, refs); err != nil { + e.t.Fatal(err) + } + return path +} + +func (e *testEnv) callTool(name string, args map[string]any) string { + e.t.Helper() + res, err := e.session.CallTool(context.Background(), &mcp.CallToolParams{Name: name, Arguments: args}) + if err != nil { + e.t.Fatalf("CallTool(%s): %v", name, err) + } + if res.IsError { + e.t.Fatalf("CallTool(%s) returned tool error: %s", name, resultText(res)) + } + return resultText(res) +} + +func (e *testEnv) callToolExpectError(name string, args map[string]any) string { + e.t.Helper() + res, err := e.session.CallTool(context.Background(), &mcp.CallToolParams{Name: name, Arguments: args}) + if err != nil { + return err.Error() + } + if !res.IsError { + e.t.Fatalf("CallTool(%s) succeeded, want error; got: %s", name, resultText(res)) + } + return resultText(res) +} + +func resultText(res *mcp.CallToolResult) string { + var b strings.Builder + for _, c := range res.Content { + if tc, ok := c.(*mcp.TextContent); ok { + b.WriteString(tc.Text) + } + } + return b.String() +} + +func wantContains(t *testing.T, got string, wants ...string) { + t.Helper() + for _, w := range wants { + if !strings.Contains(got, w) { + t.Errorf("output missing %q.\nFull output:\n%s", w, got) + } + } +} + +func wantNotContains(t *testing.T, got string, unwanted ...string) { + t.Helper() + for _, w := range unwanted { + if strings.Contains(got, w) { + t.Errorf("output unexpectedly contains %q.\nFull output:\n%s", w, got) + } + } +} + +func TestListTools(t *testing.T) { + e := setupTestEnv(t) + res, err := e.session.ListTools(context.Background(), nil) + if err != nil { + t.Fatal(err) + } + want := []string{ + "dexter_call_hierarchy", + "dexter_definition", + "dexter_file_outline", + "dexter_implementations", + "dexter_module_api", + "dexter_references", + "dexter_reindex", + "dexter_search", + "dexter_workspace", + } + if len(res.Tools) != len(want) { + t.Errorf("registered %d tools, want %d", len(res.Tools), len(want)) + } + var got []string + for _, tool := range res.Tools { + got = append(got, tool.Name) + } + for _, w := range want { + found := false + for _, g := range got { + if g == w { + found = true + } + } + if !found { + t.Errorf("tool %s not registered; got %v", w, got) + } + } + + for _, tool := range res.Tools { + a := tool.Annotations + if a == nil { + t.Errorf("tool %s has no annotations", tool.Name) + continue + } + if a.OpenWorldHint == nil || *a.OpenWorldHint { + t.Errorf("tool %s not marked closed-world", tool.Name) + } + wantReadOnly := tool.Name != "dexter_reindex" + if a.ReadOnlyHint != wantReadOnly { + t.Errorf("tool %s ReadOnlyHint = %v, want %v", tool.Name, a.ReadOnlyHint, wantReadOnly) + } + } +} diff --git a/internal/mcp/module_api.go b/internal/mcp/module_api.go new file mode 100644 index 0000000..36b64c7 --- /dev/null +++ b/internal/mcp/module_api.go @@ -0,0 +1,194 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/lsp" + "github.com/remoteoss/dexter/internal/store" +) + +type ModuleAPIParams struct { + Module string `json:"module" jsonschema:"fully-qualified module name, e.g. MyApp.Accounts (aliases are not resolved)"` + IncludePrivate bool `json:"include_private,omitempty" jsonschema:"also list defp/defmacrop definitions (default false)"` +} + +func (h *Handler) moduleAPIHandler(ctx context.Context, req *mcp.CallToolRequest, args ModuleAPIParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + if module == "" { + return nil, nil, fmt.Errorf("module must not be empty") + } + + modResults, err := h.store.LookupModule(module) + if err != nil { + return nil, nil, fmt.Errorf("looking up module: %w", err) + } + var moduleDef *store.LookupResult + implCount := 0 + isProtocol := false + for i := range modResults { + switch modResults[i].Kind { + case "defimpl": + implCount++ + case "defprotocol": + isProtocol = true + moduleDef = &modResults[i] + case "module": + if moduleDef == nil { + moduleDef = &modResults[i] + } + } + } + if moduleDef == nil { + return textResult(fmt.Sprintf("Module %s is not in the index. Use dexter_search to find the right name, or dexter_reindex if the module was just created.", module)), nil, nil + } + + var b strings.Builder + kind := "module" + if isProtocol { + kind = "protocol" + } + fmt.Fprintf(&b, "%s %s - %s:%d\n", kind, module, h.relPath(moduleDef.FilePath), moduleDef.Line) + if isProtocol && implCount > 0 { + fmt.Fprintf(&b, "%d defimpl implementation(s); list them with dexter_implementations.\n", implCount) + } + + if moduledoc := h.extractModuledoc(moduleDef.FilePath, moduleDef.Line); moduledoc != "" { + fmt.Fprintf(&b, "\n%s\n", strings.TrimRight(moduledoc, "\n")) + } + + funcs, err := h.store.ListModuleFunctions(module, !args.IncludePrivate) + if err != nil { + return nil, nil, fmt.Errorf("listing functions: %w", err) + } + callbacks, err := h.store.ListModuleCallbacks(module) + if err != nil { + return nil, nil, fmt.Errorf("listing callbacks: %w", err) + } + + // Bucket by section, preserving store order (name, arity). + sections := map[string][]store.CompletionResult{} + for _, f := range funcs { + sections[sectionFor(f.Kind)] = append(sections[sectionFor(f.Kind)], f) + } + + docs := h.newDocExtractor() + writeSection := func(title string, entries []store.CompletionResult) { + if len(entries) == 0 { + return + } + fmt.Fprintf(&b, "\n%s:\n", title) + for _, e := range entries { + sig := fmt.Sprintf("%s/%d", e.Function, e.Arity) + if e.Params != "" { + sig = fmt.Sprintf("%s(%s)", e.Function, e.Params) + } + line := fmt.Sprintf(" %s [%s:%d]", sig, h.relPath(e.FilePath), e.Line) + if e.Kind == "defdelegate" { + if target := h.delegateTarget(module, e.Function, e.Arity); target != "" { + line += " → " + target + } + } + if doc := docs.docFor(e.FilePath, e.Line); doc != "" { + line += "\n " + doc + } + b.WriteString(line + "\n") + } + } + + writeSection("Functions", sections["functions"]) + writeSection("Macros", sections["macros"]) + writeSection("Guards", sections["guards"]) + writeSection("Delegates", sections["delegates"]) + writeSection("Types", sections["types"]) + writeSection("Private functions", sections["private"]) + writeSection("Callbacks (this module is a behaviour)", callbacks) + + if subs, err := h.store.ListSubmodules(module); err == nil && len(subs) > 0 { + fmt.Fprintf(&b, "\nSubmodules (%d):\n", len(subs)) + const maxSubs = 20 + for i, s := range subs { + if i == maxSubs { + fmt.Fprintf(&b, " ... and %d more\n", len(subs)-maxSubs) + break + } + fmt.Fprintf(&b, " %s\n", s) + } + } + + if len(funcs) == 0 && len(callbacks) == 0 { + fmt.Fprintf(&b, "\nNo functions indexed for this module.\n") + } + return textResult(b.String()), nil, nil +} + +func sectionFor(kind string) string { + switch kind { + case "defmacro": + return "macros" + case "defguard": + return "guards" + case "defdelegate": + return "delegates" + case "type", "opaque": + return "types" + case "defp", "defmacrop", "defguardp": + return "private" + default: + return "functions" + } +} + +// delegateTarget renders "Target.function" for a defdelegate entry. +func (h *Handler) delegateTarget(module, function string, arity int) string { + results, err := h.store.LookupFunction(module, function) + if err != nil { + return "" + } + for _, r := range results { + if r.Kind == "defdelegate" && r.Arity == arity && r.DelegateTo != "" { + target := r.DelegateTo + "." + function + if r.DelegateAs != "" { + target = r.DelegateTo + "." + r.DelegateAs + } + return target + } + } + return "" +} + +func (h *Handler) extractModuledoc(filePath string, defLine int) string { + text, _, ok := h.lsp.ReadFileText(filePath) + if !ok { + return "" + } + return lsp.NewTokenizedFile(text).ExtractModuledoc(defLine - 1) +} + +// docExtractor extracts @doc summaries, tokenizing each source file at most once. +type docExtractor struct { + h *Handler + files map[string]*lsp.TokenizedFile +} + +func (h *Handler) newDocExtractor() *docExtractor { + return &docExtractor{h: h, files: make(map[string]*lsp.TokenizedFile)} +} + +func (d *docExtractor) docFor(filePath string, defLine int) string { + tf, ok := d.files[filePath] + if !ok { + if text, _, found := d.h.lsp.ReadFileText(filePath); found { + tf = lsp.NewTokenizedFile(text) + } + d.files[filePath] = tf // cache nil results too + } + if tf == nil { + return "" + } + doc, _ := tf.ExtractDocAbove(defLine - 1) + return firstDocLine(doc) +} diff --git a/internal/mcp/references.go b/internal/mcp/references.go new file mode 100644 index 0000000..4f27213 --- /dev/null +++ b/internal/mcp/references.go @@ -0,0 +1,67 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type ReferencesParams struct { + Module string `json:"module" jsonschema:"fully-qualified module name, e.g. MyApp.Accounts (aliases are not resolved)"` + Function string `json:"function,omitempty" jsonschema:"function name; omit to list references to the module itself (aliases, imports, uses, qualified calls)"` +} + +const maxReferenceLines = 100 + +func (h *Handler) referencesHandler(ctx context.Context, req *mcp.CallToolRequest, args ReferencesParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + if module == "" { + return nil, nil, fmt.Errorf("module must not be empty") + } + function := strings.TrimSpace(args.Function) + + refs := h.lsp.CollectReferences(module, function) + if len(refs) == 0 { + target := module + if function != "" { + target = module + "." + function + } + return textResult(fmt.Sprintf("No references to %s found in the index. If files changed recently, call dexter_reindex first.", target)), nil, nil + } + + target := module + if function != "" { + target = module + "." + function + } + + var b strings.Builder + fmt.Fprintf(&b, "%d reference(s) to %s:\n", len(refs), target) + + written := 0 + files := 0 + var lastFile string + truncated := 0 + for _, r := range refs { + if written >= maxReferenceLines { + truncated++ + continue + } + if r.FilePath != lastFile { + fmt.Fprintf(&b, "\n%s\n", h.relPath(r.FilePath)) + lastFile = r.FilePath + files++ + } + srcLine := "" + if line, ok := h.lsp.FileLine(r.FilePath, r.Line); ok { + srcLine = strings.TrimSpace(line) + } + fmt.Fprintf(&b, " %d: %s\n", r.Line, srcLine) + written++ + } + if truncated > 0 { + fmt.Fprintf(&b, "\n... and %d more reference(s) not shown. Narrow the search (e.g. pass a function name) to see the rest.\n", truncated) + } + return textResult(b.String()), nil, nil +} diff --git a/internal/mcp/reindex.go b/internal/mcp/reindex.go new file mode 100644 index 0000000..663ed1b --- /dev/null +++ b/internal/mcp/reindex.go @@ -0,0 +1,24 @@ +package mcp + +import ( + "context" + "fmt" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/version" +) + +type ReindexParams struct{} + +func (h *Handler) reindexHandler(ctx context.Context, req *mcp.CallToolRequest, args ReindexParams) (*mcp.CallToolResult, any, error) { + // A version mismatch requires a full rebuild, which must not happen under a + // live store handle; that is handled at server startup instead. + if stored := h.store.GetIndexVersion(); stored != version.IndexVersion { + return textResult(fmt.Sprintf("Index version %d does not match this binary (%d). Restart dexter mcp to rebuild the index.", stored, version.IndexVersion)), nil, nil + } + + updated, elapsed := h.lsp.ReindexWorkspace() + return textResult(fmt.Sprintf("Reindexed %d file(s) in %s. The index is up to date.", updated, elapsed.Round(time.Millisecond))), nil, nil +} diff --git a/internal/mcp/search.go b/internal/mcp/search.go new file mode 100644 index 0000000..fc4801f --- /dev/null +++ b/internal/mcp/search.go @@ -0,0 +1,39 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +type SearchParams struct { + Query string `json:"query" jsonschema:"fuzzy symbol query, e.g. 'Accounts.fetch' or 'fetch_user'"` + IncludeStdlib bool `json:"include_stdlib,omitempty" jsonschema:"also match Elixir stdlib symbols (default false)"` +} + +func (h *Handler) searchHandler(ctx context.Context, req *mcp.CallToolRequest, args SearchParams) (*mcp.CallToolResult, any, error) { + query := strings.TrimSpace(args.Query) + if query == "" { + return nil, nil, fmt.Errorf("query must not be empty") + } + + var exclude []string + if stdlibRoot := h.lsp.StdlibRoot(); !args.IncludeStdlib && stdlibRoot != "" { + exclude = append(exclude, stdlibRoot) + } + results, err := h.store.SearchSymbols(query, exclude...) + if err != nil { + return nil, nil, fmt.Errorf("searching symbols: %w", err) + } + if len(results) == 0 { + return textResult(fmt.Sprintf("No symbols matched %q. Try a shorter or less specific query; matching is fuzzy on module and function names.", query)), nil, nil + } + + var b strings.Builder + for _, r := range results { + fmt.Fprintf(&b, "%s (%s) - %s:%d\n", symbolName(r.Module, r.Function, r.Arity), r.Kind, h.relPath(r.FilePath), r.Line) + } + return textResult(b.String()), nil, nil +} diff --git a/internal/mcp/serve.go b/internal/mcp/serve.go new file mode 100644 index 0000000..d73146c --- /dev/null +++ b/internal/mcp/serve.go @@ -0,0 +1,20 @@ +package mcp + +import ( + "context" + "net/http" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// RunStdio serves MCP over stdin/stdout until ctx is canceled or the client +// disconnects. +func RunStdio(ctx context.Context, h *Handler) error { + return NewServer(h).Run(ctx, &mcp.StdioTransport{}) +} + +// HTTPHandler returns a streamable-HTTP handler serving MCP. Each session +// gets its own protocol server; they all share the Handler. +func HTTPHandler(h *Handler) http.Handler { + return mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return NewServer(h) }, nil) +} diff --git a/internal/mcp/tools_test.go b/internal/mcp/tools_test.go new file mode 100644 index 0000000..bdeabcc --- /dev/null +++ b/internal/mcp/tools_test.go @@ -0,0 +1,307 @@ +package mcp + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +const accountsSource = `defmodule MyApp.Accounts do + @moduledoc """ + The accounts context. + """ + + @doc """ + Fetches a user by id. + """ + @spec fetch_user(integer()) :: {:ok, map()} | {:error, :not_found} + def fetch_user(id) do + {:ok, %{id: id}} + end + + def list_users(opts) do + opts + end + + defp validate(id), do: id + + defdelegate create_user(attrs), to: MyApp.Accounts.Creator, as: :create + + @type user_id :: integer() +end +` + +const creatorSource = `defmodule MyApp.Accounts.Creator do + def create(attrs) do + attrs + end +end +` + +const workerSource = `defmodule MyApp.Worker do + def run do + MyApp.Accounts.fetch_user(1) + end + + def run_all do + MyApp.Accounts.list_users([]) + MyApp.Accounts.fetch_user(2) + end +end +` + +func setupProject(t *testing.T) *testEnv { + t.Helper() + e := setupTestEnv(t) + e.indexFile("mix.exs", "defmodule MyApp.MixProject do\nend\n") + e.indexFile("lib/my_app/accounts.ex", accountsSource) + e.indexFile("lib/my_app/accounts/creator.ex", creatorSource) + e.indexFile("lib/my_app/worker.ex", workerSource) + return e +} + +func TestWorkspaceTool(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_workspace", nil) + wantContains(t, out, + "Project root: "+e.root, + "mix.exs", + "definitions", + "references", + ) +} + +func TestSearchTool(t *testing.T) { + e := setupProject(t) + + out := e.callTool("dexter_search", map[string]any{"query": "fetch_user"}) + wantContains(t, out, "MyApp.Accounts.fetch_user/1", "lib/my_app/accounts.ex") + + out = e.callTool("dexter_search", map[string]any{"query": "zzz_nothing_matches"}) + wantContains(t, out, "No symbols matched") + + errText := e.callToolExpectError("dexter_search", map[string]any{"query": " "}) + wantContains(t, errText, "query must not be empty") +} + +func TestDefinitionTool_Function(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, out, + "MyApp.Accounts.fetch_user/1 (def)", + "lib/my_app/accounts.ex:10", + "@spec fetch_user(integer())", + "Fetches a user by id.", + "def fetch_user(id) do", + ) +} + +func TestDefinitionTool_FollowsDelegate(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "create_user"}) + wantContains(t, out, + "(defdelegate)", + "Delegates to MyApp.Accounts.Creator.create", + "lib/my_app/accounts/creator.ex", + ) +} + +func TestDefinitionTool_Module(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts"}) + wantContains(t, out, + "defmodule MyApp.Accounts - lib/my_app/accounts.ex:1", + "The accounts context.", + ) +} + +func TestDefinitionTool_NotFound(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Missing"}) + wantContains(t, out, "not in the index") +} + +func TestReferencesTool(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_references", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, out, + "reference(s) to MyApp.Accounts.fetch_user", + "lib/my_app/worker.ex", + "MyApp.Accounts.fetch_user(1)", + "MyApp.Accounts.fetch_user(2)", + ) +} + +func TestReferencesTool_DelegateFacade(t *testing.T) { + e := setupProject(t) + // Calls to the facade MyApp.Accounts.create_user should count as + // references to the delegate target Creator.create. + e.indexFile("lib/my_app/caller.ex", `defmodule MyApp.Caller do + def go(attrs) do + MyApp.Accounts.create_user(attrs) + end +end +`) + out := e.callTool("dexter_references", map[string]any{"module": "MyApp.Accounts.Creator", "function": "create"}) + wantContains(t, out, "lib/my_app/caller.ex") +} + +func TestReferencesTool_Truncation(t *testing.T) { + e := setupProject(t) + var b strings.Builder + b.WriteString("defmodule MyApp.Spammy do\n def go do\n") + for i := 0; i < maxReferenceLines+20; i++ { + fmt.Fprintf(&b, " MyApp.Accounts.list_users(%d)\n", i) + } + b.WriteString(" end\nend\n") + e.indexFile("lib/my_app/spammy.ex", b.String()) + + out := e.callTool("dexter_references", map[string]any{"module": "MyApp.Accounts", "function": "list_users"}) + wantContains(t, out, "more reference(s) not shown") +} + +func TestModuleAPITool(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_module_api", map[string]any{"module": "MyApp.Accounts"}) + wantContains(t, out, + "module MyApp.Accounts - lib/my_app/accounts.ex:1", + "The accounts context.", + "Functions:", + "fetch_user(id)", + "Fetches a user by id.", + "Delegates:", + "create_user(attrs)", + "→ MyApp.Accounts.Creator.create", + "Types:", + "user_id/0", + "Submodules", + "Creator", + ) + wantNotContains(t, out, "validate") + + out = e.callTool("dexter_module_api", map[string]any{"module": "MyApp.Accounts", "include_private": true}) + wantContains(t, out, "validate") +} + +func TestFileOutlineTool(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/accounts.ex"}) + wantContains(t, out, + "defmodule MyApp.Accounts (line 1)", + "def fetch_user/1", + "defp validate/1", + "defdelegate create_user/1", + "→ MyApp.Accounts.Creator.create", + "@type user_id/0", + ) + + out = e.callTool("dexter_file_outline", map[string]any{"file": "lib/nope.ex"}) + wantContains(t, out, "File not found") +} + +func TestFileOutlineTool_NestedModules(t *testing.T) { + e := setupProject(t) + e.indexFile("lib/my_app/outer.ex", `defmodule MyApp.Outer do + def outer_fun, do: :ok + + defmodule Inner do + def inner_fun, do: :ok + end +end +`) + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/outer.ex"}) + wantContains(t, out, + "defmodule MyApp.Outer (line 1)", + "defmodule MyApp.Outer.Inner (line 4)", + "def inner_fun/0", + ) +} + +func TestImplementationsTool_Behaviour(t *testing.T) { + e := setupProject(t) + e.indexFile("lib/my_app/notifier.ex", `defmodule MyApp.Notifier do + @callback deliver(map()) :: :ok | {:error, term()} +end +`) + e.indexFile("lib/my_app/email_notifier.ex", `defmodule MyApp.EmailNotifier do + @behaviour MyApp.Notifier + + @impl true + def deliver(msg) do + :ok + end +end +`) + out := e.callTool("dexter_implementations", map[string]any{"module": "MyApp.Notifier"}) + wantContains(t, out, + "Modules implementing behaviour MyApp.Notifier", + "MyApp.EmailNotifier", + "@callback deliver/1", + ) + + out = e.callTool("dexter_implementations", map[string]any{"module": "MyApp.Notifier", "function": "deliver"}) + wantContains(t, out, + "Implementations of callback MyApp.Notifier.deliver", + "MyApp.EmailNotifier.deliver/1", + "lib/my_app/email_notifier.ex", + ) +} + +func TestImplementationsTool_Protocol(t *testing.T) { + e := setupProject(t) + e.indexFile("lib/my_app/size.ex", `defprotocol MyApp.Size do + def size(data) +end +`) + e.indexFile("lib/my_app/size_impls.ex", `defimpl MyApp.Size, for: BitString do + def size(binary), do: byte_size(binary) +end + +defimpl MyApp.Size, for: Map do + def size(map), do: map_size(map) +end +`) + out := e.callTool("dexter_implementations", map[string]any{"module": "MyApp.Size"}) + wantContains(t, out, + "MyApp.Size is a protocol", + "Implementations (2)", + "lib/my_app/size_impls.ex:1", + "lib/my_app/size_impls.ex:5", + ) +} + +func TestCallHierarchyTool(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_call_hierarchy", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, out, + "Call hierarchy for MyApp.Accounts.fetch_user", + "Incoming (callers)", + "MyApp.Worker.run/0", + "lib/my_app/worker.ex", + ) + + out = e.callTool("dexter_call_hierarchy", map[string]any{"module": "MyApp.Worker", "function": "run", "direction": "outgoing"}) + wantContains(t, out, "Outgoing (callees)", "MyApp.Accounts.fetch_user") + wantNotContains(t, out, "Incoming") + + errText := e.callToolExpectError("dexter_call_hierarchy", map[string]any{"module": "MyApp.Worker", "function": "run", "direction": "sideways"}) + wantContains(t, errText, "direction must be") +} + +func TestReindexTool(t *testing.T) { + e := setupProject(t) + + // Write a new file WITHOUT indexing it: the tool must pick it up. + path := filepath.Join(e.root, "lib/my_app/fresh.ex") + if err := os.WriteFile(path, []byte("defmodule MyApp.Fresh do\n def new_fun, do: :ok\nend\n"), 0644); err != nil { + t.Fatal(err) + } + + out := e.callTool("dexter_reindex", nil) + wantContains(t, out, "Reindexed 1 file(s)") + + out = e.callTool("dexter_search", map[string]any{"query": "new_fun"}) + wantContains(t, out, "MyApp.Fresh.new_fun/0") +} diff --git a/internal/mcp/workspace.go b/internal/mcp/workspace.go new file mode 100644 index 0000000..bfb1644 --- /dev/null +++ b/internal/mcp/workspace.go @@ -0,0 +1,82 @@ +package mcp + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/version" +) + +type WorkspaceParams struct{} + +func (h *Handler) workspaceHandler(ctx context.Context, req *mcp.CallToolRequest, args WorkspaceParams) (*mcp.CallToolResult, any, error) { + var b strings.Builder + fmt.Fprintf(&b, "Dexter %s\n", version.Version) + fmt.Fprintf(&b, "Project root: %s\n", h.projectRoot) + + if projects := findMixProjects(h.projectRoot); len(projects) > 0 { + fmt.Fprintf(&b, "\nMix projects:\n") + for _, p := range projects { + fmt.Fprintf(&b, " %s\n", p) + } + } else { + fmt.Fprintf(&b, "\nNo mix.exs found at the project root. The index may cover a plain directory of Elixir files.\n") + } + + if stdlibRoot := h.lsp.StdlibRoot(); stdlibRoot != "" { + fmt.Fprintf(&b, "\nElixir stdlib: %s (indexed; stdlib symbols resolve in lookups)\n", stdlibRoot) + } else { + fmt.Fprintf(&b, "\nElixir stdlib: not detected. Set DEXTER_ELIXIR_LIB_ROOT to enable stdlib lookups.\n") + } + + st, err := h.store.Stats() + if err != nil { + return nil, nil, fmt.Errorf("reading index stats: %w", err) + } + fmt.Fprintf(&b, "\nIndex: %d files, %d definitions, %d references\n", st.Files, st.Definitions, st.References) + + if stored := h.store.GetIndexVersion(); stored != version.IndexVersion { + fmt.Fprintf(&b, "WARNING: index version %d does not match this binary (%d). Restart dexter mcp to rebuild.\n", stored, version.IndexVersion) + } + fmt.Fprintf(&b, "\nThe index updates automatically on git branch switches. After you edit, create, or delete Elixir files, call dexter_reindex before trusting lookups.\n") + + return textResult(b.String()), nil, nil +} + +// findMixProjects lists mix.exs locations relative to root: the root itself, +// umbrella apps under apps/, and direct children with their own mix.exs. +// The scan is deliberately shallow; no full tree walk. +func findMixProjects(root string) []string { + var projects []string + if _, err := os.Stat(filepath.Join(root, "mix.exs")); err == nil { + projects = append(projects, "mix.exs") + } + for _, pattern := range []string{"apps/*/mix.exs", "*/mix.exs"} { + matches, _ := filepath.Glob(filepath.Join(root, pattern)) + for _, m := range matches { + if rel, err := filepath.Rel(root, m); err == nil && rel != "mix.exs" { + projects = append(projects, rel) + } + } + } + sort.Strings(projects) + return dedupeStrings(projects) +} + +func dedupeStrings(in []string) []string { + out := in[:0] + var prev string + for i, s := range in { + if i == 0 || s != prev { + out = append(out, s) + } + prev = s + } + return out +} diff --git a/internal/store/store.go b/internal/store/store.go index 61335fc..f210137 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -677,6 +677,54 @@ func (s *Store) ListModuleFunctions(module string, publicOnly bool) ([]Completio return results, rows.Err() } +// ListModuleCallbacks returns the @callback and @macrocallback definitions of +// the given behaviour module (these are excluded from ListModuleFunctions). +func (s *Store) ListModuleCallbacks(module string) ([]CompletionResult, error) { + rows, err := s.db.Query( + "SELECT module, function, arity, kind, file_path, line, params FROM definitions WHERE module = ? AND kind IN ('callback', 'macrocallback') GROUP BY function, arity ORDER BY function, arity LIMIT 100", + module, + ) + if err != nil { + return nil, err + } + defer func() { _ = rows.Close() }() + + var results []CompletionResult + for rows.Next() { + var r CompletionResult + if err := rows.Scan(&r.Module, &r.Function, &r.Arity, &r.Kind, &r.FilePath, &r.Line, &r.Params); err != nil { + return nil, err + } + results = append(results, r) + } + return results, rows.Err() +} + +// IndexStats summarizes the size of the index. +type IndexStats struct { + Files int + Definitions int + References int +} + +// Stats returns row counts for the files, definitions, and refs tables. +func (s *Store) Stats() (IndexStats, error) { + var st IndexStats + for _, q := range []struct { + query string + dst *int + }{ + {"SELECT COUNT(*) FROM files", &st.Files}, + {"SELECT COUNT(*) FROM definitions", &st.Definitions}, + {"SELECT COUNT(*) FROM refs", &st.References}, + } { + if err := s.db.QueryRow(q.query).Scan(q.dst); err != nil { + return IndexStats{}, err + } + } + return st, nil +} + type LookupResult struct { Module string // populated by bulk queries; empty for single-module lookups FilePath string diff --git a/internal/store/store_test.go b/internal/store/store_test.go index 5c9efa0..20b4cf9 100644 --- a/internal/store/store_test.go +++ b/internal/store/store_test.go @@ -1313,3 +1313,89 @@ func TestFindProjectRoot(t *testing.T) { } }) } + +func TestListModuleCallbacks(t *testing.T) { + s, dir := setupTestStore(t) + defer func() { _ = s.Close() }() + + path := writeElixirFile(t, dir, "lib/notifier.ex", `defmodule MyApp.Notifier do + @callback deliver(map()) :: :ok | {:error, term()} + @callback name() :: String.t() + @macrocallback render(term()) :: Macro.t() + + def dispatch(msg) do + :ok + end +end +`) + + defs, _, err := parser.ParseFile(path) + if err != nil { + t.Fatal(err) + } + if err := s.IndexFile(path, defs); err != nil { + t.Fatal(err) + } + + results, err := s.ListModuleCallbacks("MyApp.Notifier") + if err != nil { + t.Fatal(err) + } + if len(results) != 3 { + t.Fatalf("expected 3 callbacks, got %d: %+v", len(results), results) + } + kinds := map[string]string{} + for _, r := range results { + kinds[r.Function] = r.Kind + if r.Function == "dispatch" { + t.Error("regular function included in callbacks") + } + } + if kinds["deliver"] != "callback" { + t.Errorf("deliver kind = %q, want callback", kinds["deliver"]) + } + if kinds["render"] != "macrocallback" { + t.Errorf("render kind = %q, want macrocallback", kinds["render"]) + } +} + +func TestStats(t *testing.T) { + s, dir := setupTestStore(t) + defer func() { _ = s.Close() }() + + st, err := s.Stats() + if err != nil { + t.Fatal(err) + } + if st.Files != 0 || st.Definitions != 0 || st.References != 0 { + t.Errorf("empty store stats = %+v, want zeros", st) + } + + path := writeElixirFile(t, dir, "lib/worker.ex", `defmodule MyApp.Worker do + def run do + MyApp.Accounts.fetch_user(1) + end +end +`) + defs, refs, err := parser.ParseFile(path) + if err != nil { + t.Fatal(err) + } + if err := s.IndexFileWithRefs(path, defs, refs); err != nil { + t.Fatal(err) + } + + st, err = s.Stats() + if err != nil { + t.Fatal(err) + } + if st.Files != 1 { + t.Errorf("Files = %d, want 1", st.Files) + } + if st.Definitions < 2 { // module + run + t.Errorf("Definitions = %d, want >= 2", st.Definitions) + } + if st.References < 1 { + t.Errorf("References = %d, want >= 1", st.References) + } +} From 012d62a0cdc9883ff50eb7aabc3cd894f7840731 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Tue, 1 Sep 2026 14:22:55 +1000 Subject: [PATCH 02/26] Watch files in MCP mode (fsnotify) Headless MCP servers get no editor LSP events, so lookups went stale until an agent chose to call dexter_reindex. Watch the project tree with fsnotify instead: file writes reindex the changed file (debounced), deletes drop entries (including whole directories), and new directories are watched and indexed as they appear. deps/, _build/, node_modules, .git, and .dexter are not watched; deps change only through mix and are covered by the startup reindex. On watcher overflow the workspace is reindexed incrementally; if watching is unavailable the server logs a warning and degrades to branch-switch detection plus dexter_reindex. --- CHANGELOG.md | 2 +- README.md | 2 +- cmd/main.go | 12 +++ go.mod | 1 + go.sum | 2 + internal/mcp/instructions.md | 6 +- internal/mcp/mcp.go | 2 +- internal/mcp/watch.go | 189 +++++++++++++++++++++++++++++++++++ internal/mcp/watch_test.go | 153 ++++++++++++++++++++++++++++ internal/mcp/workspace.go | 2 +- 10 files changed, 364 insertions(+), 7 deletions(-) create mode 100644 internal/mcp/watch.go create mode 100644 internal/mcp/watch_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index eaae6b2..7a9408a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Nine tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide +- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Nine tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. The headless server watches the project tree (fsnotify) so the index stays fresh without editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide ## [0.7.1] - 2026-06-12 diff --git a/README.md b/README.md index b2e835b..7458916 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,7 @@ Register it with your MCP client. For Claude Code: claude mcp add dexter -- dexter mcp ``` -Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use, keeps the index fresh across git branch switches, and exposes a `dexter_reindex` tool for agents to call after editing files. +Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use and keeps the index fresh by watching the project tree (fsnotify) and detecting git branch switches; a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. Useful variants: diff --git a/cmd/main.go b/cmd/main.go index 3252d15..d709ebf 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -636,6 +636,18 @@ func cmdMCP(projectRoot string, listen string) { server.ReindexWorkspace() server.WatchGitHead() + // Headless servers get no editor events, so watch the tree directly. + watcher, err := dexter_mcp.WatchFiles(s, projectRoot, func() { server.ReindexWorkspace() }) + if err != nil { + log.Printf("Warning: file watching unavailable (%v); the index updates on branch switches and via dexter_reindex", err) + } else { + defer func() { + if err := watcher.Close(); err != nil { + log.Printf("Warning: closing file watcher: %v", err) + } + }() + } + h := dexter_mcp.NewHandler(dexter_mcp.Config{LSP: server, Store: s, ProjectRoot: projectRoot}) log.Printf("Dexter MCP v%s starting (root: %s)", version.Version, projectRoot) diff --git a/go.mod b/go.mod index 94ffde1..c5ae6fd 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/remoteoss/dexter go 1.26.1 require ( + github.com/fsnotify/fsnotify v1.10.1 github.com/mattn/go-sqlite3 v1.14.38 github.com/modelcontextprotocol/go-sdk v1.6.1 github.com/spf13/cobra v1.10.2 diff --git a/go.sum b/go.sum index 6a99e0e..c8a4d7c 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/elixir-lang/tree-sitter-elixir v0.3.5 h1:Ir60dE/aHPt80uil58ukW1CTC+15l4jHax/iHBsW9HI= github.com/elixir-lang/tree-sitter-elixir v0.3.5/go.mod h1:wNBVf64kzvhSbZ8ojVtBF1jRiqGY0lsuK5Kx/60s6Z0= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md index a515005..e4de50e 100644 --- a/internal/mcp/instructions.md +++ b/internal/mcp/instructions.md @@ -16,9 +16,9 @@ Which tool for which question: - What a specific file defines: `dexter_file_outline` - Project layout and index freshness: `dexter_workspace` -After you create, edit, or delete Elixir files by any means, call -`dexter_reindex` (fast, incremental) so results stay accurate. Git branch -switches are picked up automatically. +The index updates automatically: file changes are watched (fsnotify) and git +branch switches are detected. If a lookup ever seems stale, `dexter_reindex` +forces an immediate incremental update. Elixir specifics: modules are not tied to files (use `dexter_file_outline` for a file, `dexter_definition` for a module); pass fully-qualified module names, diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index f3bd22b..96be403 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -109,7 +109,7 @@ func NewServer(h *Handler) *mcp.Server { mcp.AddTool(srv, &mcp.Tool{ Name: "dexter_reindex", Annotations: &mcp.ToolAnnotations{DestructiveHint: new(bool), IdempotentHint: true, OpenWorldHint: new(bool)}, - Description: "Update dexter's index after creating, editing, or deleting Elixir files so lookups stay accurate. Incremental and fast; the only tool that writes, and it writes only dexter's own index database.", + Description: "Force an immediate incremental reindex. The index already updates automatically as files change; use this only when a lookup seems stale. The only tool that writes, and it writes only dexter's own index database.", }, h.reindexHandler) return srv diff --git a/internal/mcp/watch.go b/internal/mcp/watch.go new file mode 100644 index 0000000..e784908 --- /dev/null +++ b/internal/mcp/watch.go @@ -0,0 +1,189 @@ +package mcp + +import ( + "errors" + "io/fs" + "log" + "os" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" + + "github.com/remoteoss/dexter/internal/parser" + "github.com/remoteoss/dexter/internal/store" +) + +// debounceWindow batches bursts of filesystem events (editor saves, git +// operations) into one reindex pass. +const debounceWindow = 300 * time.Millisecond + +// Watcher keeps the index in sync with filesystem changes. Editors drive +// index updates through LSP events, but a headless MCP server gets none, so +// it watches the project tree directly (fsnotify). +type Watcher struct { + fsw *fsnotify.Watcher + store *store.Store + root string + resync func() // full incremental reindex, used when events were lost + wg sync.WaitGroup +} + +// WatchFiles watches projectRoot recursively and incrementally reindexes +// Elixir files as they change. resync is invoked when the watcher loses +// events (queue overflow) and the whole tree must be reconciled. Callers +// should treat an error as degraded service, not fatal: the index still +// updates on startup, on git branch switches, and via dexter_reindex. +func WatchFiles(s *store.Store, projectRoot string, resync func()) (*Watcher, error) { + fsw, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + w := &Watcher{fsw: fsw, store: s, root: projectRoot, resync: resync} + if err := w.watchTree(projectRoot); err != nil { + _ = fsw.Close() + return nil, err + } + w.wg.Add(1) + go w.loop() + return w, nil +} + +// Close stops the watcher and waits for the event loop to exit. +func (w *Watcher) Close() error { + err := w.fsw.Close() + w.wg.Wait() + return err +} + +// skipDir reports whether a directory's subtree is not watched: build output, +// VCS metadata, and deps, which change only through mix and are covered by +// the startup reindex. +func skipDir(name string) bool { + switch name { + case "_build", ".git", "node_modules", "deps", ".dexter": + return true + } + return false +} + +// watchTree adds watches for root and every eligible directory below it. +func (w *Watcher) watchTree(root string) error { + return filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() { + return nil + } + if skipDir(d.Name()) { + return filepath.SkipDir + } + return w.fsw.Add(path) + }) +} + +func (w *Watcher) loop() { + defer w.wg.Done() + + pending := make(map[string]struct{}) + var timer *time.Timer + var timerC <-chan time.Time + + schedule := func() { + if timer == nil { + timer = time.NewTimer(debounceWindow) + timerC = timer.C + } else { + timer.Reset(debounceWindow) + } + } + + for { + select { + case ev, ok := <-w.fsw.Events: + if !ok { + return + } + // Directory events matter for watch maintenance; file events only + // for Elixir sources. Everything else is noise. + if parser.IsElixirFile(ev.Name) || ev.Op.Has(fsnotify.Create) || ev.Op.Has(fsnotify.Remove) || ev.Op.Has(fsnotify.Rename) { + pending[ev.Name] = struct{}{} + schedule() + } + case err, ok := <-w.fsw.Errors: + if !ok { + return + } + if errors.Is(err, fsnotify.ErrEventOverflow) && w.resync != nil { + log.Printf("Warning: file watcher overflowed, reindexing workspace") + w.resync() + continue + } + log.Printf("Warning: file watcher: %v", err) + case <-timerC: + timer = nil + timerC = nil + batch := pending + pending = make(map[string]struct{}) + w.apply(batch) + } + } +} + +// apply reconciles the index with a batch of changed paths. +func (w *Watcher) apply(batch map[string]struct{}) { + for path := range batch { + info, err := os.Stat(path) + switch { + case err != nil: + w.removePath(path) + case info.IsDir(): + if skipDir(filepath.Base(path)) { + continue + } + // New directory (e.g. git checkout, mkdir && write): watch it and + // index any Elixir files already inside, since their create events + // may predate the watch. + if err := w.watchTree(path); err != nil { + log.Printf("Warning: watching %s: %v", path, err) + } + _ = parser.WalkElixirFiles(path, func(p string, _ fs.DirEntry) error { + w.reindexFile(p) + return nil + }) + case parser.IsElixirFile(path): + w.reindexFile(path) + } + } +} + +func (w *Watcher) reindexFile(path string) { + defs, refs, err := parser.ParseFile(path) + if err != nil { + log.Printf("Warning: %s: %v", path, err) + return + } + if err := w.store.IndexFileWithRefs(path, defs, refs); err != nil { + log.Printf("Warning: %s: %v", path, err) + } +} + +// removePath drops a deleted file from the index. A deleted path may have +// been a directory, so entries under it are dropped too. +func (w *Watcher) removePath(path string) { + _ = w.store.RemoveFile(path) + stored, err := w.store.ListFilePaths() + if err != nil { + return + } + prefix := path + string(os.PathSeparator) + var under []string + for _, p := range stored { + if strings.HasPrefix(p, prefix) { + under = append(under, p) + } + } + if len(under) > 0 { + _ = w.store.RemoveFiles(under) + } +} diff --git a/internal/mcp/watch_test.go b/internal/mcp/watch_test.go new file mode 100644 index 0000000..c37cbfc --- /dev/null +++ b/internal/mcp/watch_test.go @@ -0,0 +1,153 @@ +package mcp + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/remoteoss/dexter/internal/store" +) + +// eventually polls cond until it returns true or the deadline passes. +// Filesystem notification latency varies by platform, so watcher assertions +// must poll rather than sleep. +func eventually(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(10 * time.Second) + for time.Now().Before(deadline) { + if cond() { + return + } + time.Sleep(50 * time.Millisecond) + } + t.Fatalf("timed out waiting for %s", what) +} + +func setupWatcher(t *testing.T) (*store.Store, string) { + t.Helper() + root := t.TempDir() + s, err := store.Open(root) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = s.Close() }) + + w, err := WatchFiles(s, root, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := w.Close(); err != nil { + t.Errorf("closing watcher: %v", err) + } + }) + return s, root +} + +func writeFile(t *testing.T, root, rel, content string) string { + t.Helper() + path := filepath.Join(root, rel) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } + return path +} + +func moduleIndexed(s *store.Store, module string) func() bool { + return func() bool { + results, err := s.LookupModule(module) + return err == nil && len(results) > 0 + } +} + +func TestWatcher_NewFileIndexed(t *testing.T) { + s, root := setupWatcher(t) + writeFile(t, root, "lib/fresh.ex", "defmodule MyApp.Fresh do\n def hello, do: :ok\nend\n") + eventually(t, "new file to be indexed", moduleIndexed(s, "MyApp.Fresh")) +} + +func TestWatcher_ModifiedFileReindexed(t *testing.T) { + s, root := setupWatcher(t) + path := writeFile(t, root, "lib/acc.ex", "defmodule MyApp.Acc do\n def old_fun, do: :ok\nend\n") + eventually(t, "initial index", func() bool { + r, _ := s.LookupFunction("MyApp.Acc", "old_fun") + return len(r) > 0 + }) + + if err := os.WriteFile(path, []byte("defmodule MyApp.Acc do\n def new_fun, do: :ok\nend\n"), 0644); err != nil { + t.Fatal(err) + } + eventually(t, "modified file to be reindexed", func() bool { + newR, _ := s.LookupFunction("MyApp.Acc", "new_fun") + oldR, _ := s.LookupFunction("MyApp.Acc", "old_fun") + return len(newR) > 0 && len(oldR) == 0 + }) +} + +func TestWatcher_DeletedFileRemoved(t *testing.T) { + s, root := setupWatcher(t) + path := writeFile(t, root, "lib/gone.ex", "defmodule MyApp.Gone do\nend\n") + eventually(t, "initial index", moduleIndexed(s, "MyApp.Gone")) + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + eventually(t, "deleted file to leave the index", func() bool { + results, err := s.LookupModule("MyApp.Gone") + return err == nil && len(results) == 0 + }) +} + +func TestWatcher_DeletedDirectoryRemoved(t *testing.T) { + s, root := setupWatcher(t) + writeFile(t, root, "lib/sub/a.ex", "defmodule MyApp.Sub.A do\nend\n") + writeFile(t, root, "lib/sub/b.ex", "defmodule MyApp.Sub.B do\nend\n") + eventually(t, "initial index", func() bool { + a, _ := s.LookupModule("MyApp.Sub.A") + b, _ := s.LookupModule("MyApp.Sub.B") + return len(a) > 0 && len(b) > 0 + }) + + if err := os.RemoveAll(filepath.Join(root, "lib/sub")); err != nil { + t.Fatal(err) + } + eventually(t, "deleted directory's files to leave the index", func() bool { + a, _ := s.LookupModule("MyApp.Sub.A") + b, _ := s.LookupModule("MyApp.Sub.B") + return len(a) == 0 && len(b) == 0 + }) +} + +func TestWatcher_NewDirectoryWatched(t *testing.T) { + s, root := setupWatcher(t) + // Create the directory and its file separately so the file event can only + // be seen by a watch added after the directory appeared. + if err := os.MkdirAll(filepath.Join(root, "lib/newdir"), 0755); err != nil { + t.Fatal(err) + } + time.Sleep(50 * time.Millisecond) + writeFile(t, root, "lib/newdir/mod.ex", "defmodule MyApp.NewDir.Mod do\nend\n") + eventually(t, "file in new directory to be indexed", moduleIndexed(s, "MyApp.NewDir.Mod")) +} + +func TestWatcher_SkipsDepsAndNonElixir(t *testing.T) { + s, root := setupWatcher(t) + writeFile(t, root, "deps/pkg/lib/dep.ex", "defmodule DepPkg.Ignored do\nend\n") + writeFile(t, root, "lib/notes.txt", "defmodule NotElixir do\nend\n") + + // Anchor on a real file so the negative checks below observe a watcher + // that has demonstrably processed events. + writeFile(t, root, "lib/anchor.ex", "defmodule MyApp.Anchor do\nend\n") + eventually(t, "anchor file to be indexed", moduleIndexed(s, "MyApp.Anchor")) + + if r, _ := s.LookupModule("DepPkg.Ignored"); len(r) != 0 { + t.Error("file under deps/ was indexed by the watcher") + } + if r, _ := s.LookupModule("NotElixir"); len(r) != 0 { + t.Error("non-Elixir file was indexed") + } +} diff --git a/internal/mcp/workspace.go b/internal/mcp/workspace.go index bfb1644..9e41e74 100644 --- a/internal/mcp/workspace.go +++ b/internal/mcp/workspace.go @@ -44,7 +44,7 @@ func (h *Handler) workspaceHandler(ctx context.Context, req *mcp.CallToolRequest if stored := h.store.GetIndexVersion(); stored != version.IndexVersion { fmt.Fprintf(&b, "WARNING: index version %d does not match this binary (%d). Restart dexter mcp to rebuild.\n", stored, version.IndexVersion) } - fmt.Fprintf(&b, "\nThe index updates automatically on git branch switches. After you edit, create, or delete Elixir files, call dexter_reindex before trusting lookups.\n") + fmt.Fprintf(&b, "\nThe index updates automatically as files change and on git branch switches; dexter_reindex forces an immediate update.\n") return textResult(b.String()), nil, nil } From 811a3f185f0a43bb7cd29db7eb5931c1c962fdaa Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Tue, 1 Sep 2026 16:11:48 +1000 Subject: [PATCH 03/26] Add dexter_rename_symbol MCP tool Workspace-wide rename of a module or function with the same on-disk semantics as the editor rename: changes are written to disk, files following the naming convention are moved, and the index is updated. The tool reports every file changed and moved; git provides review and revert. The exported RenameFunction/RenameModule wrappers carry the same validation as the LSP handler and reuse its machinery unchanged, except that edits the LSP would hand to an editor as TextEdits (open buffers in attached mode) are also written to disk, since an MCP caller has no editor to deliver them to. --- CHANGELOG.md | 2 +- README.md | 2 +- integration_test.go | 4 +- internal/lsp/api.go | 115 +++++++++++++++++++++++++++++++++++ internal/lsp/api_test.go | 94 ++++++++++++++++++++++++++++ internal/lsp/server.go | 49 ++++++++++++--- internal/mcp/instructions.md | 2 + internal/mcp/mcp.go | 6 ++ internal/mcp/mcp_test.go | 3 +- internal/mcp/rename.go | 55 +++++++++++++++++ internal/mcp/rename_test.go | 87 ++++++++++++++++++++++++++ 11 files changed, 404 insertions(+), 15 deletions(-) create mode 100644 internal/lsp/api_test.go create mode 100644 internal/mcp/rename.go create mode 100644 internal/mcp/rename_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a9408a..0129612 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Nine tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. The headless server watches the project tree (fsnotify) so the index stays fresh without editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide +- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. The headless server watches the project tree (fsnotify) so the index stays fresh without editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide ## [0.7.1] - 2026-06-12 diff --git a/README.md b/README.md index 7458916..25625a6 100644 --- a/README.md +++ b/README.md @@ -456,7 +456,7 @@ When running as an LSP server, dexter automatically: ## MCP server -Dexter includes a built-in [Model Context Protocol](https://modelcontextprotocol.io) server, modeled on `gopls mcp`, so AI agents can navigate Elixir codebases through the index instead of grep. Tools cover symbol search, definitions with docs and specs, references, module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, and incremental reindexing. +Dexter includes a built-in [Model Context Protocol](https://modelcontextprotocol.io) server, modeled on `gopls mcp`, so AI agents can navigate Elixir codebases through the index instead of grep. Tools cover symbol search, definitions with docs and specs, references, module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename. Register it with your MCP client. For Claude Code: diff --git a/integration_test.go b/integration_test.go index a3328a0..5c8e023 100644 --- a/integration_test.go +++ b/integration_test.go @@ -615,7 +615,7 @@ func TestIntegration_MCPStdio(t *testing.T) { for _, tool := range tools.Tools { names[tool.Name] = true } - for _, want := range []string{"dexter_workspace", "dexter_search", "dexter_definition", "dexter_references", "dexter_module_api", "dexter_file_outline", "dexter_implementations", "dexter_call_hierarchy", "dexter_reindex"} { + for _, want := range []string{"dexter_workspace", "dexter_search", "dexter_definition", "dexter_references", "dexter_module_api", "dexter_file_outline", "dexter_implementations", "dexter_call_hierarchy", "dexter_reindex", "dexter_rename_symbol"} { if !names[want] { t.Errorf("tool %s not advertised; got %v", want, names) } @@ -664,7 +664,7 @@ func TestIntegration_MCPStdio_EmptyIndexBuildsOnStartup(t *testing.T) { func TestIntegration_MCPInstructions(t *testing.T) { binary := buildDexter(t) out := runDexter(t, binary, t.TempDir(), "mcp", "--instructions") - for _, want := range []string{"dexter_workspace", "dexter_reindex"} { + for _, want := range []string{"dexter_workspace", "dexter_reindex", "dexter_rename_symbol"} { if !strings.Contains(out, want) { t.Errorf("instructions missing %q", want) } diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 433f199..2729369 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -2,7 +2,9 @@ package lsp import ( "context" + "fmt" "io" + "os" "sort" "strings" @@ -112,3 +114,116 @@ func (s *Server) CollectReferences(module, function string) []store.ReferenceRes }) return out } + +// RenameSummary reports what a rename changed on disk. +type RenameSummary struct { + FilesChanged []string + FilesMoved map[string]string // old path → new path (conventional module renames) +} + +// RenameFunction renames module.functionName to newName across the workspace, +// writing every change to disk. It performs the same validation as the LSP +// rename. Edits the LSP path would hand to an editor as TextEdits (open +// buffers in attached mode) are written to disk too, since an MCP caller has +// no editor to deliver them to. +func (s *Server) RenameFunction(module, functionName, newName string) (RenameSummary, error) { + if !isValidFunctionName(newName) { + return RenameSummary{}, fmt.Errorf("invalid function name %q: must match [a-z_][a-z0-9_?!]*", newName) + } + defs, err := s.store.LookupFunction(module, functionName) + if err != nil { + return RenameSummary{}, err + } + if len(defs) == 0 { + return RenameSummary{}, fmt.Errorf("function %s.%s not found in the index", module, functionName) + } + if existing, err := s.store.LookupFunction(module, newName); err == nil && len(existing) > 0 { + return RenameSummary{}, fmt.Errorf("function %s.%s already exists", module, newName) + } + + edit, files, err := s.renameFunctionEdits(module, functionName, newName) + if err != nil { + return RenameSummary{}, err + } + if err := s.writeEditsToDisk(edit); err != nil { + return RenameSummary{}, err + } + return RenameSummary{FilesChanged: files}, nil +} + +// RenameModule renames oldModule (and its submodules) to newModule across the +// workspace, writing changes and conventional file moves to disk. +func (s *Server) RenameModule(oldModule, newModule string) (RenameSummary, error) { + if !isValidModuleName(newModule) { + return RenameSummary{}, fmt.Errorf("invalid module name %q: must be CamelCase segments separated by dots", newModule) + } + if defs, err := s.store.LookupModule(oldModule); err != nil || len(defs) == 0 { + if err != nil { + return RenameSummary{}, err + } + return RenameSummary{}, fmt.Errorf("module %s not found in the index", oldModule) + } + + edit, moved, files, err := s.renameModuleEdits(context.Background(), oldModule, newModule, "") + if err != nil { + return RenameSummary{}, err + } + if err := s.writeEditsToDisk(edit); err != nil { + return RenameSummary{}, err + } + return RenameSummary{FilesChanged: files, FilesMoved: moved}, nil +} + +// writeEditsToDisk applies a WorkspaceEdit's TextEdits to their files on disk +// and reindexes them. The rename machinery only produces TextEdits for open +// editor buffers; headless servers have none, so this is usually a no-op. +func (s *Server) writeEditsToDisk(edit *protocol.WorkspaceEdit) error { + if edit == nil { + return nil + } + for docURI, edits := range edit.Changes { + path := uriToPath(docURI) + text, _, ok := s.ReadFileText(path) + if !ok { + return fmt.Errorf("reading %s to apply rename edits", path) + } + if err := os.WriteFile(path, []byte(applyTextEdits(text, edits)), 0644); err != nil { + return err + } + } + if len(edit.Changes) > 0 { + paths := make([]string, 0, len(edit.Changes)) + for docURI := range edit.Changes { + paths = append(paths, uriToPath(docURI)) + } + s.reindexPaths(paths) + } + return nil +} + +// applyTextEdits applies non-overlapping TextEdits to text. Positions use the +// same line/byte-column convention the rename machinery produces them in. +func applyTextEdits(text string, edits []protocol.TextEdit) string { + sorted := make([]protocol.TextEdit, len(edits)) + copy(sorted, edits) + sort.Slice(sorted, func(i, j int) bool { + a, b := sorted[i].Range.Start, sorted[j].Range.Start + if a.Line != b.Line { + return a.Line > b.Line + } + return a.Character > b.Character + }) + + lines := strings.Split(text, "\n") + for _, e := range sorted { + start, end := e.Range.Start, e.Range.End + if int(start.Line) >= len(lines) || int(end.Line) >= len(lines) { + continue + } + prefix := lines[start.Line][:start.Character] + suffix := lines[end.Line][end.Character:] + replacement := strings.Split(prefix+e.NewText+suffix, "\n") + lines = append(lines[:start.Line], append(replacement, lines[end.Line+1:]...)...) + } + return strings.Join(lines, "\n") +} diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go new file mode 100644 index 0000000..f34ea79 --- /dev/null +++ b/internal/lsp/api_test.go @@ -0,0 +1,94 @@ +package lsp + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "go.lsp.dev/protocol" + "go.lsp.dev/uri" +) + +func TestApplyTextEdits(t *testing.T) { + tests := []struct { + name string + text string + edits []protocol.TextEdit + want string + }{ + { + name: "single token on one line", + text: "def fetch_user(id) do\n fetch_user(id)\nend\n", + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 4}, End: protocol.Position{Line: 0, Character: 14}}, NewText: "get_user"}, + }, + want: "def get_user(id) do\n fetch_user(id)\nend\n", + }, + { + name: "two tokens on the same line applied right to left", + text: "fetch_user(fetch_user(1))\n", + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 0}, End: protocol.Position{Line: 0, Character: 10}}, NewText: "get_user"}, + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 11}, End: protocol.Position{Line: 0, Character: 21}}, NewText: "get_user"}, + }, + want: "get_user(get_user(1))\n", + }, + { + name: "multi-line span replacement", + text: "a\nold one\nold two\nb\n", + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 1, Character: 0}, End: protocol.Position{Line: 2, Character: 7}}, NewText: "new one\nnew two\nnew three"}, + }, + want: "a\nnew one\nnew two\nnew three\nb\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := applyTextEdits(tt.text, tt.edits); got != tt.want { + t.Errorf("got:\n%q\nwant:\n%q", got, tt.want) + } + }) + } +} + +// A rename requested through the exported API must land on disk even for +// files an editor holds open (attached mode), since the caller has no editor +// to deliver TextEdits to. +func TestRenameFunction_WritesOpenBuffers(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", `defmodule MyApp.Accounts do + def fetch_user(id), do: id +end +`) + openSrc := `defmodule MyApp.Caller do + def go(id), do: MyApp.Accounts.fetch_user(id) +end +` + indexFile(t, server.store, server.projectRoot, "lib/caller.ex", openSrc) + openPath := filepath.Join(server.projectRoot, "lib/caller.ex") + server.docs.Set(string(uri.File(openPath)), openSrc) // simulate didOpen + + summary, err := server.RenameFunction("MyApp.Accounts", "fetch_user", "get_user") + if err != nil { + t.Fatal(err) + } + server.backgroundWork.Wait() + + if len(summary.FilesChanged) != 2 { + t.Errorf("FilesChanged = %v, want both files", summary.FilesChanged) + } + data, err := os.ReadFile(openPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(data), "MyApp.Accounts.get_user(id)") { + t.Errorf("open buffer's file not written to disk:\n%s", data) + } + results, err := server.store.LookupFunction("MyApp.Accounts", "get_user") + if err != nil || len(results) == 0 { + t.Errorf("index not updated after rename: %v, %v", results, err) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 5ee9966..130de70 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -4172,7 +4173,8 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr if existing, err := s.store.LookupFunction(fullModule, params.NewName); err == nil && len(existing) > 0 { return nil, fmt.Errorf("function %s.%s already exists", fullModule, params.NewName) } - return s.renameFunctionEdits(fullModule, functionName, params.NewName) + edit, _, err := s.renameFunctionEdits(fullModule, functionName, params.NewName) + return edit, err } } else if moduleRef != "" { fullModule := resolveModule(moduleRef, aliases) @@ -4186,7 +4188,8 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr if !isValidModuleName(newModule) { return nil, fmt.Errorf("invalid module name %q: must be CamelCase segments separated by dots", params.NewName) } - return s.renameModuleEdits(ctx, fullModule, newModule, uriToPath(params.TextDocument.URI)) + edit, _, _, err := s.renameModuleEdits(ctx, fullModule, newModule, uriToPath(params.TextDocument.URI)) + return edit, err } } } @@ -4195,8 +4198,9 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr } // renameFunctionEdits builds a WorkspaceEdit renaming all occurrences of -// module.functionName to newName across the codebase. -func (s *Server) renameFunctionEdits(module, functionName, newName string) (*protocol.WorkspaceEdit, error) { +// module.functionName to newName across the codebase. The second return lists +// every file it edited. +func (s *Server) renameFunctionEdits(module, functionName, newName string) (*protocol.WorkspaceEdit, []string, error) { // Collect all (filePath, lineNumber) pairs — definitions + references type siteKey struct { filePath string @@ -4225,7 +4229,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro // Definition sites defResults, err := s.store.LookupFunction(module, functionName) if err != nil { - return nil, nil + return nil, nil, nil } for _, r := range defResults { addSite(r.FilePath, r.Line) @@ -4234,7 +4238,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro // Direct reference sites (calls, imports — skip alias/use which are module-level) refResults, err := s.store.LookupReferences(module, functionName) if err != nil { - return nil, nil + return nil, nil, nil } for _, r := range refResults { if r.Kind == "alias" || r.Kind == "use" { @@ -4322,6 +4326,11 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro edit := s.buildTextEdits(sites, functionName, newName) + changedFiles := make(map[string]bool, len(sites)) + for _, site := range sites { + changedFiles[site.filePath] = true + } + // Update defdelegate lines that forward to this function: add or update // the `as:` option so the facade keeps working after the rename. if s.followDelegates { @@ -4359,6 +4368,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro continue } + changedFiles[del.FilePath] = true fileURI := protocol.DocumentURI(uri.File(del.FilePath)) if open { if edit.Changes == nil { @@ -4383,7 +4393,12 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro } } - return edit, nil + files := make([]string, 0, len(changedFiles)) + for fp := range changedFiles { + files = append(files, fp) + } + sort.Strings(files) + return edit, files, nil } // renameModuleEdits builds a WorkspaceEdit renaming oldModule to newModule, @@ -4393,14 +4408,16 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro // parallel goroutines. Only open buffers are included in the returned // WorkspaceEdit, keeping the response small and avoiding editor freezes. // Files following the naming convention are also renamed/moved. -func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule, triggerFilePath string) (*protocol.WorkspaceEdit, error) { +// renameModuleEdits' extra returns list the files it moved (old path to new +// path, open and closed alike) and the files it edited. +func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule, triggerFilePath string) (*protocol.WorkspaceEdit, map[string]string, []string, error) { mr := s.buildModuleRename(oldModule, newModule) // Check for collisions: verify that none of the target module names // (including submodules) already exist, and that no destination file // paths are occupied. if err := mr.checkCollisions(); err != nil { - return nil, err + return nil, nil, nil, err } mr.collectSites() @@ -4436,7 +4453,19 @@ func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule, tr } } - return &protocol.WorkspaceEdit{Changes: openChanges}, nil + moved := make(map[string]string, len(movedFiles)+len(openMovedFiles)) + for from, to := range movedFiles { + moved[from] = to + } + for from, to := range openMovedFiles { + moved[from] = to + } + files := make([]string, 0, len(mr.sitesByFile)) + for fp := range mr.sitesByFile { + files = append(files, fp) + } + sort.Strings(files) + return &protocol.WorkspaceEdit{Changes: openChanges}, moved, files, nil } // moduleRename holds the state for a module rename operation. diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md index e4de50e..f1fd749 100644 --- a/internal/mcp/instructions.md +++ b/internal/mcp/instructions.md @@ -15,6 +15,8 @@ Which tool for which question: - Implementations of a behaviour or protocol: `dexter_implementations` - What a specific file defines: `dexter_file_outline` - Project layout and index freshness: `dexter_workspace` +- Rename a module or function everywhere: `dexter_rename_symbol` (writes the + changes; review with `git diff`) The index updates automatically: file changes are watched (fsnotify) and git branch switches are detected. If a lookup ever seems stale, `dexter_reindex` diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 96be403..02fb6a0 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -112,6 +112,12 @@ func NewServer(h *Handler) *mcp.Server { Description: "Force an immediate incremental reindex. The index already updates automatically as files change; use this only when a lookup seems stale. The only tool that writes, and it writes only dexter's own index database.", }, h.reindexHandler) + mcp.AddTool(srv, &mcp.Tool{ + Name: "dexter_rename_symbol", + Description: "Rename an Elixir module or function across the whole workspace, exactly like an editor rename: writes the changes to disk, moves files that follow the naming convention, and updates the index. Reports every file changed; review with git diff.", + Annotations: &mcp.ToolAnnotations{OpenWorldHint: new(bool)}, + }, h.renameHandler) + return srv } diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go index 7a3180f..90eeff8 100644 --- a/internal/mcp/mcp_test.go +++ b/internal/mcp/mcp_test.go @@ -145,6 +145,7 @@ func TestListTools(t *testing.T) { "dexter_module_api", "dexter_references", "dexter_reindex", + "dexter_rename_symbol", "dexter_search", "dexter_workspace", } @@ -176,7 +177,7 @@ func TestListTools(t *testing.T) { if a.OpenWorldHint == nil || *a.OpenWorldHint { t.Errorf("tool %s not marked closed-world", tool.Name) } - wantReadOnly := tool.Name != "dexter_reindex" + wantReadOnly := tool.Name != "dexter_reindex" && tool.Name != "dexter_rename_symbol" if a.ReadOnlyHint != wantReadOnly { t.Errorf("tool %s ReadOnlyHint = %v, want %v", tool.Name, a.ReadOnlyHint, wantReadOnly) } diff --git a/internal/mcp/rename.go b/internal/mcp/rename.go new file mode 100644 index 0000000..937f872 --- /dev/null +++ b/internal/mcp/rename.go @@ -0,0 +1,55 @@ +package mcp + +import ( + "context" + "fmt" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/lsp" +) + +type RenameParams struct { + Module string `json:"module" jsonschema:"module being renamed, or the module owning the function"` + Function string `json:"function,omitempty" jsonschema:"if set, rename this function; otherwise rename the module itself (and its submodules)"` + NewName string `json:"new_name" jsonschema:"new function name (e.g. get_user), or new fully-qualified module name (e.g. MyApp.Clients)"` +} + +func (h *Handler) renameHandler(ctx context.Context, req *mcp.CallToolRequest, args RenameParams) (*mcp.CallToolResult, any, error) { + module := strings.TrimSpace(args.Module) + function := strings.TrimSpace(args.Function) + newName := strings.TrimSpace(args.NewName) + if module == "" || newName == "" { + return nil, nil, fmt.Errorf("module and new_name must not be empty") + } + + var summary lsp.RenameSummary + var err error + var target string + if function != "" { + target = fmt.Sprintf("%s.%s to %s", module, function, newName) + summary, err = h.lsp.RenameFunction(module, function, newName) + } else { + target = fmt.Sprintf("%s to %s", module, newName) + summary, err = h.lsp.RenameModule(module, newName) + } + if err != nil { + return nil, nil, err + } + + var b strings.Builder + fmt.Fprintf(&b, "Renamed %s across %d file(s). The index is updated.\n", target, len(summary.FilesChanged)) + if len(summary.FilesMoved) > 0 { + fmt.Fprintf(&b, "\nFiles moved to follow the naming convention:\n") + for from, to := range summary.FilesMoved { + fmt.Fprintf(&b, " %s → %s\n", h.relPath(from), h.relPath(to)) + } + } + fmt.Fprintf(&b, "\nChanged files:\n") + for _, fp := range summary.FilesChanged { + fmt.Fprintf(&b, " %s\n", h.relPath(fp)) + } + fmt.Fprintf(&b, "\nReview with git diff; revert with git checkout.\n") + return textResult(b.String()), nil, nil +} diff --git a/internal/mcp/rename_test.go b/internal/mcp/rename_test.go new file mode 100644 index 0000000..742e958 --- /dev/null +++ b/internal/mcp/rename_test.go @@ -0,0 +1,87 @@ +package mcp + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func readFile(t *testing.T, root, rel string) string { + t.Helper() + b, err := os.ReadFile(filepath.Join(root, rel)) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +func TestRenameTool_Function(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Accounts", "function": "fetch_user", "new_name": "get_user", + }) + wantContains(t, out, + "Renamed MyApp.Accounts.fetch_user to get_user", + "lib/my_app/accounts.ex", + "lib/my_app/worker.ex", + "git diff", + ) + + accounts := readFile(t, e.root, "lib/my_app/accounts.ex") + wantContains(t, accounts, "def get_user(id)", "@spec get_user(integer())") + wantNotContains(t, accounts, "fetch_user") + + worker := readFile(t, e.root, "lib/my_app/worker.ex") + wantContains(t, worker, "MyApp.Accounts.get_user(1)", "MyApp.Accounts.get_user(2)") + + // The rename reindexes what it wrote: lookups resolve the new name only. + wantContains(t, e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "get_user"}), "get_user/1 (def)") + wantContains(t, e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}), "not in the index") +} + +func TestRenameTool_Module_MovesFiles(t *testing.T) { + e := setupProject(t) + out := e.callTool("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Accounts", "new_name": "MyApp.Users", + }) + wantContains(t, out, + "Renamed MyApp.Accounts to MyApp.Users", + "Files moved to follow the naming convention:", + "lib/my_app/accounts.ex → lib/my_app/users.ex", + "lib/my_app/accounts/creator.ex → lib/my_app/users/creator.ex", + ) + + if _, err := os.Stat(filepath.Join(e.root, "lib/my_app/accounts.ex")); !os.IsNotExist(err) { + t.Error("old module file still exists after rename") + } + wantContains(t, readFile(t, e.root, "lib/my_app/users.ex"), "defmodule MyApp.Users do") + wantContains(t, readFile(t, e.root, "lib/my_app/users/creator.ex"), "defmodule MyApp.Users.Creator do") + wantContains(t, readFile(t, e.root, "lib/my_app/worker.ex"), "MyApp.Users.fetch_user(1)") + + wantContains(t, e.callTool("dexter_definition", map[string]any{"module": "MyApp.Users"}), "defmodule MyApp.Users") +} + +func TestRenameTool_Errors(t *testing.T) { + e := setupProject(t) + + errText := e.callToolExpectError("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Accounts", "function": "fetch_user", "new_name": "NotValid", + }) + wantContains(t, errText, "invalid function name") + + errText = e.callToolExpectError("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Accounts", "function": "fetch_user", "new_name": "list_users", + }) + wantContains(t, errText, "already exists") + + errText = e.callToolExpectError("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Missing", "new_name": "MyApp.New", + }) + wantContains(t, errText, "not found") + + // Failed renames must not touch disk. + if s := readFile(t, e.root, "lib/my_app/accounts.ex"); !strings.Contains(s, "def fetch_user(id)") { + t.Error("failed rename modified files") + } +} From 84b938339b8ab072d1af411d5ba4ef01f5d2c0d7 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 2 Sep 2026 07:47:07 +1000 Subject: [PATCH 04/26] Serialize watcher writes and fix MCP rename delivery Three fixes for the MCP integration, none touching LSP behavior: The file watcher now holds the reindex lock while writing to the index. Without it, a file created after a concurrent workspace reindex's walk had passed its directory could be indexed by the watcher and then removed by the reindex's prune, with the create event already consumed, leaving the symbol missing until the file changed again. Open-buffer rename edits from an MCP rename are forwarded to a live LSP client as workspace/applyEdit (attached mode), so the editor applies them and stays in sync, exactly as an editor-initiated rename would. Writing those files behind the editor's back left the buffer stale and a later save would have reverted the rename. Without a client they are written to disk directly; headless servers have no open buffers. RenameFunction and RenameModule wait for the rename's background reindex before returning, so the reported "index is updated" is true when the tool call completes rather than eventually. --- cmd/main.go | 2 +- internal/lsp/api.go | 45 ++++++++++++++++++++++-------- internal/lsp/api_test.go | 57 ++++++++++++++++++++++++++++++++++++-- internal/mcp/watch.go | 23 ++++++++------- internal/mcp/watch_test.go | 40 ++++++++++++++++++++++++-- 5 files changed, 139 insertions(+), 28 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index d709ebf..ca5777d 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -637,7 +637,7 @@ func cmdMCP(projectRoot string, listen string) { server.WatchGitHead() // Headless servers get no editor events, so watch the tree directly. - watcher, err := dexter_mcp.WatchFiles(s, projectRoot, func() { server.ReindexWorkspace() }) + watcher, err := dexter_mcp.WatchFiles(server, s, projectRoot) if err != nil { log.Printf("Warning: file watching unavailable (%v); the index updates on branch switches and via dexter_reindex", err) } else { diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 2729369..c2fb938 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -115,17 +115,26 @@ func (s *Server) CollectReferences(module, function string) []store.ReferenceRes return out } +// WithReindexLock runs fn while holding the reindex lock, serializing it with +// ReindexWorkspace and the background reindexes. The MCP file watcher wraps +// its index writes in it so they cannot interleave with a concurrent +// workspace reindex's walk-and-prune. +func (s *Server) WithReindexLock(fn func()) { + s.reindexing.Lock() + defer s.reindexing.Unlock() + fn() +} + // RenameSummary reports what a rename changed on disk. type RenameSummary struct { FilesChanged []string FilesMoved map[string]string // old path → new path (conventional module renames) } -// RenameFunction renames module.functionName to newName across the workspace, -// writing every change to disk. It performs the same validation as the LSP -// rename. Edits the LSP path would hand to an editor as TextEdits (open -// buffers in attached mode) are written to disk too, since an MCP caller has -// no editor to deliver them to. +// RenameFunction renames module.functionName to newName across the workspace +// with the same validation and on-disk semantics as the LSP rename. It returns +// once the index reflects the rename. Open-buffer edits are delivered per +// deliverEdits. func (s *Server) RenameFunction(module, functionName, newName string) (RenameSummary, error) { if !isValidFunctionName(newName) { return RenameSummary{}, fmt.Errorf("invalid function name %q: must match [a-z_][a-z0-9_?!]*", newName) @@ -145,9 +154,12 @@ func (s *Server) RenameFunction(module, functionName, newName string) (RenameSum if err != nil { return RenameSummary{}, err } - if err := s.writeEditsToDisk(edit); err != nil { + if err := s.deliverEdits(edit); err != nil { return RenameSummary{}, err } + // The machinery reindexes what it wrote in the background; callers are + // promised an up-to-date index. + s.backgroundWork.Wait() return RenameSummary{FilesChanged: files}, nil } @@ -168,19 +180,28 @@ func (s *Server) RenameModule(oldModule, newModule string) (RenameSummary, error if err != nil { return RenameSummary{}, err } - if err := s.writeEditsToDisk(edit); err != nil { + if err := s.deliverEdits(edit); err != nil { return RenameSummary{}, err } + s.backgroundWork.Wait() return RenameSummary{FilesChanged: files, FilesMoved: moved}, nil } -// writeEditsToDisk applies a WorkspaceEdit's TextEdits to their files on disk -// and reindexes them. The rename machinery only produces TextEdits for open -// editor buffers; headless servers have none, so this is usually a no-op. -func (s *Server) writeEditsToDisk(edit *protocol.WorkspaceEdit) error { - if edit == nil { +// deliverEdits routes a WorkspaceEdit's TextEdits to whoever owns the +// documents. The rename machinery only produces TextEdits for open editor +// buffers; with a live LSP client (attached mode) they are forwarded as a +// workspace/applyEdit request so the editor applies them and syncs back via +// didChange, exactly as an editor-initiated rename would. Without a client +// they are written to disk directly; headless servers have no open buffers, +// so that path is a defensive no-op in practice. +func (s *Server) deliverEdits(edit *protocol.WorkspaceEdit) error { + if edit == nil || len(edit.Changes) == 0 { return nil } + if s.client != nil { + _, err := s.client.ApplyEdit(context.Background(), &protocol.ApplyWorkspaceEditParams{Edit: *edit}) + return err + } for docURI, edits := range edit.Changes { path := uriToPath(docURI) text, _, ok := s.ReadFileText(path) diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index f34ea79..dc4f6f3 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -1,6 +1,7 @@ package lsp import ( + "context" "os" "path/filepath" "strings" @@ -52,9 +53,8 @@ func TestApplyTextEdits(t *testing.T) { } } -// A rename requested through the exported API must land on disk even for -// files an editor holds open (attached mode), since the caller has no editor -// to deliver TextEdits to. +// Without a live client, a rename requested through the exported API must +// land on disk even for files marked open (the defensive fallback path). func TestRenameFunction_WritesOpenBuffers(t *testing.T) { server, cleanup := setupTestServer(t) defer cleanup() @@ -92,3 +92,54 @@ end t.Errorf("index not updated after rename: %v, %v", results, err) } } + +// fakeClient records ApplyEdit requests; other client methods are never +// called by the rename path. +type fakeClient struct { + protocol.Client + applied *protocol.WorkspaceEdit +} + +func (f *fakeClient) ApplyEdit(_ context.Context, params *protocol.ApplyWorkspaceEditParams) (bool, error) { + f.applied = ¶ms.Edit + return true, nil +} + +// With a live client (attached mode), open-buffer edits go to the editor via +// workspace/applyEdit; dexter must not write those files behind its back. +func TestRenameFunction_ForwardsOpenBufferEditsToClient(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + fc := &fakeClient{} + server.client = fc + + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", `defmodule MyApp.Accounts do + def fetch_user(id), do: id +end +`) + openSrc := `defmodule MyApp.Caller do + def go(id), do: MyApp.Accounts.fetch_user(id) +end +` + indexFile(t, server.store, server.projectRoot, "lib/caller.ex", openSrc) + openPath := filepath.Join(server.projectRoot, "lib/caller.ex") + server.docs.Set(string(uri.File(openPath)), openSrc) + + if _, err := server.RenameFunction("MyApp.Accounts", "fetch_user", "get_user"); err != nil { + t.Fatal(err) + } + + if fc.applied == nil { + t.Fatal("no workspace/applyEdit request reached the client") + } + if len(fc.applied.Changes) != 1 { + t.Errorf("ApplyEdit carried %d files, want 1", len(fc.applied.Changes)) + } + data, err := os.ReadFile(openPath) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "get_user") { + t.Error("open buffer's file was written to disk despite a live client") + } +} diff --git a/internal/mcp/watch.go b/internal/mcp/watch.go index e784908..46400a9 100644 --- a/internal/mcp/watch.go +++ b/internal/mcp/watch.go @@ -12,6 +12,7 @@ import ( "github.com/fsnotify/fsnotify" + "github.com/remoteoss/dexter/internal/lsp" "github.com/remoteoss/dexter/internal/parser" "github.com/remoteoss/dexter/internal/store" ) @@ -25,23 +26,25 @@ const debounceWindow = 300 * time.Millisecond // it watches the project tree directly (fsnotify). type Watcher struct { fsw *fsnotify.Watcher + server *lsp.Server store *store.Store root string - resync func() // full incremental reindex, used when events were lost wg sync.WaitGroup } // WatchFiles watches projectRoot recursively and incrementally reindexes -// Elixir files as they change. resync is invoked when the watcher loses -// events (queue overflow) and the whole tree must be reconciled. Callers -// should treat an error as degraded service, not fatal: the index still -// updates on startup, on git branch switches, and via dexter_reindex. -func WatchFiles(s *store.Store, projectRoot string, resync func()) (*Watcher, error) { +// Elixir files as they change. Index writes hold the server's reindex lock so +// they cannot interleave with a concurrent workspace reindex's walk-and-prune +// (which would drop a file indexed after the walk passed its directory). On +// event overflow the whole workspace is reindexed. Callers should treat an +// error as degraded service, not fatal: the index still updates on startup, +// on git branch switches, and via dexter_reindex. +func WatchFiles(server *lsp.Server, s *store.Store, projectRoot string) (*Watcher, error) { fsw, err := fsnotify.NewWatcher() if err != nil { return nil, err } - w := &Watcher{fsw: fsw, store: s, root: projectRoot, resync: resync} + w := &Watcher{fsw: fsw, server: server, store: s, root: projectRoot} if err := w.watchTree(projectRoot); err != nil { _ = fsw.Close() return nil, err @@ -114,9 +117,9 @@ func (w *Watcher) loop() { if !ok { return } - if errors.Is(err, fsnotify.ErrEventOverflow) && w.resync != nil { + if errors.Is(err, fsnotify.ErrEventOverflow) { log.Printf("Warning: file watcher overflowed, reindexing workspace") - w.resync() + w.server.ReindexWorkspace() continue } log.Printf("Warning: file watcher: %v", err) @@ -125,7 +128,7 @@ func (w *Watcher) loop() { timerC = nil batch := pending pending = make(map[string]struct{}) - w.apply(batch) + w.server.WithReindexLock(func() { w.apply(batch) }) } } } diff --git a/internal/mcp/watch_test.go b/internal/mcp/watch_test.go index c37cbfc..cdc9367 100644 --- a/internal/mcp/watch_test.go +++ b/internal/mcp/watch_test.go @@ -1,11 +1,13 @@ package mcp import ( + "fmt" "os" "path/filepath" "testing" "time" + "github.com/remoteoss/dexter/internal/lsp" "github.com/remoteoss/dexter/internal/store" ) @@ -25,6 +27,11 @@ func eventually(t *testing.T, what string, cond func() bool) { } func setupWatcher(t *testing.T) (*store.Store, string) { + s, root, _ := setupWatcherWithServer(t) + return s, root +} + +func setupWatcherWithServer(t *testing.T) (*store.Store, string, *lsp.Server) { t.Helper() root := t.TempDir() s, err := store.Open(root) @@ -33,7 +40,8 @@ func setupWatcher(t *testing.T) (*store.Store, string) { } t.Cleanup(func() { _ = s.Close() }) - w, err := WatchFiles(s, root, nil) + server := lsp.NewServer(s, root) + w, err := WatchFiles(server, s, root) if err != nil { t.Fatal(err) } @@ -42,7 +50,7 @@ func setupWatcher(t *testing.T) (*store.Store, string) { t.Errorf("closing watcher: %v", err) } }) - return s, root + return s, root, server } func writeFile(t *testing.T, root, rel, content string) string { @@ -151,3 +159,31 @@ func TestWatcher_SkipsDepsAndNonElixir(t *testing.T) { t.Error("non-Elixir file was indexed") } } + +// Watcher writes hold the reindex lock, so files created while a workspace +// reindex runs can never be indexed between its walk and its prune (which +// would remove them with their create event already consumed). +func TestWatcher_SerializesWithWorkspaceReindex(t *testing.T) { + s, root, server := setupWatcherWithServer(t) + writeFile(t, root, "lib/base.ex", "defmodule MyApp.Base do\nend\n") + eventually(t, "base file", moduleIndexed(s, "MyApp.Base")) + + const n = 12 + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < n; i++ { + server.ReindexWorkspace() + } + }() + for i := 0; i < n; i++ { + writeFile(t, root, fmt.Sprintf("lib/race_%d.ex", i), fmt.Sprintf("defmodule MyApp.Race%d do\nend\n", i)) + time.Sleep(20 * time.Millisecond) + } + <-done + + for i := 0; i < n; i++ { + mod := fmt.Sprintf("MyApp.Race%d", i) + eventually(t, mod+" to survive concurrent reindexes", moduleIndexed(s, mod)) + } +} From b5a28a6fa1e7128766b78a20dba6b18b7e71bef7 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 2 Sep 2026 08:04:51 +1000 Subject: [PATCH 05/26] Report editor-rejected rename edits as errors workspace/applyEdit responses carry an applied flag; a rename whose open-buffer edits the editor refused was still reported as complete. deliverEdits now surfaces the rejection as an error. --- internal/lsp/api.go | 5 ++++- internal/lsp/api_test.go | 27 ++++++++++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index c2fb938..f8cbbc4 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -199,7 +199,10 @@ func (s *Server) deliverEdits(edit *protocol.WorkspaceEdit) error { return nil } if s.client != nil { - _, err := s.client.ApplyEdit(context.Background(), &protocol.ApplyWorkspaceEditParams{Edit: *edit}) + applied, err := s.client.ApplyEdit(context.Background(), &protocol.ApplyWorkspaceEditParams{Edit: *edit}) + if err == nil && !applied { + err = fmt.Errorf("editor did not apply the rename edits for open files") + } return err } for docURI, edits := range edit.Changes { diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index dc4f6f3..a34ea03 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -98,11 +98,12 @@ end type fakeClient struct { protocol.Client applied *protocol.WorkspaceEdit + reject bool } func (f *fakeClient) ApplyEdit(_ context.Context, params *protocol.ApplyWorkspaceEditParams) (bool, error) { f.applied = ¶ms.Edit - return true, nil + return !f.reject, nil } // With a live client (attached mode), open-buffer edits go to the editor via @@ -143,3 +144,27 @@ end t.Error("open buffer's file was written to disk despite a live client") } } + +// An editor may refuse a workspace edit (applied: false); the rename must +// report failure, not success, when open-buffer edits were not applied. +func TestRenameFunction_ReportsRejectedApplyEdit(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + server.client = &fakeClient{reject: true} + + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", `defmodule MyApp.Accounts do + def fetch_user(id), do: id +end +`) + openSrc := `defmodule MyApp.Caller do + def go(id), do: MyApp.Accounts.fetch_user(id) +end +` + indexFile(t, server.store, server.projectRoot, "lib/caller.ex", openSrc) + openPath := filepath.Join(server.projectRoot, "lib/caller.ex") + server.docs.Set(string(uri.File(openPath)), openSrc) + + if _, err := server.RenameFunction("MyApp.Accounts", "fetch_user", "get_user"); err == nil { + t.Fatal("rename reported success despite the editor rejecting the edit") + } +} From ea514c872d394c80eca7b9a37ecf79049d28be6f Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 5 Sep 2026 18:33:02 -0400 Subject: [PATCH 06/26] Let the editor move files a module rename renames Renaming a module from the file that defines it moved that file on disk while the editor still held the buffer, and handed the editor TextEdits for the path just deleted. Neovim applied them to the stale buffer, so the next save recreated the old file holding the new module name: two files defining the same module, and the project then couldn't compile due to duplicate modules. Open files are now moved by the client, through a rename resource operation ordered right after that file's own TextEdits so the edited buffer travels to the new path; the server touches neither path. Closed files still move server-side, which is what keeps large renames off the wire. Clients without resourceOperations rename the module in place and leave the file where it is, so nothing is deleted under a live buffer. go.lsp.dev/protocol types documentChanges as []TextDocumentEdit and cannot carry resource operations, so workspace_edit.go defines the wire types and renameHandler answers textDocument/rename ahead of the generated dispatcher. Two other bugs also fixed: - `alias Old.{A, B}` names the module once as the prefix while the index records one reference per member, so a member's full name never appears on the line and the group kept pointing at the old module. - Every member on such a line resolves to the same prefix edit. TextEdits are relative to the original buffer, so emitting it once per member made the editor apply it repeatedly (Old -> NewNewNew...). Overlapping edits are now dropped; the on-disk path rewrites the line as it goes and never sees the second match. --- CHANGELOG.md | 7 + docs/architecture.md | 14 + internal/lsp/rename.go | 37 +++ internal/lsp/rename_test.go | 506 ++++++++++++++++++++++++++++++--- internal/lsp/server.go | 267 ++++++++++++----- internal/lsp/server_test.go | 3 + internal/lsp/workspace_edit.go | 113 ++++++++ 7 files changed, 834 insertions(+), 113 deletions(-) create mode 100644 internal/lsp/workspace_edit.go diff --git a/CHANGELOG.md b/CHANGELOG.md index fe4c935..2c7b299 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## [Unreleased] + +### Fixed + +- **Module rename left the old file behind** — renaming a module from the file that defines it moved that file on disk while the editor still held the buffer, so the next save recreated the old file with the new module name and the project no longer compiled (`cannot define module X because it is currently being defined in ...`). Open files are now moved by the editor, through a `rename` resource operation in the reply, and the server touches neither path +- **Grouped aliases were not renamed** — `alias Old.{A, B}` (and the `require`/`import` forms) kept pointing at the old module after a module rename, and in open buffers the shared prefix was rewritten once per member (`Old` → `NewNewNew...`) + ## [0.7.1] - 2026-06-12 ### Added diff --git a/docs/architecture.md b/docs/architecture.md index 00e2610..3097b3a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -82,6 +82,20 @@ Call sites are attributed to the **injecting module** in the store (not the defi `buildTextEdits` uses `findFunctionTokenColumns` to skip keyword-syntax occurrences (`resource_type: value`) — only `::` type separators pass through. Import-only sites use `findAllTokenColumns` since their keyword keys ARE function names. +### Who moves a file + +A module rename also moves files whose names follow the module naming convention, and who performs the move depends on whether the editor holds the file: + +- **Closed files** — the server writes the new path and deletes the old one. This keeps large renames off the wire. +- **Open files** — the client moves them, through a `rename` resource operation in the reply's `documentChanges`, ordered right after that file's own TextEdits so the edited buffer travels to the new path. The server touches neither path. Moving an open file server-side leaves the editor holding a modified buffer pointing at a deleted path, and the next save recreates the old file with the new module name — two files defining the same module. +- **Open files, client without `resourceOperations: ["rename"]`** — the module is renamed in place and the file keeps its old name. Nothing is deleted underneath a live buffer. + +`protocol.WorkspaceEdit` from `go.lsp.dev/protocol` types `documentChanges` as `[]TextDocumentEdit` and cannot carry resource operations, so `internal/lsp/workspace_edit.go` defines the wire types and `renameHandler` answers `textDocument/rename` ahead of the generated dispatcher. A client that understands `documentChanges` ignores `changes` entirely, so once one file moves, every edit in the reply goes through `documentChanges`. + +### Grouped aliases + +`alias Old.{A, B}` (and the `require`/`import` forms) names the module once, as the prefix, while the index records one reference per member — so a member's full name never appears on the line. `findGroupedAliasEdits` handles both directions: renaming the prefix rewrites the prefix, renaming a member rewrites that member inside the braces. Since every member on the line resolves to the same prefix edit, `applyEdits` drops TextEdits that overlap one already emitted for that line; the on-disk path rewrites the line as it goes and never sees the second match. + ## Key design decisions - **Tokenizer instead of tree-sitter for indexing** — a hand-rolled tokenizer + walker replaced the original regex-based parser for both file indexing and runtime `__using__` parsing. The tokenizer handles heredocs, sigils, multi-line expressions, and comments as opaque tokens, eliminating fragile line-joining heuristics. Tree-sitter is only used for scope-aware variable operations in files already opened by the editor. diff --git a/internal/lsp/rename.go b/internal/lsp/rename.go index 4a2e499..e671fcc 100644 --- a/internal/lsp/rename.go +++ b/internal/lsp/rename.go @@ -65,6 +65,43 @@ func findFunctionTokenColumns(lineText, token string) []int { return result } +// findGroupedAlias locates a grouped alias/require/import of the form +// `prefix.{A, B}` in lineText. It returns the column where prefix starts and +// the span of the text between the braces, or -1 when the line holds no such +// group for prefix. +// +// findAllTokenColumns cannot be used for this: the match would end at '{', +// whose following character is an identifier char, so the boundary check +// would reject every group. +func findGroupedAlias(lineText, prefix string) (prefixCol, groupStart, groupEnd int) { + needle := prefix + ".{" + start := 0 + for { + idx := strings.Index(lineText[start:], needle) + if idx < 0 { + return -1, -1, -1 + } + abs := start + idx + // Only the leading boundary matters — the trailing one is the brace. + if abs > 0 { + if r, _ := utf8.DecodeLastRuneInString(lineText[:abs]); r != utf8.RuneError && isRenameIdentChar(r) { + start = abs + 1 + continue + } + } + open := abs + len(needle) + end := strings.IndexByte(lineText[open:], '}') + if end < 0 { + // Multi-line group: the members continue on following lines, so + // take the rest of this one. + end = len(lineText) + } else { + end = open + end + } + return abs, open, end + } +} + // isTokenBoundary returns true when the substring [pos, pos+length) in s is // not immediately preceded or followed by an identifier character. func isTokenBoundary(s string, pos, length int) bool { diff --git a/internal/lsp/rename_test.go b/internal/lsp/rename_test.go index 9c22faa..2504e3f 100644 --- a/internal/lsp/rename_test.go +++ b/internal/lsp/rename_test.go @@ -2,11 +2,14 @@ package lsp import ( "context" + "encoding/json" "os" "path/filepath" + "sort" "strings" "testing" + "go.lsp.dev/jsonrpc2" "go.lsp.dev/protocol" "go.lsp.dev/uri" ) @@ -323,9 +326,9 @@ func TestIsValidModuleName(t *testing.T) { // === Integration helpers === -func renameAt(t *testing.T, server *Server, docURI string, line, col uint32, newName string) *protocol.WorkspaceEdit { +func renameAt(t *testing.T, server *Server, docURI string, line, col uint32, newName string) *WorkspaceEdit { t.Helper() - result, err := server.Rename(context.Background(), &protocol.RenameParams{ + result, err := server.RenameEdit(context.Background(), &protocol.RenameParams{ TextDocumentPositionParams: protocol.TextDocumentPositionParams{ TextDocument: protocol.TextDocumentIdentifier{URI: protocol.DocumentURI(docURI)}, Position: protocol.Position{Line: line, Character: col}, @@ -352,12 +355,34 @@ func prepareRenameAt(t *testing.T, server *Server, docURI string, line, col uint return result } -func collectEdits(edit *protocol.WorkspaceEdit, filePath string) []protocol.TextEdit { +func collectEdits(edit *WorkspaceEdit, filePath string) []protocol.TextEdit { if edit == nil { return nil } fileURI := protocol.DocumentURI(uri.File(filePath)) - return edit.Changes[fileURI] + if edits, ok := edit.Changes[fileURI]; ok { + return edits + } + for _, change := range edit.DocumentChanges { + if tde, ok := change.(TextDocumentEdit); ok && tde.TextDocument.URI == fileURI { + return tde.Edits + } + } + return nil +} + +// renameOp returns the rename operation for filePath in the edit, if any. +func renameOp(edit *WorkspaceEdit, filePath string) *RenameFile { + if edit == nil { + return nil + } + oldURI := protocol.DocumentURI(uri.File(filePath)) + for _, change := range edit.DocumentChanges { + if rf, ok := change.(RenameFile); ok && rf.OldURI == oldURI { + return &rf + } + } + return nil } func hasEdit(edits []protocol.TextEdit, newText string) bool { @@ -378,6 +403,50 @@ func editsContainLine(edits []protocol.TextEdit, lineNum uint32) bool { return false } +// expectClientRename asserts the edit asks the client to move oldPath to +// newPath, and that the server left both paths alone: the editor owns an open +// buffer, so it must perform the move itself. +func expectClientRename(t *testing.T, edit *WorkspaceEdit, oldPath, newPath string) { + t.Helper() + op := renameOp(edit, oldPath) + if op == nil { + t.Fatalf("expected a rename operation for %s, got %+v", oldPath, edit) + } + if want := protocol.DocumentURI(uri.File(newPath)); op.NewURI != want { + t.Errorf("rename target = %s, want %s", op.NewURI, want) + } + if _, err := os.Stat(oldPath); err != nil { + t.Errorf("server removed %s — the client has it open and must move it itself", oldPath) + } + if _, err := os.Stat(newPath); err == nil { + t.Errorf("server created %s — the client performs the move", newPath) + } +} + +// bufferAfterEdits returns what content becomes once the edit's text edits for +// filePath are applied, i.e. what the editor's buffer ends up holding. +func bufferAfterEdits(t *testing.T, edit *WorkspaceEdit, filePath, content string) string { + t.Helper() + edits := collectEdits(edit, filePath) + sorted := make([]protocol.TextEdit, len(edits)) + copy(sorted, edits) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].Range.Start.Line != sorted[j].Range.Start.Line { + return sorted[i].Range.Start.Line > sorted[j].Range.Start.Line + } + return sorted[i].Range.Start.Character > sorted[j].Range.Start.Character + }) + lines := strings.Split(content, "\n") + for _, e := range sorted { + l := int(e.Range.Start.Line) + if l >= len(lines) || e.Range.End.Line != e.Range.Start.Line { + t.Fatalf("unexpected edit range %+v", e.Range) + } + lines[l] = lines[l][:e.Range.Start.Character] + e.NewText + lines[l][e.Range.End.Character:] + } + return strings.Join(lines, "\n") +} + // fileContains checks whether the file at path contains the given substring. // Used to verify server-side writes for files not open in the editor. func fileContains(filePath, substr string) bool { @@ -390,7 +459,7 @@ func fileContains(filePath, substr string) bool { // hasRename returns true if the rename result (either in WorkspaceEdit or // written directly to disk) contains newText for the given file. -func hasRename(edit *protocol.WorkspaceEdit, filePath, newText string) bool { +func hasRename(edit *WorkspaceEdit, filePath, newText string) bool { if hasEdit(collectEdits(edit, filePath), newText) { return true } @@ -1490,16 +1559,12 @@ end t.Fatal("expected non-nil edit") } - // File should have been written to new path and old path removed + // The file is open, so the edit must ask the client to move it and the + // server must not touch either path. newPath := filepath.Join(server.projectRoot, "lib", "auth.ex") - if _, err := os.Stat(newPath); os.IsNotExist(err) { - t.Error("expected new file auth.ex to exist") - } - if _, err := os.Stat(oldPath); err == nil { - t.Error("expected old file accounts.ex to be removed") - } - if !fileContains(newPath, "defmodule MyApp.Auth") { - t.Errorf("expected new file to contain 'defmodule MyApp.Auth'") + expectClientRename(t, edit, oldPath, newPath) + if got := bufferAfterEdits(t, edit, oldPath, content); !strings.Contains(got, "defmodule MyApp.Auth") { + t.Errorf("expected buffer to contain 'defmodule MyApp.Auth', got:\n%s", got) } } @@ -1517,15 +1582,13 @@ end defURI := "file://" + oldPath server.docs.Set(defURI, content) - renameAt(t, server, defURI, 0, 20, "AuthTest") + edit := renameAt(t, server, defURI, 0, 20, "AuthTest") // File should be renamed preserving the .exs extension newPath := filepath.Join(server.projectRoot, "test", "auth_test.exs") - if _, err := os.Stat(newPath); os.IsNotExist(err) { - t.Error("expected file renamed to auth_test.exs (preserving .exs extension)") - } - if !fileContains(newPath, "defmodule MyApp.AuthTest") { - t.Errorf("expected 'defmodule MyApp.AuthTest' in new file") + expectClientRename(t, edit, oldPath, newPath) + if got := bufferAfterEdits(t, edit, oldPath, content); !strings.Contains(got, "defmodule MyApp.AuthTest") { + t.Errorf("expected buffer to contain 'defmodule MyApp.AuthTest', got:\n%s", got) } } @@ -1550,26 +1613,23 @@ end indexFile(t, server.store, server.projectRoot, "lib/docusign.ex", defContent) indexFile(t, server.store, server.projectRoot, "lib/web.ex", callerContent) - // Test 1: rename from the def file (open) + // Test 1: rename from the def file (open — the client moves it) t.Run("from def file", func(t *testing.T) { defURI := "file://" + oldPath server.docs.Set(defURI, defContent) - renameAt(t, server, defURI, 0, 16, "Docusigns") + edit := renameAt(t, server, defURI, 0, 16, "Docusigns") newPath := filepath.Join(server.projectRoot, "lib", "docusigns.ex") - if _, err := os.Stat(newPath); os.IsNotExist(err) { - t.Error("expected file renamed to docusigns.ex") - } - if !fileContains(newPath, "defmodule MyApp.Docusigns") { - data, _ := os.ReadFile(newPath) - t.Errorf("expected 'defmodule MyApp.Docusigns', got:\n%s", string(data)) + expectClientRename(t, edit, oldPath, newPath) + if got := bufferAfterEdits(t, edit, oldPath, defContent); !strings.Contains(got, "defmodule MyApp.Docusigns") { + t.Errorf("expected 'defmodule MyApp.Docusigns', got:\n%s", got) } }) - // Clean up test 1's renamed file and re-index with original content for test 2 - _ = os.Remove(filepath.Join(server.projectRoot, "lib", "docusigns.ex")) - _ = server.store.RemoveFile(filepath.Join(server.projectRoot, "lib", "docusigns.ex")) + // Re-index with original content for test 2, and close the def file so it + // takes the closed-file path (moved on disk by the server) + server.docs.Close("file://" + oldPath) indexFile(t, server.store, server.projectRoot, "lib/docusign.ex", defContent) // Test 2: rename from a caller file via alias (def file is closed) @@ -1831,12 +1891,10 @@ end end `) - renameAt(t, server, defURI, 0, 16, "Enterprises") + edit := renameAt(t, server, defURI, 0, 16, "Enterprises") - // Root file should be renamed - if _, err := os.Stat(filepath.Join(server.projectRoot, "lib", "enterprises.ex")); os.IsNotExist(err) { - t.Error("expected root module file renamed to enterprises.ex") - } + // Root file is open — the client renames it + expectClientRename(t, edit, defPath, filepath.Join(server.projectRoot, "lib", "enterprises.ex")) // Submodule file is closed — should be moved to new directory on disk newSubPath := filepath.Join(server.projectRoot, "lib", "enterprises", "do_something.ex") @@ -1938,16 +1996,14 @@ end t.Errorf("expected last segment range starting at col 16, got %d", r.Start.Character) } - renameAt(t, server, defURI, 0, 16, "CostCalculatorZ") + edit := renameAt(t, server, defURI, 0, 16, "CostCalculatorZ") - // Check the renamed file has the full qualified name + // The open file moves via the client, and its buffer gets the full + // qualified name newPath := filepath.Join(server.projectRoot, "lib", "cost_calculator_z.ex") - newContent, err := os.ReadFile(newPath) - if err != nil { - t.Fatalf("cannot read new file: %v", err) - } - if !strings.Contains(string(newContent), "defmodule MyApp.CostCalculatorZ do") { - t.Errorf("expected 'defmodule MyApp.CostCalculatorZ do', got:\n%s", newContent) + expectClientRename(t, edit, defPath, newPath) + if got := bufferAfterEdits(t, edit, defPath, content); !strings.Contains(got, "defmodule MyApp.CostCalculatorZ do") { + t.Errorf("expected 'defmodule MyApp.CostCalculatorZ do', got:\n%s", got) } } @@ -2776,3 +2832,365 @@ end t.Errorf("expected edit containing 'Approvals' in child file, got: %v", childEdits) } } + +// Regression: renaming a module from the file that defines it used to move +// that file on disk while the editor still held the buffer, and hand the +// editor TextEdits for the now-deleted path. Neovim applied the edits to the +// stale buffer, and the next save (`:w`, `:wa`, format-on-save) recreated the +// old file holding the new module name — two files defining the same module, +// and the project stopped compiling. +func TestRename_Module_OpenFileMovedByClientNotServer(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + content := `defmodule MyApp.Accounts do + def list_users, do: [] +end +` + oldPath := filepath.Join(server.projectRoot, "lib", "accounts.ex") + newPath := filepath.Join(server.projectRoot, "lib", "auth.ex") + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", content) + defURI := "file://" + oldPath + server.docs.Set(defURI, content) + + edit := renameAt(t, server, defURI, 0, 20, "Auth") + if edit == nil { + t.Fatal("expected non-nil edit") + } + + // The server must not touch a file the editor has open. + if _, err := os.Stat(oldPath); err != nil { + t.Error("server deleted accounts.ex while the editor had it open") + } + if _, err := os.Stat(newPath); err == nil { + t.Error("server created auth.ex; the client performs the move") + } + + // A client that applies documentChanges ignores changes entirely, so + // nothing may be left there. + if len(edit.Changes) != 0 { + t.Errorf("changes must be empty when documentChanges is used, got %v", edit.Changes) + } + + // The buffer's edits must come before its rename operation, so the edited + // buffer travels to the new path. + oldURI := protocol.DocumentURI(uri.File(oldPath)) + editIdx, renameIdx := -1, -1 + for i, change := range edit.DocumentChanges { + switch c := change.(type) { + case TextDocumentEdit: + if c.TextDocument.URI == oldURI { + editIdx = i + } + case RenameFile: + if c.OldURI == oldURI { + renameIdx = i + } + } + } + if editIdx < 0 { + t.Fatalf("expected text edits for %s, got %+v", oldPath, edit.DocumentChanges) + } + if renameIdx < 0 { + t.Fatalf("expected a rename operation for %s, got %+v", oldPath, edit.DocumentChanges) + } + if editIdx > renameIdx { + t.Error("text edits must precede the rename operation for the same file") + } + if got := bufferAfterEdits(t, edit, oldPath, content); !strings.Contains(got, "defmodule MyApp.Auth") { + t.Errorf("expected the buffer to become 'defmodule MyApp.Auth', got:\n%s", got) + } +} + +// A client that cannot apply rename operations gets the module renamed in +// place. The file keeps its old name, but nothing is deleted underneath an +// open buffer, so no save can resurrect a duplicate module. +func TestRename_Module_OpenFileLeftInPlaceWithoutRenameOps(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + server.renameFileOpsSupported = false + + content := `defmodule MyApp.Accounts do + def list_users, do: [] +end +` + oldPath := filepath.Join(server.projectRoot, "lib", "accounts.ex") + newPath := filepath.Join(server.projectRoot, "lib", "auth.ex") + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", content) + defURI := "file://" + oldPath + server.docs.Set(defURI, content) + + edit := renameAt(t, server, defURI, 0, 20, "Auth") + + if _, err := os.Stat(oldPath); err != nil { + t.Error("expected accounts.ex to stay in place") + } + if _, err := os.Stat(newPath); err == nil { + t.Error("expected no auth.ex — the file cannot move while the client holds it") + } + if len(edit.DocumentChanges) != 0 { + t.Errorf("expected no resource operations, got %+v", edit.DocumentChanges) + } + if got := bufferAfterEdits(t, edit, oldPath, content); !strings.Contains(got, "defmodule MyApp.Auth") { + t.Errorf("expected the buffer to become 'defmodule MyApp.Auth', got:\n%s", got) + } +} + +// The wire format is what the editor actually acts on, and a wrong JSON tag +// would be invisible to every other test here. +func TestWorkspaceEdit_RenameOperationJSON(t *testing.T) { + edit := &WorkspaceEdit{ + DocumentChanges: []interface{}{ + textDocumentEdit(protocol.DocumentURI("file:///p/lib/accounts.ex"), []protocol.TextEdit{{ + Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 16}, End: protocol.Position{Line: 0, Character: 24}}, + NewText: "Auth", + }}), + newRenameFile("/p/lib/accounts.ex", "/p/lib/auth.ex"), + }, + } + + data, err := json.Marshal(edit) + if err != nil { + t.Fatal(err) + } + var got struct { + Changes map[string]interface{} `json:"changes"` + DocumentChanges []struct { + Kind string `json:"kind"` + OldURI string `json:"oldUri"` + NewURI string `json:"newUri"` + TextDocument *struct { + URI string `json:"uri"` + Version interface{} `json:"version"` + } `json:"textDocument"` + Edits []protocol.TextEdit `json:"edits"` + Options *struct { + Overwrite bool `json:"overwrite"` + } `json:"options"` + } `json:"documentChanges"` + } + if err := json.Unmarshal(data, &got); err != nil { + t.Fatal(err) + } + if got.Changes != nil { + t.Errorf("changes must be omitted, got %v", got.Changes) + } + if len(got.DocumentChanges) != 2 { + t.Fatalf("expected 2 documentChanges, got %s", data) + } + first := got.DocumentChanges[0] + if first.TextDocument == nil || first.TextDocument.URI != "file:///p/lib/accounts.ex" { + t.Errorf("first change should be a text document edit, got %s", data) + } + if first.TextDocument != nil && first.TextDocument.Version != nil { + t.Errorf("version must be null, got %v", first.TextDocument.Version) + } + if len(first.Edits) != 1 { + t.Errorf("expected the text edit to survive, got %s", data) + } + second := got.DocumentChanges[1] + if second.Kind != "rename" { + t.Errorf("kind = %q, want \"rename\"", second.Kind) + } + if second.OldURI != "file:///p/lib/accounts.ex" || second.NewURI != "file:///p/lib/auth.ex" { + t.Errorf("rename URIs = %s → %s", second.OldURI, second.NewURI) + } + if second.Options == nil || !second.Options.Overwrite { + t.Errorf("expected overwrite:true, got %s", data) + } +} + +// The rename request must be answered by our own handler: the generated +// dispatcher marshals a protocol.WorkspaceEdit, which has nowhere to put +// resource operations, so a mis-wired handler would silently drop every file +// move. +func TestRenameHandler_RepliesWithResourceOperations(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + content := `defmodule MyApp.Accounts do + def list_users, do: [] +end +` + oldPath := filepath.Join(server.projectRoot, "lib", "accounts.ex") + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", content) + defURI := "file://" + oldPath + server.docs.Set(defURI, content) + + call, err := jsonrpc2.NewCall(jsonrpc2.NewNumberID(1), protocol.MethodTextDocumentRename, &protocol.RenameParams{ + TextDocumentPositionParams: protocol.TextDocumentPositionParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: protocol.DocumentURI(defURI)}, + Position: protocol.Position{Line: 0, Character: 20}, + }, + NewName: "Auth", + }) + if err != nil { + t.Fatal(err) + } + + var replied interface{} + nextCalled := false + handler := server.renameHandler(func(context.Context, jsonrpc2.Replier, jsonrpc2.Request) error { + nextCalled = true + return nil + }) + err = handler(context.Background(), func(_ context.Context, result interface{}, err error) error { + replied = result + return err + }, call) + if err != nil { + t.Fatal(err) + } + if nextCalled { + t.Error("rename must not fall through to the generated dispatcher") + } + + edit, ok := replied.(*WorkspaceEdit) + if !ok { + t.Fatalf("replied with %T, want *WorkspaceEdit", replied) + } + if renameOp(edit, oldPath) == nil { + t.Errorf("reply carries no rename operation: %+v", edit.DocumentChanges) + } +} + +// Regression: `alias Old.{A, B}` names the module once, as the prefix before +// the brace, while the index records one reference per member. Matching a +// member's full name against the line found nothing, so grouped aliases kept +// pointing at the old module and the project stopped compiling. +func TestRename_Module_GroupedAlias(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/shared_lib.ex", `defmodule SharedLib do + def start, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/worker.ex", `defmodule SharedLib.Worker do + def call, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/config.ex", `defmodule SharedLib.Config do + def get, do: :ok +end +`) + callerContent := `defmodule MyApp.Runner do + alias SharedLib.{Config, Worker} + require SharedLib.{Config, Worker} + + def run do + Config.get() + Worker.call() + end +end +` + callerPath := filepath.Join(server.projectRoot, "lib", "runner.ex") + indexFile(t, server.store, server.projectRoot, "lib/runner.ex", callerContent) + + defPath := filepath.Join(server.projectRoot, "lib", "shared_lib.ex") + defURI := "file://" + defPath + server.docs.Set(defURI, `defmodule SharedLib do + def start, do: :ok +end +`) + + renameAt(t, server, defURI, 0, 10, "CoreLib") + + // The caller is closed, so the server rewrites it on disk + got, err := os.ReadFile(callerPath) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{"alias CoreLib.{Config, Worker}", "require CoreLib.{Config, Worker}"} { + if !strings.Contains(string(got), want) { + t.Errorf("expected %q, got:\n%s", want, got) + } + } + if strings.Contains(string(got), "SharedLib") { + t.Errorf("SharedLib should be gone, got:\n%s", got) + } +} + +// Renaming a member of a grouped alias rewrites the member, not the prefix. +func TestRename_Module_GroupedAliasMemberRenamed(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/worker.ex", `defmodule SharedLib.Worker do + def call, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/config.ex", `defmodule SharedLib.Config do + def get, do: :ok +end +`) + callerContent := `defmodule MyApp.Runner do + alias SharedLib.{Config, Worker} + + def run, do: Worker.call() +end +` + callerPath := filepath.Join(server.projectRoot, "lib", "runner.ex") + indexFile(t, server.store, server.projectRoot, "lib/runner.ex", callerContent) + + defPath := filepath.Join(server.projectRoot, "lib", "shared_lib", "worker.ex") + defURI := "file://" + defPath + server.docs.Set(defURI, `defmodule SharedLib.Worker do + def call, do: :ok +end +`) + + renameAt(t, server, defURI, 0, 20, "Job") + + got, err := os.ReadFile(callerPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(got), "alias SharedLib.{Config, Job}") { + t.Errorf("expected 'alias SharedLib.{Config, Job}', got:\n%s", got) + } +} + +// Regression: every member of `alias Old.{A, B, C}` is its own indexed +// reference, but they share one edit to the prefix. TextEdits are all relative +// to the original buffer, so emitting the same span once per member made the +// editor apply the replacement repeatedly (Old → NewNewNew...). +func TestRename_Module_GroupedAliasInOpenBufferEditedOnce(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/shared_lib.ex", `defmodule SharedLib do + def start, do: :ok +end +`) + for _, sub := range []string{"Config", "Worker", "Job"} { + indexFile(t, server.store, server.projectRoot, + "lib/shared_lib/"+strings.ToLower(sub)+".ex", + "defmodule SharedLib."+sub+" do\n def call, do: :ok\nend\n") + } + + callerContent := `defmodule MyApp.Runner do + alias SharedLib.{Config, Job, Worker} + + def run, do: {Config.call(), Job.call(), Worker.call()} +end +` + callerPath := filepath.Join(server.projectRoot, "lib", "runner.ex") + indexFile(t, server.store, server.projectRoot, "lib/runner.ex", callerContent) + callerURI := "file://" + callerPath + server.docs.Set(callerURI, callerContent) + + defPath := filepath.Join(server.projectRoot, "lib", "shared_lib.ex") + defURI := "file://" + defPath + server.docs.Set(defURI, `defmodule SharedLib do + def start, do: :ok +end +`) + + edit := renameAt(t, server, defURI, 0, 10, "CoreLib") + + got := bufferAfterEdits(t, edit, callerPath, callerContent) + if !strings.Contains(got, "alias CoreLib.{Config, Job, Worker}") { + t.Errorf("expected 'alias CoreLib.{Config, Job, Worker}', got:\n%s", got) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 93603cb..242fc8b 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -14,6 +14,7 @@ import ( "os" "os/exec" "path/filepath" + "sort" "strconv" "strings" "sync" @@ -92,9 +93,10 @@ type Server struct { depsCache map[string]bool // dir → whether files in that dir are deps depsCacheMu sync.RWMutex - conn jsonrpc2.Conn // raw connection for server-initiated requests not on the Client interface - showDocumentSupported bool // client supports window/showDocument (LSP 3.16+) - snippetSupport bool // client supports snippet insert text in completions + conn jsonrpc2.Conn // raw connection for server-initiated requests not on the Client interface + showDocumentSupported bool // client supports window/showDocument (LSP 3.16+) + renameFileOpsSupported bool // client applies rename resource operations in a WorkspaceEdit + snippetSupport bool // client supports snippet insert text in completions reindexing sync.Mutex // serializes concurrent backgroundReindex calls notifiedOTPMismatch sync.Once // prevents repeated OTP mismatch warnings @@ -146,7 +148,7 @@ func Serve(in io.Reader, out io.Writer, s *store.Store, projectRoot string) erro server.client = protocol.ClientDispatcher(conn, logger) server.conn = conn - handler := protocol.ServerHandler(server, nil) + handler := server.renameHandler(protocol.ServerHandler(server, nil)) ctx := context.Background() conn.Go(ctx, handler) @@ -403,6 +405,17 @@ func (s *Server) Initialize(ctx context.Context, params *protocol.InitializePara if params.Capabilities.Window != nil && params.Capabilities.Window.ShowDocument != nil { s.showDocumentSupported = params.Capabilities.Window.ShowDocument.Support } + // Resource operations only exist inside documentChanges, so advertising + // them implies documentChanges support. Neovim, for one, lists + // resourceOperations without setting the separate documentChanges flag. + if ws := params.Capabilities.Workspace; ws != nil && ws.WorkspaceEdit != nil { + for _, op := range ws.WorkspaceEdit.ResourceOperations { + if op == "rename" { + s.renameFileOpsSupported = true + break + } + } + } if params.Capabilities.TextDocument != nil && params.Capabilities.TextDocument.Completion != nil && params.Capabilities.TextDocument.Completion.CompletionItem != nil { s.snippetSupport = params.Capabilities.TextDocument.Completion.CompletionItem.SnippetSupport @@ -4082,7 +4095,17 @@ func (s *Server) References(ctx context.Context, params *protocol.ReferenceParam return locations, nil } +// Rename implements the protocol.Server interface. It exists only to satisfy +// the generated dispatcher; Serve intercepts textDocument/rename before that +// dispatcher runs so the reply can carry resource operations, which +// protocol.WorkspaceEdit cannot express. func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*protocol.WorkspaceEdit, error) { + edit, err := s.RenameEdit(ctx, params) + return edit.toProtocol(), err +} + +// RenameEdit computes the workspace edit for a textDocument/rename request. +func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) (*WorkspaceEdit, error) { docURI := string(params.TextDocument.URI) text, ok := s.docs.GetOrLoad(docURI) if !ok { @@ -4122,7 +4145,7 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr NewText: params.NewName, }) } - return &protocol.WorkspaceEdit{Changes: changes}, nil + return &WorkspaceEdit{Changes: changes}, nil } } } @@ -4159,7 +4182,7 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr }) } } - return &protocol.WorkspaceEdit{Changes: changes}, nil + return &WorkspaceEdit{Changes: changes}, nil } } @@ -4193,7 +4216,7 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr if !isValidModuleName(newModule) { return nil, fmt.Errorf("invalid module name %q: must be CamelCase segments separated by dots", params.NewName) } - return s.renameModuleEdits(ctx, fullModule, newModule, uriToPath(params.TextDocument.URI)) + return s.renameModuleEdits(fullModule, newModule) } } } @@ -4203,7 +4226,7 @@ func (s *Server) Rename(ctx context.Context, params *protocol.RenameParams) (*pr // renameFunctionEdits builds a WorkspaceEdit renaming all occurrences of // module.functionName to newName across the codebase. -func (s *Server) renameFunctionEdits(module, functionName, newName string) (*protocol.WorkspaceEdit, error) { +func (s *Server) renameFunctionEdits(module, functionName, newName string) (*WorkspaceEdit, error) { // Collect all (filePath, lineNumber) pairs — definitions + references type siteKey struct { filePath string @@ -4399,8 +4422,9 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string) (*pro // Files not currently open in the editor are written directly to disk in // parallel goroutines. Only open buffers are included in the returned // WorkspaceEdit, keeping the response small and avoiding editor freezes. -// Files following the naming convention are also renamed/moved. -func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule, triggerFilePath string) (*protocol.WorkspaceEdit, error) { +// Files following the naming convention are also renamed/moved: closed ones +// by the server, open ones by the client through rename operations. +func (s *Server) renameModuleEdits(oldModule, newModule string) (*WorkspaceEdit, error) { mr := s.buildModuleRename(oldModule, newModule) // Check for collisions: verify that none of the target module names @@ -4414,36 +4438,56 @@ func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule, tr fileCache := mr.readFiles() - movedFiles, openMovedFiles, showDocumentPath := mr.moveConventionalFiles(fileCache, triggerFilePath) + movedFiles, clientRenames := mr.moveConventionalFiles(fileCache) openChanges := mr.applyEdits(fileCache, movedFiles) - mr.reindex(fileCache, movedFiles, openMovedFiles) - - // For open files that were moved: send showDocument so the editor opens - // the new path, then delete the old file in the background. - if s.showDocumentSupported && s.conn != nil { - for oldPath, newPath := range openMovedFiles { - showURI := protocol.URI(string(uri.File(newPath))) - takeFocus := newPath == showDocumentPath - go func() { - var result protocol.ShowDocumentResult - _ = protocol.Call(context.Background(), s.conn, "window/showDocument", &protocol.ShowDocumentParams{ - URI: showURI, - TakeFocus: takeFocus, - }, &result) - // Delete old file after the editor has been redirected - _ = os.Remove(oldPath) - _ = s.store.RemoveFile(oldPath) - }() + mr.reindex(fileCache, movedFiles, clientRenames) + + if len(clientRenames) == 0 { + return &WorkspaceEdit{Changes: openChanges}, nil + } + return renamesToDocumentChanges(openChanges, clientRenames), nil +} + +// renamesToDocumentChanges folds the open buffers' text edits and the file +// moves the client must perform into a single ordered documentChanges list. +// +// Each renamed file's text edits come immediately before its rename +// operation: the client edits the buffer in place and then moves it, so the +// buffer follows the file and no stale copy is left behind to be saved back +// over the rename. Text edits and moves cannot be split across `changes` and +// `documentChanges` because a client that understands documentChanges ignores +// `changes` entirely. +func renamesToDocumentChanges(openChanges map[protocol.DocumentURI][]protocol.TextEdit, clientRenames map[string]string) *WorkspaceEdit { + renamedPaths := make([]string, 0, len(clientRenames)) + for oldPath := range clientRenames { + renamedPaths = append(renamedPaths, oldPath) + } + sort.Strings(renamedPaths) + + changes := make([]interface{}, 0, len(openChanges)+len(clientRenames)) + renamedURIs := make(map[protocol.DocumentURI]bool, len(clientRenames)) + for _, oldPath := range renamedPaths { + oldURI := pathToURI(oldPath) + renamedURIs[oldURI] = true + if edits := openChanges[oldURI]; len(edits) > 0 { + changes = append(changes, textDocumentEdit(oldURI, edits)) } - } else if len(openMovedFiles) > 0 { - // Client doesn't support showDocument — still clean up old files - for oldPath := range openMovedFiles { - _ = os.Remove(oldPath) - _ = s.store.RemoveFile(oldPath) + changes = append(changes, newRenameFile(oldPath, clientRenames[oldPath])) + } + + otherURIs := make([]string, 0, len(openChanges)) + for fileURI := range openChanges { + if !renamedURIs[fileURI] { + otherURIs = append(otherURIs, string(fileURI)) } } + sort.Strings(otherURIs) + for _, u := range otherURIs { + fileURI := protocol.DocumentURI(u) + changes = append(changes, textDocumentEdit(fileURI, openChanges[fileURI])) + } - return &protocol.WorkspaceEdit{Changes: openChanges}, nil + return &WorkspaceEdit{DocumentChanges: changes} } // moduleRename holds the state for a module rename operation. @@ -4620,6 +4664,9 @@ func (mr *moduleRename) findModuleEdits(lineText string, token string) []moduleE } return results } + if results := mr.findGroupedAliasEdits(lineText, token, newToken); results != nil { + return results + } oldSuffix := token newSuffix := newToken for { @@ -4647,6 +4694,50 @@ func (mr *moduleRename) findModuleEdits(lineText string, token string) []moduleE return nil } +// findGroupedAliasEdits handles `alias Prefix.{A, B}` (and the require/import +// forms), where the module name is written once as the prefix and each member +// is indexed as its own reference — so the reference's full name never appears +// on the line. +// +// Which half moves depends on the rename: renaming the prefix rewrites the +// prefix, renaming a member rewrites that member inside the braces. Sites for +// the other members on the same line find nothing once the prefix is rewritten, +// so a group is only edited once. +func (mr *moduleRename) findGroupedAliasEdits(lineText, token, newToken string) []moduleEditResult { + dot := strings.LastIndexByte(token, '.') + if dot <= 0 { + return nil + } + prefix, member := token[:dot], token[dot+1:] + prefixCol, groupStart, groupEnd := findGroupedAlias(lineText, prefix) + if prefixCol < 0 { + return nil + } + memberCols := findAllTokenColumns(lineText[groupStart:groupEnd], member) + if len(memberCols) == 0 { + return nil + } + + newDot := strings.LastIndexByte(newToken, '.') + if newDot <= 0 { + // The member lost its namespace; a grouped alias cannot express that. + return nil + } + if newPrefix := newToken[:newDot]; newPrefix != prefix { + return []moduleEditResult{{prefixCol, len(prefix), newPrefix}} + } + + newMember := newToken[newDot+1:] + if newMember == member { + return nil + } + results := make([]moduleEditResult, 0, len(memberCols)) + for _, col := range memberCols { + results = append(results, moduleEditResult{groupStart + col, len(member), newMember}) + } + return results +} + type moduleEditResult struct { col int length int @@ -4703,14 +4794,21 @@ func (mr *moduleRename) conventionalNewPath(r store.LookupResult) (string, bool) return filepath.Join(prefix, filepath.FromSlash(newSuffix)), true } -// moveConventionalFiles moves files that follow the naming convention to their -// new paths, applying edits in the process. Open files are NOT moved on disk — -// they are left for applyEdits to handle via TextEdits so the editor buffer -// stays in sync. Returns moved files, paths that need showDocument calls -// (open files that were moved), and the path to show for the trigger file. -func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInfo, triggerFilePath string) (movedFiles map[string]string, openMovedFiles map[string]string, showDocumentPath string) { +// moveConventionalFiles moves files that follow the naming convention to +// their new paths, applying edits in the process. +// +// Files open in the editor are NOT moved here when the client can apply +// rename resource operations: the client owns the buffer, so it must move the +// file itself (see renamesToDocumentChanges). Moving it behind the client's +// back leaves the editor with a modified buffer pointing at a deleted path, +// and saving that buffer recreates the old file with the new module name. +// +// Returns the files moved on disk, the moves left to the client, the open +// files moved on disk anyway (fallback clients, which need showDocument and a +// deferred delete), and the path to show for the trigger file. +func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInfo) (movedFiles, clientRenames map[string]string) { movedFiles = make(map[string]string) - openMovedFiles = make(map[string]string) + clientRenames = make(map[string]string) for _, r := range mr.allModuleDefs { if _, ok := mr.moduleRenames[r.Module]; !ok { continue @@ -4724,26 +4822,21 @@ func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInf continue } - // Open files: write the new file to disk but DON'T delete the old one - // or mark it in movedFiles. Instead track it in openMovedFiles so that - // applyEdits still produces TextEdits for the editor buffer, and we - // send showDocument to redirect the editor to the new path. if fi.open { - updatedLines := mr.applyEditsToLines(fi.lines, mr.sitesByFile[r.FilePath]) - content := strings.Join(updatedLines, "\n") - if err := os.MkdirAll(filepath.Dir(newPath), 0755); err != nil { - log.Printf("Rename: cannot create dir for %s: %v", newPath, err) - continue - } - if err := os.WriteFile(newPath, []byte(content), 0644); err != nil { - log.Printf("Rename: cannot write %s: %v", newPath, err) + // Client applies rename operations: leave both paths untouched. + // applyEdits still emits TextEdits for the old URI, and the rename + // operation queued after them carries the edited buffer to the new + // path. + if mr.server.renameFileOpsSupported { + mr.server.debugf("Rename: %s → %s (client-applied)", r.FilePath, newPath) + clientRenames[r.FilePath] = newPath continue } - mr.server.debugf("Rename: %s → %s (open, deferred delete)", r.FilePath, newPath) - openMovedFiles[r.FilePath] = newPath - if r.FilePath == triggerFilePath && showDocumentPath == "" { - showDocumentPath = newPath - } + // Client cannot move the file and we must not do it behind its + // back: deleting a path the editor still has open leaves a buffer + // that recreates the file on the next save. Rename the contents in + // place and leave the file where it is. + log.Printf("Rename: leaving %s in place — client cannot apply rename operations and the file is open", r.FilePath) continue } @@ -4758,13 +4851,14 @@ func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInf log.Printf("Rename: cannot write %s: %v", newPath, err) continue } + if err := os.Remove(r.FilePath); err != nil { log.Printf("Rename: cannot remove %s: %v", r.FilePath, err) } mr.server.debugf("Rename: %s → %s", r.FilePath, newPath) movedFiles[r.FilePath] = newPath } - return movedFiles, openMovedFiles, showDocumentPath + return movedFiles, clientRenames } // applyEdits applies text edits to all non-moved files: open buffers get @@ -4783,12 +4877,24 @@ func (mr *moduleRename) applyEdits(fileCache map[string]moduleFileInfo, movedFil } if fi.open { fileURI := protocol.DocumentURI(uri.File(fp)) + // Each site is matched against the original line, so two sites can + // resolve to the same span — `alias Old.{A, B}` is one reference + // per member but a single edit to the shared prefix. The on-disk + // path rewrites the line as it goes and never sees the second + // match; TextEdits are all relative to the original text, so + // overlapping ones have to be dropped here or the editor applies + // the replacement twice. + claimed := make(map[int][]moduleEditResult) for _, es := range sites { if es.line-1 >= len(fi.lines) { continue } lineText := fi.lines[es.line-1] for _, e := range mr.findModuleEdits(lineText, es.token) { + if overlapsClaimed(claimed[es.line], e) { + continue + } + claimed[es.line] = append(claimed[es.line], e) openChanges[fileURI] = append(openChanges[fileURI], protocol.TextEdit{ Range: protocol.Range{ Start: protocol.Position{Line: uint32(es.line - 1), Character: uint32(e.col)}, @@ -4813,11 +4919,30 @@ func (mr *moduleRename) applyEdits(fileCache map[string]moduleFileInfo, movedFil return openChanges } +// overlapsClaimed reports whether e covers any column already taken by an +// edit on the same line. +func overlapsClaimed(claimed []moduleEditResult, e moduleEditResult) bool { + for _, c := range claimed { + if e.col < c.col+c.length && c.col < e.col+e.length { + return true + } + } + return false +} + // reindex re-parses all touched files asynchronously after the rename. -func (mr *moduleRename) reindex(fileCache map[string]moduleFileInfo, movedFiles map[string]string, openMovedFiles map[string]string) { +// +// movedFiles were moved on disk by the server, so their new paths are read +// back from disk. clientRenames have not moved yet — the client applies them +// when it receives the reply — so their new paths are indexed from the text +// the edits produce. +func (mr *moduleRename) reindex(fileCache map[string]moduleFileInfo, movedFiles, clientRenames map[string]string) { for oldPath := range movedFiles { _ = mr.server.store.RemoveFile(oldPath) } + for oldPath := range clientRenames { + _ = mr.server.store.RemoveFile(oldPath) + } var reindexPaths []string for _, newPath := range movedFiles { @@ -4839,8 +4964,8 @@ func (mr *moduleRename) reindex(fileCache map[string]moduleFileInfo, movedFiles } updatedLines := mr.applyEditsToLines(fi.lines, mr.sitesByFile[fp]) updatedText := strings.Join(updatedLines, "\n") - if newPath, moved := openMovedFiles[fp]; moved { - // Open file that was moved: reindex at the new path + if newPath, moved := clientRenames[fp]; moved { + // Open file the client is about to move: index at the new path openReindexes = append(openReindexes, textReindex{newPath, updatedText}) } else if fi.open { openReindexes = append(openReindexes, textReindex{fp, updatedText}) @@ -4849,11 +4974,15 @@ func (mr *moduleRename) reindex(fileCache map[string]moduleFileInfo, movedFiles } } - // Also reindex open moved files that had no edit sites (e.g. the file + // Also index client-renamed files that had no edit sites (e.g. the file // only contained the defmodule line which is already in allModuleDefs) - for oldPath, newPath := range openMovedFiles { - if _, hasSites := mr.sitesByFile[oldPath]; !hasSites { - reindexPaths = append(reindexPaths, newPath) + for oldPath, newPath := range clientRenames { + if _, hasSites := mr.sitesByFile[oldPath]; hasSites { + continue + } + if fi, ok := fileCache[oldPath]; ok { + updatedLines := mr.applyEditsToLines(fi.lines, nil) + openReindexes = append(openReindexes, textReindex{newPath, strings.Join(updatedLines, "\n")}) } } @@ -4880,7 +5009,7 @@ type renameSite struct { // buildTextEdits creates a WorkspaceEdit replacing all whole-token occurrences // of oldToken with newToken. Open buffers are returned in the WorkspaceEdit; // closed files are written directly to disk in parallel goroutines. -func (s *Server) buildTextEdits(sites []renameSite, oldToken, newToken string) *protocol.WorkspaceEdit { +func (s *Server) buildTextEdits(sites []renameSite, oldToken, newToken string) *WorkspaceEdit { // Group sites by file sitesByFile := make(map[string][]renameSite, len(sites)) for _, site := range sites { @@ -5012,7 +5141,7 @@ func (s *Server) buildTextEdits(sites []renameSite, oldToken, newToken string) * } }() - return &protocol.WorkspaceEdit{Changes: openChanges} + return &WorkspaceEdit{Changes: openChanges} } // reindexPaths re-parses and reindexes a specific set of files sequentially. diff --git a/internal/lsp/server_test.go b/internal/lsp/server_test.go index a9fecd8..e4e739f 100644 --- a/internal/lsp/server_test.go +++ b/internal/lsp/server_test.go @@ -29,6 +29,9 @@ func setupTestServer(t *testing.T) (*Server, func()) { server := NewServer(s, dir) server.snippetSupport = true + // Match real editors (Neovim, VS Code, Helix, Zed): they all apply rename + // resource operations. Fallback behaviour has its own tests. + server.renameFileOpsSupported = true // Resolve the mix binary so formatting tests work if p, err := exec.LookPath("mix"); err == nil { diff --git a/internal/lsp/workspace_edit.go b/internal/lsp/workspace_edit.go new file mode 100644 index 0000000..19aa39a --- /dev/null +++ b/internal/lsp/workspace_edit.go @@ -0,0 +1,113 @@ +package lsp + +import ( + "context" + "encoding/json" + "fmt" + + "go.lsp.dev/jsonrpc2" + "go.lsp.dev/protocol" + "go.lsp.dev/uri" +) + +// WorkspaceEdit is our own workspace edit type. go.lsp.dev/protocol's +// WorkspaceEdit types documentChanges as []TextDocumentEdit, so it cannot +// express resource operations (create/rename/delete file). We need rename +// operations: when a module rename moves a file that is open in the editor, +// the editor itself must move the buffer, otherwise it is left holding a +// modified buffer pointing at a path the server deleted — saving it recreates +// the old file with the new module name and the project no longer compiles. +// +// Per the LSP spec a client that supports documentChanges must ignore +// changes entirely when documentChanges is present, so the two fields are +// mutually exclusive: emit everything through documentChanges as soon as one +// resource operation is needed. +type WorkspaceEdit struct { + Changes map[protocol.DocumentURI][]protocol.TextEdit `json:"changes,omitempty"` + DocumentChanges []interface{} `json:"documentChanges,omitempty"` +} + +// TextDocumentEdit is a documentChanges entry holding text edits for one +// document. Version is always null: we never track buffer versions, and the +// spec allows a null version to mean "apply without a version check". +type TextDocumentEdit struct { + TextDocument versionedTextDocumentIdentifier `json:"textDocument"` + Edits []protocol.TextEdit `json:"edits"` +} + +type versionedTextDocumentIdentifier struct { + URI protocol.DocumentURI `json:"uri"` + Version *int `json:"version"` +} + +// RenameFile is a documentChanges entry that moves a file. Clients apply +// documentChanges in order, so a TextDocumentEdit for OldURI placed before +// this operation is applied to the buffer first and then travels with it. +type RenameFile struct { + Kind string `json:"kind"` // always "rename" + OldURI protocol.DocumentURI `json:"oldUri"` + NewURI protocol.DocumentURI `json:"newUri"` + Options *RenameFileOptions `json:"options,omitempty"` +} + +type RenameFileOptions struct { + Overwrite bool `json:"overwrite,omitempty"` + IgnoreIfExists bool `json:"ignoreIfExists,omitempty"` +} + +// pathToURI converts a filesystem path to a document URI. +func pathToURI(path string) protocol.DocumentURI { + return protocol.DocumentURI(uri.File(path)) +} + +// newRenameFile builds a rename operation for the given paths. +func newRenameFile(oldPath, newPath string) RenameFile { + return RenameFile{ + Kind: "rename", + OldURI: pathToURI(oldPath), + NewURI: pathToURI(newPath), + Options: &RenameFileOptions{Overwrite: true}, + } +} + +// textDocumentEdit builds a documentChanges entry for a single document. +func textDocumentEdit(fileURI protocol.DocumentURI, edits []protocol.TextEdit) TextDocumentEdit { + return TextDocumentEdit{ + TextDocument: versionedTextDocumentIdentifier{URI: fileURI}, + Edits: edits, + } +} + +// toProtocol degrades a WorkspaceEdit to the protocol type, dropping resource +// operations. Only used by the protocol.Server interface shim; the production +// path replies with the full type through the handler in Serve. +func (e *WorkspaceEdit) toProtocol() *protocol.WorkspaceEdit { + if e == nil { + return nil + } + return &protocol.WorkspaceEdit{Changes: e.Changes} +} + +// renameHandler intercepts textDocument/rename so the reply can carry +// resource operations. protocol.ServerHandler marshals whatever +// Server.Rename returns, and protocol.WorkspaceEdit has no field for them, +// so the request has to be answered before it reaches that dispatcher. +func (s *Server) renameHandler(next jsonrpc2.Handler) jsonrpc2.Handler { + return func(ctx context.Context, reply jsonrpc2.Replier, req jsonrpc2.Request) error { + if req.Method() != protocol.MethodTextDocumentRename { + return next(ctx, reply, req) + } + var params protocol.RenameParams + if err := json.Unmarshal(req.Params(), ¶ms); err != nil { + return reply(ctx, nil, fmt.Errorf("%w: %v", jsonrpc2.ErrParse, err)) + } + edit, err := s.RenameEdit(ctx, ¶ms) + if err != nil { + return reply(ctx, nil, err) + } + if edit == nil { + return reply(ctx, nil, nil) + } + return reply(ctx, edit, nil) + } +} From 6cd8f4c1425b7c467a25a05256d78987aba50e92 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Mon, 7 Sep 2026 17:09:54 -0400 Subject: [PATCH 07/26] Document attached MCP watcher and atomic renames --- CHANGELOG.md | 2 +- README.md | 2 +- docs/architecture.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6f6af2..bdb92fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. The headless server watches the project tree (fsnotify) so the index stays fresh without editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide +- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. Both headless and attached LSP+MCP modes watch the project tree (fsnotify) so the index stays fresh when agent edits bypass editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide ### Fixed diff --git a/README.md b/README.md index 25625a6..d859037 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,7 @@ Register it with your MCP client. For Claude Code: claude mcp add dexter -- dexter mcp ``` -Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use and keeps the index fresh by watching the project tree (fsnotify) and detecting git branch switches; a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. +Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use and keeps the index fresh by watching the project tree (fsnotify) and detecting git branch switches; a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. Attached LSP+MCP mode also watches the tree, so edits made directly by an agent are indexed even when they bypass editor notifications. Useful variants: diff --git a/docs/architecture.md b/docs/architecture.md index 276ffb1..7bc7eab 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -94,7 +94,7 @@ A module rename also moves files whose names follow the module naming convention `protocol.WorkspaceEdit` from `go.lsp.dev/protocol` types `documentChanges` as `[]TextDocumentEdit` and cannot carry resource operations, so `internal/lsp/workspace_edit.go` defines the wire types and `renameHandler` answers `textDocument/rename` ahead of the generated dispatcher. A client that understands `documentChanges` ignores `changes` entirely, so once one file moves, every edit in the reply goes through `documentChanges`. -For a rename the MCP server asked for rather than an editor, `deliverEdits` plays the part the editor would: attached to a live session it forwards the whole edit as `workspace/applyEdit` — over the raw connection, since `protocol.ApplyWorkspaceEditParams` drops resource operations for the same reason — and headless it carries out the edits and moves on disk itself. Headless has no open buffers, so it never produces a client-side move; that branch is defensive. +For a rename the MCP server asked for rather than an editor, the builders keep every affected file and move in one `WorkspaceEdit`. In attached mode, `deliverEdits` forwards that complete edit as `workspace/applyEdit` — over the raw connection, since `protocol.ApplyWorkspaceEditParams` drops resource operations — and updates buffers and the index only after the editor accepts it. A rejected edit therefore leaves closed files untouched too. In headless mode, `deliverEdits` applies the same complete edit and moves on disk itself. ### Grouped aliases From 91f63af5ad884872a6c955ec3f0af695c8ce0ae3 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Mon, 7 Sep 2026 17:24:31 -0400 Subject: [PATCH 08/26] Fix attached MCP capability handling --- cmd/main.go | 9 +++++- internal/lsp/api.go | 6 ++-- internal/lsp/api_test.go | 56 +++++++++++++++++++++++++++++++++ internal/lsp/server.go | 11 +++++-- internal/mcp/file_outline.go | 9 +++--- internal/mcp/implementations.go | 45 +++++++++++++++++++++++++- internal/mcp/mcp_test.go | 3 +- internal/mcp/tools_test.go | 33 +++++++++++++++++++ 8 files changed, 160 insertions(+), 12 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 7729c29..dcc576f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -406,7 +406,14 @@ func cmdLSP(projectRoot string, mcpListen string) { go func() { serveErrCh <- dexter_lsp.Serve(server, os.Stdin, os.Stdout) }() - <-server.Ready() + select { + case <-server.Ready(): + case err := <-serveErrCh: + if err == nil { + err = fmt.Errorf("LSP connection closed before initialization") + } + fatal(err) + } watcher, err := dexter_mcp.WatchFiles(server, s, projectRoot) if err != nil { diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 96f2226..70b9cd8 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -31,7 +31,6 @@ func Serve(server *Server, in io.Reader, out io.Writer) error { conn := jsonrpc2.NewConn(stream) server.client = protocol.ClientDispatcher(conn, logger) server.conn = conn - close(server.ready) handler := server.renameHandler(protocol.ServerHandler(server, nil)) ctx := context.Background() @@ -41,8 +40,9 @@ func Serve(server *Server, in io.Reader, out io.Writer) error { return conn.Err() } -// Ready is closed after Serve has installed the live LSP connection. Attached -// services must wait for it before accepting requests that can apply edits. +// Ready is closed after the LSP initialize request has been handled. Attached +// services must wait for it before accepting requests so client capabilities, +// stdlib discovery, and the live connection are all available. func (s *Server) Ready() <-chan struct{} { return s.ready } diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index c45b1b3..401badc 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -56,6 +56,25 @@ func TestApplyTextEdits(t *testing.T) { } } +func TestReadyWaitsForInitialize(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + select { + case <-server.Ready(): + t.Fatal("server reported ready before LSP initialization") + default: + } + if _, err := server.Initialize(context.Background(), &protocol.InitializeParams{}); err != nil { + t.Fatal(err) + } + select { + case <-server.Ready(): + default: + t.Fatal("server did not report ready after LSP initialization") + } +} + // Without a live client, a rename requested through the exported API must // land on disk even for files marked open (the defensive fallback path). func TestRenameFunction_WritesOpenBuffers(t *testing.T) { @@ -210,6 +229,7 @@ func TestRenameModule_ForwardsFileMoveToClient(t *testing.T) { defer cleanup() fc := &fakeConn{} server.conn = fc + server.renameFileOpsSupported = true src := `defmodule MyApp.Accounts do def list_users, do: [] @@ -266,6 +286,42 @@ end } } +func TestRenameModule_LeavesConventionalFileInPlaceWithoutClientMoveSupport(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + fc := &fakeConn{} + server.conn = fc + server.renameFileOpsSupported = false + + src := `defmodule MyApp.Accounts do + def list_users, do: [] +end +` + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", src) + oldPath := filepath.Join(server.projectRoot, "lib/accounts.ex") + newPath := filepath.Join(server.projectRoot, "lib/auth.ex") + + summary, err := server.RenameModule("MyApp.Accounts", "MyApp.Auth") + if err != nil { + t.Fatal(err) + } + + for _, change := range fc.applied.DocumentChanges { + if _, ok := change.(RenameFile); ok { + t.Fatal("applyEdit included a rename operation the client does not support") + } + } + if _, err := os.Stat(oldPath); err != nil { + t.Errorf("source file should remain at its old path: %v", err) + } + if _, err := os.Stat(newPath); !os.IsNotExist(err) { + t.Errorf("destination should not be created, stat error = %v", err) + } + if len(summary.FilesMoved) != 0 { + t.Errorf("summary reported unsupported moves: %v", summary.FilesMoved) + } +} + func TestRenameModule_RejectedApplyEditLeavesDiskUntouched(t *testing.T) { server, cleanup := setupTestServer(t) defer cleanup() diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 057d1ea..66b71b6 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -112,7 +112,8 @@ type Server struct { notifiedOTPMismatch sync.Once // prevents repeated OTP mismatch warnings backgroundWork sync.WaitGroup // tracks background reindex goroutines so the store isn't closed while they're running - ready chan struct{} // closed once Serve has installed the LSP connection + ready chan struct{} // closed once the LSP initialize request has completed + readyOnce sync.Once } func (s *Server) debugf(format string, args ...interface{}) { @@ -686,6 +687,7 @@ func (s *Server) Initialize(ctx context.Context, params *protocol.InitializePara }, } s.debugf("Initialize: capabilities: %+v", result.Capabilities) + s.readyOnce.Do(func() { close(s.ready) }) return result, nil } @@ -5059,7 +5061,12 @@ func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInf continue } if deliverAll { - clientRenames[r.FilePath] = newPath + // Headless callers encode moves in the edit and deliverEdits applies + // them on disk. Attached callers can forward them only when the live + // editor supports rename resource operations. + if mr.server.conn == nil || mr.server.renameFileOpsSupported { + clientRenames[r.FilePath] = newPath + } continue } diff --git a/internal/mcp/file_outline.go b/internal/mcp/file_outline.go index e74390f..c10c79a 100644 --- a/internal/mcp/file_outline.go +++ b/internal/mcp/file_outline.go @@ -3,7 +3,6 @@ package mcp import ( "context" "fmt" - "os" "sort" "strings" @@ -21,12 +20,14 @@ func (h *Handler) fileOutlineHandler(ctx context.Context, req *mcp.CallToolReque return nil, nil, fmt.Errorf("file must not be empty") } path := h.resolvePath(args.File) - if _, err := os.Stat(path); err != nil { + text, _, ok := h.lsp.ReadFileText(path) + if !ok { return textResult(fmt.Sprintf("File not found: %s", h.relPath(path))), nil, nil } - // Parse fresh from disk so the outline is correct even when the index is stale. - defs, _, err := parser.ParseFile(path) + // Parse fresh source so the outline is correct when either the index is + // stale or an attached editor has unsaved changes. + defs, _, err := parser.ParseText(path, text) if err != nil { return nil, nil, fmt.Errorf("parsing %s: %w", h.relPath(path), err) } diff --git a/internal/mcp/implementations.go b/internal/mcp/implementations.go index 6273eee..2b0b9e1 100644 --- a/internal/mcp/implementations.go +++ b/internal/mcp/implementations.go @@ -10,7 +10,7 @@ import ( type ImplementationsParams struct { Module string `json:"module" jsonschema:"behaviour or protocol module, fully qualified"` - Function string `json:"function,omitempty" jsonschema:"callback name; when set, locate its definition in each implementor"` + Function string `json:"function,omitempty" jsonschema:"callback or protocol function name; when set, locate its definition in each implementor"` } func (h *Handler) implementationsHandler(ctx context.Context, req *mcp.CallToolRequest, args ImplementationsParams) (*mcp.CallToolResult, any, error) { @@ -39,6 +39,49 @@ func (h *Handler) implementationsHandler(ctx context.Context, req *mcp.CallToolR if isProtocol { var b strings.Builder fmt.Fprintf(&b, "%s is a protocol (defprotocol at %s:%d).\n", module, h.relPath(modResults[decls[0]].FilePath), modResults[decls[0]].Line) + if function := strings.TrimSpace(args.Function); function != "" { + defs, err := h.store.LookupFunction(module, function) + if err != nil { + return nil, nil, fmt.Errorf("looking up protocol function: %w", err) + } + + // Functions in defprotocol and defimpl blocks share the protocol's + // module name in the index. Attribute each definition to the nearest + // preceding declaration in its file so a declaration and one or more + // implementations can safely coexist in the same file. + scopeKind := func(filePath string, line int) string { + kind, scopeLine := "", -1 + for _, r := range modResults { + if r.FilePath == filePath && r.Line <= line && r.Line > scopeLine { + kind, scopeLine = r.Kind, r.Line + } + } + return kind + } + arities := make(map[int]bool) + for _, d := range defs { + if scopeKind(d.FilePath, d.Line) == "defprotocol" { + arities[d.Arity] = true + } + } + if len(arities) == 0 { + return textResult(fmt.Sprintf("%s does not define a protocol function named %s. List its functions with dexter_module_api.", module, function)), nil, nil + } + + fmt.Fprintf(&b, "\nImplementations of protocol function %s.%s:\n", module, function) + found := 0 + for _, d := range defs { + if !arities[d.Arity] || scopeKind(d.FilePath, d.Line) != "defimpl" { + continue + } + fmt.Fprintf(&b, " %s - %s:%d\n", symbolName(module, function, d.Arity), h.relPath(d.FilePath), d.Line) + found++ + } + if found == 0 { + fmt.Fprintf(&b, " (no indexed defimpl defines %s)\n", function) + } + return textResult(b.String()), nil, nil + } if len(impls) == 0 { fmt.Fprintf(&b, "No defimpl implementations found in the index.\n") return textResult(b.String()), nil, nil diff --git a/internal/mcp/mcp_test.go b/internal/mcp/mcp_test.go index 90eeff8..5ae6036 100644 --- a/internal/mcp/mcp_test.go +++ b/internal/mcp/mcp_test.go @@ -22,6 +22,7 @@ import ( type testEnv struct { t *testing.T store *store.Store + lsp *lsp.Server root string session *mcp.ClientSession } @@ -56,7 +57,7 @@ func setupTestEnv(t *testing.T) *testEnv { } t.Cleanup(func() { _ = session.Close() }) - return &testEnv{t: t, store: s, root: root, session: session} + return &testEnv{t: t, store: s, lsp: server, root: root, session: session} } // indexFile writes an Elixir source file under the project root and indexes it. diff --git a/internal/mcp/tools_test.go b/internal/mcp/tools_test.go index bdeabcc..95f7194 100644 --- a/internal/mcp/tools_test.go +++ b/internal/mcp/tools_test.go @@ -1,11 +1,15 @@ package mcp import ( + "context" "fmt" "os" "path/filepath" "strings" "testing" + + "go.lsp.dev/protocol" + "go.lsp.dev/uri" ) const accountsSource = `defmodule MyApp.Accounts do @@ -219,6 +223,27 @@ end ) } +func TestFileOutlineTool_UsesOpenBuffer(t *testing.T) { + e := setupProject(t) + path := filepath.Join(e.root, "lib/my_app/accounts.ex") + buffer := `defmodule MyApp.Accounts do + def unsaved_function, do: :ok +end +` + if err := e.lsp.DidOpen(context.Background(), &protocol.DidOpenTextDocumentParams{ + TextDocument: protocol.TextDocumentItem{ + URI: protocol.DocumentURI(uri.File(path)), + Text: buffer, + }, + }); err != nil { + t.Fatal(err) + } + + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/accounts.ex"}) + wantContains(t, out, "def unsaved_function/0") + wantNotContains(t, out, "def fetch_user/1") +} + func TestImplementationsTool_Behaviour(t *testing.T) { e := setupProject(t) e.indexFile("lib/my_app/notifier.ex", `defmodule MyApp.Notifier do @@ -270,6 +295,14 @@ end "lib/my_app/size_impls.ex:1", "lib/my_app/size_impls.ex:5", ) + + out = e.callTool("dexter_implementations", map[string]any{"module": "MyApp.Size", "function": "size"}) + wantContains(t, out, + "Implementations of protocol function MyApp.Size.size", + "lib/my_app/size_impls.ex:2", + "lib/my_app/size_impls.ex:6", + ) + wantNotContains(t, out, "lib/my_app/size_impls.ex:1", "lib/my_app/size_impls.ex:5") } func TestCallHierarchyTool(t *testing.T) { From 2536cb7a9926509caab57816e1691b973a4cf784 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Mon, 7 Sep 2026 17:27:44 -0400 Subject: [PATCH 09/26] Initialize LSP in attached MCP integration test --- integration_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/integration_test.go b/integration_test.go index 7867208..f8ab35b 100644 --- a/integration_test.go +++ b/integration_test.go @@ -3,6 +3,9 @@ package main import ( "bufio" "context" + "encoding/json" + "fmt" + "io" "os" "os/exec" "path/filepath" @@ -11,6 +14,7 @@ import ( "time" sdkmcp "github.com/modelcontextprotocol/go-sdk/mcp" + "go.lsp.dev/uri" "github.com/remoteoss/dexter/internal/store" ) @@ -685,6 +689,10 @@ func TestIntegration_LSPWithMCPListen(t *testing.T) { if err != nil { t.Fatal(err) } + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } stderr, err := cmd.StderrPipe() if err != nil { t.Fatal(err) @@ -692,12 +700,31 @@ func TestIntegration_LSPWithMCPListen(t *testing.T) { if err := cmd.Start(); err != nil { t.Fatal(err) } + go func() { _, _ = io.Copy(io.Discard, stdout) }() t.Cleanup(func() { _ = stdin.Close() _ = cmd.Process.Kill() _, _ = cmd.Process.Wait() }) + // Attached MCP deliberately waits until initialize has populated the live + // client's capabilities before it starts accepting requests. + initialize, err := json.Marshal(map[string]any{ + "jsonrpc": "2.0", + "id": 1, + "method": "initialize", + "params": map[string]any{ + "rootUri": string(uri.File(root)), + "capabilities": map[string]any{}, + }, + }) + if err != nil { + t.Fatal(err) + } + if _, err := fmt.Fprintf(stdin, "Content-Length: %d\r\n\r\n%s", len(initialize), initialize); err != nil { + t.Fatal(err) + } + // Parse the bound address from stderr. addrCh := make(chan string, 1) go func() { From a5c2214b98d104dbcc765ff70bfd20bc33c887bb Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Mon, 7 Sep 2026 17:29:51 -0400 Subject: [PATCH 10/26] Stabilize sequential module rename test --- internal/lsp/rename_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/lsp/rename_test.go b/internal/lsp/rename_test.go index 2504e3f..210af25 100644 --- a/internal/lsp/rename_test.go +++ b/internal/lsp/rename_test.go @@ -1628,7 +1628,9 @@ end }) // Re-index with original content for test 2, and close the def file so it - // takes the closed-file path (moved on disk by the server) + // takes the closed-file path (moved on disk by the server). Wait for the + // first rename's asynchronous index bookkeeping before restoring it. + server.backgroundWork.Wait() server.docs.Close("file://" + oldPath) indexFile(t, server.store, server.projectRoot, "lib/docusign.ex", defContent) From 70d832ee11c9ab75e607ea73e92870cd3f0da5ee Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 09:21:12 +1000 Subject: [PATCH 11/26] Negotiate MCP workspace roots per session The headless MCP server bound its workspace to the launch directory before the session existed, indexing the wrong tree when the client started elsewhere. Without an explicit path it now obtains each session's root through MCP roots and resolves it the way the LSP resolves its own (existing .dexter index, then .git), opening the store and indexing only after resolution. Every resolved root gets one workspace, shared by sessions that resolve to it and torn down when the last one leaves, so --listen serves sessions from different projects concurrently. A roots/list_changed notification renegotiates on the session's next call. Clients that provide no usable root fall back to the launch directory; an explicit CLI path keeps today's fixed, eagerly indexed workspace, and attached mode still follows the LSP root. --- CHANGELOG.md | 2 +- README.md | 2 +- cmd/main.go | 70 +++--- internal/lsp/api.go | 9 + internal/lsp/api_test.go | 29 +++ internal/lsp/server.go | 14 +- internal/mcp/binding.go | 136 ++++++++++++ internal/mcp/mcp.go | 274 ++++++++++++++++++++--- internal/mcp/negotiation_test.go | 361 +++++++++++++++++++++++++++++++ internal/mcp/roots.go | 66 ++++++ 10 files changed, 905 insertions(+), 58 deletions(-) create mode 100644 internal/mcp/binding.go create mode 100644 internal/mcp/negotiation_test.go create mode 100644 internal/mcp/roots.go diff --git a/CHANGELOG.md b/CHANGELOG.md index bdb92fe..97fe490 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. Both headless and attached LSP+MCP modes watch the project tree (fsnotify) so the index stays fresh when agent edits bypass editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide +- **Built-in MCP server** - `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, incremental reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. Both headless and attached LSP+MCP modes watch the project tree (fsnotify) so the index stays fresh when agent edits bypass editor events. A running LSP can expose the same tools from its live session via `dexter lsp --mcp-listen=ADDR`, and `dexter mcp --instructions` prints an agent-facing usage guide. The headless server negotiates its workspace root per session through MCP roots, resolved the way the LSP resolves its own (existing `.dexter` index, then `.git`), with one workspace per resolved root in `--listen` mode; an explicit path argument overrides negotiation ### Fixed diff --git a/README.md b/README.md index d859037..3addfe0 100644 --- a/README.md +++ b/README.md @@ -464,7 +464,7 @@ Register it with your MCP client. For Claude Code: claude mcp add dexter -- dexter mcp ``` -Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server indexes the project on first use and keeps the index fresh by watching the project tree (fsnotify) and detecting git branch switches; a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. Attached LSP+MCP mode also watches the tree, so edits made directly by an agent are indexed even when they bypass editor notifications. +Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server obtains its workspace from the client through MCP roots and resolves it the way the LSP does (an existing `.dexter` index first, then the `.git` repository root), so it binds the project the client is working in rather than the directory it was launched from; clients that provide no roots get the launch directory, and an explicit path argument (`dexter mcp `) overrides negotiation entirely. In `--listen` mode each resolved root gets its own workspace, so sessions from different projects can share one server. The server indexes a workspace on first use and keeps the index fresh by watching the project tree (fsnotify) and detecting git branch switches; a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. Attached LSP+MCP mode also watches the tree, so edits made directly by an agent are indexed even when they bypass editor notifications. Useful variants: diff --git a/cmd/main.go b/cmd/main.go index dcc576f..e5c74fc 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -135,7 +135,7 @@ func main() { if err != nil { return err } - cmdMCP(projectRoot, mcpListen) + cmdMCP(projectRoot, mcpListen, len(args) > 0) return nil }, } @@ -490,43 +490,53 @@ func openStoreForServer(projectRoot string) *store.Store { } // cmdMCP starts the headless MCP server. Logs go to stderr; stdout belongs to -// the MCP stdio transport. -func cmdMCP(projectRoot string, listen string) { +// the MCP stdio transport. With an explicit path the workspace is fixed and +// indexed before serving; without one, each session's workspace root is +// negotiated through MCP roots, with projectRoot (the launch directory) as +// the fallback for clients that provide none. +func cmdMCP(projectRoot string, listen string, explicitRoot bool) { projectRoot = findProjectRoot(projectRoot) - log.SetOutput(os.Stderr) - s := openStoreForServer(projectRoot) - defer func() { - if err := s.Close(); err != nil { - fmt.Fprintf(os.Stderr, "Warning: failed to close store: %v\n", err) - } - }() - server := dexter_lsp.NewServer(s, projectRoot) - if root, ok := stdlib.Resolve(s, "", projectRoot); ok { - server.SetStdlibRoot(root) - } - - // Serve only once the index reflects the current tree: an empty index is - // built from scratch, an existing one gets a fast incremental update. - server.ReindexWorkspace() - server.WatchGitHead() - - // Headless servers get no editor events, so watch the tree directly. - watcher, err := dexter_mcp.WatchFiles(server, s, projectRoot) - if err != nil { - log.Printf("Warning: file watching unavailable (%v); the index updates on branch switches and via dexter_reindex", err) - } else { + var h *dexter_mcp.Handler + if explicitRoot { + s := openStoreForServer(projectRoot) defer func() { - if err := watcher.Close(); err != nil { - log.Printf("Warning: closing file watcher: %v", err) + if err := s.Close(); err != nil { + fmt.Fprintf(os.Stderr, "Warning: failed to close store: %v\n", err) } }() - } - h := dexter_mcp.NewHandler(dexter_mcp.Config{LSP: server, Store: s, ProjectRoot: projectRoot}) + server := dexter_lsp.NewServer(s, projectRoot) + if root, ok := stdlib.Resolve(s, "", projectRoot); ok { + server.SetStdlibRoot(root) + } - log.Printf("Dexter MCP v%s starting (root: %s)", version.Version, projectRoot) + // Serve only once the index reflects the current tree: an empty index + // is built from scratch, an existing one gets a fast incremental + // update. + server.ReindexWorkspace() + server.WatchGitHead() + + // Headless servers get no editor events, so watch the tree directly. + watcher, err := dexter_mcp.WatchFiles(server, s, projectRoot) + if err != nil { + log.Printf("Warning: file watching unavailable (%v); the index updates on branch switches and via dexter_reindex", err) + } else { + defer func() { + if err := watcher.Close(); err != nil { + log.Printf("Warning: closing file watcher: %v", err) + } + }() + } + + h = dexter_mcp.NewHandler(dexter_mcp.Config{LSP: server, Store: s, ProjectRoot: projectRoot}) + log.Printf("Dexter MCP v%s starting (root: %s)", version.Version, projectRoot) + } else { + h = dexter_mcp.NewHandler(dexter_mcp.Config{ProjectRoot: projectRoot, NegotiateRoots: true}) + defer h.Close() + log.Printf("Dexter MCP v%s starting (workspace roots negotiated per session; fallback root: %s)", version.Version, projectRoot) + } ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) defer stop() diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 70b9cd8..2e99671 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -125,6 +125,15 @@ func (s *Server) CollectReferences(module, function string) []store.ReferenceRes return out } +// StopGitHeadWatch ends the WatchGitHead goroutine and waits for it, joining +// any reindex it is mid-way through, so the store can be closed safely. The +// MCP server calls it when tearing down a workspace; an LSP session never +// does, its git-head watch runs for the life of the process. +func (s *Server) StopGitHeadWatch() { + s.gitHeadStopOnce.Do(func() { close(s.gitHeadStop) }) + s.gitHeadWG.Wait() +} + // WithReindexLock runs fn while holding the reindex lock, serializing it with // ReindexWorkspace and the background reindexes. The MCP file watcher wraps // its index writes in it so they cannot interleave with a concurrent diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index 401badc..a485e05 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "testing" + "time" "go.lsp.dev/jsonrpc2" "go.lsp.dev/protocol" @@ -382,3 +383,31 @@ end t.Errorf("summary reports moves %v, want %s → %s", summary.FilesMoved, oldPath, newPath) } } + +// StopGitHeadWatch must end the watch goroutine so a HEAD change after it can +// no longer trigger a reindex against a store the caller is about to close. +func TestStopGitHeadWatch(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + headPath := filepath.Join(server.projectRoot, ".git", "HEAD") + if err := os.MkdirAll(filepath.Dir(headPath), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(headPath, []byte("ref: refs/heads/main\n"), 0644); err != nil { + t.Fatal(err) + } + + server.WatchGitHead() + done := make(chan struct{}) + go func() { + server.StopGitHeadWatch() + server.StopGitHeadWatch() // idempotent + close(done) + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("StopGitHeadWatch did not return") + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 66b71b6..10ebae5 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -114,6 +114,10 @@ type Server struct { backgroundWork sync.WaitGroup // tracks background reindex goroutines so the store isn't closed while they're running ready chan struct{} // closed once the LSP initialize request has completed readyOnce sync.Once + + gitHeadStop chan struct{} // closed by StopGitHeadWatch to end the WatchGitHead goroutine + gitHeadStopOnce sync.Once + gitHeadWG sync.WaitGroup } func (s *Server) debugf(format string, args ...interface{}) { @@ -141,6 +145,7 @@ func NewServer(s *store.Store, projectRoot string) *Server { usingCache: make(map[string]*usingCacheEntry), depsCache: make(map[string]bool), ready: make(chan struct{}), + gitHeadStop: make(chan struct{}), } } @@ -485,7 +490,9 @@ func (s *Server) reindexWorkspace() (int, time.Duration) { // WatchGitHead polls .git/HEAD mtime and triggers reindex on branch switches. func (s *Server) WatchGitHead() { + s.gitHeadWG.Add(1) go func() { + defer s.gitHeadWG.Done() headPath := filepath.Join(s.projectRoot, ".git", "HEAD") var lastMtime int64 @@ -498,7 +505,12 @@ func (s *Server) WatchGitHead() { ticker := time.NewTicker(2 * time.Second) defer ticker.Stop() - for range ticker.C { + for { + select { + case <-s.gitHeadStop: + return + case <-ticker.C: + } info, err := os.Stat(headPath) if err != nil { continue diff --git a/internal/mcp/binding.go b/internal/mcp/binding.go new file mode 100644 index 0000000..c0937f8 --- /dev/null +++ b/internal/mcp/binding.go @@ -0,0 +1,136 @@ +package mcp + +import ( + "context" + "fmt" + "log" + "os" + "time" + + "github.com/remoteoss/dexter/internal/lsp" + "github.com/remoteoss/dexter/internal/stdlib" + "github.com/remoteoss/dexter/internal/store" + "github.com/remoteoss/dexter/internal/version" +) + +// indexWaitLimit caps how long a tool call waits for a workspace's initial +// index before reporting that it is still building. A variable so tests can +// shrink it. +var indexWaitLimit = 30 * time.Second + +// binding is one workspace a negotiating server is serving: the store, the +// headless LSP server, and the file watcher for one resolved project root. +// Sessions whose roots resolve to the same project share a binding. +type binding struct { + root string + store *store.Store + lsp *lsp.Server + watcher *Watcher + + initDone chan struct{} // closed once init finishes, successfully or not + initErr error + indexed chan struct{} // closed once the initial index pass completes +} + +// init opens the workspace. It runs once, in the call that created the +// binding; everything else waits on initDone. The initial index runs in the +// background so a cold build does not stall the session's handler queue; +// tool calls gate on it through awaitIndex. +func (b *binding) init() { + defer close(b.initDone) + s, err := openStore(b.root) + if err != nil { + b.initErr = err + return + } + b.store = s + b.lsp = lsp.NewServer(s, b.root) + if root, ok := stdlib.Resolve(s, "", b.root); ok { + b.lsp.SetStdlibRoot(root) + } + b.lsp.WatchGitHead() + if w, err := WatchFiles(b.lsp, s, b.root); err != nil { + log.Printf("Warning: file watching unavailable for %s (%v); the index updates on branch switches and via dexter_reindex", b.root, err) + } else { + b.watcher = w + } + go func() { + defer close(b.indexed) + b.lsp.ReindexWorkspace() + }() +} + +// awaitIndex blocks until the workspace is ready to answer, or reports why +// it is not. Init failures surface here as retryable errors. +func (b *binding) awaitIndex(ctx context.Context) error { + select { + case <-b.initDone: + case <-ctx.Done(): + return ctx.Err() + } + if b.initErr != nil { + return b.initErr + } + select { + case <-b.indexed: + return nil + case <-ctx.Done(): + return ctx.Err() + case <-time.After(indexWaitLimit): + return fmt.Errorf("the index for %s is still building; retry shortly", b.root) + } +} + +// close tears the workspace down: watcher first so no new index writes +// start, then the git-head watcher (joining any reindex it is running), then +// the initial index goroutine, and only then the store. +func (b *binding) close() { + <-b.initDone + if b.initErr != nil { + return + } + if b.watcher != nil { + if err := b.watcher.Close(); err != nil { + log.Printf("Warning: closing file watcher for %s: %v", b.root, err) + } + } + b.lsp.StopGitHeadWatch() + <-b.indexed + if err := b.store.Close(); err != nil { + log.Printf("Warning: closing store for %s: %v", b.root, err) + } +} + +// openStore opens the index at root with the recovery a long-running server +// needs, like cmd's openStoreForServer but returning errors instead of +// exiting: a session must survive a workspace that fails to open. A corrupt +// database or a populated index from an older format is deleted; the reopened +// empty store is then cold-built by the binding's initial index pass. +func openStore(root string) (*store.Store, error) { + s, err := store.Open(root) + if err != nil { + log.Printf("Failed to open index at %s (%v), rebuilding from scratch...", root, err) + removeIndexFiles(root) + if s, err = store.Open(root); err != nil { + return nil, fmt.Errorf("opening index at %s: %w", root, err) + } + } + if stored := s.GetIndexVersion(); stored != version.IndexVersion && !s.IsEmpty() { + log.Printf("Index version mismatch at %s (stored: %d, current: %d), rebuilding index...", root, stored, version.IndexVersion) + if err := s.Close(); err != nil { + log.Printf("Warning: closing outdated store: %v", err) + } + removeIndexFiles(root) + if s, err = store.Open(root); err != nil { + return nil, fmt.Errorf("reopening index at %s: %w", root, err) + } + } + return s, nil +} + +func removeIndexFiles(root string) { + dbPath := store.DBPath(root) + for _, p := range []string{dbPath, dbPath + "-wal", dbPath + "-shm"} { + _ = os.Remove(p) + } +} diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 02fb6a0..cf02916 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -5,10 +5,13 @@ package mcp import ( + "context" _ "embed" + "errors" "fmt" "path/filepath" "strings" + "sync" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -23,23 +26,53 @@ import ( //go:embed instructions.md var Instructions string -// Handler carries the state shared by all tool handlers. In headless mode -// (`dexter mcp`) the lsp.Server is constructed without a client connection; in -// attached mode (`dexter lsp --mcp-listen`) it is the live LSP session, so -// tools see open editor buffers and warm caches. +// Handler carries the state shared by all tool handlers. In attached mode +// (`dexter lsp --mcp-listen`) and with an explicit CLI path the workspace is +// fixed at construction; in attached mode the lsp.Server is the live LSP +// session, so tools see open editor buffers and warm caches. +// +// A negotiating handler (headless `dexter mcp` with no explicit path) has no +// fixed workspace. Each session's root is obtained through MCP roots and +// resolved the way the LSP resolves its own, and every resolved root gets one +// workspace (a binding), shared by all sessions that resolve to it. Tool +// calls run against a per-call view of the session's binding. type Handler struct { lsp *lsp.Server store *store.Store projectRoot string + + negotiate bool + fallbackRoot string // used by sessions that provide no usable root + mu sync.Mutex + bindings map[string]*binding // resolved root → workspace + sessions map[*mcp.ServerSession]*binding // session → its workspace + dirty map[*mcp.ServerSession]bool // roots changed; re-resolve on next call + watched map[*mcp.ServerSession]bool // a Wait goroutine will detach this session + closed bool } type Config struct { LSP *lsp.Server Store *store.Store ProjectRoot string + + // NegotiateRoots serves one workspace per client-provided root instead of + // the fixed LSP/Store pair, with ProjectRoot as the fallback for sessions + // that provide none. + NegotiateRoots bool } func NewHandler(cfg Config) *Handler { + if cfg.NegotiateRoots { + return &Handler{ + negotiate: true, + fallbackRoot: cfg.ProjectRoot, + bindings: make(map[string]*binding), + sessions: make(map[*mcp.ServerSession]*binding), + dirty: make(map[*mcp.ServerSession]bool), + watched: make(map[*mcp.ServerSession]bool), + } + } return &Handler{ lsp: cfg.LSP, store: cfg.Store, @@ -47,76 +80,267 @@ func NewHandler(cfg Config) *Handler { } } +var errClosed = errors.New("the MCP server is shutting down") + +// handlerFor returns the Handler a tool call should run against: the fixed +// one, or a view of the workspace bound to the call's session, waiting out +// that workspace's initial index. +func (h *Handler) handlerFor(ctx context.Context, req *mcp.CallToolRequest) (*Handler, error) { + if !h.negotiate { + return h, nil + } + b, err := h.bindingFor(ctx, req.Session) + if err != nil { + return nil, err + } + if err := b.awaitIndex(ctx); err != nil { + if b.initErr != nil { + // A workspace that failed to open is forgotten so the next call + // renegotiates from scratch instead of re-reporting a stale error. + h.forget(b) + } + return nil, err + } + return &Handler{lsp: b.lsp, store: b.store, projectRoot: b.root}, nil +} + +// bindingFor returns the session's workspace, negotiating its root first when +// the session is new or its roots changed. +func (h *Handler) bindingFor(ctx context.Context, ss *mcp.ServerSession) (*binding, error) { + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return nil, errClosed + } + b, bound := h.sessions[ss] + dirty := h.dirty[ss] + h.mu.Unlock() + if bound && !dirty { + return b, nil + } + + // Resolve outside the lock: ListRoots blocks on the client. + root, ok, err := negotiatedRoot(ctx, ss) + if err != nil { + return nil, err + } + if !ok { + root = h.fallbackRoot + } + + var created, orphan *binding + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return nil, errClosed + } + delete(h.dirty, ss) + if cur, bound := h.sessions[ss]; bound && cur.root == root { + h.mu.Unlock() + return cur, nil + } + nb, exists := h.bindings[root] + if !exists { + nb = &binding{root: root, initDone: make(chan struct{}), indexed: make(chan struct{})} + h.bindings[root] = nb + created = nb + } + if cur, bound := h.sessions[ss]; bound { + orphan = h.releaseLocked(ss, cur) + } + h.sessions[ss] = nb + if !h.watched[ss] { + h.watched[ss] = true + go func() { + _ = ss.Wait() + h.detachSession(ss) + }() + } + h.mu.Unlock() + + if orphan != nil { + go orphan.close() + } + if created != nil { + created.init() + } + return nb, nil +} + +// releaseLocked unbinds ss from b and reports b when no other session uses it +// any more, removing it from the handler; the caller closes it outside the +// lock. Callers must hold h.mu. +func (h *Handler) releaseLocked(ss *mcp.ServerSession, b *binding) (orphan *binding) { + delete(h.sessions, ss) + for _, sb := range h.sessions { + if sb == b { + return nil + } + } + if h.bindings[b.root] == b { + delete(h.bindings, b.root) + } + return b +} + +// detachSession drops everything the handler holds for a closed session, +// tearing down its workspace when no other session shares it. +func (h *Handler) detachSession(ss *mcp.ServerSession) { + h.mu.Lock() + var orphan *binding + if b, bound := h.sessions[ss]; bound { + orphan = h.releaseLocked(ss, b) + } + delete(h.dirty, ss) + delete(h.watched, ss) + h.mu.Unlock() + if orphan != nil { + orphan.close() + } +} + +// forget removes a workspace that failed to open, with every session bound to +// it, so subsequent calls renegotiate. +func (h *Handler) forget(b *binding) { + h.mu.Lock() + if h.bindings[b.root] == b { + delete(h.bindings, b.root) + } + for ss, sb := range h.sessions { + if sb == b { + delete(h.sessions, ss) + } + } + h.mu.Unlock() +} + +// onInitialized warms up a new session's workspace so the first tool call +// finds the index already building. Best-effort: failures surface on that +// first call, which renegotiates on its own context. +func (h *Handler) onInitialized(ctx context.Context, req *mcp.InitializedRequest) { + _, _ = h.bindingFor(ctx, req.Session) +} + +// onRootsChanged marks the session for renegotiation. The re-resolve happens +// on the session's next tool call, whose request context reaches the client +// reliably on every transport; if the roots still resolve to the same project +// the workspace is kept as is. +func (h *Handler) onRootsChanged(_ context.Context, req *mcp.RootsListChangedRequest) { + h.mu.Lock() + if _, bound := h.sessions[req.Session]; bound { + h.dirty[req.Session] = true + } + h.mu.Unlock() +} + +// Close tears down every workspace a negotiating handler holds. Fixed-mode +// handlers own nothing: their store and server belong to the caller. +func (h *Handler) Close() { + h.mu.Lock() + if h.closed || !h.negotiate { + h.mu.Unlock() + return + } + h.closed = true + bindings := make([]*binding, 0, len(h.bindings)) + for _, b := range h.bindings { + bindings = append(bindings, b) + } + h.bindings = nil + h.sessions = nil + h.mu.Unlock() + for _, b := range bindings { + b.close() + } +} + +// addTool registers a tool handler so each call runs against the workspace +// bound to its session (in fixed mode, always the handler itself). +func addTool[In any](srv *mcp.Server, h *Handler, t *mcp.Tool, f func(*Handler, context.Context, *mcp.CallToolRequest, In) (*mcp.CallToolResult, any, error)) { + mcp.AddTool(srv, t, func(ctx context.Context, req *mcp.CallToolRequest, args In) (*mcp.CallToolResult, any, error) { + hh, err := h.handlerFor(ctx, req) + if err != nil { + return nil, nil, err + } + return f(hh, ctx, req, args) + }) +} + // NewServer returns an MCP server with all dexter tools registered. func NewServer(h *Handler) *mcp.Server { + opts := &mcp.ServerOptions{Instructions: Instructions} + if h.negotiate { + opts.InitializedHandler = h.onInitialized + opts.RootsListChangedHandler = h.onRootsChanged + } srv := mcp.NewServer( &mcp.Implementation{Name: "dexter", Title: "Dexter Elixir language tools", Version: version.Version}, - &mcp.ServerOptions{Instructions: Instructions}, + opts, ) // The pointer hints distinguish explicit false from unset; clients must // treat unset pessimistically (destructive, open world). readOnly := &mcp.ToolAnnotations{ReadOnlyHint: true, OpenWorldHint: new(bool)} - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_workspace", Annotations: readOnly, Description: "Overview of the Elixir workspace: Mix projects, index size, stdlib status. Call once at the start of Elixir work.", - }, h.workspaceHandler) + }, (*Handler).workspaceHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_search", Annotations: readOnly, Description: "Locate Elixir modules and functions by fuzzy name match. More precise than grep for finding symbols: results are exact definitions with file:line.", - }, h.searchHandler) + }, (*Handler).searchHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_definition", Annotations: readOnly, Description: "Definition of an Elixir module or function by name: location, @doc/@spec, and source snippet, following defdelegate to the real implementation. Use instead of grep or reading files to answer where or what a symbol is.", - }, h.definitionHandler) + }, (*Handler).definitionHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_references", Annotations: readOnly, Description: "All call sites of an Elixir module or function, resolved through aliases, imports, and use-chain injection that grep cannot see. Use for any 'who calls or uses X' question.", - }, h.referencesHandler) + }, (*Handler).referencesHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_module_api", Annotations: readOnly, Description: "A module's public API in one call: moduledoc, functions with signatures and doc summaries, macros, delegates, types, callbacks, and submodules. Use before reading a module's source.", - }, h.moduleAPIHandler) + }, (*Handler).moduleAPIHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_file_outline", Annotations: readOnly, Description: "Everything an Elixir file defines: modules, functions, macros, and types with line numbers. Use instead of reading a file to map its contents; one Elixir file can define many modules.", - }, h.fileOutlineHandler) + }, (*Handler).fileOutlineHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_implementations", Annotations: readOnly, Description: "Implementations of an Elixir behaviour (@behaviour/use) or protocol (defimpl), optionally locating one callback in each implementor. Grep cannot resolve these relationships.", - }, h.implementationsHandler) + }, (*Handler).implementationsHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_call_hierarchy", Annotations: readOnly, Description: "Incoming callers and outgoing callees of an Elixir function, with file:line locations. Use to trace execution paths without reading files.", - }, h.callHierarchyHandler) + }, (*Handler).callHierarchyHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_reindex", Annotations: &mcp.ToolAnnotations{DestructiveHint: new(bool), IdempotentHint: true, OpenWorldHint: new(bool)}, Description: "Force an immediate incremental reindex. The index already updates automatically as files change; use this only when a lookup seems stale. The only tool that writes, and it writes only dexter's own index database.", - }, h.reindexHandler) + }, (*Handler).reindexHandler) - mcp.AddTool(srv, &mcp.Tool{ + addTool(srv, h, &mcp.Tool{ Name: "dexter_rename_symbol", Description: "Rename an Elixir module or function across the whole workspace, exactly like an editor rename: writes the changes to disk, moves files that follow the naming convention, and updates the index. Reports every file changed; review with git diff.", Annotations: &mcp.ToolAnnotations{OpenWorldHint: new(bool)}, - }, h.renameHandler) + }, (*Handler).renameHandler) return srv } diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go new file mode 100644 index 0000000..285a9bd --- /dev/null +++ b/internal/mcp/negotiation_test.go @@ -0,0 +1,361 @@ +package mcp + +import ( + "context" + "net/url" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/store" +) + +// negotiationEnv is a negotiating Handler with no fixed workspace, plus +// helpers to connect clients that advertise chosen roots. +type negotiationEnv struct { + t *testing.T + h *Handler + fallback string +} + +func setupNegotiation(t *testing.T) *negotiationEnv { + t.Helper() + fallback := t.TempDir() + h := NewHandler(Config{ProjectRoot: fallback, NegotiateRoots: true}) + t.Cleanup(h.Close) + return &negotiationEnv{t: t, h: h, fallback: fallback} +} + +// connect wires a new client session to the negotiating server. Roots are +// added before connecting so they are visible from the first roots/list. +func (e *negotiationEnv) connect(opts *mcp.ClientOptions, rootURIs ...string) (*mcp.ClientSession, *mcp.Client) { + e.t.Helper() + ctx := context.Background() + serverTransport, clientTransport := mcp.NewInMemoryTransports() + ss, err := NewServer(e.h).Connect(ctx, serverTransport, nil) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = ss.Close() }) + + client := mcp.NewClient(&mcp.Implementation{Name: "test-client", Version: "0.0.1"}, opts) + for _, u := range rootURIs { + client.AddRoots(&mcp.Root{URI: u}) + } + cs, err := client.Connect(ctx, clientTransport, nil) + if err != nil { + e.t.Fatal(err) + } + e.t.Cleanup(func() { _ = cs.Close() }) + return cs, client +} + +// projectDir creates a project directory containing one module and returns +// its path and file URI. +func projectDir(t *testing.T, module string) (string, string) { + t.Helper() + dir := t.TempDir() + writeSource(t, dir, "lib/mod.ex", "defmodule "+module+" do\n def hello, do: :ok\nend\n") + return dir, fileURI(dir) +} + +func writeSource(t *testing.T, dir, rel, content string) { + t.Helper() + path := filepath.Join(dir, rel) + if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0644); err != nil { + t.Fatal(err) + } +} + +func fileURI(path string) string { + return (&url.URL{Scheme: "file", Path: filepath.ToSlash(path)}).String() +} + +func toolText(t *testing.T, cs *mcp.ClientSession, name string, args map[string]any) (string, bool) { + t.Helper() + res, err := cs.CallTool(context.Background(), &mcp.CallToolParams{Name: name, Arguments: args}) + if err != nil { + return err.Error(), false + } + return resultText(res), !res.IsError +} + +func mustTool(t *testing.T, cs *mcp.ClientSession, name string, args map[string]any) string { + t.Helper() + out, ok := toolText(t, cs, name, args) + if !ok { + t.Fatalf("CallTool(%s) failed: %s", name, out) + } + return out +} + +func hasIndex(root string) bool { + _, err := os.Stat(filepath.Join(root, ".dexter", "dexter.db")) + return err == nil +} + +func TestNegotiation_BindsClientRoot(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t, "NegBind.Hello") + cs, _ := e.connect(nil, uri) + + out := mustTool(t, cs, "dexter_search", map[string]any{"query": "NegBind"}) + wantContains(t, out, "NegBind.Hello") + + if !hasIndex(root) { + t.Error("no index created under the negotiated root") + } + if hasIndex(e.fallback) { + t.Error("index created under the fallback root despite a negotiated root") + } + wantContains(t, mustTool(t, cs, "dexter_workspace", nil), root) +} + +func TestNegotiation_FallsBackWithoutUsableRoots(t *testing.T) { + cases := []struct { + name string + opts *mcp.ClientOptions + uris []string + }{ + // A default go-sdk client advertises roots with an empty list; this is + // what most clients look like, not an edge case. + {name: "empty roots list", opts: nil}, + {name: "roots capability off", opts: &mcp.ClientOptions{Capabilities: &mcp.ClientCapabilities{}}}, + {name: "non-file roots only", opts: nil, uris: []string{"https://example.com/project"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + e := setupNegotiation(t) + writeSource(t, e.fallback, "lib/mod.ex", "defmodule NegFall.Hello do\nend\n") + cs, _ := e.connect(tc.opts, tc.uris...) + + out := mustTool(t, cs, "dexter_search", map[string]any{"query": "NegFall"}) + wantContains(t, out, "NegFall.Hello") + if !hasIndex(e.fallback) { + t.Error("no index created under the fallback root") + } + }) + } +} + +// A root inside a repository resolves upward to the repository, exactly like +// the LSP's Initialize: .dexter/dexter.db or .git win, and a nested mix.exs +// does not stop the walk. +func TestNegotiation_ResolvesRootLikeLSP(t *testing.T) { + e := setupNegotiation(t) + repo := t.TempDir() + if err := os.MkdirAll(filepath.Join(repo, ".git"), 0755); err != nil { + t.Fatal(err) + } + writeSource(t, repo, "apps/web/mix.exs", "defmodule Web.MixProject do\nend\n") + writeSource(t, repo, "lib/top.ex", "defmodule NegRepo.Top do\nend\n") + cs, _ := e.connect(nil, fileURI(filepath.Join(repo, "apps", "web"))) + + // A module outside the advertised subdirectory is indexed, proving the + // workspace anchored on the repository root. + out := mustTool(t, cs, "dexter_search", map[string]any{"query": "NegRepo"}) + wantContains(t, out, "NegRepo.Top") + if !hasIndex(repo) { + t.Error("no index at the repository root") + } + if hasIndex(filepath.Join(repo, "apps", "web")) { + t.Error("index created at the subdirectory instead of the repository root") + } +} + +func TestNegotiation_BadRootIsRetryable(t *testing.T) { + e := setupNegotiation(t) + badURI := "file:///nonexistent/dexter-negotiation-test" + cs, client := e.connect(nil, badURI) + + out, ok := toolText(t, cs, "dexter_search", map[string]any{"query": "x"}) + if ok { + t.Fatalf("tool call succeeded against a nonexistent root: %s", out) + } + if !strings.Contains(out, "not a directory") { + t.Errorf("error does not name the problem: %s", out) + } + + // The failure is not cached: with the roots fixed, the same session works. + root, goodURI := projectDir(t, "NegRetry.Hello") + client.RemoveRoots(badURI) + client.AddRoots(&mcp.Root{URI: goodURI}) + eventually(t, "session to bind the corrected root", func() bool { + out, ok := toolText(t, cs, "dexter_search", map[string]any{"query": "NegRetry"}) + return ok && strings.Contains(out, "NegRetry.Hello") + }) + if !hasIndex(root) { + t.Error("no index created under the corrected root") + } +} + +// Sessions with different roots work concurrently against their own +// workspaces; sessions with the same root share one. +func TestNegotiation_MultipleRoots(t *testing.T) { + e := setupNegotiation(t) + rootA, uriA := projectDir(t, "NegMultiA.Mod") + rootB, uriB := projectDir(t, "NegMultiB.Mod") + csA, _ := e.connect(nil, uriA) + csB, _ := e.connect(nil, uriB) + csA2, _ := e.connect(nil, uriA) + + outA := mustTool(t, csA, "dexter_search", map[string]any{"query": "NegMulti"}) + wantContains(t, outA, "NegMultiA.Mod") + wantNotContains(t, outA, "NegMultiB.Mod") + + outB := mustTool(t, csB, "dexter_search", map[string]any{"query": "NegMulti"}) + wantContains(t, outB, "NegMultiB.Mod") + wantNotContains(t, outB, "NegMultiA.Mod") + + wantContains(t, mustTool(t, csA2, "dexter_search", map[string]any{"query": "NegMulti"}), "NegMultiA.Mod") + + if !hasIndex(rootA) || !hasIndex(rootB) { + t.Error("expected an index under each negotiated root") + } + e.h.mu.Lock() + nbindings := len(e.h.bindings) + e.h.mu.Unlock() + if nbindings != 2 { + t.Errorf("3 sessions over 2 roots hold %d workspaces, want 2", nbindings) + } +} + +func TestNegotiation_RootsChangedSwapsWorkspace(t *testing.T) { + e := setupNegotiation(t) + rootA, uriA := projectDir(t, "NegSwapA.Mod") + rootB, uriB := projectDir(t, "NegSwapB.Mod") + cs, client := e.connect(nil, uriA) + + wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "NegSwapA"}), "NegSwapA.Mod") + + client.RemoveRoots(uriA) + client.AddRoots(&mcp.Root{URI: uriB}) + eventually(t, "session to move to the new root", func() bool { + out, ok := toolText(t, cs, "dexter_search", map[string]any{"query": "NegSwapB"}) + return ok && strings.Contains(out, "NegSwapB.Mod") + }) + wantContains(t, mustTool(t, cs, "dexter_workspace", nil), rootB) + + // The old workspace is torn down: its watcher no longer indexes new files + // into its store. + e.h.mu.Lock() + _, oldBound := e.h.bindings[rootA] + e.h.mu.Unlock() + if oldBound { + t.Error("old workspace still held after the swap") + } + writeSource(t, rootA, "lib/late.ex", "defmodule NegSwapA.Late do\nend\n") + time.Sleep(4 * debounceWindow) + oldStore, err := store.Open(rootA) + if err != nil { + t.Fatal(err) + } + defer oldStore.Close() + if results, err := oldStore.LookupModule("NegSwapA.Late"); err != nil || len(results) != 0 { + t.Errorf("old workspace's watcher still indexing after teardown: %v, %v", results, err) + } +} + +// A roots change that resolves to the same project keeps the workspace: no +// teardown, no rebuild. +func TestNegotiation_SameRootChangeIsNoop(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t, "NegNoop.Mod") + cs, client := e.connect(nil, uri) + mustTool(t, cs, "dexter_search", map[string]any{"query": "NegNoop"}) + + e.h.mu.Lock() + before := e.h.bindings[root] + e.h.mu.Unlock() + + // Same project, different advertised directory: the first call binds it, + // so the subdirectory resolves upward via .dexter/dexter.db. + subdir := filepath.Join(root, "lib") + client.AddRoots(&mcp.Root{URI: fileURI(subdir)}) + eventually(t, "roots change notification to arrive", func() bool { + e.h.mu.Lock() + defer e.h.mu.Unlock() + for _, d := range e.h.dirty { + if d { + return true + } + } + return false + }) + mustTool(t, cs, "dexter_search", map[string]any{"query": "NegNoop"}) // renegotiates + + e.h.mu.Lock() + after := e.h.bindings[root] + e.h.mu.Unlock() + if before != after { + t.Error("workspace was rebuilt for a change that resolves to the same root") + } +} + +// A workspace root with characters that URI-encode (spaces) binds correctly. +func TestNegotiation_RootWithSpaces(t *testing.T) { + e := setupNegotiation(t) + root := filepath.Join(t.TempDir(), "my project") + writeSource(t, root, "lib/mod.ex", "defmodule NegSpace.Mod do\nend\n") + uri := fileURI(root) + if !strings.Contains(uri, "%20") { + t.Fatalf("test URI %q does not exercise percent-encoding", uri) + } + cs, _ := e.connect(nil, uri) + wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "NegSpace"}), "NegSpace.Mod") + if !hasIndex(root) { + t.Error("no index created under the percent-encoded root") + } +} + +// A tool call during a long initial index reports that the workspace is still +// building instead of hanging. +func TestNegotiation_ReportsInitializing(t *testing.T) { + prev := indexWaitLimit + indexWaitLimit = 10 * time.Millisecond + defer func() { indexWaitLimit = prev }() + + e := setupNegotiation(t) + // A pre-installed workspace whose initial index never finishes. + b := &binding{root: e.fallback, initDone: make(chan struct{}), indexed: make(chan struct{})} + close(b.initDone) + e.h.mu.Lock() + e.h.bindings[e.fallback] = b + e.h.mu.Unlock() + + cs, _ := e.connect(nil) + out, ok := toolText(t, cs, "dexter_search", map[string]any{"query": "x"}) + if ok { + t.Fatalf("tool call succeeded against an unindexed workspace: %s", out) + } + if !strings.Contains(out, "still building") { + t.Errorf("error does not report the index build: %s", out) + } + close(b.indexed) // let Close tear it down without blocking + b.initErr = context.Canceled +} + +// A session disconnecting releases its workspace. +func TestNegotiation_SessionCloseReleasesWorkspace(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t, "NegClose.Mod") + cs, _ := e.connect(nil, uri) + mustTool(t, cs, "dexter_search", map[string]any{"query": "NegClose"}) + + if err := cs.Close(); err != nil { + t.Fatal(err) + } + eventually(t, "workspace to be released", func() bool { + e.h.mu.Lock() + defer e.h.mu.Unlock() + _, held := e.h.bindings[root] + return !held && len(e.h.sessions) == 0 + }) +} diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go new file mode 100644 index 0000000..459ac01 --- /dev/null +++ b/internal/mcp/roots.go @@ -0,0 +1,66 @@ +package mcp + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/modelcontextprotocol/go-sdk/mcp" + + "github.com/remoteoss/dexter/internal/store" +) + +// fileURIToPath converts a file:// URI to an absolute filesystem path. +func fileURIToPath(raw string) (string, error) { + u, err := url.Parse(raw) + if err != nil { + return "", fmt.Errorf("invalid root URI %q: %w", raw, err) + } + if u.Host != "" && u.Host != "localhost" { + return "", fmt.Errorf("root URI %q names a remote host", raw) + } + path := filepath.FromSlash(u.Path) + if !filepath.IsAbs(path) { + return "", fmt.Errorf("root URI %q has no absolute path", raw) + } + return path, nil +} + +// negotiatedRoot resolves a session's workspace root from the MCP roots the +// client advertises. ok is false when the client offers no usable root (no +// roots capability, an empty list, or no file:// root): callers fall back to +// the launch-directory root. A transport failure or an unusable file:// root +// is an error the caller should surface and retry, not cache. +// +// A usable root resolves like the LSP's Initialize does: upward from the +// given directory to an existing index (.dexter/dexter.db) or repository +// marker (.git), so an existing index is reused and a subdirectory root +// still lands on the project. +func negotiatedRoot(ctx context.Context, ss *mcp.ServerSession) (root string, ok bool, err error) { + params := ss.InitializeParams() + if params == nil || params.Capabilities == nil || params.Capabilities.RootsV2 == nil { + return "", false, nil + } + res, err := ss.ListRoots(ctx, nil) + if err != nil { + return "", false, fmt.Errorf("listing client roots: %w", err) + } + for _, r := range res.Roots { + if !strings.HasPrefix(r.URI, "file:") { + continue + } + path, err := fileURIToPath(r.URI) + if err != nil { + return "", false, err + } + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return "", false, fmt.Errorf("client root %q is not a directory", path) + } + return store.FindProjectRoot(path), true, nil + } + return "", false, nil +} From 36790b2a2b407ea9f673fb91df431d845be1811d Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 12:48:19 +1000 Subject: [PATCH 12/26] Log MCP workspace bind and teardown The negotiating server decides per session which directory it serves, so the stderr log records each decision (root and whether it came from client roots or the fallback) and each workspace teardown. --- internal/mcp/binding.go | 1 + internal/mcp/mcp.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/internal/mcp/binding.go b/internal/mcp/binding.go index c0937f8..271a919 100644 --- a/internal/mcp/binding.go +++ b/internal/mcp/binding.go @@ -99,6 +99,7 @@ func (b *binding) close() { if err := b.store.Close(); err != nil { log.Printf("Warning: closing store for %s: %v", b.root, err) } + log.Printf("MCP workspace closed: %s", b.root) } // openStore opens the index at root with the recovery a long-running server diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index cf02916..623393d 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -9,6 +9,7 @@ import ( _ "embed" "errors" "fmt" + "log" "path/filepath" "strings" "sync" @@ -124,8 +125,10 @@ func (h *Handler) bindingFor(ctx context.Context, ss *mcp.ServerSession) (*bindi if err != nil { return nil, err } + source := "client roots" if !ok { root = h.fallbackRoot + source = "fallback" } var created, orphan *binding @@ -149,6 +152,7 @@ func (h *Handler) bindingFor(ctx context.Context, ss *mcp.ServerSession) (*bindi orphan = h.releaseLocked(ss, cur) } h.sessions[ss] = nb + log.Printf("MCP session workspace: %s (%s)", root, source) if !h.watched[ss] { h.watched[ss] = true go func() { From 676539d0a660b2a116e113811dbb32da9c856243 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 13:17:48 +1000 Subject: [PATCH 13/26] Check store close error in negotiation test --- internal/mcp/negotiation_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 285a9bd..63c43a5 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -257,7 +257,7 @@ func TestNegotiation_RootsChangedSwapsWorkspace(t *testing.T) { if err != nil { t.Fatal(err) } - defer oldStore.Close() + defer func() { _ = oldStore.Close() }() if results, err := oldStore.LookupModule("NegSwapA.Late"); err != nil || len(results) != 0 { t.Errorf("old workspace's watcher still indexing after teardown: %v, %v", results, err) } From 4c0ae5c02609f05be4de83fa61815c5c0371712f Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 14:01:48 +1000 Subject: [PATCH 14/26] Address review findings on root negotiation Canonicalize negotiated root paths so URI variants of one directory cannot key two workspaces onto the same database, stop the git-head watcher on explicit-root shutdown before the store closes, and skip text edits whose columns fall outside the target line instead of panicking. --- cmd/main.go | 1 + internal/lsp/api.go | 3 +++ internal/lsp/api_test.go | 8 ++++++++ internal/mcp/negotiation_test.go | 27 +++++++++++++++++++++++++++ internal/mcp/roots.go | 2 +- 5 files changed, 40 insertions(+), 1 deletion(-) diff --git a/cmd/main.go b/cmd/main.go index e5c74fc..8b28f67 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -517,6 +517,7 @@ func cmdMCP(projectRoot string, listen string, explicitRoot bool) { // update. server.ReindexWorkspace() server.WatchGitHead() + defer server.StopGitHeadWatch() // Headless servers get no editor events, so watch the tree directly. watcher, err := dexter_mcp.WatchFiles(server, s, projectRoot) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 2e99671..8c31a9a 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -363,6 +363,9 @@ func applyTextEdits(text string, edits []protocol.TextEdit) string { if int(start.Line) >= len(lines) || int(end.Line) >= len(lines) { continue } + if int(start.Character) > len(lines[start.Line]) || int(end.Character) > len(lines[end.Line]) { + continue + } prefix := lines[start.Line][:start.Character] suffix := lines[end.Line][end.Character:] replacement := strings.Split(prefix+e.NewText+suffix, "\n") diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index a485e05..5402ffc 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -39,6 +39,14 @@ func TestApplyTextEdits(t *testing.T) { }, want: "get_user(get_user(1))\n", }, + { + name: "out-of-range column is skipped, not a panic", + text: "short\n", + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 40}, End: protocol.Position{Line: 0, Character: 50}}, NewText: "x"}, + }, + want: "short\n", + }, { name: "multi-line span replacement", text: "a\nold one\nold two\nb\n", diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 63c43a5..4c4fcd8 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -96,6 +96,33 @@ func mustTool(t *testing.T, cs *mcp.ClientSession, name string, args map[string] return out } +func TestFileURIToPath(t *testing.T) { + cases := []struct { + uri string + want string // "" means an error is expected + }{ + {"file:///a/b", "/a/b"}, + {"file:///a/b/", "/a/b"}, // trailing slash must not key a second workspace + {"file://localhost/a/b", "/a/b"}, + {"file:///a/my%20project", "/a/my project"}, + {"file://otherhost/a", ""}, + {"file://a", ""}, // host form, no path + {"file:relative", ""}, + } + for _, tc := range cases { + got, err := fileURIToPath(tc.uri) + if tc.want == "" { + if err == nil { + t.Errorf("fileURIToPath(%q) = %q, want error", tc.uri, got) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("fileURIToPath(%q) = %q, %v; want %q", tc.uri, got, err, tc.want) + } + } +} + func hasIndex(root string) bool { _, err := os.Stat(filepath.Join(root, ".dexter", "dexter.db")) return err == nil diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go index 459ac01..fac5c0f 100644 --- a/internal/mcp/roots.go +++ b/internal/mcp/roots.go @@ -22,7 +22,7 @@ func fileURIToPath(raw string) (string, error) { if u.Host != "" && u.Host != "localhost" { return "", fmt.Errorf("root URI %q names a remote host", raw) } - path := filepath.FromSlash(u.Path) + path := filepath.Clean(filepath.FromSlash(u.Path)) if !filepath.IsAbs(path) { return "", fmt.Errorf("root URI %q has no absolute path", raw) } From f9f5b2cca688985e8ba21983150749e4b910f2a7 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 17:44:12 +1000 Subject: [PATCH 15/26] Serialize same-root workspace turnover and canonicalize roots An orphaned workspace closes in the background, and its close can wait out a running initial index build. Rebinding that root meanwhile opened a second store over the same database while the first was still bulk writing, violating the indexer's single-writer contract. Roots now drain: a rebind waits for the old workspace's close to finish. Resolved roots are also symlink-canonicalized, since path aliases of one directory (such as /tmp vs /private/tmp) would otherwise key two live workspaces onto one database with the same effect. --- internal/mcp/mcp.go | 113 +++++++++++++++++++++---------- internal/mcp/negotiation_test.go | 83 +++++++++++++++++++++-- internal/mcp/roots.go | 12 +++- 3 files changed, 166 insertions(+), 42 deletions(-) diff --git a/internal/mcp/mcp.go b/internal/mcp/mcp.go index 623393d..beb3824 100644 --- a/internal/mcp/mcp.go +++ b/internal/mcp/mcp.go @@ -49,6 +49,7 @@ type Handler struct { sessions map[*mcp.ServerSession]*binding // session → its workspace dirty map[*mcp.ServerSession]bool // roots changed; re-resolve on next call watched map[*mcp.ServerSession]bool // a Wait goroutine will detach this session + draining map[string]chan struct{} // roots whose last workspace is still closing closed bool } @@ -67,11 +68,12 @@ func NewHandler(cfg Config) *Handler { if cfg.NegotiateRoots { return &Handler{ negotiate: true, - fallbackRoot: cfg.ProjectRoot, + fallbackRoot: canonicalRoot(cfg.ProjectRoot), bindings: make(map[string]*binding), sessions: make(map[*mcp.ServerSession]*binding), dirty: make(map[*mcp.ServerSession]bool), watched: make(map[*mcp.ServerSession]bool), + draining: make(map[string]chan struct{}), } } return &Handler{ @@ -131,44 +133,73 @@ func (h *Handler) bindingFor(ctx context.Context, ss *mcp.ServerSession) (*bindi source = "fallback" } - var created, orphan *binding - h.mu.Lock() - if h.closed { - h.mu.Unlock() - return nil, errClosed - } - delete(h.dirty, ss) - if cur, bound := h.sessions[ss]; bound && cur.root == root { + for { + var created, orphan *binding + h.mu.Lock() + if h.closed { + h.mu.Unlock() + return nil, errClosed + } + // The last workspace for this root may still be tearing down; opening + // a second store over the same database would race its final writes. + if ch, ok := h.draining[root]; ok { + h.mu.Unlock() + select { + case <-ch: + continue + case <-ctx.Done(): + return nil, ctx.Err() + } + } + delete(h.dirty, ss) + if cur, bound := h.sessions[ss]; bound && cur.root == root { + h.mu.Unlock() + return cur, nil + } + nb, exists := h.bindings[root] + if !exists { + nb = &binding{root: root, initDone: make(chan struct{}), indexed: make(chan struct{})} + h.bindings[root] = nb + created = nb + } + if cur, bound := h.sessions[ss]; bound { + orphan = h.releaseLocked(ss, cur) + } + h.sessions[ss] = nb + log.Printf("MCP session workspace: %s (%s)", root, source) + if !h.watched[ss] { + h.watched[ss] = true + go func() { + _ = ss.Wait() + h.detachSession(ss) + }() + } h.mu.Unlock() - return cur, nil - } - nb, exists := h.bindings[root] - if !exists { - nb = &binding{root: root, initDone: make(chan struct{}), indexed: make(chan struct{})} - h.bindings[root] = nb - created = nb - } - if cur, bound := h.sessions[ss]; bound { - orphan = h.releaseLocked(ss, cur) - } - h.sessions[ss] = nb - log.Printf("MCP session workspace: %s (%s)", root, source) - if !h.watched[ss] { - h.watched[ss] = true - go func() { - _ = ss.Wait() - h.detachSession(ss) - }() - } - h.mu.Unlock() - if orphan != nil { - go orphan.close() - } - if created != nil { - created.init() + if orphan != nil { + h.drainOrphan(orphan) + } + if created != nil { + created.init() + } + return nb, nil } - return nb, nil +} + +// drainOrphan closes an unbound workspace in the background, keeping its root +// marked as draining until the close finishes so no new workspace opens over +// the same database in the meantime. releaseLocked marked the root. +func (h *Handler) drainOrphan(b *binding) { + go func() { + b.close() + h.mu.Lock() + ch := h.draining[b.root] + delete(h.draining, b.root) + h.mu.Unlock() + if ch != nil { + close(ch) + } + }() } // releaseLocked unbinds ss from b and reports b when no other session uses it @@ -184,6 +215,7 @@ func (h *Handler) releaseLocked(ss *mcp.ServerSession, b *binding) (orphan *bind if h.bindings[b.root] == b { delete(h.bindings, b.root) } + h.draining[b.root] = make(chan struct{}) return b } @@ -199,7 +231,7 @@ func (h *Handler) detachSession(ss *mcp.ServerSession) { delete(h.watched, ss) h.mu.Unlock() if orphan != nil { - orphan.close() + h.drainOrphan(orphan) } } @@ -252,10 +284,17 @@ func (h *Handler) Close() { } h.bindings = nil h.sessions = nil + draining := make([]chan struct{}, 0, len(h.draining)) + for _, ch := range h.draining { + draining = append(draining, ch) + } h.mu.Unlock() for _, b := range bindings { b.close() } + for _, ch := range draining { + <-ch + } } // addTool registers a tool handler so each call runs against the workspace diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 4c4fcd8..8819cf0 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -24,12 +24,24 @@ type negotiationEnv struct { func setupNegotiation(t *testing.T) *negotiationEnv { t.Helper() - fallback := t.TempDir() + fallback := canonTempDir(t) h := NewHandler(Config{ProjectRoot: fallback, NegotiateRoots: true}) t.Cleanup(h.Close) return &negotiationEnv{t: t, h: h, fallback: fallback} } +// canonTempDir returns a symlink-free temp dir: negotiated roots are +// canonicalized, so expectations must be built from canonical paths +// (t.TempDir itself is symlinked on macOS). +func canonTempDir(t *testing.T) string { + t.Helper() + dir, err := filepath.EvalSymlinks(t.TempDir()) + if err != nil { + t.Fatal(err) + } + return dir +} + // connect wires a new client session to the negotiating server. Roots are // added before connecting so they are visible from the first roots/list. func (e *negotiationEnv) connect(opts *mcp.ClientOptions, rootURIs ...string) (*mcp.ClientSession, *mcp.Client) { @@ -58,7 +70,7 @@ func (e *negotiationEnv) connect(opts *mcp.ClientOptions, rootURIs ...string) (* // its path and file URI. func projectDir(t *testing.T, module string) (string, string) { t.Helper() - dir := t.TempDir() + dir := canonTempDir(t) writeSource(t, dir, "lib/mod.ex", "defmodule "+module+" do\n def hello, do: :ok\nend\n") return dir, fileURI(dir) } @@ -177,7 +189,7 @@ func TestNegotiation_FallsBackWithoutUsableRoots(t *testing.T) { // does not stop the walk. func TestNegotiation_ResolvesRootLikeLSP(t *testing.T) { e := setupNegotiation(t) - repo := t.TempDir() + repo := canonTempDir(t) if err := os.MkdirAll(filepath.Join(repo, ".git"), 0755); err != nil { t.Fatal(err) } @@ -329,7 +341,7 @@ func TestNegotiation_SameRootChangeIsNoop(t *testing.T) { // A workspace root with characters that URI-encode (spaces) binds correctly. func TestNegotiation_RootWithSpaces(t *testing.T) { e := setupNegotiation(t) - root := filepath.Join(t.TempDir(), "my project") + root := filepath.Join(canonTempDir(t), "my project") writeSource(t, root, "lib/mod.ex", "defmodule NegSpace.Mod do\nend\n") uri := fileURI(root) if !strings.Contains(uri, "%20") { @@ -369,6 +381,69 @@ func TestNegotiation_ReportsInitializing(t *testing.T) { b.initErr = context.Canceled } +// Symlink aliases of one directory must share a workspace: two live +// workspaces over one database would race each other's index writes. +func TestNegotiation_SymlinkedRootsShareWorkspace(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t, "NegLink.Mod") + link := filepath.Join(canonTempDir(t), "link") + if err := os.Symlink(root, link); err != nil { + t.Fatal(err) + } + cs1, _ := e.connect(nil, uri) + cs2, _ := e.connect(nil, fileURI(link)) + + wantContains(t, mustTool(t, cs1, "dexter_search", map[string]any{"query": "NegLink"}), "NegLink.Mod") + wantContains(t, mustTool(t, cs2, "dexter_search", map[string]any{"query": "NegLink"}), "NegLink.Mod") + + e.h.mu.Lock() + nbindings := len(e.h.bindings) + e.h.mu.Unlock() + if nbindings != 1 { + t.Errorf("symlink alias created %d workspaces, want 1", nbindings) + } +} + +// While a root's last workspace is still tearing down, a new session for that +// root must wait it out instead of opening a second store over the same +// database mid-teardown. +func TestNegotiation_WaitsForDrainingWorkspace(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t, "NegDrain.Mod") + + drain := make(chan struct{}) + e.h.mu.Lock() + e.h.draining[root] = drain + e.h.mu.Unlock() + + cs, _ := e.connect(nil, uri) + result := make(chan string, 1) + go func() { + out, _ := toolText(t, cs, "dexter_search", map[string]any{"query": "NegDrain"}) + result <- out + }() + + select { + case out := <-result: + t.Fatalf("call proceeded while the workspace was draining: %s", out) + case <-time.After(300 * time.Millisecond): + } + + e.h.mu.Lock() + delete(e.h.draining, root) + e.h.mu.Unlock() + close(drain) + + select { + case out := <-result: + if !strings.Contains(out, "NegDrain.Mod") { + t.Errorf("call after drain did not find the module: %s", out) + } + case <-time.After(10 * time.Second): + t.Fatal("call never completed after the drain finished") + } +} + // A session disconnecting releases its workspace. func TestNegotiation_SessionCloseReleasesWorkspace(t *testing.T) { e := setupNegotiation(t) diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go index fac5c0f..5c1e2ea 100644 --- a/internal/mcp/roots.go +++ b/internal/mcp/roots.go @@ -60,7 +60,17 @@ func negotiatedRoot(ctx context.Context, ss *mcp.ServerSession) (root string, ok if err != nil || !info.IsDir() { return "", false, fmt.Errorf("client root %q is not a directory", path) } - return store.FindProjectRoot(path), true, nil + return store.FindProjectRoot(canonicalRoot(path)), true, nil } return "", false, nil } + +// canonicalRoot resolves symlinks so every alias of a directory keys the same +// workspace; two live workspaces over one database would race each other's +// index writes. +func canonicalRoot(path string) string { + if resolved, err := filepath.EvalSymlinks(path); err == nil { + return resolved + } + return path +} From 6e0334629a24b68cfc6ecbc68a41e2c7e98d3ead Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 17:52:47 +1000 Subject: [PATCH 16/26] Resolve the fallback root's symlinks before the marker walk Negotiated roots resolve symlinks before walking for project markers, but the fallback walked the launch directory's logical path first and canonicalized after, so a marker above a symlink's target was invisible and the two mechanisms could key different workspaces for one directory. --- cmd/main.go | 9 +++++++++ internal/mcp/negotiation_test.go | 24 ++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/cmd/main.go b/cmd/main.go index 8b28f67..f65a58f 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -495,6 +495,15 @@ func openStoreForServer(projectRoot string) *store.Store { // negotiated through MCP roots, with projectRoot (the launch directory) as // the fallback for clients that provide none. func cmdMCP(projectRoot string, listen string, explicitRoot bool) { + if !explicitRoot { + // The fallback root must key the same workspace a negotiated root + // would, and negotiated roots resolve symlinks before walking for + // project markers: a marker above a symlink's target is invisible + // from the symlink's logical parents. + if resolved, err := filepath.EvalSymlinks(projectRoot); err == nil { + projectRoot = resolved + } + } projectRoot = findProjectRoot(projectRoot) log.SetOutput(os.Stderr) diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 8819cf0..610cf91 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -404,6 +404,30 @@ func TestNegotiation_SymlinkedRootsShareWorkspace(t *testing.T) { } } +// A symlinked fallback root and a negotiated root for the same directory must +// key one workspace, so no-roots and roots-advertising sessions share it. +func TestNegotiation_SymlinkedFallbackSharesWorkspace(t *testing.T) { + root, uri := projectDir(t, "NegFallLink.Mod") + link := filepath.Join(canonTempDir(t), "link") + if err := os.Symlink(root, link); err != nil { + t.Fatal(err) + } + e := &negotiationEnv{t: t, h: NewHandler(Config{ProjectRoot: link, NegotiateRoots: true}), fallback: root} + t.Cleanup(e.h.Close) + + noRoots, _ := e.connect(&mcp.ClientOptions{Capabilities: &mcp.ClientCapabilities{}}) + withRoots, _ := e.connect(nil, uri) + wantContains(t, mustTool(t, noRoots, "dexter_search", map[string]any{"query": "NegFallLink"}), "NegFallLink.Mod") + wantContains(t, mustTool(t, withRoots, "dexter_search", map[string]any{"query": "NegFallLink"}), "NegFallLink.Mod") + + e.h.mu.Lock() + nbindings := len(e.h.bindings) + e.h.mu.Unlock() + if nbindings != 1 { + t.Errorf("fallback and negotiated sessions hold %d workspaces, want 1 shared", nbindings) + } +} + // While a root's last workspace is still tearing down, a new session for that // root must wait it out instead of opening a second store over the same // database mid-teardown. From c4faac720efd4e47318b9d307b9fc5ea57ca4bca Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Wed, 9 Sep 2026 18:00:31 +1000 Subject: [PATCH 17/26] Resolve the fallback root exactly like a negotiated root The fallback still walked with the CLI's extra mix.exs marker while negotiated roots use the LSP's markers only, so a no-roots session and a roots session for one directory could key different workspaces in a markerless tree. Explicit paths keep the CLI resolution. --- cmd/main.go | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index f65a58f..a0b4144 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -495,16 +495,18 @@ func openStoreForServer(projectRoot string) *store.Store { // negotiated through MCP roots, with projectRoot (the launch directory) as // the fallback for clients that provide none. func cmdMCP(projectRoot string, listen string, explicitRoot bool) { - if !explicitRoot { - // The fallback root must key the same workspace a negotiated root - // would, and negotiated roots resolve symlinks before walking for - // project markers: a marker above a symlink's target is invisible - // from the symlink's logical parents. + if explicitRoot { + projectRoot = findProjectRoot(projectRoot) + } else { + // The fallback root must key the same workspace a negotiated root for + // the launch directory would, so it resolves identically: symlinks + // first (a marker above a symlink's target is invisible from the + // symlink's logical parents), then the LSP's marker walk. if resolved, err := filepath.EvalSymlinks(projectRoot); err == nil { projectRoot = resolved } + projectRoot = store.FindProjectRoot(projectRoot) } - projectRoot = findProjectRoot(projectRoot) log.SetOutput(os.Stderr) var h *dexter_mcp.Handler From 5967ffafd1bf4dd7f7908fe03c861af2a6f7b5df Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Fri, 11 Sep 2026 13:10:18 +1000 Subject: [PATCH 18/26] Decode text edit columns from the client encoding before slicing applyTextEdits sliced lines by Position.Character as a byte offset, but the rename machinery emits columns in the client's encoding, UTF-16 by default since the encoding negotiation landed. A non-ASCII character left of an edited token made the headless rename write the wrong span, and the attached-mode index snapshot diverge from what the editor applied. Columns now convert through the same helpers every inbound position uses, whose clamping also covers out-of-range columns. --- internal/lsp/api.go | 19 +++++++++++-------- internal/lsp/api_test.go | 29 +++++++++++++++++++++++++++-- 2 files changed, 38 insertions(+), 10 deletions(-) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 8c31a9a..a9b0ba0 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -246,7 +246,7 @@ func (s *Server) deliverEdits(edit *WorkspaceEdit) error { if !ok { return fmt.Errorf("reading %s to apply rename edits", path) } - if err := os.WriteFile(path, []byte(applyTextEdits(text, fileEdits)), 0644); err != nil { + if err := os.WriteFile(path, []byte(s.applyTextEdits(text, fileEdits)), 0644); err != nil { return err } } @@ -308,7 +308,7 @@ func (s *Server) prepareDeliveredEdit(edit *WorkspaceEdit) ([]deliveredFile, err return nil, fmt.Errorf("reading %s to prepare rename edits", path) } if fileEdits := edits[path]; len(fileEdits) > 0 { - text = applyTextEdits(text, fileEdits) + text = s.applyTextEdits(text, fileEdits) } newPath := path if renamed, ok := renames[path]; ok { @@ -344,9 +344,10 @@ func (s *Server) recordDeliveredEdit(files []deliveredFile) { } } -// applyTextEdits applies non-overlapping TextEdits to text. Positions use the -// same line/byte-column convention the rename machinery produces them in. -func applyTextEdits(text string, edits []protocol.TextEdit) string { +// applyTextEdits applies non-overlapping TextEdits to text. The rename +// machinery emits Position.Character in the client's encoding (outPos), so +// columns are converted back to byte offsets before slicing. +func (s *Server) applyTextEdits(text string, edits []protocol.TextEdit) string { sorted := make([]protocol.TextEdit, len(edits)) copy(sorted, edits) sort.Slice(sorted, func(i, j int) bool { @@ -363,11 +364,13 @@ func applyTextEdits(text string, edits []protocol.TextEdit) string { if int(start.Line) >= len(lines) || int(end.Line) >= len(lines) { continue } - if int(start.Character) > len(lines[start.Line]) || int(end.Character) > len(lines[end.Line]) { + startCol := s.inCol(lines, int(start.Line), start.Character) + endCol := s.inCol(lines, int(end.Line), end.Character) + if startCol > len(lines[start.Line]) || endCol > len(lines[end.Line]) { continue } - prefix := lines[start.Line][:start.Character] - suffix := lines[end.Line][end.Character:] + prefix := lines[start.Line][:startCol] + suffix := lines[end.Line][endCol:] replacement := strings.Split(prefix+e.NewText+suffix, "\n") lines = append(lines[:start.Line], append(replacement, lines[end.Line+1:]...)...) } diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index 5402ffc..e4628ec 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -19,9 +19,20 @@ func TestApplyTextEdits(t *testing.T) { tests := []struct { name string text string + enc PositionEncoding // zero value means UTF-16, the default edits []protocol.TextEdit want string }{ + { + name: "UTF-16 columns after a multi-byte character", + text: "ü = fetch_user(1)\n", + // "fetch_user" starts at UTF-16 column 4 but byte offset 5: + // slicing by the wire column would replace the wrong span. + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 4}, End: protocol.Position{Line: 0, Character: 14}}, NewText: "get_user"}, + }, + want: "ü = get_user(1)\n", + }, { name: "single token on one line", text: "def fetch_user(id) do\n fetch_user(id)\nend\n", @@ -40,8 +51,17 @@ func TestApplyTextEdits(t *testing.T) { want: "get_user(get_user(1))\n", }, { - name: "out-of-range column is skipped, not a panic", + name: "out-of-range column clamps to line end, not a panic", + text: "short\n", + edits: []protocol.TextEdit{ + {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 40}, End: protocol.Position{Line: 0, Character: 50}}, NewText: "x"}, + }, + want: "shortx\n", + }, + { + name: "out-of-range byte column is skipped, not a panic", text: "short\n", + enc: EncodingUTF8, edits: []protocol.TextEdit{ {Range: protocol.Range{Start: protocol.Position{Line: 0, Character: 40}, End: protocol.Position{Line: 0, Character: 50}}, NewText: "x"}, }, @@ -58,7 +78,12 @@ func TestApplyTextEdits(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := applyTextEdits(tt.text, tt.edits); got != tt.want { + enc := tt.enc + if enc == "" { + enc = EncodingUTF16 + } + srv := &Server{positionEncoding: enc} + if got := srv.applyTextEdits(tt.text, tt.edits); got != tt.want { t.Errorf("got:\n%q\nwant:\n%q", got, tt.want) } }) From 59a1aa7882de25725da1ee345da98dd5e10ea736 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Tue, 22 Sep 2026 08:31:19 +1000 Subject: [PATCH 19/26] Address review findings on rename delivery and reference parity Index writes from delivered edits now hold the reindex lock so a concurrent workspace reindex's prune cannot drop rows written after its walk. CollectReferences gained the injected-alias collection the References handler acquired upstream, restoring the mirrored-results promise. moveConventionalFiles checks the same-path guard before the deliverAll branch, so a namespace-only rename no longer encodes a path-to-itself move. --- internal/lsp/api.go | 53 ++++++++++++++++++--------- internal/lsp/api_test.go | 77 ++++++++++++++++++++++++++++++++++++++++ internal/lsp/server.go | 19 +++++----- 3 files changed, 123 insertions(+), 26 deletions(-) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index a9b0ba0..7a78df2 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -73,6 +73,19 @@ func (s *Server) CollectReferences(module, function string) []store.ReferenceRes return nil } + // Sites written through an alias that a __using__ block injects. The file + // holding them declares no alias of its own, so the index has them under + // the bare short name and the direct lookup cannot see them. + moduleKindRefs := refResults + if function != "" { + if refs, err := s.store.LookupReferences(module, ""); err == nil { + moduleKindRefs = refs + } else { + moduleKindRefs = nil + } + } + refResults = append(refResults, s.injectedAliasReferences(module, function, moduleKindRefs)...) + if function != "" { // Transitive refs via static __using__ import chains. Call sites of // use-injected functions are attributed to the injecting module in the @@ -276,7 +289,9 @@ func (s *Server) deliverEdits(edit *WorkspaceEdit) error { } } if len(paths) > 0 { - s.reindexPaths(paths) + // Under the reindex lock so a concurrent workspace reindex's + // walk-and-prune cannot drop rows written after its walk passed. + s.WithReindexLock(func() { s.reindexPaths(paths) }) } return nil } @@ -321,27 +336,31 @@ func (s *Server) prepareDeliveredEdit(edit *WorkspaceEdit) ([]deliveredFile, err // recordDeliveredEdit makes MCP reads and index queries reflect an accepted // editor edit immediately, without waiting for subsequent didChange events. +// It holds the reindex lock so a concurrent workspace reindex's +// walk-and-prune cannot drop rows written after its walk passed. func (s *Server) recordDeliveredEdit(files []deliveredFile) { - s.indexWrites.RLock() - defer s.indexWrites.RUnlock() - if s.indexUnavailable { - return - } - for _, file := range files { - if file.oldPath != file.newPath { - _ = s.store.RemoveFile(file.oldPath) + s.WithReindexLock(func() { + s.indexWrites.RLock() + defer s.indexWrites.RUnlock() + if s.indexUnavailable { + return } - defs, refs, err := parser.ParseText(file.newPath, file.text) - if err == nil { - _ = s.store.IndexFileWithRefs(file.newPath, defs, refs) - } - if file.open { + for _, file := range files { if file.oldPath != file.newPath { - s.docs.Close(string(uri.File(file.oldPath))) + _ = s.store.RemoveFile(file.oldPath) + } + defs, refs, err := parser.ParseText(file.newPath, file.text) + if err == nil { + _ = s.store.IndexFileWithRefs(file.newPath, defs, refs) + } + if file.open { + if file.oldPath != file.newPath { + s.docs.Close(string(uri.File(file.oldPath))) + } + s.docs.Set(string(uri.File(file.newPath)), file.text) } - s.docs.Set(string(uri.File(file.newPath)), file.text) } - } + }) } // applyTextEdits applies non-overlapping TextEdits to text. The rename diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index e4628ec..ac1351b 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -444,3 +444,80 @@ func TestStopGitHeadWatch(t *testing.T) { t.Fatal("StopGitHeadWatch did not return") } } + +// CollectReferences must mirror the References handler's collection: call +// sites written through a __using__-injected alias are indexed under the bare +// short name and only reachable through injectedAliasReferences. +func TestCollectReferences_ViaUseInjectedAlias(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/my_app/repo.ex", `defmodule MyApp.Repo do + defmacro __using__(_) do + quote do + alias MyApp.Repo + end + end + + def all(queryable), do: queryable +end +`) + indexFile(t, server.store, server.projectRoot, "lib/my_app/accounts.ex", `defmodule MyApp.Accounts do + use MyApp.Repo + + def list_users do + Repo.all(User) + end +end +`) + // Same short name, no `use` — a different module entirely. + indexFile(t, server.store, server.projectRoot, "lib/my_app/unrelated.ex", `defmodule MyApp.Unrelated do + def count do + Repo.aggregate(:count) + end +end +`) + + refs := server.CollectReferences("MyApp.Repo", "all") + accountsPath := filepath.Join(server.projectRoot, "lib/my_app/accounts.ex") + unrelatedPath := filepath.Join(server.projectRoot, "lib/my_app/unrelated.ex") + found := make(map[string]bool, len(refs)) + for _, r := range refs { + found[r.FilePath] = true + } + if !found[accountsPath] { + t.Errorf("Repo.all call through the injected alias not collected: %+v", refs) + } + if found[unrelatedPath] { + t.Errorf("bare Repo. in a file that does not use MyApp.Repo was collected: %+v", refs) + } +} + +// A namespace-only module rename keeps the conventional file path, so the +// edit must carry no rename operation: moving a path onto itself would fail +// after the text edits are already prepared. +func TestRenameModule_NamespaceOnlyKeepsPath(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + path := filepath.Join(server.projectRoot, "lib", "user.ex") + indexFile(t, server.store, server.projectRoot, "lib/user.ex", `defmodule MyApp.Accounts.User do + def name(u), do: u.name +end +`) + + summary, err := server.RenameModule("MyApp.Accounts.User", "MyApp.Billing.User") + if err != nil { + t.Fatal(err) + } + if len(summary.FilesMoved) != 0 { + t.Errorf("namespace-only rename reported moves: %v", summary.FilesMoved) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatalf("file left its conventional path: %v", err) + } + if !strings.Contains(string(data), "defmodule MyApp.Billing.User") { + t.Errorf("module not renamed in place:\n%s", data) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 73260ba..b9c2236 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -6224,15 +6224,6 @@ func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInf if !hasContent { continue } - if deliverAll { - // Headless callers encode moves in the edit and deliverEdits applies - // them on disk. Attached callers can forward them only when the live - // editor supports rename resource operations. - if mr.server.conn == nil || mr.server.renameFileOpsSupported { - clientRenames[r.FilePath] = newPath - } - continue - } // The conventional path is the module's last segment inside the // directory the file already sits in, so a rename that leaves that @@ -6247,6 +6238,16 @@ func (mr *moduleRename) moveConventionalFiles(fileCache map[string]moduleFileInf continue } + if deliverAll { + // Headless callers encode moves in the edit and deliverEdits applies + // them on disk. Attached callers can forward them only when the live + // editor supports rename resource operations. + if mr.server.conn == nil || mr.server.renameFileOpsSupported { + clientRenames[r.FilePath] = newPath + } + continue + } + if fi.open { // Client applies rename operations: leave both paths untouched. // applyEdits still emits TextEdits for the old URI, and the rename From bb4852f2613d88d986cbc25b010be3c7ec18e833 Mon Sep 17 00:00:00 2001 From: "shane.hull" Date: Tue, 22 Sep 2026 08:47:22 +1000 Subject: [PATCH 20/26] Do not retake the index write lock inside WithReindexLock WithReindexLock already holds indexWrites for reading; reindexPaths and recordDeliveredEdit retook it on the same goroutine, which deadlocks once a writer is waiting (RWMutex is not reentrant). They now rely on the wrapper's locks, using indexOneFileLocked like other pre-locked callers. --- internal/lsp/api.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 7a78df2..c98702c 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -336,15 +336,12 @@ func (s *Server) prepareDeliveredEdit(edit *WorkspaceEdit) ([]deliveredFile, err // recordDeliveredEdit makes MCP reads and index queries reflect an accepted // editor edit immediately, without waiting for subsequent didChange events. -// It holds the reindex lock so a concurrent workspace reindex's -// walk-and-prune cannot drop rows written after its walk passed. +// WithReindexLock supplies both locks the writes need: the reindex lock so a +// concurrent workspace reindex's walk-and-prune cannot drop rows written +// after its walk passed, and the indexWrites read lock (not retaken here, +// RWMutex is not reentrant). func (s *Server) recordDeliveredEdit(files []deliveredFile) { s.WithReindexLock(func() { - s.indexWrites.RLock() - defer s.indexWrites.RUnlock() - if s.indexUnavailable { - return - } for _, file := range files { if file.oldPath != file.newPath { _ = s.store.RemoveFile(file.oldPath) @@ -396,8 +393,10 @@ func (s *Server) applyTextEdits(text string, edits []protocol.TextEdit) string { return strings.Join(lines, "\n") } +// reindexPaths must run inside WithReindexLock, which already holds the +// indexWrites read lock indexOneFile would retake (RWMutex is not reentrant). func (s *Server) reindexPaths(paths []string) { for _, path := range paths { - s.indexOneFile(path) + s.indexOneFileLocked(path) } } From dde480e8ae147a537bdecd8eb2ae9160efdc1915 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 3 Oct 2026 22:01:32 -0400 Subject: [PATCH 21/26] Fix review findings on grouped aliases, root URIs, and MCP roots - Module rename: when only one member of `alias Prefix.{A, B}` moves to another namespace, the member leaves the group and gets its own alias line. Before, the shared prefix was rewritten, so the other members named modules that do not exist. - Module rename: a group whose members continue on the next lines is rewritten on its opening line when the prefix module is renamed. Before, the opening line had no member, so the prefix stayed. - MCP roots: on Windows, a file URI keeps the drive letter after a leading slash (file:///C:/project). The slash is removed, so the root is an absolute path. - MCP roots: the fallback root and client roots go through one function (mcpConfig) with the CLI's project-root search. Tests: regression tests for each item, an MCP rename with non-ASCII text left of the name, dexter_references through an alias that __using__ injects, and `dexter mcp --listen` over HTTP. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 2 + cmd/main.go | 30 +++++++---- cmd/main_test.go | 53 +++++++++++++++++++ docs/architecture.md | 2 +- integration_test.go | 61 ++++++++++++++++++++++ internal/lsp/rename_test.go | 87 ++++++++++++++++++++++++++++++++ internal/lsp/server.go | 66 ++++++++++++++++++------ internal/mcp/negotiation_test.go | 20 ++++++++ internal/mcp/rename_test.go | 16 ++++++ internal/mcp/roots.go | 16 +++++- internal/mcp/tools_test.go | 29 ++++++++++- 11 files changed, 355 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d28861..292f054 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ ### Fixed +- **Grouped aliases in a module rename** — moving one member of `alias Old.{A, B}` to another namespace rewrote the shared prefix, so the other members named modules that do not exist; the moved member now leaves the group and gets its own `alias`. A group whose members continue on the next lines (`alias Old.{` on its own line) kept the old prefix when the prefix module was renamed; it is now rewritten + - **A worktree moved into the project, or a directory that cannot be read, is no longer indexed by mistake** — `git worktree move` renames the directory and then writes its `.git` file again in place, so a watcher or a walk could find the file empty and index the whole worktree. Such a directory is now treated as a worktree until git is done, by the watchers and by both index walks, and each check reads the `.git` file once: two reads could see the empty file and then the complete one, and answer that a worktree was neither a worktree nor still being written. This happened with git 2.48 and later, which write the file again after the move; older git does not. A directory that the watcher cannot read (for example when the process has no file descriptor left) used to be skipped silently: it was not watched, not retried, and its files were reported as if it were a plain directory. It is now marked as not covered, so the coverage report says so and the retry reads it again, and its files are indexed only once Dexter knows what it is - **A def inside a macro's `quote` is no longer indexed as a function of the macro's module** — `defmacro route(...) do quote do def handle(...) end end` made the index say that the DSL module defines `handle/2`, which it does not. A consumer that imports the DSL then resolved `Consumer.handle` into the macro's body. Such a def is now skipped, so the call goes to the line in the consumer that declared it, from the compiled BEAM. What `__using__` injects, and a quote in a helper function, are still indexed as before. The index is rebuilt once after the upgrade diff --git a/cmd/main.go b/cmd/main.go index 696cdb4..fe12cbb 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -766,6 +766,25 @@ func fatal(err error) { os.Exit(1) } +// mcpConfig resolves the MCP frontend's workspace roots. The fallback root and +// the roots that clients give go through the same project-root search as the +// CLI, so every frontend reaches the same daemon for one directory. +func mcpConfig(launchDir string, explicitRoot bool) (dexter_mcp.Config, error) { + root, err := projectRootFor(launchDir) + if err != nil { + return dexter_mcp.Config{}, err + } + cfg := dexter_mcp.Config{Root: root, Fixed: explicitRoot, ResolveRoot: projectRootFor} + if !explicitRoot { + // The launch directory is only a guess at the workspace. Starting a + // daemon on a directory that is not a project would index all of it. + if err := nonProjectRootError(root); err != nil { + cfg.FallbackErr = fmt.Errorf("%w. The MCP client gave no workspace root; configure the server with the project path (`dexter mcp `) or start it in the project", err) + } + } + return cfg, nil +} + // cmdMCP serves MCP to an agent. Like `dexter lsp`, it is a frontend of the // shared workspace daemon: it opens no index and starts no watcher, and every // tool call is answered by the daemon of the session's workspace, which it @@ -778,18 +797,11 @@ func fatal(err error) { // fallback for clients that give no root. func cmdMCP(projectRoot string, listen string, explicitRoot bool) { log.SetOutput(os.Stderr) - root, err := projectRootFor(projectRoot) + cfg, err := mcpConfig(projectRoot, explicitRoot) if err != nil { fatal(err) } - cfg := dexter_mcp.Config{Root: root, Fixed: explicitRoot, ResolveRoot: projectRootFor} - if !explicitRoot { - // The launch directory is only a guess at the workspace. Starting a - // daemon on a directory that is not a project would index all of it. - if err := nonProjectRootError(root); err != nil { - cfg.FallbackErr = fmt.Errorf("%w. The MCP client gave no workspace root; configure the server with the project path (`dexter mcp `) or start it in the project", err) - } - } + root := cfg.Root frontend := dexter_mcp.NewFrontend(cfg) defer frontend.Close() if explicitRoot { diff --git a/cmd/main_test.go b/cmd/main_test.go index 84dea2d..e8d3639 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -123,3 +123,56 @@ func TestFileFailureDoesNotHideTheBuildingNote(t *testing.T) { t.Errorf("the building note is hidden: %q", got) } } + +// Regression: the MCP fallback root and the roots that MCP clients give were +// resolved by two different searches (one treated mix.exs as a marker, one did +// not), so a session with roots and a session without them could start two +// daemons for one directory. Both now use the CLI's search. +func TestMCPRootsResolveLikeCLI(t *testing.T) { + project := t.TempDir() + if err := os.WriteFile(filepath.Join(project, "mix.exs"), []byte("defmodule App.MixProject do\nend\n"), 0o644); err != nil { + t.Fatal(err) + } + sub := filepath.Join(project, "lib") + if err := os.MkdirAll(sub, 0o755); err != nil { + t.Fatal(err) + } + + cfg, err := mcpConfig(sub, false) + if err != nil { + t.Fatal(err) + } + want := findProjectRoot(sub) + if cfg.Root != want { + t.Errorf("fallback root = %q, want the CLI root %q", cfg.Root, want) + } + negotiated, err := cfg.ResolveRoot(sub) + if err != nil { + t.Fatal(err) + } + if negotiated != want { + t.Errorf("negotiated root = %q, want the CLI root %q", negotiated, want) + } + if cfg.FallbackErr != nil { + t.Errorf("a Mix project was refused as the fallback root: %v", cfg.FallbackErr) + } +} + +// A launch directory that is not a project is not indexed when the MCP client +// gives no root. +func TestMCPFallbackRefusesNonProject(t *testing.T) { + cfg, err := mcpConfig(t.TempDir(), false) + if err != nil { + t.Fatal(err) + } + if cfg.FallbackErr == nil || !strings.Contains(cfg.FallbackErr.Error(), "does not look like an Elixir project") { + t.Errorf("FallbackErr = %v, want a refusal", cfg.FallbackErr) + } + explicit, err := mcpConfig(t.TempDir(), true) + if err != nil { + t.Fatal(err) + } + if explicit.FallbackErr != nil { + t.Errorf("an explicit root was refused: %v", explicit.FallbackErr) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index e7c4597..0bccf91 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -207,7 +207,7 @@ The MCP rename tool runs in the workspace daemon on its headless language servic ### Grouped aliases -`alias Old.{A, B}` (and the `require`/`import` forms) names the module once, as the prefix, while the index records one reference per member — so a member's full name never appears on the line. `findGroupedAliasEdits` handles both directions: renaming the prefix rewrites the prefix, renaming a member rewrites that member inside the braces. Since every member on the line resolves to the same prefix edit, `applyEdits` drops TextEdits that overlap one already emitted for that line; the on-disk path rewrites the line as it goes and never sees the second match. +`alias Old.{A, B}` (and the `require`/`import` forms) names the module once, as the prefix, while the index records one reference per member — so a member's full name never appears on the line. `findGroupedAliasEdits` handles both directions: when the renamed module is the prefix or one of its ancestors, the prefix is rewritten (also on the opening line of a group whose members continue on the next lines, where the index records every member); renaming a member rewrites that member inside the braces, and a member that moves to another namespace leaves the group and gets its own `alias` line, so the other members keep their prefix. Since every member on the line resolves to the same prefix edit, `applyEdits` drops TextEdits that overlap one already emitted for that line; the on-disk path rewrites the line as it goes and never sees the second match. ## Indexing throughput diff --git a/integration_test.go b/integration_test.go index 67733e2..7445700 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1,6 +1,7 @@ package main import ( + "bufio" "context" "os" "os/exec" @@ -1131,3 +1132,63 @@ func TestIntegration_MCPSharesWorkspaceDaemon(t *testing.T) { t.Errorf("daemon pid changed from %d to %d: a second daemon served the workspace", status.PID, after.PID) } } + +// TestIntegration_MCPListenHTTP serves MCP over streamable HTTP and calls a +// tool, which the workspace daemon answers. +func TestIntegration_MCPListenHTTP(t *testing.T) { + binary := buildDexter(t) + root := scaffoldProject(t) + runDexter(t, binary, root, "init", root) + + cmd := exec.Command(binary, "mcp", "--listen=localhost:0", root) + cmd.Dir = root + cmd.Env = append(os.Environ(), "DEXTER_DAEMON_IDLE_TIMEOUT=1s", "PWD="+mustAbs(t, root)) + stderr, err := cmd.StderrPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = cmd.Process.Signal(os.Interrupt) + _, _ = cmd.Process.Wait() + }) + + addrCh := make(chan string, 1) + go func() { + scanner := bufio.NewScanner(stderr) + for scanner.Scan() { + if i := strings.Index(scanner.Text(), "MCP server listening on "); i >= 0 { + addrCh <- strings.TrimSpace(scanner.Text()[i+len("MCP server listening on "):]) + break + } + } + // Keep draining so the child never blocks on a full stderr pipe. + for scanner.Scan() { + } + }() + var addr string + select { + case addr = <-addrCh: + case <-time.After(30 * time.Second): + t.Fatal("timed out waiting for the MCP listen address on stderr") + } + + client := sdkmcp.NewClient(&sdkmcp.Implementation{Name: "integration-test", Version: "0.0.1"}, nil) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + session, err := client.Connect(ctx, &sdkmcp.StreamableClientTransport{Endpoint: "http://" + addr}, nil) + if err != nil { + t.Fatalf("connecting to the MCP HTTP server: %v", err) + } + defer func() { _ = session.Close() }() + + res, err := session.CallTool(ctx, &sdkmcp.CallToolParams{Name: "dexter_definition", Arguments: map[string]any{"module": "MyApp.Repo", "function": "get"}}) + if err != nil { + t.Fatal(err) + } + if out := mcpToolText(t, res); !strings.Contains(out, "lib/my_app/repo.ex") { + t.Errorf("definition over HTTP missing location:\n%s", out) + } +} diff --git a/internal/lsp/rename_test.go b/internal/lsp/rename_test.go index a5874ec..400ceed 100644 --- a/internal/lsp/rename_test.go +++ b/internal/lsp/rename_test.go @@ -3567,3 +3567,90 @@ func TestRename_Module_ConventionalPathUnchangedOpenFile(t *testing.T) { t.Errorf("expected 'defmodule MyApp.AbTest', got:\n%s", got) } } + +// Moving one member of a grouped alias to another namespace must not rewrite +// the shared prefix: the other members would then resolve to modules that do +// not exist. The moved member leaves the group and gets its own alias. +func TestRename_Module_GroupedAliasMemberMovedToOtherNamespace(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/worker.ex", `defmodule SharedLib.Worker do + def call, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/config.ex", `defmodule SharedLib.Config do + def get, do: :ok +end +`) + callerPath := filepath.Join(server.projectRoot, "lib", "runner.ex") + indexFile(t, server.store, server.projectRoot, "lib/runner.ex", `defmodule MyApp.Runner do + alias SharedLib.{Config, Worker} + + def run, do: {Config.get(), Worker.call()} +end +`) + + if _, err := server.RenameModule("SharedLib.Worker", "OtherLib.Worker"); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(callerPath) + if err != nil { + t.Fatal(err) + } + for _, want := range []string{" alias SharedLib.{Config}\n", " alias OtherLib.Worker\n"} { + if !strings.Contains(string(got), want) { + t.Errorf("expected %q, got:\n%s", want, got) + } + } + if strings.Contains(string(got), "OtherLib.{") { + t.Errorf("the shared prefix was rewritten, so Config now names OtherLib.Config:\n%s", got) + } +} + +// A grouped alias whose members continue on the next lines names the prefix +// only on the opening line, where the index records every member. Renaming +// the prefix module must rewrite that line. +func TestRename_Module_MultilineGroupedAlias(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + + indexFile(t, server.store, server.projectRoot, "lib/shared_lib.ex", `defmodule SharedLib do + def start, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/worker.ex", `defmodule SharedLib.Worker do + def call, do: :ok +end +`) + indexFile(t, server.store, server.projectRoot, "lib/shared_lib/config.ex", `defmodule SharedLib.Config do + def get, do: :ok +end +`) + callerPath := filepath.Join(server.projectRoot, "lib", "runner.ex") + indexFile(t, server.store, server.projectRoot, "lib/runner.ex", `defmodule MyApp.Runner do + alias SharedLib.{ + Config, + Worker + } + + def run, do: {Config.get(), Worker.call()} +end +`) + + if _, err := server.RenameModule("SharedLib", "CoreLib"); err != nil { + t.Fatal(err) + } + + got, err := os.ReadFile(callerPath) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(got), " alias CoreLib.{\n Config,\n Worker\n }") { + t.Errorf("expected the multi-line group to use CoreLib, got:\n%s", got) + } + if strings.Contains(string(got), "SharedLib") { + t.Errorf("SharedLib should be gone, got:\n%s", got) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 1378c92..e979218 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -6281,10 +6281,14 @@ func (mr *moduleRename) findModuleEdits(lineText string, token string) []moduleE // is indexed as its own reference — so the reference's full name never appears // on the line. // -// Which half moves depends on the rename: renaming the prefix rewrites the -// prefix, renaming a member rewrites that member inside the braces. Sites for -// the other members on the same line find nothing once the prefix is rewritten, -// so a group is only edited once. +// Which half moves depends on the rename. When the renamed module is the +// prefix or one of its ancestors, every member moves with it, so the prefix is +// rewritten; this is also correct for a group whose members continue on the +// next lines, where the index records every member on the opening line. Sites +// for the other members on the same line find nothing once the prefix is +// rewritten, so a group is only edited once. When only the member is renamed, +// the member is rewritten inside the braces, or, when it moves to another +// namespace, it leaves the group and gets its own alias. func (mr *moduleRename) findGroupedAliasEdits(lineText, token, newToken string) []moduleEditResult { dot := strings.LastIndexByte(token, '.') if dot <= 0 { @@ -6295,29 +6299,61 @@ func (mr *moduleRename) findGroupedAliasEdits(lineText, token, newToken string) if prefixCol < 0 { return nil } - memberCols := findAllTokenColumns(lineText[groupStart:groupEnd], member) - if len(memberCols) == 0 { - return nil - } newDot := strings.LastIndexByte(newToken, '.') if newDot <= 0 { // The member lost its namespace; a grouped alias cannot express that. return nil } - if newPrefix := newToken[:newDot]; newPrefix != prefix { + newPrefix, newMember := newToken[:newDot], newToken[newDot+1:] + if prefix == mr.oldModule || strings.HasPrefix(prefix, mr.oldModule+".") { + if newPrefix == prefix { + return nil + } return []moduleEditResult{{prefixCol, len(prefix), newPrefix}} } - newMember := newToken[newDot+1:] - if newMember == member { + memberCols := findAllTokenColumns(lineText[groupStart:groupEnd], member) + if len(memberCols) == 0 { + return nil + } + if newPrefix == prefix { + if newMember == member { + return nil + } + results := make([]moduleEditResult, 0, len(memberCols)) + for _, col := range memberCols { + results = append(results, moduleEditResult{groupStart + col, len(member), newMember}) + } + return results + } + return splitGroupedAlias(lineText, prefix, member, newToken, prefixCol, groupStart, groupEnd) +} + +// splitGroupedAlias moves one member out of `Prefix.{A, B}` to its own +// `alias New.A` line after the group, because only that member changes +// namespace. Rewriting the shared prefix instead would move the other members +// too. A group with no other member gets the new name in place. A group that +// continues on the next lines is left unchanged: the members are not on this +// line. +func splitGroupedAlias(lineText, prefix, member, newToken string, prefixCol, groupStart, groupEnd int) []moduleEditResult { + if groupEnd >= len(lineText) || lineText[groupEnd] != '}' { return nil } - results := make([]moduleEditResult, 0, len(memberCols)) - for _, col := range memberCols { - results = append(results, moduleEditResult{groupStart + col, len(member), newMember}) + var rest []string + for _, m := range strings.Split(lineText[groupStart:groupEnd], ",") { + if m = strings.TrimSpace(m); m != "" && m != member { + rest = append(rest, m) + } } - return results + span := groupEnd + 1 - prefixCol + if len(rest) == 0 { + return []moduleEditResult{{prefixCol, span, newToken}} + } + // The new line repeats what comes before the prefix: the indentation and + // the alias, require, or import keyword. + replacement := prefix + ".{" + strings.Join(rest, ", ") + "}\n" + lineText[:prefixCol] + newToken + return []moduleEditResult{{prefixCol, span, replacement}} } type moduleEditResult struct { diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 60cb90f..10a7993 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -417,3 +417,23 @@ func TestFixedRootIgnoresClientRoots(t *testing.T) { cs, _ := e.connect(nil, uri) wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}), "root="+fixed) } + +// A Windows file URI carries the drive letter after a leading slash. Without +// removing that slash, `C:/project` is not an absolute path on Windows and the +// negotiation fails. Other systems keep the path as it is. +func TestURIPathDriveLetter(t *testing.T) { + cases := []struct { + path, goos, want string + }{ + {"/C:/project", "windows", "C:/project"}, + {"/c:/my project", "windows", "c:/my project"}, + {"/project", "windows", "/project"}, + {"/C:/project", "linux", "/C:/project"}, + {"/C:/project", "darwin", "/C:/project"}, + } + for _, tc := range cases { + if got := uriPath(tc.path, tc.goos); got != tc.want { + t.Errorf("uriPath(%q, %s) = %q, want %q", tc.path, tc.goos, got, tc.want) + } + } +} diff --git a/internal/mcp/rename_test.go b/internal/mcp/rename_test.go index 742e958..84dcc26 100644 --- a/internal/mcp/rename_test.go +++ b/internal/mcp/rename_test.go @@ -85,3 +85,19 @@ func TestRenameTool_Errors(t *testing.T) { t.Error("failed rename modified files") } } + +// Regression: the rename once applied editor positions (UTF-16 columns) as +// byte offsets on disk, so a non-ASCII character left of the name moved the +// edit and corrupted the line. +func TestRenameTool_NonASCIIBeforeName(t *testing.T) { + e := setupProject(t) + e.indexFile("lib/my_app/greeter.ex", `defmodule MyApp.Greeter do + def greet(id), do: {"héllo wörld ✓", MyApp.Accounts.fetch_user(id)} +end +`) + e.callTool("dexter_rename_symbol", map[string]any{ + "module": "MyApp.Accounts", "function": "fetch_user", "new_name": "get_user", + }) + wantContains(t, readFile(t, e.root, "lib/my_app/greeter.ex"), + ` def greet(id), do: {"héllo wörld ✓", MyApp.Accounts.get_user(id)}`) +} diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go index 0957114..d35c1e0 100644 --- a/internal/mcp/roots.go +++ b/internal/mcp/roots.go @@ -6,6 +6,7 @@ import ( "net/url" "os" "path/filepath" + "runtime" "strings" "github.com/modelcontextprotocol/go-sdk/mcp" @@ -23,13 +24,26 @@ func fileURIToPath(raw string) (string, error) { if u.Host != "" && u.Host != "localhost" { return "", fmt.Errorf("root URI %q names a remote host", raw) } - path := filepath.Clean(filepath.FromSlash(u.Path)) + path := filepath.Clean(filepath.FromSlash(uriPath(u.Path, runtime.GOOS))) if !filepath.IsAbs(path) { return "", fmt.Errorf("root URI %q has no absolute path", raw) } return path, nil } +// uriPath returns the filesystem form of a file URI's path. On Windows, +// file:///C:/project carries the drive letter after a leading slash. +func uriPath(p, goos string) string { + if goos == "windows" && len(p) >= 3 && p[0] == '/' && p[2] == ':' && isDriveLetter(p[1]) { + return p[1:] + } + return p +} + +func isDriveLetter(c byte) bool { + return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') +} + // negotiatedRoot resolves a session's workspace root from the MCP roots the // client advertises. ok is false when the client offers no usable root (no // roots capability, an empty list, or no file:// root): callers fall back to diff --git a/internal/mcp/tools_test.go b/internal/mcp/tools_test.go index 55dbae4..e6d9be8 100644 --- a/internal/mcp/tools_test.go +++ b/internal/mcp/tools_test.go @@ -344,6 +344,9 @@ func TestReindexTool(t *testing.T) { func TestToolNotesIndexStillBuilding(t *testing.T) { root := t.TempDir() release := make(chan struct{}) + rt := openTestRuntime(t, root, workspace.Options{BeforeInitialReconcile: func() { <-release }}) + // Registered after the runtime, so it runs before the runtime closes: a + // close waits for the held reconciliation. t.Cleanup(func() { select { case <-release: @@ -351,7 +354,6 @@ func TestToolNotesIndexStillBuilding(t *testing.T) { close(release) } }) - rt := openTestRuntime(t, root, workspace.Options{BeforeInitialReconcile: func() { <-release }}) h := NewHandler(rt, rt.LanguageServices()) out, err := h.Call(context.Background(), "dexter_search", json.RawMessage(`{"query":"x"}`), 10*time.Millisecond) @@ -392,3 +394,28 @@ func TestToolNotesDegradedIndex(t *testing.T) { out = e.callTool("dexter_workspace", nil) wantContains(t, out, "Workspace conditions:", "warning: 2 files could not be indexed") } + +// Regression: a call written through an alias that a `__using__` block +// injects is indexed under the short name. The editor's find-references found +// it, and dexter_references must find it too. +func TestReferencesTool_ViaUseInjectedAlias(t *testing.T) { + e := setupProject(t) + e.indexFile("lib/my_app/repo.ex", `defmodule MyApp.Repo do + defmacro __using__(_) do + quote do + alias MyApp.Repo + end + end + + def all(q), do: q +end +`) + e.indexFile("lib/my_app/users.ex", `defmodule MyApp.Users do + use MyApp.Repo + + def list, do: Repo.all(:users) +end +`) + out := e.callTool("dexter_references", map[string]any{"module": "MyApp.Repo", "function": "all"}) + wantContains(t, out, "lib/my_app/users.ex", "4: def list, do: Repo.all(:users)") +} From 02dc12cdc8575777ce956631bc7e446d66213447 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 3 Oct 2026 22:08:15 -0400 Subject: [PATCH 22/26] Skip an unusable MCP root when a later root is usable A client can list a stale or deleted directory, or a file, before the project root. Root negotiation stopped at the first unusable file:// root, so the session failed although a later root was a real workspace. The first usable root now wins; when no file:// root is usable, the first error is reported, as before. Co-Authored-By: Claude Opus 5.5 --- internal/mcp/negotiation_test.go | 13 +++++++++++ internal/mcp/roots.go | 39 +++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 13 deletions(-) diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 10a7993..97346ed 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -437,3 +437,16 @@ func TestURIPathDriveLetter(t *testing.T) { } } } + +// An unusable root before a usable one must not fail the session: the client +// can list a stale or deleted directory first. +func TestNegotiation_SkipsUnusableRoot(t *testing.T) { + e := setupNegotiation(t) + root, uri := projectDir(t) + file := filepath.Join(t.TempDir(), "not-a-dir.txt") + if err := os.WriteFile(file, nil, 0o644); err != nil { + t.Fatal(err) + } + cs, _ := e.connect(nil, "file:///nonexistent/dexter-negotiation-test", fileURI(file), uri) + wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}), "root="+root) +} diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go index d35c1e0..bbda015 100644 --- a/internal/mcp/roots.go +++ b/internal/mcp/roots.go @@ -47,8 +47,8 @@ func isDriveLetter(c byte) bool { // negotiatedRoot resolves a session's workspace root from the MCP roots the // client advertises. ok is false when the client offers no usable root (no // roots capability, an empty list, or no file:// root): callers fall back to -// the launch-directory root. A transport failure or an unusable file:// root -// is an error the caller should surface and retry, not cache. +// the launch-directory root. A transport failure, or file:// roots that are +// all unusable, is an error the caller should surface and retry, not cache. // // A usable root goes through resolve, which finds the project root the same // way for every frontend. The spelling the client used is kept: the daemon @@ -63,27 +63,40 @@ func negotiatedRoot(ctx context.Context, ss *mcp.ServerSession, resolve func(str if err != nil { return "", false, fmt.Errorf("listing client roots: %w", err) } + // The first usable file:// root wins. An unusable one (a stale or deleted + // directory, a file) is skipped, so it cannot hide a usable root after + // it; when no file:// root is usable, the first error is reported. + var firstErr error for _, r := range res.Roots { if !strings.HasPrefix(r.URI, "file:") { continue } - path, err := fileURIToPath(r.URI) - if err != nil { - return "", false, err + root, err := usableRoot(r.URI, resolve) + if err == nil { + return root, true, nil } - info, err := os.Stat(path) - if err != nil || !info.IsDir() { - return "", false, fmt.Errorf("client root %q is not a directory", path) + if firstErr == nil { + firstErr = err } - root, err := resolve(path) - if err != nil { - return "", false, err - } - return root, true, nil + } + if firstErr != nil { + return "", false, firstErr } return "", false, nil } +func usableRoot(uri string, resolve func(string) (string, error)) (string, error) { + path, err := fileURIToPath(uri) + if err != nil { + return "", err + } + info, err := os.Stat(path) + if err != nil || !info.IsDir() { + return "", fmt.Errorf("client root %q is not a directory", path) + } + return resolve(path) +} + // defaultResolveRoot finds the project root above dir with the store's marker // search (an existing index, then a repository). func defaultResolveRoot(dir string) (string, error) { From ec2b01654fb034b4e47988f880107888e13c5592 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 3 Oct 2026 23:23:33 -0400 Subject: [PATCH 23/26] Fix MCP review findings: rename races, file reads, cancel, HTTP Renames: a workspace-wide renameSerial mutex on IndexCoordinator keeps editor and MCP renames one at a time, from the first read of the affected files to the end of the writes. Two parallel renames no longer write over each other's edits. File reads: dexter_file_outline accepts only a path inside the project root after symlinks are resolved, and only a regular file of at most 10 MB, so /dev/zero, a FIFO, or a file outside the project is refused. Tools that read file text use the newest unsaved buffer of an attached editor when it differs from the disk, map index lines into it, and say so in the answer. dexter_definition shows at most 20 clauses and reads and tokenizes each file once per call. Cancel: the control protocol gets $/cancel and a context per request, so a canceled call frees its slot and its index wait ends. The MCP frontend runs at most 32 calls per workspace connection, below the daemon's 64. A canceled rename, or a call that a roots change ended, says what happened. ContractVersion is 4, so an older daemon is replaced instead of used with a protocol it does not know. HTTP: --listen accepts only a loopback address unless --listen-unsafe is given; cross-origin requests are refused, bodies are capped at 4 MB, and an idle session closes after 30 minutes. Roots: a client root that is not a project, or is the home directory, is refused like the launch directory, and a non-project root is not warmed. Docs: recovery hints say dexter stop --force; the changelog, the agent instructions, the README, and docs/daemon.md match the code. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 +- README.md | 9 +- cmd/main.go | 54 +++-- cmd/main_test.go | 20 ++ docs/architecture.md | 2 +- docs/daemon.md | 34 +++- integration_test.go | 17 ++ internal/daemon/client.go | 7 + internal/daemon/endpoint.go | 2 +- internal/daemon/registry.go | 4 +- internal/daemon/server.go | 63 +++++- internal/daemon/server_test.go | 84 ++++++++ internal/lsp/api.go | 8 + internal/lsp/documents.go | 22 ++- internal/lsp/server.go | 11 ++ internal/mcp/daemon.go | 84 +++++++- internal/mcp/daemon_test.go | 215 ++++++++++++++++++++ internal/mcp/definition.go | 55 ++++-- internal/mcp/file_outline.go | 16 +- internal/mcp/frontend.go | 22 ++- internal/mcp/http_test.go | 116 +++++++++++ internal/mcp/instructions.md | 10 +- internal/mcp/module_api.go | 20 +- internal/mcp/negotiation_test.go | 47 +++++ internal/mcp/references.go | 8 +- internal/mcp/rename_test.go | 52 +++++ internal/mcp/roots.go | 9 +- internal/mcp/serve.go | 47 ++++- internal/mcp/source.go | 330 +++++++++++++++++++++++++++++++ internal/mcp/source_test.go | 138 +++++++++++++ internal/mcp/source_unix_test.go | 18 ++ internal/mcp/tools.go | 25 +-- internal/mcp/workspace.go | 2 +- internal/store/project.go | 18 ++ internal/workspace/runtime.go | 25 +++ 35 files changed, 1502 insertions(+), 96 deletions(-) create mode 100644 internal/mcp/daemon_test.go create mode 100644 internal/mcp/http_test.go create mode 100644 internal/mcp/source.go create mode 100644 internal/mcp/source_test.go create mode 100644 internal/mcp/source_unix_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 292f054..4f07685 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### Added -- **Built-in MCP server** — `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. Like `dexter lsp` and the CLI, `dexter mcp` is a frontend of the workspace daemon: it opens no index and starts no watcher of its own, and every tool runs in the daemon against the same index, watchers, and caches as the editor, so an agent and an editor never index the same tree twice. Definitions and references use the same name navigation as go-to-definition, find-references, and `dexter lookup`. A tool call waits for a cold index for a short time, then answers with a note that the index is still building; a rebuilding or degraded index is stated in the answer, and a rename is refused until the index is complete. Each session's workspace root comes from MCP roots and resolves like the CLI's (with the launch directory as the fallback, refused when it is not a project); an explicit path argument (`dexter mcp `) or `--root` fixes it. `dexter mcp --instructions` prints an agent-facing usage guide +- **Built-in MCP server** — `dexter mcp` serves the index to AI agents over the Model Context Protocol (stdio, or streamable HTTP with `--listen`), modeled on `gopls mcp`. Ten tools cover workspace overview, fuzzy symbol search, definitions with docs and specs, references (including use-chain injected call sites), module API summaries, file outlines, behaviour/protocol implementations, call hierarchy, reindexing, and workspace-wide rename with the same on-disk semantics as the editor rename. Like `dexter lsp` and the CLI, `dexter mcp` is a frontend of the workspace daemon: it opens no index and starts no watcher of its own, and every tool runs in the daemon against the same index, watchers, and caches as the editor, so an agent and an editor never index the same tree twice. Definitions and references use the same name navigation as go-to-definition, find-references, and `dexter lookup`. A tool call waits for a cold index for a short time, then answers with a note that the index is still building; a rebuilding or degraded index is stated in the answer, and a rename is refused until the index is complete. Each session's workspace root comes from MCP roots and resolves like the CLI's (with the launch directory as the fallback; a root that is not a project, or is the home directory, is refused); an explicit path argument (`dexter mcp `) or `--root` fixes it. Tools read the newest unsaved editor buffer of a file when an attached editor holds one, and say so. `--listen` serves only a loopback address unless `--listen-unsafe` is given. `dexter mcp --instructions` prints an agent-facing usage guide - **Go-to-definition reaches the line that declared a generated function** — a function a macro generated used to resolve to the top of its module. Dexter now reads the line from the compiled module's debug info, which is standard compiler output, so no framework is special-cased. A generator that expands each function at the line of the call that declared it, or stamps it with `@file {file, line}`, sends definition, call hierarchy, the references declaration, and `dexter lookup` (including `--strict`) to that line. The line is used only when it was compiled from the file being opened; a BEAM older than the source still gives its line, because Dexter cannot compile the project and the last compile's line is closer than the module line; only edits to the declaring file move it, and the next compile makes it exact. A function whose only recorded line is the module line, such as one a `@before_compile` hook made, goes to the call in its module that declares it by name. When several calls spell the name, as an Ash action and the code interface that runs it do, the macro whose calls name the most of the module's generated functions wins, and a tie keeps the module line. A function with a clause per DSL call goes to every clause. A line past the end of the file is never returned. A module compiled without debug info falls back to its Docs chunk annotation, which is often, but not always, the same line. A generated module with no source of its own, such as one `Module.create` made or a Spark DSL entity, goes to the file it was compiled from, rebased onto the project when it was built elsewhere, and so does go-to-definition on its name. A module a macro made with `defmodule` and a name it computed records no line of its own, so its name goes to its first function's line. A bare call to a generated function of an imported module now resolves. Ash code interfaces go to their `define` line with released Ash, and from the recorded line with an Ash release that includes [ash-project/ash#2971](https://github.com/ash-project/ash/pull/2971) ([#108](https://github.com/remoteoss/dexter/issues/108)) @@ -20,7 +20,7 @@ - **`dexter stop`** — stops a workspace daemon on demand instead of hunting for its pid. It finds the process by workspace, reports whether one was running, and is a no-op when nothing was. A plain stop refuses while editors or CLI clients are attached, so a shared workspace is not yanked out from under them; `--force` skips the handshake and locates the daemon process directly, escalating to a kill if it will not exit, which is the manual way out for a daemon that is stuck or built by an older version -- **One workspace daemon, shared by every frontend** — the editor and the CLI now attach to a single per-workspace process that owns the index, the file and Git watchers, and the language caches, instead of each keeping its own copy of all three and indexing the same tree twice. It is also the foundation an MCP frontend will attach to, so MCP can drop its own store, watchers, and caches rather than index the same tree a second time. The daemon belongs to the workspace rather than to whichever frontend started it, and it starts on demand: it listens on a socket under `/tmp/dexter-` and exits after 15 minutes with no clients; `DEXTER_DAEMON_IDLE_TIMEOUT` changes that (`0` keeps it forever). Ownership is an advisory kernel lock held for the process lifetime, so a crash or `kill -9` releases it at once — there is no stale lock to clear and no PID file to go wrong. The socket binds before the first index pass, so opening an editor never waits on a cold build, and a CLI call reuses whatever caches the editor already warmed. The handshake carries a contract version: when an upgrade leaves a daemon from an older build owning the workspace, the first current-build frontend replaces it automatically — the old daemon shuts itself down when it understands the contract, or is signaled by the pid its refusal carries when it predates it — and a frontend older than the daemon is refused with a message telling its user to restart it. `IndexVersion` remains the store's own rebuild trigger, bumped together with the contract. See `docs/daemon.md` +- **One workspace daemon, shared by every frontend** — the editor and the CLI now attach to a single per-workspace process that owns the index, the file and Git watchers, and the language caches, instead of each keeping its own copy of all three and indexing the same tree twice. The daemon belongs to the workspace rather than to whichever frontend started it, and it starts on demand: it listens on a socket under `/tmp/dexter-` and exits after 15 minutes with no clients; `DEXTER_DAEMON_IDLE_TIMEOUT` changes that (`0` keeps it forever). Ownership is an advisory kernel lock held for the process lifetime, so a crash or `kill -9` releases it at once — there is no stale lock to clear and no PID file to go wrong. The socket binds before the first index pass, so opening an editor never waits on a cold build, and a CLI call reuses whatever caches the editor already warmed. The handshake carries a contract version: when an upgrade leaves a daemon from an older build owning the workspace, the first current-build frontend replaces it automatically — the old daemon shuts itself down when it understands the contract, or is signaled by the pid its refusal carries when it predates it — and a frontend older than the daemon is refused with a message telling its user to restart it. `IndexVersion` remains the store's own rebuild trigger, bumped together with the contract. See `docs/daemon.md` - **Completion and hover for macro-generated functions** — Dexter now reads compiled BEAM exports and documentation to surface public functions and macros that do not exist in source. This includes generated functions and macros in application modules, entirely generated application modules such as Phoenix route helpers (including `alias ..., as: Routes`), Oban constructors hidden from generated documentation, exported introspection APIs such as `__schema__`, and generated dependency DSLs. Source indexing remains authoritative and compilation remains optional: stale BEAMs can contribute genuinely generated names, while an absent BEAM leaves the existing source-only behavior unchanged. Generated Spark/Ash DSL macros are resolved from persisted extension attributes and narrowed to the modules in scope at the cursor's nested block path: section macros at module level, an entity's macro inside its section body, and an entity's own option macros inside its body — including when a language form such as `for` or `if` sits in the block path. Hover on any of them renders the compiled signature. Both the OTP 24–27 and OTP 28+ atom-table layouts are supported. In a monorepo whose libraries are compiled as path dependencies of another Mix project, Dexter finds the workspace's builds and looks for each library's application in them, so a library without a `_build` of its own, and modules generated beneath its modules, still resolve; when a library is compiled in more than one build, the most recently compiled BEAM is used diff --git a/README.md b/README.md index 4bca37f..ce44d5f 100644 --- a/README.md +++ b/README.md @@ -487,14 +487,15 @@ Register it with your MCP client. For Claude Code: claude mcp add dexter -- dexter mcp ``` -Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server obtains its workspace from the client through MCP roots and resolves it the way the CLI does, so it binds the project the client is working in rather than the directory it was launched from. Clients that provide no roots get the launch directory (when it is a project), and an explicit path argument (`dexter mcp `) overrides negotiation entirely. In `--listen` mode each resolved root gets its own workspace, so sessions from different projects can share one server. +Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server obtains its workspace from the client through MCP roots and resolves it the way the CLI does, so it binds the project the client is working in rather than the directory it was launched from. Clients that provide no roots get the launch directory (when it is a project), and an explicit path argument (`dexter mcp `) overrides negotiation entirely. One session serves one workspace: when a client gives several roots, the session uses the first one that is a usable project (an existing directory inside an Elixir project, not the home directory) and ignores the others. In `--listen` mode each resolved root gets its own workspace, so sessions from different projects can share one server. -`dexter mcp` is a frontend of the workspace daemon, like `dexter lsp` and the CLI: it starts the daemon when necessary and keeps no index of its own. The tools answer from the same index, watchers, and caches as the editor, so edits made directly by an agent are indexed by the daemon's file watcher, and a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. A tool that answers from an index that is still building or degraded says so in its answer. +`dexter mcp` is a frontend of the workspace daemon, like `dexter lsp` and the CLI: it starts the daemon when necessary and keeps no index of its own. The tools answer from the same index, watchers, and caches as the editor, so edits made directly by an agent are indexed by the daemon's file watcher, and a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. A tool that answers from an index that is still building or degraded says so in its answer. The tools read file text the way the user sees it: when an editor attached to the same daemon holds a file open with unsaved changes, outlines, definition snippets, and reference lines come from that buffer (with its line numbers), and the answer says so. The rename tool writes on disk and does not yet look at editor buffers, so save your editor's changes before an agent renames. Useful variants: ```sh -# Serve over streamable HTTP instead of stdio +# Serve over streamable HTTP instead of stdio (loopback addresses only; the +# server has no authentication, so another address needs --listen-unsafe) dexter mcp --listen localhost:8092 # Print the agent-facing usage guide (save as context for clients that want it) @@ -685,7 +686,7 @@ dexter init --force ~/code/my-elixir-project If the issue persists, enable debug mode to get verbose logs. You can do this in two ways: 1. Set the `debug` option in your editor's LSP `initializationOptions` (see [LSP options](#lsp-options)). It applies to that editor session as soon as it connects. -2. Or set the `DEXTER_DEBUG=true` environment variable for the editor or CLI command that starts the workspace daemon. The daemon reads it when it starts, so if one is already running, run `dexter stop` first. This is also how to debug CLI commands such as `dexter lookup`. +2. Or set the `DEXTER_DEBUG=true` environment variable for the editor or CLI command that starts the workspace daemon. The daemon reads it when it starts, so if one is already running, run `dexter stop --force` first (a plain stop is refused while an editor or an MCP session is attached). This is also how to debug CLI commands such as `dexter lookup`. Debug mode logs timing and resolution details for every definition, hover, references, and rename request. Each editor receives the lines for its own requests in its LSP log (in Neovim usually `~/.local/state/nvim/lsp.log`, in VS Code Output > Dexter). Every editor and CLI command for a workspace shares one daemon, and all of its lines, including those for CLI commands, also go to the daemon's log file: `.log` in its runtime directory (`/tmp/dexter-` on macOS and Linux; see [docs/daemon.md](docs/daemon.md)). The first line `dexter lsp` writes to your editor's log names that file. diff --git a/cmd/main.go b/cmd/main.go index fe12cbb..b750d49 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -157,6 +157,7 @@ func main() { } var mcpListen string + var mcpListenUnsafe bool var mcpInstructions bool mcpCmd := &cobra.Command{ Use: "mcp [path]", @@ -171,11 +172,12 @@ func main() { if err != nil { return err } - cmdMCP(projectRoot, mcpListen, len(args) > 0 || rootDir != "") + cmdMCP(projectRoot, mcpListen, mcpListenUnsafe, len(args) > 0 || rootDir != "") return nil }, } - mcpCmd.Flags().StringVar(&mcpListen, "listen", "", "Serve MCP over streamable HTTP on this address instead of stdio") + mcpCmd.Flags().StringVar(&mcpListen, "listen", "", "Serve MCP over streamable HTTP on this loopback address instead of stdio") + mcpCmd.Flags().BoolVar(&mcpListenUnsafe, "listen-unsafe", false, "Allow --listen on an address that other machines can reach (the server has no authentication)") mcpCmd.Flags().BoolVar(&mcpInstructions, "instructions", false, "Print the MCP instructions file and exit") var daemonIdleTimeout time.Duration @@ -318,26 +320,11 @@ func requireProjectRoot(dir string, allowNonProject bool) { if allowNonProject { return } - if err := nonProjectRootError(dir); err != nil { + if err := store.NonProjectRootError(dir); err != nil { fatal(fmt.Errorf("%w\nhint: run from a project, pass --root , or pass -y/--yes to index it anyway", err)) } } -// nonProjectRootError explains why dir is not a workspace to index, or returns -// nil when it is one. -func nonProjectRootError(dir string) error { - if store.IsHomeDir(dir) { - if store.HasIndex(dir) { - return nil - } - return fmt.Errorf("refusing to use %s as a workspace: it is your home directory, not a project", dir) - } - if store.LooksLikeProject(dir) { - return nil - } - return fmt.Errorf("refusing to use %s as a workspace: no mix.exs, .git, or Dexter database found, so it does not look like an Elixir project", dir) -} - // defaultIdleTimeout resolves the daemon idle timeout. DEXTER_DAEMON_IDLE_TIMEOUT // overrides the built-in default for every daemon this machine spawns, including // ones an editor starts, so it can be set once in a shell profile. @@ -774,17 +761,32 @@ func mcpConfig(launchDir string, explicitRoot bool) (dexter_mcp.Config, error) { if err != nil { return dexter_mcp.Config{}, err } - cfg := dexter_mcp.Config{Root: root, Fixed: explicitRoot, ResolveRoot: projectRootFor} + cfg := dexter_mcp.Config{Root: root, Fixed: explicitRoot, ResolveRoot: mcpClientRoot} if !explicitRoot { // The launch directory is only a guess at the workspace. Starting a // daemon on a directory that is not a project would index all of it. - if err := nonProjectRootError(root); err != nil { + if err := store.NonProjectRootError(root); err != nil { cfg.FallbackErr = fmt.Errorf("%w. The MCP client gave no workspace root; configure the server with the project path (`dexter mcp `) or start it in the project", err) } } return cfg, nil } +// mcpClientRoot resolves a directory that an MCP client gives as a root, with +// the same refusal as the launch directory: a client root that is not a +// project, or that is the home directory, would start a daemon that indexes +// all of it. The MCP frontend treats a refused root as unusable. +func mcpClientRoot(dir string) (string, error) { + root, err := projectRootFor(dir) + if err != nil { + return "", err + } + if err := store.NonProjectRootError(root); err != nil { + return "", err + } + return root, nil +} + // cmdMCP serves MCP to an agent. Like `dexter lsp`, it is a frontend of the // shared workspace daemon: it opens no index and starts no watcher, and every // tool call is answered by the daemon of the session's workspace, which it @@ -795,8 +797,15 @@ func mcpConfig(launchDir string, explicitRoot bool) (dexter_mcp.Config, error) { // Without one, each session's root is negotiated through MCP roots and // resolved like the CLI resolves its own, with the launch directory as the // fallback for clients that give no root. -func cmdMCP(projectRoot string, listen string, explicitRoot bool) { +func cmdMCP(projectRoot string, listen string, listenUnsafe bool, explicitRoot bool) { log.SetOutput(os.Stderr) + if listen != "" { + if err := dexter_mcp.CheckListenAddr(listen, listenUnsafe); err != nil { + fatal(err) + } + } else if listenUnsafe { + fatal(fmt.Errorf("--listen-unsafe needs --listen")) + } cfg, err := mcpConfig(projectRoot, explicitRoot) if err != nil { fatal(err) @@ -819,6 +828,9 @@ func cmdMCP(projectRoot string, listen string, explicitRoot bool) { fatal(err) } log.Printf("MCP server listening on %s", ln.Addr()) + if err := dexter_mcp.CheckListenAddr(listen, false); err != nil { + log.Printf("WARNING: --listen-unsafe: the MCP server on %s has no authentication; anyone who can reach it can read this code and rename symbols", ln.Addr()) + } httpSrv := &http.Server{Handler: dexter_mcp.HTTPHandler(frontend), ReadHeaderTimeout: 10 * time.Second} go func() { <-ctx.Done() diff --git a/cmd/main_test.go b/cmd/main_test.go index e8d3639..46b2b7e 100644 --- a/cmd/main_test.go +++ b/cmd/main_test.go @@ -176,3 +176,23 @@ func TestMCPFallbackRefusesNonProject(t *testing.T) { t.Errorf("an explicit root was refused: %v", explicit.FallbackErr) } } + +// Regression: a client root that was not a project (or was the home +// directory) started a daemon that indexed all of it. It is refused like the +// launch directory, so the frontend skips it as unusable. +func TestMCPClientRootRefusesNonProject(t *testing.T) { + cfg, err := mcpConfig(t.TempDir(), true) + if err != nil { + t.Fatal(err) + } + plain := t.TempDir() + if _, err := cfg.ResolveRoot(plain); err == nil || !strings.Contains(err.Error(), "does not look like an Elixir project") { + t.Errorf("ResolveRoot(%s) error = %v, want a refusal", plain, err) + } + + home := t.TempDir() + t.Setenv("HOME", home) + if _, err := cfg.ResolveRoot(home); err == nil || !strings.Contains(err.Error(), "home directory") { + t.Errorf("ResolveRoot(home) error = %v, want a refusal", err) + } +} diff --git a/docs/architecture.md b/docs/architecture.md index 0bccf91..24abb45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,7 +203,7 @@ A module rename also moves files whose names follow the module naming convention `protocol.WorkspaceEdit` from `go.lsp.dev/protocol` types `documentChanges` as `[]TextDocumentEdit` and cannot carry resource operations, so `internal/lsp/workspace_edit.go` defines the wire types and `renameHandler` answers `textDocument/rename` ahead of the generated dispatcher. A client that understands `documentChanges` ignores `changes` entirely, so once one file moves, every edit in the reply goes through `documentChanges`. -The MCP rename tool runs in the workspace daemon on its headless language service (`RenameFunction`/`RenameModule` in `internal/lsp/api.go`). That service holds no editor buffers, so the builders write every affected file and move closed files themselves, exactly as for closed files above, and the summary lists the files changed, moved, and not written. These writes do not yet look at buffers that an editor attached to the same daemon has open; the MCP rename will adopt the shared rule for frontends without an editor (never write over unsaved editor work) when it lands in `internal/lsp`. +The MCP rename tool runs in the workspace daemon on its headless language service (`RenameFunction`/`RenameModule` in `internal/lsp/api.go`). That service holds no editor buffers, so the builders write every affected file and move closed files themselves, exactly as for closed files above, and the summary lists the files changed, moved, and not written. Renames run one at a time across the workspace: `IndexCoordinator.renameSerial`, which editor and MCP renames both take, is held from the first read of the affected files to the end of the writes, so two renames cannot write over each other's edits. These writes do not yet look at buffers that an editor attached to the same daemon has open; the MCP rename will adopt the shared rule for frontends without an editor (never write over unsaved editor work) when it lands in `internal/lsp`. ### Grouped aliases diff --git a/docs/daemon.md b/docs/daemon.md index 0a2fad8..7a8bab1 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -77,6 +77,15 @@ responses carry an id, and the daemon may push notifications (a method, no id) between them. One connection therefore multiplexes concurrent calls and subscriptions, and a slow reindex cannot block a lookup. +A connection runs at most 64 requests at once; one more is refused at once +instead of blocking the reader. Each request has its own context, derived from +the connection's. A client that stops waiting for a request sends +`{"id":0,"method":"$/cancel","params":{"id":N}}`, the pattern of LSP's +`$/cancelRequest`: the reader handles it outside the request limit and cancels +request `N`, so its index waits end and its slot is free again. The canceled +request still answers; the client has dropped it. A subscription made by +`workspace/watch` belongs to the connection, not to the request that made it. + Each message is one line of at most 16 MiB, newline included. A writer refuses a longer line before sending anything, so the stream stays in step: a result that is too large fails only its own call, and a `workspace/changed` @@ -289,6 +298,7 @@ Built-in control surface: | `workspace/references` | semantic references through the shared language service | | `workspace/reindex` | whole workspace or one path, returning after the barrier | | `workspace/watch`, `workspace/unwatch` | subscribe to coalesced index changes, pushed as `workspace/changed` notifications | +| `$/cancel` | cancel one in-flight request of this connection; sent with id 0, no response | ## The MCP frontend @@ -318,8 +328,30 @@ runs no LSP lifecycle of its own. or error condition, ends with a note that says so. The rename tool refuses until the index is complete, because a rename from a partial index would change some call sites and leave others with the old name. +- **Editor buffers.** Tools that read file text (outlines, definition and + module docs, reference lines) use the newest buffer that any attached editor + session holds open when it differs from the disk, and the disk otherwise. + The answer names the files that came from unsaved buffers. The index + positions refer to the saved file, so a line is mapped into the buffer + through the lines that both share at the start and at the end; a position + in the changed part shows the saved line, marked as such. Paths that the + agent gives must be inside the project root after symlinks are resolved, and + must name a regular file of at most 10 MB. +- **Limits and cancellation.** One frontend runs at most 32 tool calls at once + on a workspace connection, below the daemon's 64, and a call past the limit + waits for a slot. A call that the MCP client cancels sends `$/cancel`, so the + daemon stops waiting for the index for it. A canceled rename says that it may + have been applied. When the client's roots change to another project during + a call, the call ends with an error that says so. +- **HTTP.** `--listen` accepts only a loopback address unless + `--listen-unsafe` is given, because the server has no authentication. The + SDK refuses a non-loopback `Host` on a loopback connection (DNS rebinding), + cross-origin browser requests are refused, a request body is capped at + 4 MB, and a session with no request for 30 minutes is closed, so a client + that went away does not keep its daemon alive. - **Reconnects.** When the daemon goes away (an upgrade replaced it, or - `dexter stop`), the next tool call connects again, which starts a new daemon. + `dexter stop --force`; a plain `dexter stop` is refused while MCP is + attached), the next tool call connects again, which starts a new daemon. A read-only call that was in flight is sent once more; a rename is never repeated. - **Rename.** The rename runs on the headless language service and writes the diff --git a/integration_test.go b/integration_test.go index 7445700..00aedb4 100644 --- a/integration_test.go +++ b/integration_test.go @@ -1192,3 +1192,20 @@ func TestIntegration_MCPListenHTTP(t *testing.T) { t.Errorf("definition over HTTP missing location:\n%s", out) } } + +// Regression: --listen accepted any address, and the server has no +// authentication, so a wildcard address exposed the code and the rename tool +// to the network. +func TestIntegration_MCPListenRefusesNonLoopback(t *testing.T) { + binary := buildDexter(t) + root := scaffoldProject(t) + cmd := exec.Command(binary, "mcp", "--listen=0.0.0.0:0", root) + cmd.Dir = root + out, err := cmd.CombinedOutput() + if err == nil { + t.Fatalf("dexter mcp --listen=0.0.0.0:0 started:\n%s", out) + } + if !strings.Contains(string(out), "refusing to listen") || !strings.Contains(string(out), "--listen-unsafe") { + t.Errorf("refusal does not explain itself:\n%s", out) + } +} diff --git a/internal/daemon/client.go b/internal/daemon/client.go index 56601fb..5460686 100644 --- a/internal/daemon/client.go +++ b/internal/daemon/client.go @@ -367,6 +367,13 @@ func (c *Client) Call(ctx context.Context, method string, params, result any) er c.pendingMu.Lock() delete(c.pending, id) c.pendingMu.Unlock() + // Tell the daemon, so the request stops waiting and frees its slot. + // Best effort: a connection that cannot take it is ending anyway. + if cancelParams, err := json.Marshal(CancelParams{ID: id}); err == nil { + c.writeMu.Lock() + _ = writeJSONLine(c.conn, request{Method: MethodCancel, Params: cancelParams}) + c.writeMu.Unlock() + } return ctx.Err() } } diff --git a/internal/daemon/endpoint.go b/internal/daemon/endpoint.go index 7234094..8457f9a 100644 --- a/internal/daemon/endpoint.go +++ b/internal/daemon/endpoint.go @@ -18,7 +18,7 @@ import ( // running daemon, whose startup then rebuilds a populated index whose // IndexVersion differs. Routine changes that leave frontends and daemons // compatible do not bump it, so a running daemon is left alone. -const ContractVersion = 3 +const ContractVersion = 4 // maxSocketPath keeps a workspace socket inside sockaddr_un on every supported // platform (about 104 bytes on macOS, 108 on Linux), including the NUL. diff --git a/internal/daemon/registry.go b/internal/daemon/registry.go index 178e329..a6028ce 100644 --- a/internal/daemon/registry.go +++ b/internal/daemon/registry.go @@ -49,6 +49,8 @@ type Frontend interface { // MethodContext is what a registered control method may use. type MethodContext struct { + // Context is canceled when the client cancels this request (MethodCancel), + // when the connection ends, or when the daemon stops. Context context.Context Runtime *workspace.Runtime Session string @@ -129,7 +131,7 @@ func lookupMethod(name string) (MethodHandler, bool) { func isBuiltinMethod(name string) bool { switch name { case MethodStatus, MethodShutdown, MethodWorkspaceStatus, MethodLookup, - MethodReferences, MethodReindex, MethodWatch, MethodUnwatch: + MethodReferences, MethodReindex, MethodWatch, MethodUnwatch, MethodCancel: return true } return false diff --git a/internal/daemon/server.go b/internal/daemon/server.go index a32804d..58150fd 100644 --- a/internal/daemon/server.go +++ b/internal/daemon/server.go @@ -64,8 +64,19 @@ const ( MethodReindex = "workspace/reindex" MethodWatch = "workspace/watch" MethodUnwatch = "workspace/unwatch" + + // MethodCancel cancels one in-flight request of the same connection, like + // $/cancelRequest in LSP. It is sent with id 0 and gets no response; the + // canceled request still answers, usually with a context error, and the + // client has stopped waiting for it. + MethodCancel = "$/cancel" ) +// CancelParams names the request that MethodCancel cancels. +type CancelParams struct { + ID uint64 `json:"id"` +} + // Status describes the daemon serving a workspace. type Status struct { Root string `json:"root"` @@ -579,6 +590,11 @@ type conn struct { sem chan struct{} requests sync.WaitGroup + // inflight maps the id of each running request to the cancel func of its + // context, for MethodCancel. + inflightMu sync.Mutex + inflight map[uint64]context.CancelFunc + writeMu sync.Mutex subsMu sync.Mutex subs map[string]func() @@ -772,7 +788,7 @@ func (s lspStream) Close() error { return s.conn.Close() } // serveControl handles requests concurrently so a long reindex cannot block a // lookup, and serializes only the writes. func (s *server) serveControl(c *conn, reader *bufio.Reader, sessionID string) error { - mc := MethodContext{ + base := MethodContext{ Context: c.ctx, Runtime: s.runtime, Session: sessionID, @@ -785,6 +801,15 @@ func (s *server) serveControl(c *conn, reader *bufio.Reader, sessionID string) e if err := readJSONLine(reader, &req); err != nil { return err } + if req.Method == MethodCancel { + // Handled on the reader, outside the request limit, so a cancel + // gets through when every slot is taken. + var params CancelParams + if err := json.Unmarshal(req.Params, ¶ms); err == nil { + c.cancelRequest(params.ID) + } + continue + } select { case c.sem <- struct{}{}: case <-c.ctx.Done(): @@ -794,6 +819,12 @@ func (s *server) serveControl(c *conn, reader *bufio.Reader, sessionID string) e continue } c.requests.Add(1) + // Each request has its own context, so a cancel ends its waits and + // frees its slot before the connection ends. + mc := base + var cancel context.CancelFunc + mc.Context, cancel = context.WithCancel(c.ctx) + c.trackRequest(req.ID, cancel) go func(req request) { res := response{ID: req.ID} defer func() { @@ -808,6 +839,8 @@ func (s *server) serveControl(c *conn, reader *bufio.Reader, sessionID string) e log.Printf("Daemon control method %q: %v", req.Method, err) _ = c.write(response{ID: req.ID, Error: fmt.Sprintf("%s result is too large to send (%d bytes, limit %d); narrow the query", req.Method, len(res.Result), maxProtocolLine)}) } + c.untrackRequest(req.ID) + cancel() <-c.sem c.requests.Done() }() @@ -824,6 +857,30 @@ func (s *server) serveControl(c *conn, reader *bufio.Reader, sessionID string) e } } +func (c *conn) trackRequest(id uint64, cancel context.CancelFunc) { + c.inflightMu.Lock() + if c.inflight == nil { + c.inflight = make(map[uint64]context.CancelFunc) + } + c.inflight[id] = cancel + c.inflightMu.Unlock() +} + +func (c *conn) untrackRequest(id uint64) { + c.inflightMu.Lock() + delete(c.inflight, id) + c.inflightMu.Unlock() +} + +func (c *conn) cancelRequest(id uint64) { + c.inflightMu.Lock() + cancel := c.inflight[id] + c.inflightMu.Unlock() + if cancel != nil { + cancel() + } +} + func (s *server) handleRequest(c *conn, mc MethodContext, req request) (any, error) { switch req.Method { case MethodStatus: @@ -958,7 +1015,9 @@ func (s *server) handleRequest(c *conn, mc MethodContext, req request) (any, err // the only watchers, so a frontend uses this instead of watching the tree again. func (s *server) watch(c *conn, mc MethodContext, params WatchParams) (any, error) { changes, cancel := s.runtime.Subscribe(params.Buffer) - ctx, stop := context.WithCancel(mc.Context) + // The subscription outlives the request that made it: it ends with the + // connection or an unwatch. + ctx, stop := context.WithCancel(c.ctx) id := c.addSub(func() { stop() cancel() diff --git a/internal/daemon/server_test.go b/internal/daemon/server_test.go index 3b67eac..288334b 100644 --- a/internal/daemon/server_test.go +++ b/internal/daemon/server_test.go @@ -615,6 +615,10 @@ func init() { RegisterMethod("srvtest/panic", func(MethodContext, json.RawMessage) (any, error) { panic("method exploded on purpose") }) + RegisterMethod("srvtest/wait", func(mc MethodContext, _ json.RawMessage) (any, error) { + <-mc.Context.Done() + return nil, mc.Context.Err() + }) RegisterMethod("srvtest/block", func(mc MethodContext, _ json.RawMessage) (any, error) { blockingMethodMu.Lock() state := blockingMethod @@ -838,3 +842,83 @@ func TestDaemonExitsWhenItsSocketDisappears(t *testing.T) { t.Fatal("daemon kept running after its socket disappeared") } } + +// pipeControlClient serves one control connection over an in-memory pipe. +func pipeControlClient(t *testing.T) *Client { + t.Helper() + quietEnv(t) + root := t.TempDir() + s, endpoint := pipeServer(t, root) + serverConn, clientConn := net.Pipe() + go func() { _ = s.serveConn(serverConn) }() + reader := bufio.NewReader(clientConn) + if err := writeJSONLine(clientConn, hello{ + Contract: ContractVersion, + Kind: kindControl, + Root: endpoint.Root, + Identity: endpoint.Identity, + }); err != nil { + t.Fatal(err) + } + var response helloResponse + if err := readJSONLine(reader, &response); err != nil || !response.OK { + t.Fatalf("handshake = %+v, %v", response, err) + } + client := newClient(clientConn, reader) + t.Cleanup(func() { _ = client.Close() }) + return client +} + +// Regression: a call that the client canceled kept running in the daemon +// until the connection ended, so after 64 canceled calls every new call on the +// connection failed with "too many concurrent control requests". +func TestCanceledCallsFreeTheirSlots(t *testing.T) { + client := pipeControlClient(t) + for i := 0; i < 3*maxConcurrentRequests; i++ { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + err := client.Call(ctx, "srvtest/wait", struct{}{}, nil) + cancel() + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("call %d: error = %v, want the deadline", i, err) + } + } + if _, err := client.DaemonStatus(context.Background()); err != nil { + t.Fatalf("connection unusable after canceled calls: %v", err) + } +} + +// lastContractWithoutCancel is the contract of the builds whose daemons do not +// know $/cancel. A frontend that sends it must not attach to such a daemon: +// its canceled requests would keep their slots. +const lastContractWithoutCancel = 3 + +// The contract was bumped with $/cancel, so the two builds refuse each other +// and the newer one replaces the older daemon (TestEnsureReplacesAnOlderDaemon) +// instead of sharing it. +func TestContractBumpedForRequestCancel(t *testing.T) { + if ContractVersion <= lastContractWithoutCancel { + t.Fatalf("ContractVersion = %d; $/cancel needs a contract above %d", ContractVersion, lastContractWithoutCancel) + } + quietEnv(t) + root := t.TempDir() + s, endpoint := pipeServer(t, root) + serverConn, clientConn := net.Pipe() + go func() { _ = s.serveConn(serverConn) }() + defer func() { _ = clientConn.Close() }() + reader := bufio.NewReader(clientConn) + if err := writeJSONLine(clientConn, hello{ + Contract: lastContractWithoutCancel, + Kind: kindControl, + Root: endpoint.Root, + Identity: endpoint.Identity, + }); err != nil { + t.Fatal(err) + } + var response helloResponse + if err := readJSONLine(reader, &response); err != nil { + t.Fatal(err) + } + if response.OK || !response.Incompatible || response.Exiting { + t.Fatalf("handshake from contract %d = %+v, want an incompatible refusal that keeps the daemon", lastContractWithoutCancel, response) + } +} diff --git a/internal/lsp/api.go b/internal/lsp/api.go index f1b23eb..6552a76 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -4,6 +4,8 @@ import ( "fmt" "sort" "sync" + + "go.lsp.dev/uri" ) // This file is the exported, name-based surface of the LSP server for callers @@ -21,6 +23,12 @@ func (s *Server) FileLine(filePath string, lineNum int) (string, bool) { return s.getFileLine(filePath, lineNum) } +// OpenBuffer returns the text of filePath when an editor holds it open in this +// session. seq orders buffers across sessions: the higher one changed later. +func (s *Server) OpenBuffer(filePath string) (text string, seq uint64, ok bool) { + return s.docs.GetOpenSeq(string(uri.File(filePath))) +} + // RenameSummary reports what a rename changed on disk. type RenameSummary struct { FilesChanged []string diff --git a/internal/lsp/documents.go b/internal/lsp/documents.go index 8fec7ec..511da5b 100644 --- a/internal/lsp/documents.go +++ b/internal/lsp/documents.go @@ -5,6 +5,7 @@ import ( "os" "strings" "sync" + "sync/atomic" tree_sitter "github.com/tree-sitter/go-tree-sitter" tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go" @@ -31,8 +32,15 @@ type cachedDoc struct { // LRU and evicted once the transient cap is reached. Editor-owned // entries (created via Set) are never transient and never evicted. transient bool + // seq orders editor-owned entries across every store in the process: + // the higher one was set later. A frontend without an editor uses it to + // pick the newest unsaved buffer when several editors hold one file. + seq uint64 } +// docSeq numbers editor-owned entries; see cachedDoc.seq. +var docSeq atomic.Uint64 + // refTree wraps a tree-sitter parse tree with refcounting so that // concurrent handlers walking the tree (RootNode, queries) aren't racing // with eviction or replacement, which would free the underlying C memory @@ -144,7 +152,7 @@ func (ds *DocumentStore) Set(uri string, text string) { } // Editor took ownership of this URI - drop any LRU tracking for it. ds.removeFromLRULocked(uri) - ds.docs[uri] = &cachedDoc{text: text} + ds.docs[uri] = &cachedDoc{text: text, seq: docSeq.Add(1)} } func (ds *DocumentStore) Close(uri string) { @@ -206,6 +214,18 @@ func (ds *DocumentStore) GetIfOpen(uri string) (string, bool) { return doc.text, true } +// GetOpenSeq is GetIfOpen that also returns the entry's sequence number; see +// cachedDoc.seq. +func (ds *DocumentStore) GetOpenSeq(uri string) (string, uint64, bool) { + ds.mu.RLock() + defer ds.mu.RUnlock() + doc, ok := ds.docs[uri] + if !ok || doc.transient { + return "", 0, false + } + return doc.text, doc.seq, true +} + // GetOrLoad returns the text for the given URI, falling back to a disk // read if no editor has opened the document. Disk-loaded entries are // marked transient and tracked in an LRU; if the transient population diff --git a/internal/lsp/server.go b/internal/lsp/server.go index e979218..352ae12 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -97,6 +97,13 @@ type IndexCoordinator struct { stdlibMu sync.RWMutex stdlibRoot string + // renameSerial keeps renames one at a time across every session of the + // workspace, editor and headless alike. A rename reads the affected files + // and writes them back; two at once would each write over the other's + // edits. It is held from the first read of the affected files to the end + // of the writes, and no other request takes it. + renameSerial sync.Mutex + // writes is held for writing by a cold full build and for reading by every // single-file write. The bulk path is insert-only and cannot overlap any // incremental mutation. @@ -5763,6 +5770,8 @@ func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) // module.functionName to newName across the codebase. When report is not nil, // it receives the files the rename changed and the files it could not change. func (s *Server) renameFunctionEdits(module, functionName, newName string, report *RenameSummary) (*WorkspaceEdit, error) { + s.index.renameSerial.Lock() + defer s.index.renameSerial.Unlock() // Collect all (filePath, lineNumber) pairs — definitions + references type siteKey struct { filePath string @@ -5967,6 +5976,8 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string, repor // report is not nil, it receives the files the rename changed or moved and the // files it could not change. func (s *Server) renameModuleEdits(oldModule, newModule string, report *RenameSummary) (*WorkspaceEdit, error) { + s.index.renameSerial.Lock() + defer s.index.renameSerial.Unlock() mr := s.buildModuleRename(oldModule, newModule) // Check for collisions: verify that none of the target module names diff --git a/internal/mcp/daemon.go b/internal/mcp/daemon.go index 8dd3ac4..9c3095a 100644 --- a/internal/mcp/daemon.go +++ b/internal/mcp/daemon.go @@ -75,22 +75,74 @@ type warmer interface { // control connection, which keeps the daemon alive while the MCP session is // open, and connects again when the daemon goes away. func DaemonBackend(root string) Backend { - return &daemonBackend{root: root} + return newDaemonBackend(root, func(ctx context.Context, root string) (controlClient, error) { + client, err := daemon.Ensure(ctx, root) + if err != nil { + // Not a typed nil in an interface. + return nil, err + } + return client, nil + }) +} + +func newDaemonBackend(root string, ensure func(ctx context.Context, root string) (controlClient, error)) *daemonBackend { + return &daemonBackend{root: root, ensure: ensure, slots: make(chan struct{}, maxConcurrentToolCalls)} +} + +// controlClient is the part of *daemon.Client that a daemonBackend uses. +type controlClient interface { + Call(ctx context.Context, method string, params, result any) error + Done() <-chan struct{} + Close() error } +// maxConcurrentToolCalls bounds the tool calls that one workspace connection +// runs at the same time. It is below the daemon's limit of concurrent +// requests per connection, so the MCP frontend waits for a slot instead of +// getting refusals from the daemon. +const maxConcurrentToolCalls = 32 + type daemonBackend struct { mu sync.Mutex root string - client *daemon.Client + ensure func(ctx context.Context, root string) (controlClient, error) + client controlClient closed bool + // closeReason is the error that calls get after Close. + closeReason error + + slots chan struct{} +} + +// retire closes the backend with the error that its calls in flight get. +func (b *daemonBackend) retire(reason error) error { + b.mu.Lock() + b.closeReason = reason + b.mu.Unlock() + return b.Close() +} + +func (b *daemonBackend) closedErr() error { + b.mu.Lock() + defer b.mu.Unlock() + if !b.closed { + return nil + } + if b.closeReason != nil { + return b.closeReason + } + return errClosed } // connection returns the live control connection, connecting when there is // none or the last one ended. -func (b *daemonBackend) connection(ctx context.Context) (*daemon.Client, error) { +func (b *daemonBackend) connection(ctx context.Context) (controlClient, error) { b.mu.Lock() defer b.mu.Unlock() if b.closed { + if b.closeReason != nil { + return nil, b.closeReason + } return nil, errClosed } if b.client != nil { @@ -102,7 +154,7 @@ func (b *daemonBackend) connection(ctx context.Context) (*daemon.Client, error) return b.client, nil } } - client, err := daemon.Ensure(ctx, b.root) + client, err := b.ensure(ctx, b.root) var mismatch *daemon.RootMismatchError if errors.As(err, &mismatch) { // A daemon already serves this workspace through another spelling of @@ -110,7 +162,7 @@ func (b *daemonBackend) connection(ctx context.Context) (*daemon.Client, error) // gets paths in that spelling, which name the same files. log.Printf("MCP: workspace %s is served as %s; using that root", b.root, mismatch.Daemon) b.root = mismatch.Daemon - client, err = daemon.Ensure(ctx, b.root) + client, err = b.ensure(ctx, b.root) } if err != nil { return nil, err @@ -126,6 +178,12 @@ func (b *daemonBackend) warm(ctx context.Context) { } func (b *daemonBackend) CallTool(ctx context.Context, name string, args json.RawMessage) (string, error) { + select { + case b.slots <- struct{}{}: + defer func() { <-b.slots }() + case <-ctx.Done(): + return "", ctx.Err() + } params := ToolParams{Name: name, Arguments: args, WaitReadyMs: int(indexWaitLimit.Milliseconds())} for attempt := 0; ; attempt++ { client, err := b.connection(ctx) @@ -138,7 +196,21 @@ func (b *daemonBackend) CallTool(ctx context.Context, name string, args json.Raw return res.Text, nil } if strings.Contains(err.Error(), fmt.Sprintf("unknown daemon method %q", MethodTool)) { - return "", fmt.Errorf("the dexter daemon for %s was started by a build without MCP tools; run `dexter stop` in the project, then retry", b.root) + return "", fmt.Errorf("the dexter daemon for %s was started by a build without MCP tools; run `dexter stop --force` in the project, then retry", b.root) + } + if name == renameToolName && ctx.Err() != nil { + // A client that canceled may not read the answer, so the log + // keeps it too. + log.Printf("MCP: a rename was canceled while it ran in %s; it may have been applied, so check git status", b.root) + return "", fmt.Errorf("the rename was canceled while it ran, so it may have been applied; check git status (%w)", ctx.Err()) + } + if reason := b.closedErr(); reason != nil { + // The frontend closed this connection during the call, for + // example because the client's roots changed. + if name == renameToolName { + return "", fmt.Errorf("%w; the rename may have been applied before the connection closed, so check git status", reason) + } + return "", reason } select { case <-client.Done(): diff --git a/internal/mcp/daemon_test.go b/internal/mcp/daemon_test.go new file mode 100644 index 0000000..efe02c2 --- /dev/null +++ b/internal/mcp/daemon_test.go @@ -0,0 +1,215 @@ +package mcp + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +// blockingClient is a daemon connection whose calls run until the caller +// cancels them or the connection closes, like a tool call that waits for a +// cold index or a rename that is writing files. +type blockingClient struct { + done chan struct{} + closeOnce sync.Once + inflight atomic.Int32 + peak atomic.Int32 +} + +func newBlockingClient() *blockingClient { return &blockingClient{done: make(chan struct{})} } + +func (c *blockingClient) Call(ctx context.Context, method string, params, result any) error { + n := c.inflight.Add(1) + defer c.inflight.Add(-1) + for { + p := c.peak.Load() + if n <= p || c.peak.CompareAndSwap(p, n) { + break + } + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-c.done: + return errors.New("daemon connection closed") + } +} + +func (c *blockingClient) Done() <-chan struct{} { return c.done } + +func (c *blockingClient) Close() error { + c.closeOnce.Do(func() { close(c.done) }) + return nil +} + +func backendOver(client *blockingClient) *daemonBackend { + return newDaemonBackend("/project", func(context.Context, string) (controlClient, error) { return client, nil }) +} + +// Regression: a rename that the client canceled reported a bare context +// error, although the daemon can have written some files already. +func TestDaemonBackend_CanceledRenameSaysItMayBeApplied(t *testing.T) { + b := backendOver(newBlockingClient()) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + _, err := b.CallTool(ctx, renameToolName, nil) + if err == nil || !strings.Contains(err.Error(), "may have been applied; check git status") { + t.Fatalf("error = %v, want a note that the rename may have been applied", err) + } + + // Other tools keep the plain context error. + ctx2, cancel2 := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel2() + if _, err := b.CallTool(ctx2, "dexter_search", nil); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("error = %v, want the deadline", err) + } +} + +// Regression: when the client's roots changed during a call, the frontend +// closed the old workspace connection, and the call said "the MCP server is +// shutting down". +func TestDaemonBackend_RootsChangeDuringCall(t *testing.T) { + for _, tool := range []string{"dexter_search", renameToolName} { + b := backendOver(newBlockingClient()) + errc := make(chan error, 1) + go func() { + _, err := b.CallTool(context.Background(), tool, nil) + errc <- err + }() + time.Sleep(10 * time.Millisecond) + if err := b.retire(errRootsChanged); err != nil { + t.Fatal(err) + } + var err error + select { + case err = <-errc: + case <-time.After(5 * time.Second): + t.Fatal("call did not end") + } + if err == nil || !strings.Contains(err.Error(), "roots changed") || strings.Contains(err.Error(), "shutting down") { + t.Fatalf("%s: error = %v, want a roots-changed error", tool, err) + } + if tool == renameToolName && !strings.Contains(err.Error(), "may have been applied") { + t.Fatalf("rename error = %v, want a note that it may have been applied", err) + } + } +} + +// One MCP frontend must not take every request slot of the daemon +// connection: calls past the frontend's limit wait for a slot. +func TestDaemonBackend_LimitsConcurrentCalls(t *testing.T) { + client := newBlockingClient() + b := backendOver(client) + ctx, cancel := context.WithCancel(context.Background()) + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { + defer wg.Done() + _, _ = b.CallTool(ctx, "dexter_search", nil) + }() + } + deadline := time.Now().Add(5 * time.Second) + for client.inflight.Load() < maxConcurrentToolCalls && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + time.Sleep(50 * time.Millisecond) + cancel() + wg.Wait() + if peak := client.peak.Load(); peak != maxConcurrentToolCalls { + t.Fatalf("peak concurrent daemon calls = %d, want %d", peak, maxConcurrentToolCalls) + } + if maxConcurrentToolCalls >= 64 { + t.Fatalf("frontend limit %d is not below the daemon's 64", maxConcurrentToolCalls) + } +} + +// The frontend retires the old workspace connection with errRootsChanged, so +// a call still running on it says why it ended. +func TestFrontend_RootsChangeEndsCallWithRootsChangedError(t *testing.T) { + var mu sync.Mutex + clients := map[string]*blockingClient{} + cfg := Config{Root: t.TempDir(), Connect: func(root string) Backend { + mu.Lock() + defer mu.Unlock() + c := newBlockingClient() + clients[root] = c + return backendOver(c) + }} + e := setupNegotiationWith(t, cfg) + e.f.cfg.Connect = cfg.Connect // setupNegotiationWith installs fake backends + rootA, uriA := projectDir(t) + _, uriB := projectDir(t) + cs, client := e.connect(nil, uriA) + t.Cleanup(func() { + // Calls still waiting on a fake daemon end before the session closes. + mu.Lock() + defer mu.Unlock() + for _, c := range clients { + _ = c.Close() + } + }) + + first := make(chan string, 1) + go func() { + out, _ := toolText(t, cs, "dexter_search", map[string]any{"query": "x"}) + first <- out + }() + eventually(t, "the first call to reach the daemon", func() bool { + mu.Lock() + defer mu.Unlock() + c := clients[rootA] + return c != nil && c.inflight.Load() == 1 + }) + + client.RemoveRoots(uriA) + client.AddRoots(&mcp.Root{URI: uriB}) + go func() { + // The next call negotiates again and moves the session to rootB. + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _, _ = cs.CallTool(ctx, &mcp.CallToolParams{Name: "dexter_search", Arguments: map[string]any{"query": "x"}}) + }() + select { + case out := <-first: + wantContains(t, out, "roots changed") + wantNotContains(t, out, "shutting down") + case <-time.After(10 * time.Second): + t.Fatal("the first call did not end") + } +} + +// errorClient is a daemon connection whose calls fail with one error. +type errorClient struct { + err error + done chan struct{} +} + +func (c *errorClient) Call(context.Context, string, any, any) error { return c.err } +func (c *errorClient) Done() <-chan struct{} { return c.done } +func (c *errorClient) Close() error { return nil } + +// Regression: recovery hints said to run a plain dexter stop, which a daemon +// refuses while this MCP session is attached. +func TestRecoveryHintsSayForce(t *testing.T) { + b := newDaemonBackend("/project", func(context.Context, string) (controlClient, error) { + return &errorClient{err: errors.New(`unknown daemon method "mcp/tool"`), done: make(chan struct{})}, nil + }) + _, err := b.CallTool(context.Background(), "dexter_search", nil) + if err == nil || !strings.Contains(err.Error(), "dexter stop --force") { + t.Errorf("old-daemon error = %v, want the --force hint", err) + } + + e := setupProject(t) + if err := e.store.SetIndexVersion(1); err != nil { + t.Fatal(err) + } + out := e.callTool("dexter_workspace", nil) + wantContains(t, out, "dexter stop --force") +} diff --git a/internal/mcp/definition.go b/internal/mcp/definition.go index 148ca95..3e34774 100644 --- a/internal/mcp/definition.go +++ b/internal/mcp/definition.go @@ -42,15 +42,23 @@ func (h *Handler) definition(ctx context.Context, args DefinitionParams) (string return "", fmt.Errorf("looking up function: %w", err) } if len(resolved) == 0 { - return fmt.Sprintf("%s.%s is not in the index. It can be defined in a quote block that the index cannot see, or misspelled. Try dexter_search or dexter_module_api %s.", module, function, module), nil + return fmt.Sprintf("%s.%s is not in the index. It can be generated by a macro that the index cannot see, or misspelled. Try dexter_search or dexter_module_api %s.", module, function, module), nil } - for _, r := range resolved { + for i, r := range resolved { + if i == maxDefinitionClauses { + writeMoreClauses(&b, len(resolved)-i) + break + } h.writeDefinition(&b, module, function, r) } return b.String(), nil } - for _, r := range direct { + for i, r := range direct { + if i == maxDefinitionClauses { + writeMoreClauses(&b, len(direct)-i) + break + } h.writeDefinition(&b, module, function, r) if r.Kind != "defdelegate" { continue @@ -64,16 +72,31 @@ func (h *Handler) definition(ctx context.Context, args DefinitionParams) (string continue } fmt.Fprintf(&b, "\nDelegates to %s.%s:\n", targetModule, targetFunction) - for _, t := range targets { + written := 0 + for j, t := range targets { if t.FilePath == r.FilePath && t.Line == r.Line { continue } + if written == maxDefinitionClauses { + writeMoreClauses(&b, len(targets)-j) + break + } h.writeDefinition(&b, targetModule, targetFunction, t) + written++ } } return b.String(), nil } +// maxDefinitionClauses caps the clauses that one definition answer shows. A +// generated function can have thousands of clauses, and each one shown costs +// its docs and its head line. +const maxDefinitionClauses = 20 + +func writeMoreClauses(b *strings.Builder, n int) { + fmt.Fprintf(b, "\n… and %d more clause(s) not shown.\n", n) +} + // delegateTargetAt names the function that the defdelegate at r forwards to. func (h *Handler) delegateTargetAt(module, function string, r lsp.NameLocation) (targetModule, targetFunction string, ok bool) { defs, err := h.store.LookupFunction(module, function) @@ -104,12 +127,15 @@ func (h *Handler) moduleDefinition(module string) (string, error) { var b strings.Builder for _, r := range results { - fmt.Fprintf(&b, "%s %s - %s:%d\n", moduleKindLabel(r.Kind), module, h.relPath(r.FilePath), r.Line) + a, ok := h.at(r.FilePath, r.Line) + if !ok { + fmt.Fprintf(&b, "%s %s - %s:%d\n", moduleKindLabel(r.Kind), module, h.relPath(r.FilePath), r.Line) + continue + } + fmt.Fprintf(&b, "%s %s - %s:%d%s\n", moduleKindLabel(r.Kind), module, h.relPath(r.FilePath), a.line, a.label()) if r.Kind != "defimpl" { - if text, _, ok := h.lsp.ReadFileText(r.FilePath); ok { - if doc := lsp.NewTokenizedFile(text).ExtractModuledoc(r.Line - 1); doc != "" { - fmt.Fprintf(&b, "\n%s\n", strings.TrimRight(doc, "\n")) - } + if doc := a.tokenized().ExtractModuledoc(a.line - 1); doc != "" { + fmt.Fprintf(&b, "\n%s\n", strings.TrimRight(doc, "\n")) } } } @@ -123,18 +149,17 @@ func (h *Handler) writeDefinition(b *strings.Builder, module, function string, r if kind == "" { kind = "def" } - fmt.Fprintf(b, "%s (%s) - %s:%d\n", symbolName(module, function, r.Arity), kind, h.relPath(r.FilePath), r.Line) - - text, _, ok := h.lsp.ReadFileText(r.FilePath) + a, ok := h.at(r.FilePath, r.Line) if !ok { + fmt.Fprintf(b, "%s (%s) - %s:%d\n", symbolName(module, function, r.Arity), kind, h.relPath(r.FilePath), r.Line) return } - tf := lsp.NewTokenizedFile(text) - doc, spec := tf.ExtractDocAbove(r.Line - 1) + fmt.Fprintf(b, "%s (%s) - %s:%d%s\n", symbolName(module, function, r.Arity), kind, h.relPath(r.FilePath), a.line, a.label()) + doc, spec := a.tokenized().ExtractDocAbove(a.line - 1) if spec != "" { fmt.Fprintf(b, "%s\n", spec) } - if head, ok := h.lsp.FileLine(r.FilePath, r.Line); ok { + if head, ok := a.lineText(); ok { fmt.Fprintf(b, "%s\n", strings.TrimRight(head, " \t")) } if doc != "" { diff --git a/internal/mcp/file_outline.go b/internal/mcp/file_outline.go index bb605b4..5f5eae7 100644 --- a/internal/mcp/file_outline.go +++ b/internal/mcp/file_outline.go @@ -2,6 +2,7 @@ package mcp import ( "context" + "errors" "fmt" "sort" "strings" @@ -17,11 +18,20 @@ func (h *Handler) fileOutline(ctx context.Context, args FileOutlineParams) (stri if strings.TrimSpace(args.File) == "" { return "", fmt.Errorf("file must not be empty") } - path := h.resolvePath(args.File) - text, _, ok := h.lsp.ReadFileText(path) - if !ok { + path, err := h.userPath(args.File) + if errors.Is(err, errNotFound) { return fmt.Sprintf("File not found: %s", h.relPath(path)), nil } + if err != nil { + return "", err + } + text, err := h.readSource(path) + if errors.Is(err, errNotFound) { + return fmt.Sprintf("File not found: %s", h.relPath(path)), nil + } + if err != nil { + return "", err + } // Parse fresh source so the outline is correct when either the index is // stale or an attached editor has unsaved changes. diff --git a/internal/mcp/frontend.go b/internal/mcp/frontend.go index 397d693..0fa5df9 100644 --- a/internal/mcp/frontend.go +++ b/internal/mcp/frontend.go @@ -9,11 +9,21 @@ import ( "github.com/modelcontextprotocol/go-sdk/mcp" + "github.com/remoteoss/dexter/internal/store" "github.com/remoteoss/dexter/internal/version" ) var errClosed = errors.New("the MCP server is shutting down") +// errRootsChanged is the error of a call whose workspace connection closed +// because the client's roots changed to another project. +var errRootsChanged = errors.New("the MCP client's workspace roots changed during the call, so the call ended; retry it") + +// retirer is a Backend that can close with the error its calls in flight get. +type retirer interface { + retire(reason error) error +} + // Config configures a Frontend. type Config struct { // Root is the workspace root for sessions that give no usable MCP root. @@ -124,9 +134,17 @@ func (f *Frontend) backendFor(ctx context.Context, ss *mcp.ServerSession) (Backe }() } if orphan != nil { - closeBackend(orphan) + if r, ok := orphan.(retirer); ok { + if err := r.retire(errRootsChanged); err != nil { + log.Printf("MCP: closing workspace connection: %v", err) + } + } else { + closeBackend(orphan) + } } - if w, ok := c.backend.(warmer); ok && !exists { + // Warm only a project: a root that the user gave explicitly but that is + // not one starts its daemon at the first tool call, not before. + if w, ok := c.backend.(warmer); ok && !exists && store.NonProjectRootError(root) == nil { go w.warm(context.WithoutCancel(ctx)) } return c.backend, nil diff --git a/internal/mcp/http_test.go b/internal/mcp/http_test.go new file mode 100644 index 0000000..c23d7e1 --- /dev/null +++ b/internal/mcp/http_test.go @@ -0,0 +1,116 @@ +package mcp + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/modelcontextprotocol/go-sdk/mcp" +) + +func TestCheckListenAddr(t *testing.T) { + for addr, ok := range map[string]bool{ + "localhost:0": true, + "127.0.0.1:8080": true, + "127.1.2.3:8080": true, + "[::1]:8080": true, + ":8080": false, + "0.0.0.0:8080": false, + "[::]:8080": false, + "192.168.1.20:8080": false, + "example.com:8080": false, + "localhost": false, // no port + } { + err := CheckListenAddr(addr, false) + if (err == nil) != ok { + t.Errorf("CheckListenAddr(%q) = %v, want ok=%v", addr, err, ok) + } + } + if err := CheckListenAddr("0.0.0.0:8080", true); err != nil { + t.Errorf("an explicit unsafe listen was refused: %v", err) + } +} + +const initializeBody = `{"jsonrpc":"2.0","id":1,"method":"initialize","params":{"protocolVersion":"2025-06-18","capabilities":{},"clientInfo":{"name":"t","version":"1"}}}` + +func postMCP(t *testing.T, url string, body io.Reader, header map[string]string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodPost, url, body) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json, text/event-stream") + for k, v := range header { + req.Header.Set(k, v) + } + res, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = res.Body.Close() }) + return res +} + +func httpTestServer(t *testing.T) (*httptest.Server, *negotiationEnv) { + t.Helper() + e := setupNegotiationWith(t, Config{Root: t.TempDir(), Fixed: true}) + srv := httptest.NewServer(HTTPHandler(e.f)) + t.Cleanup(srv.Close) + return srv, e +} + +// Regression: a request body had no size limit; a 300 MB POST made the server +// hold gigabytes. +func TestHTTPHandler_CapsBodySize(t *testing.T) { + srv, _ := httpTestServer(t) + if res := postMCP(t, srv.URL, strings.NewReader(initializeBody), nil); res.StatusCode != http.StatusOK { + t.Fatalf("a small initialize got status %d", res.StatusCode) + } + // Valid JSON, padded past the limit. + padded := initializeBody + strings.Repeat(" ", maxHTTPBodyBytes) + if res := postMCP(t, srv.URL, bytes.NewReader([]byte(padded)), nil); res.StatusCode == http.StatusOK { + t.Fatalf("a %d-byte body was accepted", len(padded)) + } +} + +// A web page in the user's browser must not drive the tools. +func TestHTTPHandler_RefusesCrossOriginRequests(t *testing.T) { + srv, _ := httpTestServer(t) + for _, header := range []map[string]string{ + {"Sec-Fetch-Site": "cross-site"}, + {"Origin": "http://attacker.example"}, + } { + if res := postMCP(t, srv.URL, strings.NewReader(initializeBody), header); res.StatusCode != http.StatusForbidden { + t.Errorf("cross-origin request %v got status %d, want 403", header, res.StatusCode) + } + } +} + +// Regression: HTTP sessions never timed out, so a client that went away +// without closing its session kept its workspace connection, and the daemon, +// alive forever. +func TestHTTPHandler_IdleSessionReleasesWorkspace(t *testing.T) { + old := httpSessionTimeout + httpSessionTimeout = 200 * time.Millisecond + t.Cleanup(func() { httpSessionTimeout = old }) + srv, e := httpTestServer(t) + + client := mcp.NewClient(&mcp.Implementation{Name: "t", Version: "1"}, nil) + cs, err := client.Connect(context.Background(), &mcp.StreamableClientTransport{Endpoint: srv.URL, DisableStandaloneSSE: true}, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = cs.Close() }) + mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}) + if n := e.connCount(); n != 1 { + t.Fatalf("%d workspace connections, want 1", n) + } + // The client stays silent; the session times out and lets go. + eventually(t, "the idle session to release its workspace connection", func() bool { return e.connCount() == 0 }) +} diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md index 9e28cb1..e1b6afe 100644 --- a/internal/mcp/instructions.md +++ b/internal/mcp/instructions.md @@ -26,6 +26,10 @@ not be indexed) can be incomplete; retry after the note goes away. Elixir specifics: modules are not tied to files (use `dexter_file_outline` for a file, `dexter_definition` for a module); pass fully-qualified module names, -not aliases; function names take no arity; functions defined inside a -`__using__` quote block may not be indexed, so an empty lookup can mean -macro-generated code. +not aliases; function names take no arity. Functions that a `__using__` +quote block injects resolve through the use chain, but code generated in other +ways (other macros, `unquote` names) can be missing from the index, so an empty +lookup can mean macro-generated code. + +Answers read files the way the user sees them: a file open in an attached +editor with unsaved changes is read from that buffer, and the answer says so. diff --git a/internal/mcp/module_api.go b/internal/mcp/module_api.go index 73c61d2..89cfcc2 100644 --- a/internal/mcp/module_api.go +++ b/internal/mcp/module_api.go @@ -5,7 +5,6 @@ import ( "fmt" "strings" - "github.com/remoteoss/dexter/internal/lsp" "github.com/remoteoss/dexter/internal/store" ) @@ -159,34 +158,27 @@ func (h *Handler) delegateTarget(module, function string, arity int) string { } func (h *Handler) extractModuledoc(filePath string, defLine int) string { - text, _, ok := h.lsp.ReadFileText(filePath) + a, ok := h.at(filePath, defLine) if !ok { return "" } - return lsp.NewTokenizedFile(text).ExtractModuledoc(defLine - 1) + return a.tokenized().ExtractModuledoc(a.line - 1) } // docExtractor extracts @doc summaries, tokenizing each source file at most once. type docExtractor struct { - h *Handler - files map[string]*lsp.TokenizedFile + h *Handler } func (h *Handler) newDocExtractor() *docExtractor { - return &docExtractor{h: h, files: make(map[string]*lsp.TokenizedFile)} + return &docExtractor{h: h} } func (d *docExtractor) docFor(filePath string, defLine int) string { - tf, ok := d.files[filePath] + a, ok := d.h.at(filePath, defLine) if !ok { - if text, _, found := d.h.lsp.ReadFileText(filePath); found { - tf = lsp.NewTokenizedFile(text) - } - d.files[filePath] = tf // cache nil results too - } - if tf == nil { return "" } - doc, _ := tf.ExtractDocAbove(defLine - 1) + doc, _ := a.tokenized().ExtractDocAbove(a.line - 1) return firstDocLine(doc) } diff --git a/internal/mcp/negotiation_test.go b/internal/mcp/negotiation_test.go index 97346ed..3c333a9 100644 --- a/internal/mcp/negotiation_test.go +++ b/internal/mcp/negotiation_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "testing" "time" @@ -450,3 +451,49 @@ func TestNegotiation_SkipsUnusableRoot(t *testing.T) { cs, _ := e.connect(nil, "file:///nonexistent/dexter-negotiation-test", fileURI(file), uri) wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}), "root="+root) } + +// warmCounter is a fake backend that counts warm calls. +type warmCounter struct { + fakeBackend + warms *atomic.Int32 +} + +func (w *warmCounter) warm(context.Context) { w.warms.Add(1) } + +// Regression: a session's workspace was warmed (its daemon started and +// indexing) even when the root was not an Elixir project. +func TestFrontend_WarmsOnlyProjects(t *testing.T) { + for _, tc := range []struct { + name string + project bool + }{{"project", true}, {"plain directory", false}} { + t.Run(tc.name, func(t *testing.T) { + root := t.TempDir() + if tc.project { + writeSource(t, root, "mix.exs", "defmodule App.MixProject do\nend\n") + } + var warms atomic.Int32 + f := NewFrontend(Config{Root: root, Fixed: true, Connect: func(root string) Backend { + return &warmCounter{fakeBackend: fakeBackend{root: root}, warms: &warms} + }}) + cs := connectFrontend(t, f) + mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}) + time.Sleep(50 * time.Millisecond) // warm runs in a goroutine + if got := warms.Load() > 0; got != tc.project { + t.Errorf("warmed = %v, want %v", got, tc.project) + } + }) + } +} + +// A client root that is not a project is unusable: a later project root wins. +func TestNegotiation_SkipsNonProjectRoot(t *testing.T) { + e := setupNegotiation(t) + plain := t.TempDir() + project, projectURI := projectDir(t) + cs, _ := e.connect(nil, fileURI(plain), projectURI) + wantContains(t, mustTool(t, cs, "dexter_search", map[string]any{"query": "x"}), "root="+project) + if got := e.backendsFor(plain); len(got) != 0 { + t.Errorf("a backend was opened for the non-project root %s", plain) + } +} diff --git a/internal/mcp/references.go b/internal/mcp/references.go index 48e639c..458eea9 100644 --- a/internal/mcp/references.go +++ b/internal/mcp/references.go @@ -63,11 +63,11 @@ func (h *Handler) references(ctx context.Context, args ReferencesParams) (string fmt.Fprintf(&b, "\n%s\n", h.relPath(r.FilePath)) lastFile = r.FilePath } - srcLine := "" - if line, ok := h.lsp.FileLine(r.FilePath, r.Line); ok { - srcLine = strings.TrimSpace(line) + srcLine, line, label, ok := h.sourceLine(r.FilePath, r.Line) + if !ok { + line = r.Line } - fmt.Fprintf(&b, " %d: %s\n", r.Line, srcLine) + fmt.Fprintf(&b, " %d: %s%s\n", line, strings.TrimSpace(srcLine), label) written++ } if truncated > 0 { diff --git a/internal/mcp/rename_test.go b/internal/mcp/rename_test.go index 84dcc26..e4a8a20 100644 --- a/internal/mcp/rename_test.go +++ b/internal/mcp/rename_test.go @@ -1,9 +1,13 @@ package mcp import ( + "context" + "encoding/json" + "fmt" "os" "path/filepath" "strings" + "sync" "testing" ) @@ -101,3 +105,51 @@ end wantContains(t, readFile(t, e.root, "lib/my_app/greeter.ex"), ` def greet(id), do: {"héllo wörld ✓", MyApp.Accounts.get_user(id)}`) } + +// Regression: two renames at the same time each read the affected files and +// wrote them back, so the later write dropped the other rename's edits while +// both reported success. +func TestRenameTool_ConcurrentRenamesKeepBothEdits(t *testing.T) { + e := setupProject(t) + for i := 0; i < 40; i++ { + e.indexFile(fmt.Sprintf("lib/my_app/caller_%d.ex", i), fmt.Sprintf(`defmodule MyApp.Caller%d do + def run do + MyApp.Accounts.fetch_user(1) + MyApp.Accounts.list_users([]) + end +end +`, i)) + } + h := NewHandler(e.rt, e.lsp) + pairs := [][2][2]string{ + {{"fetch_user", "get_user"}, {"list_users", "all_users"}}, + {{"get_user", "fetch_user"}, {"all_users", "list_users"}}, + } + for round := 0; round < 6; round++ { + renames := pairs[round%2] + var wg sync.WaitGroup + errs := make([]error, 2) + for i, r := range renames { + wg.Add(1) + go func() { + defer wg.Done() + args, _ := json.Marshal(map[string]any{"module": "MyApp.Accounts", "function": r[0], "new_name": r[1]}) + _, errs[i] = h.Call(context.Background(), renameToolName, args, 0) + }() + } + wg.Wait() + for _, err := range errs { + if err != nil { + t.Fatalf("round %d: rename failed: %v", round, err) + } + } + for i := 0; i < 40; i++ { + text := readFile(t, e.root, fmt.Sprintf("lib/my_app/caller_%d.ex", i)) + for _, r := range renames { + if !strings.Contains(text, "MyApp.Accounts."+r[1]+"(") || strings.Contains(text, "MyApp.Accounts."+r[0]+"(") { + t.Fatalf("round %d: caller_%d.ex lost the rename %s → %s:\n%s", round, i, r[0], r[1], text) + } + } + } + } +} diff --git a/internal/mcp/roots.go b/internal/mcp/roots.go index bbda015..2f93b18 100644 --- a/internal/mcp/roots.go +++ b/internal/mcp/roots.go @@ -98,7 +98,12 @@ func usableRoot(uri string, resolve func(string) (string, error)) (string, error } // defaultResolveRoot finds the project root above dir with the store's marker -// search (an existing index, then a repository). +// search (an existing index, then a repository), and refuses a directory that +// is not a project. func defaultResolveRoot(dir string) (string, error) { - return store.FindProjectRoot(dir), nil + root := store.FindProjectRoot(dir) + if err := store.NonProjectRootError(root); err != nil { + return "", err + } + return root, nil } diff --git a/internal/mcp/serve.go b/internal/mcp/serve.go index 35e864b..f029c42 100644 --- a/internal/mcp/serve.go +++ b/internal/mcp/serve.go @@ -2,7 +2,10 @@ package mcp import ( "context" + "fmt" + "net" "net/http" + "time" "github.com/modelcontextprotocol/go-sdk/mcp" ) @@ -13,8 +16,50 @@ func RunStdio(ctx context.Context, f *Frontend) error { return NewServer(f).Run(ctx, &mcp.StdioTransport{}) } +// httpSessionTimeout closes an HTTP session that sends no request for this +// long. A client that goes away without closing its session would otherwise +// keep its workspace connection, and with it the daemon, alive forever. A +// variable so tests can shrink it. +var httpSessionTimeout = 30 * time.Minute + +// maxHTTPBodyBytes caps one HTTP request body. MCP requests from an agent are +// small; the cap keeps one request from filling the server's memory. +const maxHTTPBodyBytes = 4 << 20 + // HTTPHandler returns a streamable-HTTP handler serving MCP. Each session // gets its own protocol server; they all share the Frontend. +// +// The SDK refuses a request that reaches a loopback address with a Host that +// is not loopback (DNS rebinding). Cross-origin browser requests are refused +// too, so a web page cannot drive the tools, and bodies are capped. func HTTPHandler(f *Frontend) http.Handler { - return mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return NewServer(f) }, nil) + h := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return NewServer(f) }, &mcp.StreamableHTTPOptions{ + SessionTimeout: httpSessionTimeout, + }) + return http.MaxBytesHandler(http.NewCrossOriginProtection().Handler(h), maxHTTPBodyBytes) +} + +// CheckListenAddr refuses an HTTP listen address that other machines can +// reach, because the server has no authentication: anyone who reaches it can +// read the code and rename symbols. allowRemote is the explicit way through. +func CheckListenAddr(addr string, allowRemote bool) error { + if allowRemote { + return nil + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("invalid listen address %q: %w", addr, err) + } + if isLoopbackHost(host) { + return nil + } + return fmt.Errorf("refusing to listen on %q: the MCP HTTP server has no authentication, so it listens only on a loopback address (for example localhost:8080 or 127.0.0.1:8080). Pass --listen-unsafe to listen on %q anyway", addr, addr) +} + +func isLoopbackHost(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() } diff --git a/internal/mcp/source.go b/internal/mcp/source.go new file mode 100644 index 0000000..334970b --- /dev/null +++ b/internal/mcp/source.go @@ -0,0 +1,330 @@ +package mcp + +import ( + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/remoteoss/dexter/internal/lsp" +) + +// maxSourceBytes caps the size of one file that a tool reads. Elixir source +// files are far smaller; the cap keeps a wrong path from filling the daemon's +// memory. +const maxSourceBytes = 10 << 20 + +// sourceCache holds the file text that one tool call read, so that each file +// is read and tokenized at most once per call, and records which files came +// from unsaved editor buffers. +type sourceCache struct { + mu sync.Mutex + views map[string]*sourceView // nil for a file that cannot be read + unsaved map[string]struct{} +} + +// errNotFound reports a file that does not exist. +var errNotFound = errors.New("file not found") + +// userPath resolves a file path that the agent gave. A relative path is +// resolved against the project root. The path, with every symlink resolved, +// must be inside the project root, also with its symlinks resolved. The +// result is spelled under the root as the daemon spells it, so that it names +// the same file as the index and the editors. +func (h *Handler) userPath(p string) (string, error) { + candidate := p + if !filepath.IsAbs(candidate) { + candidate = filepath.Join(h.projectRoot, candidate) + } + candidate = filepath.Clean(candidate) + root, err := filepath.EvalSymlinks(h.projectRoot) + if err != nil { + return "", fmt.Errorf("resolving the project root: %w", err) + } + real, err := filepath.EvalSymlinks(candidate) + if errors.Is(err, fs.ErrNotExist) { + // A file that is not on disk can still be open in an editor. Only a + // path inside the root as spelled can name one. + if rel, ok := inside(h.projectRoot, candidate); ok { + path := filepath.Join(h.projectRoot, rel) + if _, open := h.rt.EditorBuffer(path); open { + return path, nil + } + return path, errNotFound + } + return "", outsideRootError(p, h.projectRoot) + } + if err != nil { + return "", err + } + rel, ok := inside(root, real) + if !ok { + return "", outsideRootError(p, h.projectRoot) + } + return filepath.Join(h.projectRoot, rel), nil +} + +func outsideRootError(p, root string) error { + return fmt.Errorf("%s is outside the project root %s; dexter reads only files inside the project", p, root) +} + +// inside returns path relative to root when it is root or under it. +func inside(root, path string) (string, bool) { + rel, err := filepath.Rel(root, path) + if err != nil || filepath.IsAbs(rel) || rel == ".." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) { + return "", false + } + return rel, true +} + +// readRegularFile reads a regular file of at most maxSourceBytes. Devices, +// FIFOs, sockets, and directories are refused before they are opened, so a +// read can neither block nor run without end. +func readRegularFile(path string) (string, error) { + info, err := os.Stat(path) + if err != nil { + if errors.Is(err, fs.ErrNotExist) { + return "", errNotFound + } + return "", err + } + if !info.Mode().IsRegular() { + return "", fmt.Errorf("%s is not a regular file", path) + } + if info.Size() > maxSourceBytes { + return "", fmt.Errorf("%s is %d bytes, more than the %d MB limit for one file", path, info.Size(), maxSourceBytes>>20) + } + f, err := os.Open(path) + if err != nil { + return "", err + } + defer func() { _ = f.Close() }() + // The path can change between the stat and the open. + if opened, err := f.Stat(); err != nil || !os.SameFile(info, opened) { + return "", fmt.Errorf("%s changed while it was read; retry", path) + } + data, err := io.ReadAll(io.LimitReader(f, maxSourceBytes+1)) + if err != nil { + return "", err + } + if len(data) > maxSourceBytes { + return "", fmt.Errorf("%s is more than the %d MB limit for one file", path, maxSourceBytes>>20) + } + return string(data), nil +} + +// sourceView is one file as a tool call reads it. When an attached editor +// holds the file open with changes that are not saved, text is that buffer +// and disk is the saved file, which the index positions refer to. +type sourceView struct { + text string + unsaved bool + disk string + hasDisk bool + // The lines that the buffer and the saved file share at the start and at + // the end. An index line in one of them is found in the buffer; a line + // between them was changed in the buffer. + prefix, suffix int + diskLines, textLines int + + tf, diskTF *lsp.TokenizedFile +} + +func newSourceView(text string, unsaved bool, disk string, hasDisk bool) *sourceView { + v := &sourceView{text: text, unsaved: unsaved, disk: disk, hasDisk: hasDisk} + if !unsaved || !hasDisk { + return v + } + a, b := strings.Split(disk, "\n"), strings.Split(text, "\n") + v.diskLines, v.textLines = len(a), len(b) + for v.prefix < len(a) && v.prefix < len(b) && a[v.prefix] == b[v.prefix] { + v.prefix++ + } + for v.suffix < len(a)-v.prefix && v.suffix < len(b)-v.prefix && a[len(a)-1-v.suffix] == b[len(b)-1-v.suffix] { + v.suffix++ + } + return v +} + +// locate maps a 1-based line of the saved file, where the index puts a +// definition or a reference, to the same line in the text. inText is false +// when the buffer changed that line. +func (v *sourceView) locate(indexLine int) (line int, inText bool) { + if !v.unsaved || !v.hasDisk || indexLine <= v.prefix { + return indexLine, true + } + if indexLine > v.diskLines-v.suffix { + return indexLine + v.textLines - v.diskLines, true + } + return indexLine, false +} + +func (v *sourceView) tokenized(inText bool) *lsp.TokenizedFile { + if inText { + if v.tf == nil { + v.tf = lsp.NewTokenizedFile(v.text) + } + return v.tf + } + if v.diskTF == nil { + v.diskTF = lsp.NewTokenizedFile(v.disk) + } + return v.diskTF +} + +// sourceAt is one index position in a file as the user sees it. +type sourceAt struct { + v *sourceView + line int // the 1-based line to show + // changed is true when an unsaved buffer changed this line: the text and + // the line are the saved file's. + changed bool +} + +func (a sourceAt) lineText() (string, bool) { + if a.changed { + return nthLine(a.v.disk, a.line) + } + return nthLine(a.v.text, a.line) +} + +func (a sourceAt) tokenized() *lsp.TokenizedFile { return a.v.tokenized(!a.changed) } + +// label is the suffix of a location whose text did not come from the buffer +// the user sees. +func (a sourceAt) label() string { + if a.changed { + return " (saved text; this part is changed in an unsaved editor buffer)" + } + return "" +} + +// readSource returns the text of path as the user sees it: the newest buffer +// that an attached editor holds open when it differs from the disk, otherwise +// the file on disk. Text from an unsaved buffer is recorded, so the answer +// says so. +func (h *Handler) readSource(path string) (string, error) { + v, err := h.view(path) + if err != nil { + return "", err + } + return v.text, nil +} + +// at finds the index position path:indexLine in the text the user sees. +func (h *Handler) at(path string, indexLine int) (sourceAt, bool) { + v, err := h.view(path) + if err != nil { + return sourceAt{}, false + } + line, inText := v.locate(indexLine) + return sourceAt{v: v, line: line, changed: !inText}, true +} + +// view reads path once per tool call. +func (h *Handler) view(path string) (*sourceView, error) { + c := &h.sources + c.mu.Lock() + if cached, ok := c.views[path]; ok { + c.mu.Unlock() + if cached == nil { + return nil, errNotFound + } + return cached, nil + } + c.mu.Unlock() + + v, err := h.loadSource(path) + + c.mu.Lock() + defer c.mu.Unlock() + if c.views == nil { + c.views = make(map[string]*sourceView) + } + c.views[path] = v + if v != nil && v.unsaved { + if c.unsaved == nil { + c.unsaved = make(map[string]struct{}) + } + c.unsaved[path] = struct{}{} + } + return v, err +} + +func (h *Handler) loadSource(path string) (*sourceView, error) { + disk, diskErr := readRegularFile(path) + if buffer, open := h.rt.EditorBuffer(path); open { + if diskErr == nil && disk == buffer { + return newSourceView(disk, false, "", false), nil + } + return newSourceView(buffer, true, disk, diskErr == nil), nil + } + if diskErr != nil { + return nil, diskErr + } + return newSourceView(disk, false, "", false), nil +} + +// sourceLine returns the text of one index position. A file that no editor +// holds open is scanned only up to the line, so a reference list does not +// read whole files. +func (h *Handler) sourceLine(path string, indexLine int) (text string, line int, label string, ok bool) { + c := &h.sources + c.mu.Lock() + _, cached := c.views[path] + c.mu.Unlock() + if !cached { + if _, open := h.rt.EditorBuffer(path); !open { + text, ok := h.lsp.FileLine(path, indexLine) + return text, indexLine, "", ok + } + } + a, ok := h.at(path, indexLine) + if !ok { + return "", 0, "", false + } + text, ok = a.lineText() + return text, a.line, a.label(), ok +} + +// nthLine returns the 1-based line n of text. +func nthLine(text string, n int) (string, bool) { + if n < 1 { + return "", false + } + for i := 1; ; i++ { + end := strings.IndexByte(text, '\n') + if i == n { + if end < 0 { + return strings.TrimSuffix(text, "\r"), true + } + return strings.TrimSuffix(text[:end], "\r"), true + } + if end < 0 { + return "", false + } + text = text[end+1:] + } +} + +// unsavedNote names the files whose text came from unsaved editor buffers in +// this call, or is empty when there are none. +func (h *Handler) unsavedNote() string { + c := &h.sources + c.mu.Lock() + paths := make([]string, 0, len(c.unsaved)) + for p := range c.unsaved { + paths = append(paths, h.relPath(p)) + } + c.mu.Unlock() + if len(paths) == 0 { + return "" + } + sort.Strings(paths) + return "Note: read from unsaved editor buffers (the files on disk differ; line numbers are the buffer's): " + strings.Join(paths, ", ") +} diff --git a/internal/mcp/source_test.go b/internal/mcp/source_test.go new file mode 100644 index 0000000..1126489 --- /dev/null +++ b/internal/mcp/source_test.go @@ -0,0 +1,138 @@ +package mcp + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "testing" + + "go.lsp.dev/protocol" + "go.lsp.dev/uri" +) + +// Regression: dexter_file_outline read any path the agent gave, so an +// absolute path or one with .. could read every file the user can read, and +// /dev/zero filled the daemon's memory. +func TestFileOutline_RefusesPathsOutsideRoot(t *testing.T) { + e := setupProject(t) + outside := filepath.Join(t.TempDir(), "secret.ex") + if err := os.WriteFile(outside, []byte("defmodule Secret do\n def key, do: 1\nend\n"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(e.root, "lib", "link.ex")); err != nil { + t.Fatal(err) + } + if err := os.Symlink("/dev/zero", filepath.Join(e.root, "lib", "zero.ex")); err != nil { + t.Fatal(err) + } + rel, err := filepath.Rel(e.root, outside) + if err != nil { + t.Fatal(err) + } + for _, p := range []string{outside, rel, "lib/link.ex", "/dev/zero", "lib/zero.ex", "../" + filepath.Base(e.root) + "x/a.ex"} { + errText := e.callToolExpectError("dexter_file_outline", map[string]any{"file": p}) + wantContains(t, errText, "outside the project root") + wantNotContains(t, errText, "Secret") + } + + // A symlink that stays inside the root is still fine. + if err := os.Symlink(filepath.Join(e.root, "lib/my_app/worker.ex"), filepath.Join(e.root, "lib", "alias.ex")); err != nil { + t.Fatal(err) + } + wantContains(t, e.callTool("dexter_file_outline", map[string]any{"file": "lib/alias.ex"}), "defmodule MyApp.Worker") +} + +func TestFileOutline_RefusesLargeFilesAndDirectories(t *testing.T) { + e := setupProject(t) + big := filepath.Join(e.root, "lib", "big.ex") + if err := os.WriteFile(big, make([]byte, maxSourceBytes+1), 0644); err != nil { + t.Fatal(err) + } + wantContains(t, e.callToolExpectError("dexter_file_outline", map[string]any{"file": "lib/big.ex"}), "MB limit") + wantContains(t, e.callToolExpectError("dexter_file_outline", map[string]any{"file": "lib"}), "not a regular file") + wantContains(t, e.callTool("dexter_file_outline", map[string]any{"file": "lib/missing.ex"}), "File not found: lib/missing.ex") +} + +// Regression: a function with thousands of clauses gave an answer of hundreds +// of kilobytes, and each clause read and tokenized the file again. +func TestDefinitionTool_CapsClauses(t *testing.T) { + e := setupProject(t) + var src strings.Builder + src.WriteString("defmodule MyApp.Big do\n") + for i := 0; i < 300; i++ { + fmt.Fprintf(&src, " def code(%d), do: %d\n", i, i) + } + src.WriteString("end\n") + e.indexFile("lib/my_app/big.ex", src.String()) + + out := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Big", "function": "code"}) + if got := strings.Count(out, "MyApp.Big.code/1 (def)"); got != 20 { + t.Errorf("answer shows %d clauses, want 20:\n%s", got, out) + } + wantContains(t, out, "… and 280 more clause(s) not shown.") +} + +// MCP answers from what the user sees: a buffer that an attached editor holds +// open, with changes not yet saved, wins over the disk, and the answer says so. +// The index positions refer to the saved file, so they are mapped into the +// buffer. +func TestTools_SeeUnsavedEditorBuffers(t *testing.T) { + e := setupProject(t) + _, session, release := e.rt.AttachLSPSession() + t.Cleanup(release) + + change := func(rel, text string) { + t.Helper() + path := filepath.Join(e.root, rel) + if err := session.DidChange(context.Background(), &protocol.DidChangeTextDocumentParams{ + TextDocument: protocol.VersionedTextDocumentIdentifier{TextDocumentIdentifier: protocol.TextDocumentIdentifier{URI: protocol.DocumentURI(uri.File(path))}}, + ContentChanges: []protocol.TextDocumentContentChangeEvent{{Text: text}}, + }); err != nil { + t.Fatal(err) + } + } + + // A new module at the end of the buffer shows in the outline. + change("lib/my_app/worker.ex", workerSource+"\ndefmodule MyApp.Draft do\n def draft_only(x), do: x\nend\n") + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantContains(t, out, "defmodule MyApp.Draft", "draft_only/1", "unsaved editor buffers", "lib/my_app/worker.ex") + + // Two lines inserted at the top move every reference down by two. + change("lib/my_app/worker.ex", "# draft\n# notes\n"+workerSource) + refs := e.callTool("dexter_references", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, refs, "5: MyApp.Accounts.fetch_user(1)", "10: MyApp.Accounts.fetch_user(2)", "unsaved editor buffers") + + // A reference on a line that the buffer changed shows the saved line, + // marked as such. + change("lib/my_app/worker.ex", strings.Replace(workerSource, "MyApp.Accounts.fetch_user(1)", "MyApp.Accounts.fetch_user(id)", 1)) + refs = e.callTool("dexter_references", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, refs, "3: MyApp.Accounts.fetch_user(1) (saved text; this part is changed in an unsaved editor buffer)", "8: MyApp.Accounts.fetch_user(2)\n") + + // A definition below an inserted function: the head, @spec, and @doc come + // from the buffer at the moved line. + change("lib/my_app/accounts.ex", strings.Replace(accountsSource, " @doc \"\"\"\n Fetches", " def helper, do: :ok\n\n @doc \"\"\"\n Fetches a user, unsaved.\n Fetches", 1)) + def := e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, def, "lib/my_app/accounts.ex:13", "@spec fetch_user(integer())", "def fetch_user(id) do", "Fetches a user, unsaved.", "unsaved editor buffers") + + // A buffer that matches the disk is not unsaved. + change("lib/my_app/worker.ex", workerSource) + change("lib/my_app/accounts.ex", accountsSource) + out = e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantNotContains(t, out, "MyApp.Draft", "unsaved") + def = e.callTool("dexter_definition", map[string]any{"module": "MyApp.Accounts", "function": "fetch_user"}) + wantContains(t, def, "lib/my_app/accounts.ex:10", "def fetch_user(id) do") + wantNotContains(t, def, "unsaved") +} + +func TestSourceViewLocate(t *testing.T) { + disk := "a\nb\nc\nd\ne" + v := newSourceView("a\nX\nY\nc\nd\ne", true, disk, true) // b replaced by X, Y + for line, want := range map[int][2]int{1: {1, 1}, 2: {2, 0}, 3: {4, 1}, 5: {6, 1}} { + got, in := v.locate(line) + if in != (want[1] == 1) || (in && got != want[0]) { + t.Errorf("locate(%d) = %d, %v; want %v", line, got, in, want) + } + } +} diff --git a/internal/mcp/source_unix_test.go b/internal/mcp/source_unix_test.go new file mode 100644 index 0000000..18049dc --- /dev/null +++ b/internal/mcp/source_unix_test.go @@ -0,0 +1,18 @@ +//go:build unix + +package mcp + +import ( + "path/filepath" + "syscall" + "testing" +) + +// Regression: a FIFO blocked the read forever. +func TestFileOutline_RefusesFIFO(t *testing.T) { + e := setupProject(t) + if err := syscall.Mkfifo(filepath.Join(e.root, "lib", "pipe.ex"), 0644); err != nil { + t.Fatal(err) + } + wantContains(t, e.callToolExpectError("dexter_file_outline", map[string]any{"file": "lib/pipe.ex"}), "not a regular file") +} diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index cbebc02..ebef002 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -41,6 +41,10 @@ type Handler struct { lsp *lsp.Server store *store.Store projectRoot string + + // sources caches the files that one tool call reads. Call gives each call + // its own Handler, so the cache never outlives the call. + sources sourceCache } // NewHandler returns a Handler over the runtime's store and the given language @@ -163,11 +167,18 @@ func (h *Handler) Call(ctx context.Context, name string, args json.RawMessage, w cancel() } - text, err := spec.run(h, ctx, args) - notes := "" + call := &Handler{rt: h.rt, lsp: h.lsp, store: h.store, projectRoot: h.projectRoot} + text, err := spec.run(call, ctx, args) + var noteList []string if !spec.ownStatus { - notes = h.indexNotes() + if n := h.indexNotes(); n != "" { + noteList = append(noteList, n) + } + } + if n := call.unsavedNote(); n != "" { + noteList = append(noteList, n) } + notes := strings.Join(noteList, "\n") if err != nil { if notes != "" { return "", fmt.Errorf("%w\n%s", err, notes) @@ -221,14 +232,6 @@ func (h *Handler) relPath(p string) string { return p } -// resolvePath interprets a user-supplied path against the project root. -func (h *Handler) resolvePath(p string) string { - if filepath.IsAbs(p) { - return p - } - return filepath.Join(h.projectRoot, p) -} - // symbolName renders Module.function/arity (or just the module name). func symbolName(module, function string, arity int) string { if function == "" { diff --git a/internal/mcp/workspace.go b/internal/mcp/workspace.go index 016eedc..bc4a7f9 100644 --- a/internal/mcp/workspace.go +++ b/internal/mcp/workspace.go @@ -44,7 +44,7 @@ func (h *Handler) workspace(ctx context.Context, args WorkspaceParams) (string, fmt.Fprintf(&b, "Index state: still building; answers can be incomplete until it is ready\n") } if st.IndexVersion != st.ExpectedIndexVersion && st.Ready { - fmt.Fprintf(&b, "WARNING: index version %d does not match this binary (%d). Run `dexter stop` in the project so the next call starts a current daemon.\n", st.IndexVersion, st.ExpectedIndexVersion) + fmt.Fprintf(&b, "WARNING: index version %d does not match this binary (%d). Run `dexter stop --force` in the project so the next call starts a current daemon (a plain stop is refused while this MCP session is attached).\n", st.IndexVersion, st.ExpectedIndexVersion) } if st.Watching { fmt.Fprintf(&b, "\nThe index updates automatically as files change and on git branch switches; dexter_reindex forces an immediate update.\n") diff --git a/internal/store/project.go b/internal/store/project.go index a02aaf4..16f8dca 100644 --- a/internal/store/project.go +++ b/internal/store/project.go @@ -1,6 +1,7 @@ package store import ( + "fmt" "os" "path/filepath" ) @@ -47,3 +48,20 @@ func gitMarker(path string) bool { info, err := os.Stat(path) return err == nil && (info.IsDir() || info.Mode().IsRegular()) } + +// NonProjectRootError explains why dir is not a workspace to index, or returns +// nil when it is one. Every frontend refuses such a directory unless the user +// insists, because indexing it (the home directory, a mistyped path) would +// read every file under it. +func NonProjectRootError(dir string) error { + if IsHomeDir(dir) { + if HasIndex(dir) { + return nil + } + return fmt.Errorf("refusing to use %s as a workspace: it is your home directory, not a project", dir) + } + if LooksLikeProject(dir) { + return nil + } + return fmt.Errorf("refusing to use %s as a workspace: no mix.exs, .git, or Dexter database found, so it does not look like an Elixir project", dir) +} diff --git a/internal/workspace/runtime.go b/internal/workspace/runtime.go index c083bdf..5eae084 100644 --- a/internal/workspace/runtime.go +++ b/internal/workspace/runtime.go @@ -222,6 +222,31 @@ func (r *Runtime) Session(id string) (*lsp.Server, bool) { return s.server, true } +// EditorBuffer returns the newest buffer that an attached editor holds open +// for path, or false when no editor has it open. A frontend without an editor, +// such as MCP, uses it to read what the user sees instead of the disk. +func (r *Runtime) EditorBuffer(path string) (string, bool) { + r.sessMu.Lock() + if len(r.sessions) == 0 { + r.sessMu.Unlock() + return "", false + } + servers := make([]*lsp.Server, 0, len(r.sessions)) + for _, s := range r.sessions { + servers = append(servers, s.server) + } + r.sessMu.Unlock() + var text string + var newest uint64 + found := false + for _, srv := range servers { + if t, seq, ok := srv.OpenBuffer(path); ok && (!found || seq > newest) { + text, newest, found = t, seq, true + } + } + return text, found +} + // SessionInfo describes one attached editor session for status output. type SessionInfo struct { ID string From 5856a02b05aa091c0e8034743a6212950b6d0e10 Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 3 Oct 2026 23:36:39 -0400 Subject: [PATCH 24/26] Stop canceled renames and same-name rename races A call canceled while it waited for the index no longer runs, and the rename tool checks the context once more before its first write, so a rename does not start after the client was told it may have been applied. A rename now waits, under renameSerial, until the index shows the last rename, and checks again that the new function name is free. Two renames to one new name could both pass the check that ran before the lock and both write. Co-Authored-By: Claude Opus 5.5 --- docs/architecture.md | 2 +- internal/lsp/server.go | 28 +++++++++++++++----- internal/mcp/rename.go | 4 +++ internal/mcp/rename_test.go | 52 +++++++++++++++++++++++++++++++++++++ internal/mcp/tools.go | 5 ++++ 5 files changed, 84 insertions(+), 7 deletions(-) diff --git a/docs/architecture.md b/docs/architecture.md index 24abb45..606d89e 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -203,7 +203,7 @@ A module rename also moves files whose names follow the module naming convention `protocol.WorkspaceEdit` from `go.lsp.dev/protocol` types `documentChanges` as `[]TextDocumentEdit` and cannot carry resource operations, so `internal/lsp/workspace_edit.go` defines the wire types and `renameHandler` answers `textDocument/rename` ahead of the generated dispatcher. A client that understands `documentChanges` ignores `changes` entirely, so once one file moves, every edit in the reply goes through `documentChanges`. -The MCP rename tool runs in the workspace daemon on its headless language service (`RenameFunction`/`RenameModule` in `internal/lsp/api.go`). That service holds no editor buffers, so the builders write every affected file and move closed files themselves, exactly as for closed files above, and the summary lists the files changed, moved, and not written. Renames run one at a time across the workspace: `IndexCoordinator.renameSerial`, which editor and MCP renames both take, is held from the first read of the affected files to the end of the writes, so two renames cannot write over each other's edits. These writes do not yet look at buffers that an editor attached to the same daemon has open; the MCP rename will adopt the shared rule for frontends without an editor (never write over unsaved editor work) when it lands in `internal/lsp`. +The MCP rename tool runs in the workspace daemon on its headless language service (`RenameFunction`/`RenameModule` in `internal/lsp/api.go`). That service holds no editor buffers, so the builders write every affected file and move closed files themselves, exactly as for closed files above, and the summary lists the files changed, moved, and not written. Renames run one at a time across the workspace: `IndexCoordinator.renameSerial`, which editor and MCP renames both take, is held from the first read of the affected files to the end of the writes, so two renames cannot write over each other's edits. A rename that takes it first waits until the index shows the previous rename, so its check that the new name is free sees that rename. These writes do not yet look at buffers that an editor attached to the same daemon has open; the MCP rename will adopt the shared rule for frontends without an editor (never write over unsaved editor work) when it lands in `internal/lsp`. ### Grouped aliases diff --git a/internal/lsp/server.go b/internal/lsp/server.go index 352ae12..a7132b4 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -101,8 +101,11 @@ type IndexCoordinator struct { // workspace, editor and headless alike. A rename reads the affected files // and writes them back; two at once would each write over the other's // edits. It is held from the first read of the affected files to the end - // of the writes, and no other request takes it. - renameSerial sync.Mutex + // of the writes, and no other request takes it. renameIndexed, guarded by + // it, closes when the index shows the last rename, so the next rename + // checks its new name against a current index. + renameSerial sync.Mutex + renameIndexed <-chan struct{} // writes is held for writing by a cold full build and for reading by every // single-file write. The bulk path is insert-only and cannot overlap any @@ -127,6 +130,16 @@ type IndexCoordinator struct { cancelWork context.CancelFunc } +// lockRename takes renameSerial and waits until the index shows the last +// rename. It returns the unlock. +func (ic *IndexCoordinator) lockRename() func() { + ic.renameSerial.Lock() + if ic.renameIndexed != nil { + <-ic.renameIndexed + } + return ic.renameSerial.Unlock +} + // CancelWork stops the reconciliation in flight and makes every later one // return at once. The workspace calls it first when it shuts down, so a pass // over a large change set cannot keep the process (and its workspace lock) @@ -5770,8 +5783,11 @@ func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) // module.functionName to newName across the codebase. When report is not nil, // it receives the files the rename changed and the files it could not change. func (s *Server) renameFunctionEdits(module, functionName, newName string, report *RenameSummary) (*WorkspaceEdit, error) { - s.index.renameSerial.Lock() - defer s.index.renameSerial.Unlock() + defer s.index.lockRename()() + // The check before the lock can race another rename to the same name. + if existing, err := s.store.LookupFunction(module, newName); err == nil && len(existing) > 0 { + return nil, fmt.Errorf("function %s.%s already exists", module, newName) + } // Collect all (filePath, lineNumber) pairs — definitions + references type siteKey struct { filePath string @@ -5976,8 +5992,7 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string, repor // report is not nil, it receives the files the rename changed or moved and the // files it could not change. func (s *Server) renameModuleEdits(oldModule, newModule string, report *RenameSummary) (*WorkspaceEdit, error) { - s.index.renameSerial.Lock() - defer s.index.renameSerial.Unlock() + defer s.index.lockRename()() mr := s.buildModuleRename(oldModule, newModule) // Check for collisions: verify that none of the target module names @@ -6804,6 +6819,7 @@ func (s *Server) buildTextEdits(sites []renameSite, oldToken, newToken string, r // The returned channel is closed when the index shows the rename. func (s *Server) reindexAfterRename(removePaths, diskPaths []string, textPaths []textReindex) <-chan struct{} { done := make(chan struct{}) + s.index.renameIndexed = done // the caller holds renameSerial s.index.backgroundWork.Add(1) go func() { defer s.index.backgroundWork.Done() diff --git a/internal/mcp/rename.go b/internal/mcp/rename.go index fe26ee5..27416b0 100644 --- a/internal/mcp/rename.go +++ b/internal/mcp/rename.go @@ -30,6 +30,10 @@ func (h *Handler) rename(ctx context.Context, args RenameParams) (string, error) if function != "" { target = fmt.Sprintf("%s.%s to %s", module, function, newName) } + // The last point where a canceled rename has changed nothing. + if err := ctx.Err(); err != nil { + return "", fmt.Errorf("the rename was canceled before it changed any file: %w", err) + } summary, err := renameSymbol(h.lsp, module, function, newName) if err != nil { return "", err diff --git a/internal/mcp/rename_test.go b/internal/mcp/rename_test.go index e4a8a20..b0d3507 100644 --- a/internal/mcp/rename_test.go +++ b/internal/mcp/rename_test.go @@ -9,6 +9,7 @@ import ( "strings" "sync" "testing" + "time" ) func readFile(t *testing.T, root, rel string) string { @@ -153,3 +154,54 @@ end } } } + +// Regression: a call canceled while it waited for the index still ran, so a +// rename could start its writes after the client was told that it may have +// been applied. +func TestRenameTool_CanceledCallChangesNothing(t *testing.T) { + e := setupProject(t) + h := NewHandler(e.rt, e.lsp) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + args, _ := json.Marshal(map[string]any{"module": "MyApp.Accounts", "function": "fetch_user", "new_name": "get_user"}) + if _, err := h.Call(ctx, renameToolName, args, time.Second); err == nil { + t.Fatal("a canceled rename succeeded") + } + wantContains(t, readFile(t, e.root, "lib/my_app/worker.ex"), "MyApp.Accounts.fetch_user(1)") + wantNotContains(t, readFile(t, e.root, "lib/my_app/accounts.ex"), "get_user") +} + +// Regression: the check that the new name is free ran before the rename +// lock, so two renames to the same new name could both pass it and both +// write. +func TestRenameTool_ConcurrentRenamesToOneNameOnlyOneWins(t *testing.T) { + e := setupProject(t) + h := NewHandler(e.rt, e.lsp) + current := [2]string{"fetch_user", "list_users"} + for round := 0; round < 8; round++ { + target := fmt.Sprintf("taken_%d", round) + var wg sync.WaitGroup + errs := make([]error, 2) + for i := range current { + wg.Add(1) + go func() { + defer wg.Done() + args, _ := json.Marshal(map[string]any{"module": "MyApp.Accounts", "function": current[i], "new_name": target}) + _, errs[i] = h.Call(context.Background(), renameToolName, args, 0) + }() + } + wg.Wait() + wins := 0 + for i, err := range errs { + if err == nil { + wins++ + current[i] = target + } else if !strings.Contains(err.Error(), "already exists") { + t.Fatalf("round %d: unexpected error: %v", round, err) + } + } + if wins != 1 { + t.Fatalf("round %d: %d renames to %s succeeded, want 1:\n%s", round, wins, target, readFile(t, e.root, "lib/my_app/accounts.ex")) + } + } +} diff --git a/internal/mcp/tools.go b/internal/mcp/tools.go index ebef002..3172c60 100644 --- a/internal/mcp/tools.go +++ b/internal/mcp/tools.go @@ -167,6 +167,11 @@ func (h *Handler) Call(ctx context.Context, name string, args json.RawMessage, w cancel() } + // A call canceled during the wait does not run: a rename must not start + // its writes after the client was told that it may have been applied. + if err := ctx.Err(); err != nil { + return "", err + } call := &Handler{rt: h.rt, lsp: h.lsp, store: h.store, projectRoot: h.projectRoot} text, err := spec.run(call, ctx, args) var noteList []string From b8067ee670fbe6e39a3d1228fee77855c386e26b Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sat, 3 Oct 2026 23:47:19 -0400 Subject: [PATCH 25/26] Let a canceled rename stop while it waits for the rename lock The wait for the previous rename's index update now ends with the request context, and a rename that gets renameSerial after its context ended returns without a write. RenameFunctionContext and RenameModuleContext carry the context; the MCP rename tool uses them, and the editor rename passes its request context. Co-Authored-By: Claude Opus 5.5 --- internal/lsp/api.go | 18 ++++++++-- internal/lsp/api_test.go | 75 ++++++++++++++++++++++++++++++++++++++++ internal/lsp/server.go | 36 +++++++++++++------ internal/mcp/rename.go | 8 ++--- 4 files changed, 121 insertions(+), 16 deletions(-) diff --git a/internal/lsp/api.go b/internal/lsp/api.go index 6552a76..bfc443c 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -1,6 +1,7 @@ package lsp import ( + "context" "fmt" "sort" "sync" @@ -169,6 +170,13 @@ func sortedUnique(paths []string) []string { // is written on disk. The workspace daemon calls it on its headless language // service, which holds no editor buffers. func (s *Server) RenameFunction(module, functionName, newName string) (*RenameSummary, error) { + return s.RenameFunctionContext(context.Background(), module, functionName, newName) +} + +// RenameFunctionContext is RenameFunction for a caller that can cancel. A +// rename canceled before it starts to write changes nothing; one canceled +// during its writes completes. +func (s *Server) RenameFunctionContext(ctx context.Context, module, functionName, newName string) (*RenameSummary, error) { if !isValidFunctionName(newName) { return nil, fmt.Errorf("invalid function name %q: must match [a-z_][a-z0-9_?!]*", newName) } @@ -184,7 +192,7 @@ func (s *Server) RenameFunction(module, functionName, newName string) (*RenameSu } summary := &RenameSummary{} - edit, err := s.renameFunctionEdits(module, functionName, newName, summary) + edit, err := s.renameFunctionEdits(ctx, module, functionName, newName, summary) summary.finish() if err != nil { return nil, err @@ -200,6 +208,12 @@ func (s *Server) RenameFunction(module, functionName, newName string) (*RenameSu // including the moves of files that follow the naming convention. It returns // when the index shows the rename. func (s *Server) RenameModule(oldModule, newModule string) (*RenameSummary, error) { + return s.RenameModuleContext(context.Background(), oldModule, newModule) +} + +// RenameModuleContext is RenameModule for a caller that can cancel, with the +// same rule as RenameFunctionContext. +func (s *Server) RenameModuleContext(ctx context.Context, oldModule, newModule string) (*RenameSummary, error) { if !isValidModuleName(newModule) { return nil, fmt.Errorf("invalid module name %q: must be CamelCase segments separated by dots", newModule) } @@ -212,7 +226,7 @@ func (s *Server) RenameModule(oldModule, newModule string) (*RenameSummary, erro } summary := &RenameSummary{} - edit, err := s.renameModuleEdits(oldModule, newModule, summary) + edit, err := s.renameModuleEdits(ctx, oldModule, newModule, summary) summary.finish() if err != nil { return nil, err diff --git a/internal/lsp/api_test.go b/internal/lsp/api_test.go index 87fbc6c..2dea81a 100644 --- a/internal/lsp/api_test.go +++ b/internal/lsp/api_test.go @@ -1,11 +1,13 @@ package lsp import ( + "context" "os" "path/filepath" "runtime" "strings" "testing" + "time" ) // A rename without an editor writes every changed file, reports the files, and @@ -177,3 +179,76 @@ end }) } } + +// Regression: a rename that waited for the rename lock, or for the index to +// show the previous rename, ignored its context, so a rename that the client +// had canceled could still start to write. +func TestRenameContext_CanceledWhileWaitingChangesNothing(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", "defmodule MyApp.Accounts do\n def fetch_user(id), do: id\nend\n") + path := filepath.Join(server.projectRoot, "lib/accounts.ex") + + // Another rename holds the lock, and its index update has not finished. + pending := make(chan struct{}) + server.index.renameSerial.Lock() + server.index.renameIndexed = pending + + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + _, err := server.RenameFunctionContext(ctx, "MyApp.Accounts", "fetch_user", "get_user") + errc <- err + }() + time.Sleep(20 * time.Millisecond) + cancel() + server.index.renameSerial.Unlock() // the other rename's writes end; its index update does not + + select { + case err := <-errc: + if err == nil || !strings.Contains(err.Error(), "canceled before it changed any file") { + t.Fatalf("error = %v, want a cancellation", err) + } + case <-time.After(5 * time.Second): + t.Fatal("the canceled rename kept waiting for the previous rename's index update") + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "get_user") { + t.Fatalf("a canceled rename wrote the file:\n%s", data) + } + close(pending) +} + +// A rename canceled while another rename held the lock does not write once +// it gets the lock. +func TestRenameContext_CanceledWhileLockedChangesNothing(t *testing.T) { + server, cleanup := setupTestServer(t) + defer cleanup() + indexFile(t, server.store, server.projectRoot, "lib/accounts.ex", "defmodule MyApp.Accounts do\n def fetch_user(id), do: id\nend\n") + path := filepath.Join(server.projectRoot, "lib/accounts.ex") + + server.index.renameSerial.Lock() + ctx, cancel := context.WithCancel(context.Background()) + errc := make(chan error, 1) + go func() { + _, err := server.RenameModuleContext(ctx, "MyApp.Accounts", "MyApp.Users") + errc <- err + }() + time.Sleep(20 * time.Millisecond) + cancel() + server.index.renameSerial.Unlock() + + if err := <-errc; err == nil || !strings.Contains(err.Error(), "canceled before it changed any file") { + t.Fatalf("error = %v, want a cancellation", err) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(data), "MyApp.Users") { + t.Fatalf("a canceled rename wrote the file:\n%s", data) + } +} diff --git a/internal/lsp/server.go b/internal/lsp/server.go index a7132b4..dae643d 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -131,13 +131,21 @@ type IndexCoordinator struct { } // lockRename takes renameSerial and waits until the index shows the last -// rename. It returns the unlock. -func (ic *IndexCoordinator) lockRename() func() { +// rename. It returns the unlock. When ctx ends first, it returns ctx's error +// and holds nothing, so a canceled rename does not start to write. +func (ic *IndexCoordinator) lockRename(ctx context.Context) (func(), error) { ic.renameSerial.Lock() if ic.renameIndexed != nil { - <-ic.renameIndexed + select { + case <-ic.renameIndexed: + case <-ctx.Done(): + } + } + if err := ctx.Err(); err != nil { + ic.renameSerial.Unlock() + return nil, fmt.Errorf("the rename was canceled before it changed any file: %w", err) } - return ic.renameSerial.Unlock + return ic.renameSerial.Unlock, nil } // CancelWork stops the reconciliation in flight and makes every later one @@ -5757,7 +5765,7 @@ func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) if existing, err := s.store.LookupFunction(fullModule, params.NewName); err == nil && len(existing) > 0 { return nil, fmt.Errorf("function %s.%s already exists", fullModule, params.NewName) } - return s.renameFunctionEdits(fullModule, functionName, params.NewName, nil) + return s.renameFunctionEdits(ctx, fullModule, functionName, params.NewName, nil) } } else if moduleRef != "" { fullModule := resolveModule(moduleRef, aliases) @@ -5771,7 +5779,7 @@ func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) if !isValidModuleName(newModule) { return nil, fmt.Errorf("invalid module name %q: must be CamelCase segments separated by dots", params.NewName) } - return s.renameModuleEdits(fullModule, newModule, nil) + return s.renameModuleEdits(ctx, fullModule, newModule, nil) } } } @@ -5782,8 +5790,12 @@ func (s *Server) RenameEdit(ctx context.Context, params *protocol.RenameParams) // renameFunctionEdits builds a WorkspaceEdit renaming all occurrences of // module.functionName to newName across the codebase. When report is not nil, // it receives the files the rename changed and the files it could not change. -func (s *Server) renameFunctionEdits(module, functionName, newName string, report *RenameSummary) (*WorkspaceEdit, error) { - defer s.index.lockRename()() +func (s *Server) renameFunctionEdits(ctx context.Context, module, functionName, newName string, report *RenameSummary) (*WorkspaceEdit, error) { + unlock, err := s.index.lockRename(ctx) + if err != nil { + return nil, err + } + defer unlock() // The check before the lock can race another rename to the same name. if existing, err := s.store.LookupFunction(module, newName); err == nil && len(existing) > 0 { return nil, fmt.Errorf("function %s.%s already exists", module, newName) @@ -5991,8 +6003,12 @@ func (s *Server) renameFunctionEdits(module, functionName, newName string, repor // by the server, open ones by the client through rename operations. When // report is not nil, it receives the files the rename changed or moved and the // files it could not change. -func (s *Server) renameModuleEdits(oldModule, newModule string, report *RenameSummary) (*WorkspaceEdit, error) { - defer s.index.lockRename()() +func (s *Server) renameModuleEdits(ctx context.Context, oldModule, newModule string, report *RenameSummary) (*WorkspaceEdit, error) { + unlock, err := s.index.lockRename(ctx) + if err != nil { + return nil, err + } + defer unlock() mr := s.buildModuleRename(oldModule, newModule) // Check for collisions: verify that none of the target module names diff --git a/internal/mcp/rename.go b/internal/mcp/rename.go index 27416b0..9a83c99 100644 --- a/internal/mcp/rename.go +++ b/internal/mcp/rename.go @@ -34,7 +34,7 @@ func (h *Handler) rename(ctx context.Context, args RenameParams) (string, error) if err := ctx.Err(); err != nil { return "", fmt.Errorf("the rename was canceled before it changed any file: %w", err) } - summary, err := renameSymbol(h.lsp, module, function, newName) + summary, err := renameSymbol(ctx, h.lsp, module, function, newName) if err != nil { return "", err } @@ -74,9 +74,9 @@ func (h *Handler) rename(ctx context.Context, args RenameParams) (string, error) // write over unsaved editor buffers: write closed files and clean open // files, and refuse with an actionable error when an affected file has unsaved // changes in an editor), call its headless entry point here instead. -func renameSymbol(server *lsp.Server, module, function, newName string) (*lsp.RenameSummary, error) { +func renameSymbol(ctx context.Context, server *lsp.Server, module, function, newName string) (*lsp.RenameSummary, error) { if function != "" { - return server.RenameFunction(module, function, newName) + return server.RenameFunctionContext(ctx, module, function, newName) } - return server.RenameModule(module, newName) + return server.RenameModuleContext(ctx, module, newName) } From 36e9fe17db0777e2a1fad91776d26e8b2248796e Mon Sep 17 00:00:00 2001 From: Jesse Herrick Date: Sun, 4 Oct 2026 00:35:00 -0400 Subject: [PATCH 26/26] Read only dirty editor buffers; map lines with a diff; 413 for big bodies A buffer that an editor holds open replaced the disk whenever the two differed, so a clean buffer that the editor had not reloaded yet hid the agent's own edits and was reported as unsaved work. The document store now marks a buffer dirty on didChange, with the time of the change, and clean on didOpen and didSave. MCP tools read a buffer only when it is dirty and the file on disk did not change after it. When both changed, the disk is read and the answer warns that the editor's unsaved changes may conflict. Index lines are mapped into a buffer with a bounded Myers line diff that ignores line endings, so only changed lines are marked and unchanged lines map exactly. A 10,000-line file costs about 0.3 ms. An HTTP body over 4 MB now gets 413 instead of 400. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- docs/daemon.md | 22 +-- internal/lsp/api.go | 10 +- internal/lsp/documents.go | 46 +++++- internal/lsp/server.go | 3 +- internal/mcp/http_test.go | 12 +- internal/mcp/instructions.md | 3 +- internal/mcp/linemap.go | 125 +++++++++++++++++ internal/mcp/serve.go | 31 +++- internal/mcp/source.go | 119 +++++++++------- internal/mcp/source_test.go | 257 ++++++++++++++++++++++++++++++++++ internal/workspace/runtime.go | 27 ++-- 12 files changed, 572 insertions(+), 85 deletions(-) create mode 100644 internal/mcp/linemap.go diff --git a/README.md b/README.md index ce44d5f..4ca6181 100644 --- a/README.md +++ b/README.md @@ -489,7 +489,7 @@ claude mcp add dexter -- dexter mcp Any client that speaks MCP over stdio works the same way: point it at `dexter mcp`. The server obtains its workspace from the client through MCP roots and resolves it the way the CLI does, so it binds the project the client is working in rather than the directory it was launched from. Clients that provide no roots get the launch directory (when it is a project), and an explicit path argument (`dexter mcp `) overrides negotiation entirely. One session serves one workspace: when a client gives several roots, the session uses the first one that is a usable project (an existing directory inside an Elixir project, not the home directory) and ignores the others. In `--listen` mode each resolved root gets its own workspace, so sessions from different projects can share one server. -`dexter mcp` is a frontend of the workspace daemon, like `dexter lsp` and the CLI: it starts the daemon when necessary and keeps no index of its own. The tools answer from the same index, watchers, and caches as the editor, so edits made directly by an agent are indexed by the daemon's file watcher, and a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. A tool that answers from an index that is still building or degraded says so in its answer. The tools read file text the way the user sees it: when an editor attached to the same daemon holds a file open with unsaved changes, outlines, definition snippets, and reference lines come from that buffer (with its line numbers), and the answer says so. The rename tool writes on disk and does not yet look at editor buffers, so save your editor's changes before an agent renames. +`dexter mcp` is a frontend of the workspace daemon, like `dexter lsp` and the CLI: it starts the daemon when necessary and keeps no index of its own. The tools answer from the same index, watchers, and caches as the editor, so edits made directly by an agent are indexed by the daemon's file watcher, and a `dexter_reindex` tool forces an immediate update if a lookup ever seems stale. A tool that answers from an index that is still building or degraded says so in its answer. The tools read file text the way the user sees it: when an editor attached to the same daemon holds a file open with changes it has not saved (and the file on disk has not changed since), outlines, definition snippets, and reference lines come from that buffer (with its line numbers), and the answer says so. The rename tool writes on disk and does not yet look at editor buffers, so save your editor's changes before an agent renames. Useful variants: diff --git a/docs/daemon.md b/docs/daemon.md index 7a8bab1..2b47a18 100644 --- a/docs/daemon.md +++ b/docs/daemon.md @@ -329,12 +329,18 @@ runs no LSP lifecycle of its own. until the index is complete, because a rename from a partial index would change some call sites and leave others with the old name. - **Editor buffers.** Tools that read file text (outlines, definition and - module docs, reference lines) use the newest buffer that any attached editor - session holds open when it differs from the disk, and the disk otherwise. - The answer names the files that came from unsaved buffers. The index - positions refer to the saved file, so a line is mapped into the buffer - through the lines that both share at the start and at the end; a position - in the changed part shows the saved line, marked as such. Paths that the + module docs, reference lines) use the newest buffer that an attached editor + session holds open with unsaved changes: the editor changed it after its + last open or save (`didChange` after `didOpen`/`didSave`). A buffer with no + unsaved changes can be older than the disk (an editor that has not reloaded + the file after an agent wrote it), so the disk is used. When the file on + disk changed after the buffer's last change, the disk is used too, and the + answer warns that the editor's unsaved changes may conflict. The answer + names the files that came from unsaved buffers. The index positions refer + to the saved file, so a line is mapped into the buffer with a line diff + (Myers, line endings ignored; bounded, with a fallback to the lines that + both texts share at the start and at the end); a position on a changed line + shows the saved line, marked as such. Paths that the agent gives must be inside the project root after symlinks are resolved, and must name a regular file of at most 10 MB. - **Limits and cancellation.** One frontend runs at most 32 tool calls at once @@ -346,8 +352,8 @@ runs no LSP lifecycle of its own. - **HTTP.** `--listen` accepts only a loopback address unless `--listen-unsafe` is given, because the server has no authentication. The SDK refuses a non-loopback `Host` on a loopback connection (DNS rebinding), - cross-origin browser requests are refused, a request body is capped at - 4 MB, and a session with no request for 30 minutes is closed, so a client + cross-origin browser requests are refused, a request body over 4 MB gets + 413, and a session with no request for 30 minutes is closed, so a client that went away does not keep its daemon alive. - **Reconnects.** When the daemon goes away (an upgrade replaced it, or `dexter stop --force`; a plain `dexter stop` is refused while MCP is diff --git a/internal/lsp/api.go b/internal/lsp/api.go index bfc443c..5764bda 100644 --- a/internal/lsp/api.go +++ b/internal/lsp/api.go @@ -5,6 +5,7 @@ import ( "fmt" "sort" "sync" + "time" "go.lsp.dev/uri" ) @@ -24,10 +25,11 @@ func (s *Server) FileLine(filePath string, lineNum int) (string, bool) { return s.getFileLine(filePath, lineNum) } -// OpenBuffer returns the text of filePath when an editor holds it open in this -// session. seq orders buffers across sessions: the higher one changed later. -func (s *Server) OpenBuffer(filePath string) (text string, seq uint64, ok bool) { - return s.docs.GetOpenSeq(string(uri.File(filePath))) +// UnsavedBuffer returns the text of filePath when an editor holds it open in +// this session with changes that it has not saved, and the time of the last +// change. seq orders buffers across sessions: the higher one changed later. +func (s *Server) UnsavedBuffer(filePath string) (text string, seq uint64, changedAt time.Time, ok bool) { + return s.docs.UnsavedBuffer(string(uri.File(filePath))) } // RenameSummary reports what a rename changed on disk. diff --git a/internal/lsp/documents.go b/internal/lsp/documents.go index 511da5b..2a55cc4 100644 --- a/internal/lsp/documents.go +++ b/internal/lsp/documents.go @@ -6,6 +6,7 @@ import ( "strings" "sync" "sync/atomic" + "time" tree_sitter "github.com/tree-sitter/go-tree-sitter" tree_sitter_elixir "github.com/tree-sitter/tree-sitter-elixir/bindings/go" @@ -36,6 +37,11 @@ type cachedDoc struct { // the higher one was set later. A frontend without an editor uses it to // pick the newest unsaved buffer when several editors hold one file. seq uint64 + // dirty is true when the editor changed the buffer after its last open + // or save, at changedAt. A clean buffer can be older than the disk (an + // editor that has not reloaded yet), so only a dirty one is unsaved work. + dirty bool + changedAt time.Time } // docSeq numbers editor-owned entries; see cachedDoc.seq. @@ -155,6 +161,31 @@ func (ds *DocumentStore) Set(uri string, text string) { ds.docs[uri] = &cachedDoc{text: text, seq: docSeq.Add(1)} } +// SetChanged is Set for an edit in the editor: the buffer has unsaved changes +// until MarkSaved. +func (ds *DocumentStore) SetChanged(uri string, text string) { + ds.mu.Lock() + defer ds.mu.Unlock() + if ds.closed { + return + } + if doc, ok := ds.docs[uri]; ok { + doc.tree.retireLocked() + } + ds.removeFromLRULocked(uri) + ds.docs[uri] = &cachedDoc{text: text, seq: docSeq.Add(1), dirty: true, changedAt: time.Now()} +} + +// MarkSaved records that the editor saved the buffer, so it has no unsaved +// changes. +func (ds *DocumentStore) MarkSaved(uri string) { + ds.mu.Lock() + defer ds.mu.Unlock() + if doc, ok := ds.docs[uri]; ok && !doc.transient { + doc.dirty = false + } +} + func (ds *DocumentStore) Close(uri string) { ds.mu.Lock() defer ds.mu.Unlock() @@ -214,16 +245,17 @@ func (ds *DocumentStore) GetIfOpen(uri string) (string, bool) { return doc.text, true } -// GetOpenSeq is GetIfOpen that also returns the entry's sequence number; see -// cachedDoc.seq. -func (ds *DocumentStore) GetOpenSeq(uri string) (string, uint64, bool) { +// UnsavedBuffer returns an editor-owned entry that has changes the editor has +// not saved, with its sequence number (see cachedDoc.seq) and the time of its +// last change. +func (ds *DocumentStore) UnsavedBuffer(uri string) (text string, seq uint64, changedAt time.Time, ok bool) { ds.mu.RLock() defer ds.mu.RUnlock() - doc, ok := ds.docs[uri] - if !ok || doc.transient { - return "", 0, false + doc, found := ds.docs[uri] + if !found || doc.transient || !doc.dirty { + return "", 0, time.Time{}, false } - return doc.text, doc.seq, true + return doc.text, doc.seq, doc.changedAt, true } // GetOrLoad returns the text for the given URI, falling back to a disk diff --git a/internal/lsp/server.go b/internal/lsp/server.go index dae643d..b375fb5 100644 --- a/internal/lsp/server.go +++ b/internal/lsp/server.go @@ -1117,7 +1117,7 @@ func (s *Server) DidChange(ctx context.Context, params *protocol.DidChangeTextDo if len(params.ContentChanges) > 0 { // Full sync mode — last change contains the full text text := params.ContentChanges[len(params.ContentChanges)-1].Text - s.docs.Set(string(params.TextDocument.URI), text) + s.docs.SetChanged(string(params.TextDocument.URI), text) } return nil } @@ -1143,6 +1143,7 @@ func (s *Server) restartBeamForFormatterConfig(path string) { } func (s *Server) DidSave(ctx context.Context, params *protocol.DidSaveTextDocumentParams) error { + s.docs.MarkSaved(string(params.TextDocument.URI)) path := uriToPath(params.TextDocument.URI) if path == "" { return nil diff --git a/internal/mcp/http_test.go b/internal/mcp/http_test.go index c23d7e1..31158ea 100644 --- a/internal/mcp/http_test.go +++ b/internal/mcp/http_test.go @@ -74,8 +74,16 @@ func TestHTTPHandler_CapsBodySize(t *testing.T) { } // Valid JSON, padded past the limit. padded := initializeBody + strings.Repeat(" ", maxHTTPBodyBytes) - if res := postMCP(t, srv.URL, bytes.NewReader([]byte(padded)), nil); res.StatusCode == http.StatusOK { - t.Fatalf("a %d-byte body was accepted", len(padded)) + if res := postMCP(t, srv.URL, bytes.NewReader([]byte(padded)), nil); res.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("a %d-byte body got status %d, want 413", len(padded), res.StatusCode) + } + // A chunked body has no Content-Length; its limit shows while it is read. + chunked := io.MultiReader(strings.NewReader(padded)) + if res := postMCP(t, srv.URL, chunked, nil); res.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("a chunked %d-byte body got status %d, want 413", len(padded), res.StatusCode) + } + if res := postMCP(t, srv.URL, io.MultiReader(strings.NewReader(initializeBody)), nil); res.StatusCode != http.StatusOK { + t.Fatalf("a small chunked initialize got status %d", res.StatusCode) } } diff --git a/internal/mcp/instructions.md b/internal/mcp/instructions.md index e1b6afe..027c520 100644 --- a/internal/mcp/instructions.md +++ b/internal/mcp/instructions.md @@ -32,4 +32,5 @@ ways (other macros, `unquote` names) can be missing from the index, so an empty lookup can mean macro-generated code. Answers read files the way the user sees them: a file open in an attached -editor with unsaved changes is read from that buffer, and the answer says so. +editor with unsaved changes is read from that buffer, unless the file on disk +changed later, and the answer says so. diff --git a/internal/mcp/linemap.go b/internal/mcp/linemap.go new file mode 100644 index 0000000..e061b4f --- /dev/null +++ b/internal/mcp/linemap.go @@ -0,0 +1,125 @@ +package mcp + +import "strings" + +// Bounds of the line diff. On a 10,000-line file, a few edits cost about +// 0.3 ms and 900 edits about 2 ms and 5 MB; outside the bounds lineMap falls +// back to the lines that both texts share at the start and at the end. +const ( + maxDiffLines = 50000 // lines of both texts after the shared start and end + maxDiffEdits = 1000 // inserted plus deleted lines +) + +// lineMap maps each 1-based line of the saved file (where the index puts +// definitions and references) to the same line in an editor buffer, or to 0 +// when the buffer changed or deleted that line. A line counts as equal +// whatever its line ending, so a buffer that differs from the disk only in +// CRLF against LF changes no line. +type lineMap struct { + to []int32 // to[i] is the buffer line of saved line i; index 0 is unused +} + +func newLineMap(disk, text string) lineMap { + a, b := splitLines(disk), splitLines(text) + m := lineMap{to: make([]int32, len(a)+1)} + prefix := 0 + for prefix < len(a) && prefix < len(b) && a[prefix] == b[prefix] { + m.to[prefix+1] = int32(prefix + 1) + prefix++ + } + suffix := 0 + for suffix < len(a)-prefix && suffix < len(b)-prefix && a[len(a)-1-suffix] == b[len(b)-1-suffix] { + m.to[len(a)-suffix] = int32(len(b) - suffix) + suffix++ + } + midA, midB := a[prefix:len(a)-suffix], b[prefix:len(b)-suffix] + if len(midA) == 0 || len(midB) == 0 || len(midA)+len(midB) > maxDiffLines { + return m + } + matchLines(midA, midB, func(i, j int) { + m.to[prefix+i+1] = int32(prefix + j + 1) + }) + return m +} + +// locate returns the buffer line of saved line n, or false when the buffer +// changed it. +func (m lineMap) locate(n int) (int, bool) { + if n < 1 || n >= len(m.to) || m.to[n] == 0 { + return n, false + } + return int(m.to[n]), true +} + +// splitLines splits text into lines without their line endings. +func splitLines(text string) []string { + lines := strings.Split(text, "\n") + for i, l := range lines { + lines[i] = strings.TrimSuffix(l, "\r") + } + return lines +} + +// matchLines reports the pairs of equal lines of a shortest edit script from a +// to b (Myers, "An O(ND) Difference Algorithm"), in order. It reports nothing +// when the script needs more than maxDiffEdits edits. +func matchLines(a, b []string, match func(i, j int)) { + n, m := len(a), len(b) + maxD := n + m + if maxD > maxDiffEdits { + maxD = maxDiffEdits + } + off := maxD + 1 + v := make([]int32, 2*maxD+3) + // trace[d] is v[off-d-1 : off+d+2] before step d, for the backtrack. + var trace [][]int32 + for d := 0; d <= maxD; d++ { + trace = append(trace, append([]int32(nil), v[off-d-1:off+d+2]...)) + for k := -d; k <= d; k += 2 { + var x int + if k == -d || (k != d && v[off+k-1] < v[off+k+1]) { + x = int(v[off+k+1]) + } else { + x = int(v[off+k-1]) + 1 + } + y := x - k + for x < n && y < m && a[x] == b[y] { + x++ + y++ + } + v[off+k] = int32(x) + if x >= n && y >= m { + backtrack(trace, n, m, match) + return + } + } + } +} + +func backtrack(trace [][]int32, n, m int, match func(i, j int)) { + type pair struct{ i, j int } + var pairs []pair + x, y := n, m + for d := len(trace) - 1; d >= 0; d-- { + v := trace[d] // v[k] is at index k+d+1 + at := func(k int) int { return int(v[k+d+1]) } + k := x - y + var prevK int + if k == -d || (k != d && at(k-1) < at(k+1)) { + prevK = k + 1 + } else { + prevK = k - 1 + } + prevX := at(prevK) + prevY := prevX - prevK + for x > prevX && y > prevY { + x-- + y-- + pairs = append(pairs, pair{x, y}) + } + x, y = prevX, prevY + } + for i := len(pairs) - 1; i >= 0; i-- { + match(pairs[i].i, pairs[i].j) + } +} diff --git a/internal/mcp/serve.go b/internal/mcp/serve.go index f029c42..0b148b5 100644 --- a/internal/mcp/serve.go +++ b/internal/mcp/serve.go @@ -1,8 +1,11 @@ package mcp import ( + "bytes" "context" + "errors" "fmt" + "io" "net" "net/http" "time" @@ -36,7 +39,33 @@ func HTTPHandler(f *Frontend) http.Handler { h := mcp.NewStreamableHTTPHandler(func(*http.Request) *mcp.Server { return NewServer(f) }, &mcp.StreamableHTTPOptions{ SessionTimeout: httpSessionTimeout, }) - return http.MaxBytesHandler(http.NewCrossOriginProtection().Handler(h), maxHTTPBodyBytes) + return limitBody(http.NewCrossOriginProtection().Handler(h), maxHTTPBodyBytes) +} + +// limitBody reads a request body of at most limit bytes before next sees the +// request, and answers a longer one with 413. The SDK turns a read error into +// 400, so the limit cannot be left to a reader that next drains. +func limitBody(next http.Handler, limit int64) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.ContentLength > limit { + http.Error(w, "request body too large", http.StatusRequestEntityTooLarge) + return + } + if r.Body != nil && r.Body != http.NoBody { + body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, limit)) + if err != nil { + var tooLarge *http.MaxBytesError + if errors.As(err, &tooLarge) { + http.Error(w, "request body too large", http.StatusRequestEntityTooLarge) + return + } + http.Error(w, "failed to read the request body", http.StatusBadRequest) + return + } + r.Body = io.NopCloser(bytes.NewReader(body)) + } + next.ServeHTTP(w, r) + }) } // CheckListenAddr refuses an HTTP listen address that other machines can diff --git a/internal/mcp/source.go b/internal/mcp/source.go index 334970b..6353c6a 100644 --- a/internal/mcp/source.go +++ b/internal/mcp/source.go @@ -10,6 +10,7 @@ import ( "sort" "strings" "sync" + "time" "github.com/remoteoss/dexter/internal/lsp" ) @@ -26,6 +27,9 @@ type sourceCache struct { mu sync.Mutex views map[string]*sourceView // nil for a file that cannot be read unsaved map[string]struct{} + // conflicts are files read from disk that also have older unsaved + // changes in an editor. + conflicts map[string]struct{} } // errNotFound reports a file that does not exist. @@ -52,7 +56,7 @@ func (h *Handler) userPath(p string) (string, error) { // path inside the root as spelled can name one. if rel, ok := inside(h.projectRoot, candidate); ok { path := filepath.Join(h.projectRoot, rel) - if _, open := h.rt.EditorBuffer(path); open { + if _, open := h.rt.UnsavedBuffer(path); open { return path, nil } return path, errNotFound @@ -85,37 +89,38 @@ func inside(root, path string) (string, bool) { // readRegularFile reads a regular file of at most maxSourceBytes. Devices, // FIFOs, sockets, and directories are refused before they are opened, so a // read can neither block nor run without end. -func readRegularFile(path string) (string, error) { +func readRegularFile(path string) (string, time.Time, error) { info, err := os.Stat(path) if err != nil { if errors.Is(err, fs.ErrNotExist) { - return "", errNotFound + return "", time.Time{}, errNotFound } - return "", err + return "", time.Time{}, err } if !info.Mode().IsRegular() { - return "", fmt.Errorf("%s is not a regular file", path) + return "", time.Time{}, fmt.Errorf("%s is not a regular file", path) } if info.Size() > maxSourceBytes { - return "", fmt.Errorf("%s is %d bytes, more than the %d MB limit for one file", path, info.Size(), maxSourceBytes>>20) + return "", time.Time{}, fmt.Errorf("%s is %d bytes, more than the %d MB limit for one file", path, info.Size(), maxSourceBytes>>20) } f, err := os.Open(path) if err != nil { - return "", err + return "", time.Time{}, err } defer func() { _ = f.Close() }() // The path can change between the stat and the open. - if opened, err := f.Stat(); err != nil || !os.SameFile(info, opened) { - return "", fmt.Errorf("%s changed while it was read; retry", path) + opened, err := f.Stat() + if err != nil || !os.SameFile(info, opened) { + return "", time.Time{}, fmt.Errorf("%s changed while it was read; retry", path) } data, err := io.ReadAll(io.LimitReader(f, maxSourceBytes+1)) if err != nil { - return "", err + return "", time.Time{}, err } if len(data) > maxSourceBytes { - return "", fmt.Errorf("%s is more than the %d MB limit for one file", path, maxSourceBytes>>20) + return "", time.Time{}, fmt.Errorf("%s is more than the %d MB limit for one file", path, maxSourceBytes>>20) } - return string(data), nil + return string(data), opened.ModTime(), nil } // sourceView is one file as a tool call reads it. When an attached editor @@ -126,27 +131,17 @@ type sourceView struct { unsaved bool disk string hasDisk bool - // The lines that the buffer and the saved file share at the start and at - // the end. An index line in one of them is found in the buffer; a line - // between them was changed in the buffer. - prefix, suffix int - diskLines, textLines int + // lines maps the saved file's lines, which the index positions refer + // to, into the buffer. + lines lineMap tf, diskTF *lsp.TokenizedFile } func newSourceView(text string, unsaved bool, disk string, hasDisk bool) *sourceView { v := &sourceView{text: text, unsaved: unsaved, disk: disk, hasDisk: hasDisk} - if !unsaved || !hasDisk { - return v - } - a, b := strings.Split(disk, "\n"), strings.Split(text, "\n") - v.diskLines, v.textLines = len(a), len(b) - for v.prefix < len(a) && v.prefix < len(b) && a[v.prefix] == b[v.prefix] { - v.prefix++ - } - for v.suffix < len(a)-v.prefix && v.suffix < len(b)-v.prefix && a[len(a)-1-v.suffix] == b[len(b)-1-v.suffix] { - v.suffix++ + if unsaved && hasDisk { + v.lines = newLineMap(disk, text) } return v } @@ -155,13 +150,10 @@ func newSourceView(text string, unsaved bool, disk string, hasDisk bool) *source // definition or a reference, to the same line in the text. inText is false // when the buffer changed that line. func (v *sourceView) locate(indexLine int) (line int, inText bool) { - if !v.unsaved || !v.hasDisk || indexLine <= v.prefix { + if !v.unsaved || !v.hasDisk { return indexLine, true } - if indexLine > v.diskLines-v.suffix { - return indexLine + v.textLines - v.diskLines, true - } - return indexLine, false + return v.lines.locate(indexLine) } func (v *sourceView) tokenized(inText bool) *lsp.TokenizedFile { @@ -239,7 +231,7 @@ func (h *Handler) view(path string) (*sourceView, error) { } c.mu.Unlock() - v, err := h.loadSource(path) + v, conflict, err := h.loadSource(path) c.mu.Lock() defer c.mu.Unlock() @@ -253,21 +245,35 @@ func (h *Handler) view(path string) (*sourceView, error) { } c.unsaved[path] = struct{}{} } + if conflict { + if c.conflicts == nil { + c.conflicts = make(map[string]struct{}) + } + c.conflicts[path] = struct{}{} + } return v, err } -func (h *Handler) loadSource(path string) (*sourceView, error) { - disk, diskErr := readRegularFile(path) - if buffer, open := h.rt.EditorBuffer(path); open { - if diskErr == nil && disk == buffer { - return newSourceView(disk, false, "", false), nil +// loadSource reads path as the user sees it. A buffer that an editor holds +// open is used only when it has changes that the editor has not saved and the +// file on disk did not change after them. When both changed, the disk wins, +// because the index follows the disk, and conflict is true. +func (h *Handler) loadSource(path string) (v *sourceView, conflict bool, err error) { + disk, mtime, diskErr := readRegularFile(path) + if buffer, open := h.rt.UnsavedBuffer(path); open { + switch { + case diskErr == nil && disk == buffer.Text: + return newSourceView(disk, false, "", false), false, nil + case diskErr == nil && mtime.After(buffer.ChangedAt): + return newSourceView(disk, false, "", false), true, nil + default: + return newSourceView(buffer.Text, true, disk, diskErr == nil), false, nil } - return newSourceView(buffer, true, disk, diskErr == nil), nil } if diskErr != nil { - return nil, diskErr + return nil, false, diskErr } - return newSourceView(disk, false, "", false), nil + return newSourceView(disk, false, "", false), false, nil } // sourceLine returns the text of one index position. A file that no editor @@ -279,7 +285,7 @@ func (h *Handler) sourceLine(path string, indexLine int) (text string, line int, _, cached := c.views[path] c.mu.Unlock() if !cached { - if _, open := h.rt.EditorBuffer(path); !open { + if _, open := h.rt.UnsavedBuffer(path); !open { text, ok := h.lsp.FileLine(path, indexLine) return text, indexLine, "", ok } @@ -313,18 +319,29 @@ func nthLine(text string, n int) (string, bool) { } // unsavedNote names the files whose text came from unsaved editor buffers in -// this call, or is empty when there are none. +// this call, and the files that changed on disk after an editor's unsaved +// changes to them, or is empty when there are none. func (h *Handler) unsavedNote() string { c := &h.sources c.mu.Lock() - paths := make([]string, 0, len(c.unsaved)) - for p := range c.unsaved { - paths = append(paths, h.relPath(p)) - } + unsaved := h.relPaths(c.unsaved) + conflicts := h.relPaths(c.conflicts) c.mu.Unlock() - if len(paths) == 0 { - return "" + var notes []string + if len(unsaved) > 0 { + notes = append(notes, "Note: read from unsaved editor buffers (the files on disk differ; line numbers are the buffer's): "+strings.Join(unsaved, ", ")) + } + if len(conflicts) > 0 { + notes = append(notes, "Note: read from disk, but an editor also has unsaved changes to these files, made before the files on disk changed; the two may conflict: "+strings.Join(conflicts, ", ")) + } + return strings.Join(notes, "\n") +} + +func (h *Handler) relPaths(set map[string]struct{}) []string { + out := make([]string, 0, len(set)) + for p := range set { + out = append(out, h.relPath(p)) } - sort.Strings(paths) - return "Note: read from unsaved editor buffers (the files on disk differ; line numbers are the buffer's): " + strings.Join(paths, ", ") + sort.Strings(out) + return out } diff --git a/internal/mcp/source_test.go b/internal/mcp/source_test.go index 1126489..5789a02 100644 --- a/internal/mcp/source_test.go +++ b/internal/mcp/source_test.go @@ -3,13 +3,17 @@ package mcp import ( "context" "fmt" + "math/rand/v2" "os" "path/filepath" "strings" "testing" + "time" "go.lsp.dev/protocol" "go.lsp.dev/uri" + + "github.com/remoteoss/dexter/internal/lsp" ) // Regression: dexter_file_outline read any path the agent gave, so an @@ -136,3 +140,256 @@ func TestSourceViewLocate(t *testing.T) { } } } + +// editorSession is an attached editor session that a test drives. +type editorSession struct { + t *testing.T + root string + srv *lsp.Server +} + +func attachEditor(t *testing.T, e *testEnv) *editorSession { + t.Helper() + _, session, release := e.rt.AttachLSPSession() + t.Cleanup(release) + return &editorSession{t: t, root: e.root, srv: session} +} + +func (s *editorSession) uri(rel string) protocol.DocumentURI { + return protocol.DocumentURI(uri.File(filepath.Join(s.root, rel))) +} + +func (s *editorSession) open(rel, text string) { + s.t.Helper() + if err := s.srv.DidOpen(context.Background(), &protocol.DidOpenTextDocumentParams{ + TextDocument: protocol.TextDocumentItem{URI: s.uri(rel), LanguageID: "elixir", Version: 1, Text: text}, + }); err != nil { + s.t.Fatal(err) + } +} + +func (s *editorSession) change(rel, text string) { + s.t.Helper() + if err := s.srv.DidChange(context.Background(), &protocol.DidChangeTextDocumentParams{ + TextDocument: protocol.VersionedTextDocumentIdentifier{TextDocumentIdentifier: protocol.TextDocumentIdentifier{URI: s.uri(rel)}}, + ContentChanges: []protocol.TextDocumentContentChangeEvent{{Text: text}}, + }); err != nil { + s.t.Fatal(err) + } +} + +func (s *editorSession) save(rel string) { + s.t.Helper() + if err := s.srv.DidSave(context.Background(), &protocol.DidSaveTextDocumentParams{ + TextDocument: protocol.TextDocumentIdentifier{URI: s.uri(rel)}, + }); err != nil { + s.t.Fatal(err) + } +} + +// writeLater writes a file with a modification time after every editor change +// so far, as an agent's write that follows them does. +func writeLater(t *testing.T, path, text string) { + t.Helper() + time.Sleep(20 * time.Millisecond) + if err := os.WriteFile(path, []byte(text), 0644); err != nil { + t.Fatal(err) + } +} + +const agentWorkerSource = "# agent\n# notes\n" + workerSource + "\ndefmodule MyApp.AgentAdded do\n def added(x), do: x\nend\n" + +// Regression: an editor buffer that the user never changed, but that is older +// than the disk (the editor has not reloaded the file after the agent wrote +// it), hid the agent's edits and was reported as unsaved work. +func TestTools_CleanStaleBufferReadsDisk(t *testing.T) { + e := setupProject(t) + ed := attachEditor(t, e) + ed.open("lib/my_app/worker.ex", workerSource) + path := filepath.Join(e.root, "lib/my_app/worker.ex") + writeLater(t, path, agentWorkerSource) + + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantContains(t, out, "defmodule MyApp.AgentAdded") + wantNotContains(t, out, "unsaved") +} + +// A buffer with unsaved changes, made after the disk last changed, is what the +// user sees, so it wins. +func TestTools_DirtyBufferWins(t *testing.T) { + e := setupProject(t) + ed := attachEditor(t, e) + ed.open("lib/my_app/worker.ex", workerSource) + ed.change("lib/my_app/worker.ex", workerSource+"\ndefmodule MyApp.Typed do\nend\n") + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantContains(t, out, "defmodule MyApp.Typed", "read from unsaved editor buffers") +} + +// When both the buffer and the disk changed, and the disk changed later, the +// disk wins (the index follows it), and the answer warns about the editor's +// older unsaved changes. +func TestTools_DirtyBufferOlderThanDiskReadsDiskWithNote(t *testing.T) { + e := setupProject(t) + ed := attachEditor(t, e) + ed.open("lib/my_app/worker.ex", workerSource) + ed.change("lib/my_app/worker.ex", workerSource+"\ndefmodule MyApp.Typed do\nend\n") + writeLater(t, filepath.Join(e.root, "lib/my_app/worker.ex"), agentWorkerSource) + + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantContains(t, out, "defmodule MyApp.AgentAdded", "an editor also has unsaved changes", "may conflict") + wantNotContains(t, out, "MyApp.Typed", "read from unsaved editor buffers") +} + +// A save ends the buffer's unsaved changes: a later write by the agent is +// read from disk with no warning. +func TestTools_SaveClearsUnsavedState(t *testing.T) { + e := setupProject(t) + ed := attachEditor(t, e) + path := filepath.Join(e.root, "lib/my_app/worker.ex") + ed.open("lib/my_app/worker.ex", workerSource) + edited := workerSource + "\ndefmodule MyApp.Typed do\nend\n" + ed.change("lib/my_app/worker.ex", edited) + if err := os.WriteFile(path, []byte(edited), 0644); err != nil { + t.Fatal(err) + } + ed.save("lib/my_app/worker.ex") + writeLater(t, path, agentWorkerSource) + + out := e.callTool("dexter_file_outline", map[string]any{"file": "lib/my_app/worker.ex"}) + wantContains(t, out, "defmodule MyApp.AgentAdded") + wantNotContains(t, out, "unsaved") +} + +// Regression: lines were mapped through the lines that both texts share at +// the start and at the end, so an edit at the top and one at the bottom +// marked every line between them as changed, and a buffer with CRLF line +// endings marked every line. +func TestLineMap(t *testing.T) { + disk := "a\nb\nc\nd\ne\nf" + m := newLineMap(disk, "TOP\na\nb\nc\nd\ne\nBOTTOM") // f changed, line added at top + for line, want := range map[int]int{1: 2, 2: 3, 3: 4, 4: 5, 5: 6} { + if got, ok := m.locate(line); !ok || got != want { + t.Errorf("locate(%d) = %d, %v; want %d", line, got, ok, want) + } + } + if _, ok := m.locate(6); ok { + t.Error("the changed last line is mapped") + } + + crlf := newLineMap(disk, strings.ReplaceAll(disk, "\n", "\r\n")) + for line := 1; line <= 6; line++ { + if got, ok := crlf.locate(line); !ok || got != line { + t.Errorf("CRLF: locate(%d) = %d, %v; want %d", line, got, ok, line) + } + } + + // A deleted line in the middle, and a changed one. + mid := newLineMap(disk, "a\nc\nD\ne\nf") + for line, want := range map[int]int{1: 1, 3: 2, 5: 4, 6: 5} { + if got, ok := mid.locate(line); !ok || got != want { + t.Errorf("mid: locate(%d) = %d, %v; want %d", line, got, ok, want) + } + } + for _, line := range []int{2, 4} { + if _, ok := mid.locate(line); ok { + t.Errorf("mid: changed line %d is mapped", line) + } + } +} + +// The diff is exact on random edits: every mapped pair is equal, mapped lines +// keep their order, and no more lines are left unmapped than a shortest edit +// script deletes. +func TestLineMapRandomEdits(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + for round := 0; round < 200; round++ { + var a []string + for i := 0; i < 1+rng.IntN(60); i++ { + a = append(a, fmt.Sprintf("l%d", rng.IntN(8))) + } + var b []string + for _, l := range a { + switch rng.IntN(6) { + case 0: // delete + case 1: + b = append(b, "new", l) + case 2: + b = append(b, "changed") + default: + b = append(b, l) + } + } + m := newLineMap(strings.Join(a, "\n"), strings.Join(b, "\n")) + last := 0 + mapped := 0 + for i := 1; i <= len(a); i++ { + j, ok := m.locate(i) + if !ok { + continue + } + mapped++ + if j <= last || a[i-1] != b[j-1] { + t.Fatalf("round %d: line %d mapped to %d (last %d): %q vs %q", round, i, j, last, a[i-1], b[j-1]) + } + last = j + } + if want := lcsLen(a, b); mapped != want { + t.Fatalf("round %d: %d lines mapped, want the LCS length %d", round, mapped, want) + } + } +} + +func lcsLen(a, b []string) int { + dp := make([][]int, len(a)+1) + for i := range dp { + dp[i] = make([]int, len(b)+1) + } + for i := 1; i <= len(a); i++ { + for j := 1; j <= len(b); j++ { + if a[i-1] == b[j-1] { + dp[i][j] = dp[i-1][j-1] + 1 + } else { + dp[i][j] = max(dp[i-1][j], dp[i][j-1]) + } + } + } + return dp[len(a)][len(b)] +} + +// BenchmarkLineMap10k measures a 10,000-line file with edits at the top, in +// the middle, and at the bottom. +func BenchmarkLineMap10k(b *testing.B) { + var lines []string + for i := 0; i < 10000; i++ { + lines = append(lines, fmt.Sprintf(" def f%d(x), do: x + %d", i, i)) + } + disk := strings.Join(lines, "\n") + edited := append([]string{"# top"}, lines...) + edited[5000] = " # changed" + edited = append(edited[:7000], append([]string{" def added(x), do: x"}, edited[7000:]...)...) + edited[len(edited)-1] = "# bottom" + text := strings.Join(edited, "\n") + b.ResetTimer() + for i := 0; i < b.N; i++ { + newLineMap(disk, text) + } +} + +// BenchmarkLineMap10kManyEdits is a 10,000-line file with edits near the +// bound (about 900 inserted and deleted lines). +func BenchmarkLineMap10kManyEdits(b *testing.B) { + var lines []string + for i := 0; i < 10000; i++ { + lines = append(lines, fmt.Sprintf(" def f%d(x), do: x + %d", i, i)) + } + disk := strings.Join(lines, "\n") + edited := append([]string(nil), lines...) + for i := 0; i < len(edited); i += 22 { + edited[i] = " # changed" + } + text := strings.Join(edited, "\n") + b.ResetTimer() + for i := 0; i < b.N; i++ { + newLineMap(disk, text) + } +} diff --git a/internal/workspace/runtime.go b/internal/workspace/runtime.go index 5eae084..fd8c1f2 100644 --- a/internal/workspace/runtime.go +++ b/internal/workspace/runtime.go @@ -222,29 +222,38 @@ func (r *Runtime) Session(id string) (*lsp.Server, bool) { return s.server, true } -// EditorBuffer returns the newest buffer that an attached editor holds open -// for path, or false when no editor has it open. A frontend without an editor, -// such as MCP, uses it to read what the user sees instead of the disk. -func (r *Runtime) EditorBuffer(path string) (string, bool) { +// EditorBuffer is a buffer that an attached editor holds open with changes it +// has not saved. +type EditorBuffer struct { + Text string + ChangedAt time.Time // the time of the last change in the editor +} + +// UnsavedBuffer returns the newest buffer for path that an attached editor +// holds open with unsaved changes, or false when there is none. A frontend +// without an editor, such as MCP, uses it to read what the user is editing +// instead of the disk. A buffer the editor has not changed since it opened or +// saved it is not returned: it can be older than the disk. +func (r *Runtime) UnsavedBuffer(path string) (EditorBuffer, bool) { r.sessMu.Lock() if len(r.sessions) == 0 { r.sessMu.Unlock() - return "", false + return EditorBuffer{}, false } servers := make([]*lsp.Server, 0, len(r.sessions)) for _, s := range r.sessions { servers = append(servers, s.server) } r.sessMu.Unlock() - var text string + var out EditorBuffer var newest uint64 found := false for _, srv := range servers { - if t, seq, ok := srv.OpenBuffer(path); ok && (!found || seq > newest) { - text, newest, found = t, seq, true + if t, seq, at, ok := srv.UnsavedBuffer(path); ok && (!found || seq > newest) { + out, newest, found = EditorBuffer{Text: t, ChangedAt: at}, seq, true } } - return text, found + return out, found } // SessionInfo describes one attached editor session for status output.