From ef84ea6fdbadac7e14c0dd8d5ae016d3aa077c6f Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:12:39 +0800 Subject: [PATCH 01/13] feat(wiki): collect and extract Scala files in the code graph (#981) .scala files were silently dropped by the code collector, so Java/Scala mixed projects produced a Java-only graph. Whitelist the extension, map it to the scala language, and add a regex heuristic extractor following the Swift extractor's shape: classes/objects/enums and defs as components, traits as interfaces, Error/Exception-suffix types as errors, sys.env/System.getenv reads as configs, and imports (plain, brace, `_` and `*` wildcards) as relation facts. Also add scala to the extension strips in call-chain-tracer and import-repo cross-repo matching, mark Main.scala/App.scala as key files, and sync the skill-data language map and entry-file list. Fixes #981 Co-Authored-By: Claude Code --- .../references/agents/kb-doc-generator.md | 1 + skill-data/wiki/scripts/scan_repo.py | 2 +- src/__tests__/scala-extractor.test.ts | 159 ++++++++++++++++++ src/import-repo.ts | 4 +- src/wiki-engine/call-chain-tracer.ts | 8 +- .../code-knowledge/code-collector.ts | 7 +- .../code-knowledge/extractors/index.ts | 3 + .../code-knowledge/extractors/scala.ts | 87 ++++++++++ 8 files changed, 261 insertions(+), 10 deletions(-) create mode 100644 src/__tests__/scala-extractor.test.ts create mode 100644 src/wiki-engine/code-knowledge/extractors/scala.ts diff --git a/skill-data/wiki/references/agents/kb-doc-generator.md b/skill-data/wiki/references/agents/kb-doc-generator.md index 448c7afe4..da308d242 100644 --- a/skill-data/wiki/references/agents/kb-doc-generator.md +++ b/skill-data/wiki/references/agents/kb-doc-generator.md @@ -131,6 +131,7 @@ Use the `Glob → Grep → Read` three-step method (**adapt to the language of t TypeScript: app.ts / index.ts / main.ts / server.ts Rust: main.rs / src/main.rs Swift: main.swift / App.swift + Scala: Main.scala / App.scala 2. Grep: locate the core Handlers/Routers (choose the pattern by language + framework) Go: grep -rn 'func.*Handler\|\.GET\|\.POST\|router\.\|@handler' diff --git a/skill-data/wiki/scripts/scan_repo.py b/skill-data/wiki/scripts/scan_repo.py index 08691ab2b..ca965ae18 100644 --- a/skill-data/wiki/scripts/scan_repo.py +++ b/skill-data/wiki/scripts/scan_repo.py @@ -53,7 +53,7 @@ # Language extension map LANG_MAP = { ".py": "Python", ".go": "Go", ".js": "JavaScript", ".ts": "TypeScript", - ".java": "Java", ".rs": "Rust", ".swift": "Swift", ".rb": "Ruby", ".php": "PHP", + ".java": "Java", ".rs": "Rust", ".swift": "Swift", ".scala": "Scala", ".rb": "Ruby", ".php": "PHP", ".c": "C", ".cpp": "C++", ".h": "C/C++ Header", ".proto": "Protobuf", ".thrift": "Thrift", ".graphql": "GraphQL", ".sql": "SQL", ".sh": "Shell", ".bash": "Shell", diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts new file mode 100644 index 000000000..0c7a1bf58 --- /dev/null +++ b/src/__tests__/scala-extractor.test.ts @@ -0,0 +1,159 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { CodeCollectedFile } from '../wiki-engine/code-knowledge/code-collector.js'; +import { collectCode } from '../wiki-engine/code-knowledge/code-collector.js'; +import { extractScala } from '../wiki-engine/code-knowledge/extractors/scala.js'; +import { extractForLanguage, supportedLanguages } from '../wiki-engine/code-knowledge/extractors/index.js'; + +function scalaFile(content: string, relativePath = 'src/main/scala/com/payments/App.scala'): CodeCollectedFile { + return { + path: `/virtual/${relativePath}`, + relativePath, + language: 'scala', + sha256: 'test', + content, + }; +} + +/** Collect the `kind:name` pairs the extractor produced. */ +function extracted(content: string): string[] { + return extractScala([scalaFile(content)]).map((f) => `${f.kind}:${f.name}`); +} + +describe('Scala heuristic extractor', () => { + it('is registered for the scala language', () => { + expect(supportedLanguages()).toContain('scala'); + expect(extractForLanguage('scala', [scalaFile('object App { }\n')])).not.toEqual([]); + }); + + it('extracts classes, objects, traits, enums and defs', () => { + const facts = extracted( + [ + 'package com.payments', + '', + 'trait PaymentGateway {', + ' def charge(order: Order): Receipt', + '}', + '', + 'final case class Invoice(id: Int, amount: BigDecimal)', + '', + 'case object Paid', + '', + 'class OrderRepository(cache: Cache)', + '', + 'enum Color {', + ' case Red, Blue', + '}', + '', + 'object Main extends App {', + ' def run(): Unit = ()', + '}', + ].join('\n'), + ); + + expect(facts).toContain('interface:PaymentGateway'); + expect(facts).toContain('component:charge'); + expect(facts).toContain('component:Invoice'); + expect(facts).toContain('component:Paid'); + expect(facts).toContain('component:OrderRepository'); + expect(facts).toContain('component:Color'); + expect(facts).toContain('component:Main'); + expect(facts).toContain('component:run'); + }); + + it('sees past annotations and modifiers, including a `case` match clause', () => { + const facts = extracted( + [ + '@Singleton', + 'final case class Invoice(id: Int)', + '', + 'sealed abstract class Payment(val id: String)', + '', + 'private[payments] object PaymentRegistry', + '', + 'implicit class RichString(s: String) {', + ' def quoted: String = s', + '}', + '', + 'transparent inline def show(x: Any): String = x.toString', + '', + 'val label = payload match {', + ' case Invoice(id) => id', + ' case other => 0', + '}', + ].join('\n'), + ); + + // `case Invoice(id) =>` is a match clause, not a declaration. + expect(facts.filter((f) => f === 'component:Invoice')).toHaveLength(1); + expect(facts).toContain('component:Payment'); + expect(facts).toContain('component:PaymentRegistry'); + expect(facts).toContain('component:RichString'); + expect(facts).toContain('component:quoted'); + expect(facts).toContain('component:show'); + }); + + it('extracts import relations regardless of the import form', () => { + const facts = extracted( + [ + 'import com.payments.gateway', + 'import java.util.concurrent.TimeUnit', + 'import scala.collection.mutable.{Map => MMap}', + 'import com.payments.events._', + 'import com.payments.core.*', + ].join('\n'), + ); + + expect(facts).toEqual([ + 'relation:com.payments.gateway', + 'relation:java.util.concurrent.TimeUnit', + 'relation:scala.collection.mutable', + 'relation:com.payments.events', + 'relation:com.payments.core', + ]); + }); + + it('infers error types from the Error/Exception suffix and reads environment config', () => { + const facts = extracted( + [ + 'case class PaymentError(msg: String) extends RuntimeException("boom")', + '', + 'sealed trait AppException extends Exception', + '', + 'object Config {', + ' val apiKey = sys.env("PAYMENTS_API_KEY")', + ' val region = System.getenv("AWS_REGION")', + '}', + ].join('\n'), + ); + + expect(facts).toContain('error:PaymentError'); + expect(facts).toContain('error:AppException'); + expect(facts).toContain('config:PAYMENTS_API_KEY'); + expect(facts).toContain('config:AWS_REGION'); + }); +}); + +describe('Scala source collection', () => { + it('collects .scala files with the scala language and key-file marking', async () => { + const root = mkdtempSync(path.join(os.tmpdir(), 'teamai-scala-collect-')); + try { + mkdirSync(path.join(root, 'src', 'main', 'scala'), { recursive: true }); + writeFileSync(path.join(root, 'src', 'main', 'scala', 'Main.scala'), 'object Main extends App\n'); + writeFileSync(path.join(root, 'src', 'main', 'scala', 'Helper.scala'), 'object Helper { }\n'); + + const { manifest } = await collectCode({ root }); + const byPath = new Map(manifest.files.map((f) => [f.relativePath, f])); + + expect(byPath.get('src/main/scala/Main.scala')?.language).toBe('scala'); + expect(byPath.get('src/main/scala/Main.scala')?.isKeyFile).toBe(true); + expect(byPath.get('src/main/scala/Helper.scala')?.language).toBe('scala'); + expect(byPath.get('src/main/scala/Helper.scala')?.isKeyFile).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/import-repo.ts b/src/import-repo.ts index f7b8fd360..4d559c479 100644 --- a/src/import-repo.ts +++ b/src/import-repo.ts @@ -119,7 +119,7 @@ export function detectCrossRepoEdges( for (const edge of overlay.edges) { if (edge.relation !== 'imports') continue; const segments = edge.to.split('/'); - const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift)$/, '') ?? ''; + const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/, '') ?? ''; const pascalName = fileName.split(/[-_]/).map(s => s.charAt(0).toUpperCase() + s.slice(1)).join(''); const match = existingIndex.get(pascalName.toLowerCase()); @@ -146,7 +146,7 @@ export function detectCrossRepoEdges( for (const edge of existing.edges) { if (edge.relation !== 'imports') continue; const segments = edge.to.split('/'); - const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift)$/, '') ?? ''; + const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/, '') ?? ''; const pascalName = fileName.split(/[-_]/).map(s => s.charAt(0).toUpperCase() + s.slice(1)).join(''); const match = overlayIndex.get(pascalName.toLowerCase()); diff --git a/src/wiki-engine/call-chain-tracer.ts b/src/wiki-engine/call-chain-tracer.ts index 0332dbe25..71929891f 100644 --- a/src/wiki-engine/call-chain-tracer.ts +++ b/src/wiki-engine/call-chain-tracer.ts @@ -24,9 +24,9 @@ const ENTRY_PATTERNS = [ /route/i, /controller/i, /endpoint/i, - /main\.(ts|go|py|rs|java|swift)$/, - /server\.(ts|go|py|rs|java|swift)$/, - /app\.(ts|go|py|rs|java|swift)$/, + /main\.(ts|go|py|rs|java|swift|scala)$/, + /server\.(ts|go|py|rs|java|swift|scala)$/, + /app\.(ts|go|py|rs|java|swift|scala)$/, ]; const ORCHESTRATION_PATTERNS = [ @@ -237,7 +237,7 @@ function resolveRelationTarget(importPath: string, filesByModule: Map = { java: [/Application\.java$/, /Controller\.java$/, /Service\.java$/], typescript: [/index\.ts$/, /server\.ts$/, /app\.ts$/, /router\.ts$/], rust: [/main\.rs$/, /lib\.rs$/, /mod\.rs$/], - swift: [/main\.swift$/, /App\.swift$/, /Package\.swift$/] + swift: [/main\.swift$/, /App\.swift$/, /Package\.swift$/], + scala: [/Main\.scala$/, /App\.scala$/] }; export function isKeyFile(relativePath: string, language: string): boolean { @@ -132,7 +133,7 @@ async function walk(directory: string, results: string[], includeTests: boolean) } function isCodeFile(filePath: string): boolean { - return [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".swift", ".json", ".yaml", ".yml", ".toml", ".sql", ".conf", ".ini"].includes( + return [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".swift", ".scala", ".json", ".yaml", ".yml", ".toml", ".sql", ".conf", ".ini"].includes( path.extname(filePath).toLowerCase() ); } @@ -145,7 +146,7 @@ function languageFor(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); const map: Record = { ".ts": "typescript", ".tsx": "typescript", ".js": "javascript", ".jsx": "javascript", - ".py": "python", ".go": "go", ".rs": "rust", ".java": "java", ".swift": "swift", + ".py": "python", ".go": "go", ".rs": "rust", ".java": "java", ".swift": "swift", ".scala": "scala", ".json": "json", ".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".sql": "sql", ".conf": "toml", ".ini": "toml", }; diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index 913e9e6f1..7fd7324ff 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -5,6 +5,7 @@ import { extractGo } from "./go.js"; import { extractJava } from "./java.js"; import { extractPython } from "./python.js"; import { extractRust } from "./rust.js"; +import { extractScala } from "./scala.js"; import { extractSwift } from "./swift.js"; import { extractTypescript } from "./typescript.js"; @@ -20,6 +21,7 @@ const EXTRACTOR_REGISTRY: Record = { python: extractPython, java: extractJava, rust: extractRust, + scala: extractScala, swift: extractSwift, toml: extractToml, sql: extractSql, @@ -48,5 +50,6 @@ export { extractGo } from "./go.js"; export { extractJava } from "./java.js"; export { extractPython } from "./python.js"; export { extractRust } from "./rust.js"; +export { extractScala } from "./scala.js"; export { extractSwift } from "./swift.js"; export { extractTypescript } from "./typescript.js"; diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts new file mode 100644 index 000000000..a3e61a8f7 --- /dev/null +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -0,0 +1,87 @@ +import { type CodeCollectedFile } from "../code-collector.js"; +import { type CodeFact, type CodeFactKind, mapKindToEvidenceType } from "../code-extractors.js"; + +/** + * Scala extractor. + * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. + */ +export function extractScala(files: CodeCollectedFile[]): CodeFact[] { + const facts: CodeFact[] = []; + + for (const file of files) { + const lines = file.content.split(/\r?\n/); + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNumber = i + 1; + + // Annotations (`@deprecated`), modifiers (`final case`), and Scala 3 + // modifiers (`transparent inline`) precede the keyword on the same line. + const decl = stripLeadingModifiers(line); + + // --- Components --- + const typeDecl = /^(class|object|enum)\s+([A-Z]\w*)/u.exec(decl); + if (typeDecl) { + facts.push(makeFact("component", typeDecl[2], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + const defDecl = /^def\s+([a-z_]\w*)/u.exec(decl); + if (defDecl) { + facts.push(makeFact("component", defDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Interfaces --- + const traitDecl = /^trait\s+([A-Z]\w*)/u.exec(decl); + if (traitDecl) { + facts.push(makeFact("interface", traitDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Configs --- + const envRead = /\b(?:sys\.env|System\.getenv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)/u.exec(line); + if (envRead) { + facts.push(makeFact("config", envRead[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Errors --- + const typeName = typeDecl?.[2] ?? traitDecl?.[1]; + if (typeName && /(?:Error|Exception)$/u.test(typeName)) { + facts.push(makeFact("error", typeName, file.relativePath, lineNumber, line, "INFERRED")); + } + + // --- Relations --- + // `import com.foo.Bar`, `import com.foo.{Bar, Baz}`, `import com.foo._` + // (Scala 2) and `import com.foo.*` (Scala 3) all narrow to `com.foo`. + const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)/u.exec(decl); + if (importDecl) { + facts.push(makeFact("relation", importDecl[1].replace(/\._$/u, ""), file.relativePath, lineNumber, line, "EXTRACTED")); + } + } + } + + return facts; +} + +/** + * `case` is the one token that is both a declaration modifier (`case class`) + * and a `match` clause (`case Invoice(id) =>`). It is stripped either way, but + * a clause line then starts with a capitalized name — no declaration keyword — + * so it cannot be mistaken for a declaration. + */ +const MODIFIER_PATTERN = + /^\s*(?:@\w+(?:\[[^\]]*\])?(?:\([^)]*\))?\s+)*(?:(?:private|protected)(?:\[[^\]]*\])?\s+|(?:final|sealed|abstract|case|implicit|lazy|override|open|transparent|inline|infix|erased)\s+)*/u; + +/** Anything after the leading annotations/modifiers is the declaration itself. */ +function stripLeadingModifiers(line: string): string { + return line.replace(MODIFIER_PATTERN, ""); +} + +function makeFact( + kind: CodeFactKind, + name: string, + file: string, + lineStart: number, + rawLine: string, + confidence: CodeFact["confidence"] +): CodeFact { + return { kind, name, file, lineStart, detail: rawLine.trim(), confidence, evidenceType: mapKindToEvidenceType(kind) }; +} From 0a33a3dd1fef9e75cb92daab867748975fef8071 Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 20:52:04 +0800 Subject: [PATCH 02/13] fix(wiki): resolve Scala import relations into graph edges (#996 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Relation facts carried dotted package paths (com.foo.Bar), which neither buildCodeGraph's fuzzy file match nor the call-chain tracer's module map could match against slash-separated file paths — an internal Scala import produced no DEPENDS_ON edge. Emit the import target as a path (com/foo/Bar) instead; both consumers then resolve it (exact, /index and basename lookups all split on /). Add a regression test for the edge. Also make the main/server/app entry-file patterns case-insensitive so Scala's Main.scala / App.scala key files are selected as call-chain entry points. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 30 +++++++++++++++---- src/wiki-engine/call-chain-tracer.ts | 8 +++-- .../code-knowledge/extractors/scala.ts | 7 ++++- 3 files changed, 36 insertions(+), 9 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 0c7a1bf58..b735f4d40 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -5,6 +5,7 @@ import path from 'node:path'; import type { CodeCollectedFile } from '../wiki-engine/code-knowledge/code-collector.js'; import { collectCode } from '../wiki-engine/code-knowledge/code-collector.js'; +import { buildCodeGraph } from '../wiki-engine/code-knowledge/code-graph.js'; import { extractScala } from '../wiki-engine/code-knowledge/extractors/scala.js'; import { extractForLanguage, supportedLanguages } from '../wiki-engine/code-knowledge/extractors/index.js'; @@ -107,15 +108,34 @@ describe('Scala heuristic extractor', () => { ].join('\n'), ); + // Relation names are slash-separated paths: buildCodeGraph and the + // call-chain tracer both match against file paths, not dotted packages. expect(facts).toEqual([ - 'relation:com.payments.gateway', - 'relation:java.util.concurrent.TimeUnit', - 'relation:scala.collection.mutable', - 'relation:com.payments.events', - 'relation:com.payments.core', + 'relation:com/payments/gateway', + 'relation:java/util/concurrent/TimeUnit', + 'relation:scala/collection/mutable', + 'relation:com/payments/events', + 'relation:com/payments/core', ]); }); + it('produces a dependency edge for an internal import', () => { + const gateway = scalaFile( + ['package com.demo.payments', '', 'import com.demo.core.Invoice', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + + const graph = buildCodeGraph([...extractScala([gateway]), ...extractScala([invoice])]); + const edge = graph.edges.find((e) => e.relation === 'DEPENDS_ON'); + + expect(edge?.from).toBe('src/main/scala/com/demo/payments/Gateway.scala'); + expect(edge?.to).toBe('src/main/scala/com/demo/core/Invoice.scala'); + }); + it('infers error types from the Error/Exception suffix and reads environment config', () => { const facts = extracted( [ diff --git a/src/wiki-engine/call-chain-tracer.ts b/src/wiki-engine/call-chain-tracer.ts index 71929891f..b816edd06 100644 --- a/src/wiki-engine/call-chain-tracer.ts +++ b/src/wiki-engine/call-chain-tracer.ts @@ -24,9 +24,11 @@ const ENTRY_PATTERNS = [ /route/i, /controller/i, /endpoint/i, - /main\.(ts|go|py|rs|java|swift|scala)$/, - /server\.(ts|go|py|rs|java|swift|scala)$/, - /app\.(ts|go|py|rs|java|swift|scala)$/, + // Case-insensitive: Scala's key files are Main.scala / App.scala, and a + // capitalized App.ts or Main.go is just as much an entry point. + /main\.(ts|go|py|rs|java|swift|scala)$/i, + /server\.(ts|go|py|rs|java|swift|scala)$/i, + /app\.(ts|go|py|rs|java|swift|scala)$/i, ]; const ORCHESTRATION_PATTERNS = [ diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index a3e61a8f7..27262b08d 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -51,9 +51,14 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { // --- Relations --- // `import com.foo.Bar`, `import com.foo.{Bar, Baz}`, `import com.foo._` // (Scala 2) and `import com.foo.*` (Scala 3) all narrow to `com.foo`. + // Dots are package separators; both relation consumers match + // slash-separated paths (buildCodeGraph's fuzzy file match, the call-chain + // tracer's module map), so emit the import as a path: `com.foo.Bar` → + // `com/foo/Bar`. const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)/u.exec(decl); if (importDecl) { - facts.push(makeFact("relation", importDecl[1].replace(/\._$/u, ""), file.relativePath, lineNumber, line, "EXTRACTED")); + const target = importDecl[1].replace(/\._$/u, "").replace(/\./gu, "/"); + facts.push(makeFact("relation", target, file.relativePath, lineNumber, line, "EXTRACTED")); } } } From 4aa533b3b8b5e7eac018c2a4db18a60336e56d5a Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:00:44 +0800 Subject: [PATCH 03/13] fix(wiki): expand Scala brace imports to per-symbol relations (#996 review) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A brace selector was discarded and only the package path emitted, so buildCodeGraph tied the importer to every file in the directory via substring matching while the call-chain tracer resolved nothing (no module is named after a package segment). Emit one relation per imported symbol — com.foo.{Bar, Baz => B} becomes com/foo/Bar and com/foo/Baz; renames take the name before =>, and the in-brace _ wildcard / nested selectors fall back to the package path, which is what a wildcard honestly names. Add regression coverage for both consumers: the graph edge test asserts a {Invoice} import links exactly Invoice.scala and not a sibling in the same package, and a traceCallChains test asserts the entry step resolves Invoice and never touches the sibling. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 54 ++++++++++++++++++- .../code-knowledge/extractors/scala.ts | 41 ++++++++++---- 2 files changed, 85 insertions(+), 10 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index b735f4d40..2a2ea884d 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -6,6 +6,7 @@ import path from 'node:path'; import type { CodeCollectedFile } from '../wiki-engine/code-knowledge/code-collector.js'; import { collectCode } from '../wiki-engine/code-knowledge/code-collector.js'; import { buildCodeGraph } from '../wiki-engine/code-knowledge/code-graph.js'; +import { traceCallChains } from '../wiki-engine/call-chain-tracer.js'; import { extractScala } from '../wiki-engine/code-knowledge/extractors/scala.js'; import { extractForLanguage, supportedLanguages } from '../wiki-engine/code-knowledge/extractors/index.js'; @@ -105,17 +106,21 @@ describe('Scala heuristic extractor', () => { 'import scala.collection.mutable.{Map => MMap}', 'import com.payments.events._', 'import com.payments.core.*', + 'import com.payments.model.{Invoice, Order => O}', ].join('\n'), ); // Relation names are slash-separated paths: buildCodeGraph and the // call-chain tracer both match against file paths, not dotted packages. + // Brace selectors expand to one relation per imported symbol. expect(facts).toEqual([ 'relation:com/payments/gateway', 'relation:java/util/concurrent/TimeUnit', - 'relation:scala/collection/mutable', + 'relation:scala/collection/mutable/Map', 'relation:com/payments/events', 'relation:com/payments/core', + 'relation:com/payments/model/Invoice', + 'relation:com/payments/model/Order', ]); }); @@ -136,6 +141,53 @@ describe('Scala heuristic extractor', () => { expect(edge?.to).toBe('src/main/scala/com/demo/core/Invoice.scala'); }); + it('scopes a brace import to the selected symbol, not the whole package directory', () => { + const gateway = scalaFile( + ['package com.demo.payments', '', 'import com.demo.core.{Invoice}', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + const admin = scalaFile( + ['package com.demo.core', '', 'object InternalAdmin'].join('\n'), + 'src/main/scala/com/demo/core/InternalAdmin.scala', + ); + + const graph = buildCodeGraph([...extractScala([gateway]), ...extractScala([invoice]), ...extractScala([admin])]); + const deps = graph.edges.filter((e) => e.relation === 'DEPENDS_ON'); + + expect(deps).toHaveLength(1); + expect(deps[0].from).toBe('src/main/scala/com/demo/payments/Gateway.scala'); + expect(deps[0].to).toBe('src/main/scala/com/demo/core/Invoice.scala'); + }); + + it('resolves a brace import in the call-chain tracer', () => { + const main = scalaFile( + ['package com.demo.app', '', 'import com.demo.core.{Invoice}', '', 'object Main extends App {', ' val i: Invoice = null', '}'].join('\n'), + 'src/main/scala/com/demo/app/Main.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + const admin = scalaFile( + ['package com.demo.core', '', 'object InternalAdmin'].join('\n'), + 'src/main/scala/com/demo/core/InternalAdmin.scala', + ); + const files = [main, invoice, admin]; + const facts = files.flatMap((f) => extractScala([f])); + + const chains = traceCallChains(facts, files); + const mainChain = chains.find((c) => c.entryPoint.includes('Main.scala')); + + // The brace import resolves to Invoice.scala's component; the sibling + // InternalAdmin.scala in the same package directory is not pulled in. + expect(mainChain?.steps[0]?.callsTo).toContain('Invoice'); + expect(mainChain?.steps.some((s) => s.file.endsWith('InternalAdmin.scala'))).toBe(false); + }); + it('infers error types from the Error/Exception suffix and reads environment config', () => { const facts = extracted( [ diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 27262b08d..45f0fd456 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -49,16 +49,21 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { } // --- Relations --- - // `import com.foo.Bar`, `import com.foo.{Bar, Baz}`, `import com.foo._` - // (Scala 2) and `import com.foo.*` (Scala 3) all narrow to `com.foo`. - // Dots are package separators; both relation consumers match - // slash-separated paths (buildCodeGraph's fuzzy file match, the call-chain - // tracer's module map), so emit the import as a path: `com.foo.Bar` → - // `com/foo/Bar`. - const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)/u.exec(decl); + // One relation per imported symbol, as a slash-separated path + // (`com.foo.{Bar, Baz => B}` → `com/foo/Bar`, `com/foo/Baz`). Both + // relation consumers match paths, not dotted packages: buildCodeGraph + // fuzzy-matches file paths and the call-chain tracer resolves the + // basename. Expanding the selector keeps the edge on the named file — + // a bare package path would tie the importer to every file in the + // directory. + const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)(?:\s*\.\s*\{([^}]*)\})?/u.exec(decl); if (importDecl) { - const target = importDecl[1].replace(/\._$/u, "").replace(/\./gu, "/"); - facts.push(makeFact("relation", target, file.relativePath, lineNumber, line, "EXTRACTED")); + const packagePath = toPath(importDecl[1]); + const symbols = importDecl[2] ? selectedSymbols(importDecl[2]) : []; + const targets = symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]; + for (const target of targets) { + facts.push(makeFact("relation", target, file.relativePath, lineNumber, line, "EXTRACTED")); + } } } } @@ -80,6 +85,24 @@ function stripLeadingModifiers(line: string): string { return line.replace(MODIFIER_PATTERN, ""); } +/** `com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ +function toPath(dotted: string): string { + return dotted.replace(/[._*]+$/u, "").replace(/\./gu, "/"); +} + +/** + * `Bar, Baz => B` → `["Bar", "Baz"]`: the name a selector imports is the part + * before `=>`. Anything that is not a plain identifier — the in-brace `_` + * wildcard, nested selectors — is dropped; the package path covers it. + */ +function selectedSymbols(selector: string): string[] { + const names = selector + .split(",") + .map((item) => item.split("=>")[0].trim()) + .filter((name) => /^[A-Za-z]\w*$/u.test(name)); + return [...new Set(names)]; +} + function makeFact( kind: CodeFactKind, name: string, From df6231f3418e9eb78d7b9390ffa7413f0fcc7bbf Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:10:17 +0800 Subject: [PATCH 04/13] fix(wiki): handle Scala import hiding, `as` renames and member selectors MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A `{Invoice => _, _}` selector inverted its meaning: the hidden name got its own relation (a dependency on the excluded file) while the wildcard's package-wide import produced nothing. Parse the selector into a wholePackage flag plus named symbols — `_` marks the package, an alias of `_` hides its name, Scala 3 `as` renames like `=>` take the name before the alias — and emit only the package path when the selector imports wholesale. Member imports (`com.foo.Bar.apply`) named a path no file has; trim the lowercase tail after the last type name, since the file that defines the type is what the consumers match. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 18 ++++- .../code-knowledge/extractors/scala.ts | 65 +++++++++++++++---- 2 files changed, 69 insertions(+), 14 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 2a2ea884d..33b8254e5 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -106,13 +106,15 @@ describe('Scala heuristic extractor', () => { 'import scala.collection.mutable.{Map => MMap}', 'import com.payments.events._', 'import com.payments.core.*', - 'import com.payments.model.{Invoice, Order => O}', + 'import com.payments.model.{Invoice as Inv, Order => O}', + 'import com.demo.core.Invoice.apply', ].join('\n'), ); // Relation names are slash-separated paths: buildCodeGraph and the // call-chain tracer both match against file paths, not dotted packages. - // Brace selectors expand to one relation per imported symbol. + // Brace selectors expand to one relation per imported symbol; a member + // import narrows to the type's path. expect(facts).toEqual([ 'relation:com/payments/gateway', 'relation:java/util/concurrent/TimeUnit', @@ -121,9 +123,21 @@ describe('Scala heuristic extractor', () => { 'relation:com/payments/core', 'relation:com/payments/model/Invoice', 'relation:com/payments/model/Order', + 'relation:com/demo/core/Invoice', ]); }); + it('treats a wildcard selector as the whole package and never a hidden name', () => { + const facts = extracted( + ['import com.demo.core.{Invoice => _, _}', 'import com.demo.core.{Order => _}'].join('\n'), + ); + + // `Invoice => _` hides Invoice — no per-symbol relation may name it; with + // the wildcard the package is imported wholesale, and a selector of only + // hidden names imports the package minus those names. + expect(facts).toEqual(['relation:com/demo/core', 'relation:com/demo/core']); + }); + it('produces a dependency edge for an internal import', () => { const gateway = scalaFile( ['package com.demo.payments', '', 'import com.demo.core.Invoice', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 45f0fd456..dfd876f32 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -55,12 +55,15 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { // fuzzy-matches file paths and the call-chain tracer resolves the // basename. Expanding the selector keeps the edge on the named file — // a bare package path would tie the importer to every file in the - // directory. + // directory. A wildcard selector (`_`, `{Bar => _, _}`) imports the + // whole package and keeps the package path, with no per-symbol relation + // for a name the selector explicitly hides. const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)(?:\s*\.\s*\{([^}]*)\})?/u.exec(decl); if (importDecl) { const packagePath = toPath(importDecl[1]); - const symbols = importDecl[2] ? selectedSymbols(importDecl[2]) : []; - const targets = symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]; + const selection = importDecl[2] ? parseSelectors(importDecl[2]) : WHOLE_PACKAGE; + const symbols = selection.wholePackage ? [] : selection.symbols; + const targets = [...new Set((symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]).map(toModulePath))]; for (const target of targets) { facts.push(makeFact("relation", target, file.relativePath, lineNumber, line, "EXTRACTED")); } @@ -91,16 +94,54 @@ function toPath(dotted: string): string { } /** - * `Bar, Baz => B` → `["Bar", "Baz"]`: the name a selector imports is the part - * before `=>`. Anything that is not a plain identifier — the in-brace `_` - * wildcard, nested selectors — is dropped; the package path covers it. + * `com/demo/core/Invoice/apply` → `com/demo/core/Invoice`: a lowercase tail + * after a type names a member of it, and the file that defines the type is + * the file the consumers can match. */ -function selectedSymbols(selector: string): string[] { - const names = selector - .split(",") - .map((item) => item.split("=>")[0].trim()) - .filter((name) => /^[A-Za-z]\w*$/u.test(name)); - return [...new Set(names)]; +function toModulePath(path: string): string { + const segments = path.split("/"); + let lastType = -1; + for (let i = 0; i < segments.length; i++) { + if (/^[A-Z]/u.test(segments[i])) { + lastType = i; + } + } + return lastType === -1 ? path : segments.slice(0, lastType + 1).join("/"); +} + +interface ImportSelection { + /** The whole package is imported (a `_` wildcard, or only hidden names). */ + wholePackage: boolean; + /** The symbols the selector names, excluding renames to `_` (hidden). */ + symbols: string[]; +} + +const WHOLE_PACKAGE: ImportSelection = { wholePackage: true, symbols: [] }; + +/** + * `{Invoice as Inv, Order => O}` → symbols `["Invoice", "Order"]`: the name an + * entry imports is the part before the rename (`=>` in Scala 2, `as` in + * Scala 3). `{Invoice => _, _}` → wholePackage: the wildcard subsumes the + * names, and an alias of `_` hides its name entirely — neither gets a + * per-symbol relation. Anything that is not a plain identifier (nested + * selectors) is dropped; the package path covers it. + */ +function parseSelectors(selector: string): ImportSelection { + const symbols: string[] = []; + let wholePackage = false; + for (const entry of selector.split(",")) { + const item = entry.trim(); + const renamed = /^([A-Za-z_]\w*)\s*(?:=>|\bas\b)\s*(.+)$/u.exec(item); + const alias = renamed?.[2].trim(); + if (item === "_") { + wholePackage = true; + } else if (renamed && alias && alias !== "_" && /^[A-Za-z_]\w*$/u.test(alias)) { + symbols.push(renamed[1]); + } else if (!renamed && /^[A-Za-z_]\w*$/u.test(item)) { + symbols.push(item); + } + } + return { wholePackage: wholePackage || symbols.length === 0, symbols: [...new Set(symbols)] }; } function makeFact( From e1b95f0f9312e299b20e0a67ac31c8ee2d204f05 Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:19:56 +0800 Subject: [PATCH 05/13] fix(wiki): name the files a Scala wildcard imports, and parse wrapped imports MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wildcard import still emitted the package path, so buildCodeGraph's substring match tied the importer to every file under the package — including a name the selector hides — and the call-chain tracer resolved nothing (no module is named after a package segment). Expand a wildcard over the collected files of the package instead, skipping hidden names, and keep the package path only when no collected file matches (an external package). Also parse the import forms the line-based scan mishandled: a brace selector wrapped across lines by scalafmt opened as a bare package import, and a comma-separated `import a.B, c.D` dropped every clause after the first. Track a wrapped selector until its `}`, and split a line into clauses (brace contents keep their own commas). Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 40 +++++- .../code-knowledge/extractors/scala.ts | 130 +++++++++++++----- 2 files changed, 133 insertions(+), 37 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 33b8254e5..7903cd29a 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -127,17 +127,49 @@ describe('Scala heuristic extractor', () => { ]); }); - it('treats a wildcard selector as the whole package and never a hidden name', () => { + it('falls back to the package path when no collected file matches the import', () => { const facts = extracted( ['import com.demo.core.{Invoice => _, _}', 'import com.demo.core.{Order => _}'].join('\n'), ); - // `Invoice => _` hides Invoice — no per-symbol relation may name it; with - // the wildcard the package is imported wholesale, and a selector of only - // hidden names imports the package minus those names. + // The fixture file sits outside com/demo/core, so the wildcard has no + // collected file to name; a selector of only hidden names likewise leaves + // nothing but the package. Neither may produce a relation for `Invoice`. expect(facts).toEqual(['relation:com/demo/core', 'relation:com/demo/core']); }); + it('reads a brace import scalafmt wraps across lines', () => { + const facts = extracted(['import com.demo.core.{', ' Invoice,', ' Order => O', '}'].join('\n')); + + expect(facts).toEqual(['relation:com/demo/core/Invoice', 'relation:com/demo/core/Order']); + }); + + it('reads every clause of a comma-separated import', () => { + const facts = extracted('import com.demo.A, com.demo.B'); + + expect(facts).toEqual(['relation:com/demo/A', 'relation:com/demo/B']); + }); + + it('expands a wildcard over the package files and skips hidden names', () => { + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + + const names = extractScala([gateway, invoice, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala']); + }); + + it('expands a plain wildcard over the package files', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core\n\nobject InternalAdmin\n', 'src/main/scala/com/demo/core/InternalAdmin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + }); + it('produces a dependency edge for an internal import', () => { const gateway = scalaFile( ['package com.demo.payments', '', 'import com.demo.core.Invoice', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index dfd876f32..4361757eb 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -10,11 +10,23 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { for (const file of files) { const lines = file.content.split(/\r?\n/); + // A brace selector scalafmt wraps across lines stays open until its `}`. + let openImport: { line: number; prefix: string; selector: string } | undefined; for (let i = 0; i < lines.length; i++) { const line = lines[i]; const lineNumber = i + 1; + if (openImport) { + openImport.selector += ` ${line}`; + if (openImport.selector.includes("}")) { + const { line: start, prefix, selector } = openImport; + openImport = undefined; + pushRelations(facts, files, prefix, selector.split("}")[0], file.relativePath, start, line); + } + continue; + } + // Annotations (`@deprecated`), modifiers (`final case`), and Scala 3 // modifiers (`transparent inline`) precede the keyword on the same line. const decl = stripLeadingModifiers(line); @@ -49,23 +61,26 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { } // --- Relations --- - // One relation per imported symbol, as a slash-separated path - // (`com.foo.{Bar, Baz => B}` → `com/foo/Bar`, `com/foo/Baz`). Both - // relation consumers match paths, not dotted packages: buildCodeGraph - // fuzzy-matches file paths and the call-chain tracer resolves the - // basename. Expanding the selector keeps the edge on the named file — - // a bare package path would tie the importer to every file in the - // directory. A wildcard selector (`_`, `{Bar => _, _}`) imports the - // whole package and keeps the package path, with no per-symbol relation - // for a name the selector explicitly hides. - const importDecl = /^import\s+([A-Za-z_]\w*(?:\.\w+)*)(?:\s*\.\s*\{([^}]*)\})?/u.exec(decl); - if (importDecl) { - const packagePath = toPath(importDecl[1]); - const selection = importDecl[2] ? parseSelectors(importDecl[2]) : WHOLE_PACKAGE; - const symbols = selection.wholePackage ? [] : selection.symbols; - const targets = [...new Set((symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]).map(toModulePath))]; - for (const target of targets) { - facts.push(makeFact("relation", target, file.relativePath, lineNumber, line, "EXTRACTED")); + // One clause per imported symbol, as a path both relation consumers can + // match: buildCodeGraph compares file paths, and the call-chain tracer + // looks files up by path and basename. Several imports may share one + // line (`import a.B, c.D`); a brace selector may continue on the lines + // below, in which case it opens and waits for its `}`. + const importHead = /^import\s+(.+)$/u.exec(decl); + if (importHead) { + let rest = importHead[1]; + while (rest.length > 0) { + const clause = /^([A-Za-z_]\w*(?:\.\w+)*)(\s*\.\s*\{([^}]*)\})?/u.exec(rest); + if (!clause) { + break; + } + const after = rest.slice(clause[0].length); + if (/^\s*\.\s*\{/u.test(after)) { + openImport = { line: lineNumber, prefix: clause[1], selector: after.replace(/^\s*\.\s*\{/u, "") }; + break; + } + pushRelations(facts, files, clause[1], clause[3], file.relativePath, lineNumber, line); + rest = after.replace(/^\s*,\s*/u, ""); } } } @@ -110,38 +125,87 @@ function toModulePath(path: string): string { } interface ImportSelection { - /** The whole package is imported (a `_` wildcard, or only hidden names). */ - wholePackage: boolean; - /** The symbols the selector names, excluding renames to `_` (hidden). */ + /** A `_` entry imports everything in the package. */ + wildcard: boolean; + /** The symbols the selector names (`Invoice as Inv` imports `Invoice`). */ symbols: string[]; + /** Names renamed to `_` — imported only to be hidden. */ + hidden: string[]; } -const WHOLE_PACKAGE: ImportSelection = { wholePackage: true, symbols: [] }; - /** * `{Invoice as Inv, Order => O}` → symbols `["Invoice", "Order"]`: the name an * entry imports is the part before the rename (`=>` in Scala 2, `as` in - * Scala 3). `{Invoice => _, _}` → wholePackage: the wildcard subsumes the - * names, and an alias of `_` hides its name entirely — neither gets a - * per-symbol relation. Anything that is not a plain identifier (nested - * selectors) is dropped; the package path covers it. + * Scala 3). An alias of `_` hides its name, and `_` alone imports the whole + * package. Anything that is not a plain identifier (nested selectors) is + * dropped; the package covers it. */ function parseSelectors(selector: string): ImportSelection { - const symbols: string[] = []; - let wholePackage = false; + const selection: ImportSelection = { wildcard: false, symbols: [], hidden: [] }; for (const entry of selector.split(",")) { const item = entry.trim(); const renamed = /^([A-Za-z_]\w*)\s*(?:=>|\bas\b)\s*(.+)$/u.exec(item); const alias = renamed?.[2].trim(); if (item === "_") { - wholePackage = true; - } else if (renamed && alias && alias !== "_" && /^[A-Za-z_]\w*$/u.test(alias)) { - symbols.push(renamed[1]); + selection.wildcard = true; + } else if (renamed && alias === "_") { + selection.hidden.push(renamed[1]); + } else if (renamed && alias && /^[A-Za-z_]\w*$/u.test(alias)) { + selection.symbols.push(renamed[1]); } else if (!renamed && /^[A-Za-z_]\w*$/u.test(item)) { - symbols.push(item); + selection.symbols.push(item); } } - return { wholePackage: wholePackage || symbols.length === 0, symbols: [...new Set(symbols)] }; + selection.symbols = [...new Set(selection.symbols)]; + return selection; +} + +/** + * Emit one relation per imported symbol, as a slash-separated path + * (`com.foo.{Bar, Baz => B}` → `com/foo/Bar`, `com/foo/Baz`) — a bare package + * path would tie the importer to every file in the directory. A wildcard + * (a trailing `._`/`.*` or a `_` entry) names the collected files of the + * package itself, minus the names the selector hides, so the excluded file + * gets no edge; with no collected file to name (an external package) the + * package path is all that is left to say. + */ +function pushRelations( + facts: CodeFact[], + files: CodeCollectedFile[], + dottedPrefix: string, + selector: string | undefined, + file: string, + lineNumber: number, + rawLine: string +): void { + const packagePath = toPath(dottedPrefix); + const selection = selector !== undefined ? parseSelectors(selector) : undefined; + const wildcard = /[._*]$/u.test(dottedPrefix) || (selection?.wildcard ?? false); + let targets: string[]; + if (wildcard) { + const hidden = new Set(selection?.hidden ?? []); + const expanded = expandWildcard(packagePath, files, hidden); + targets = expanded.length > 0 ? expanded : [packagePath]; + } else { + const symbols = selection?.symbols ?? []; + targets = [...new Set((symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]).map(toModulePath))]; + } + for (const target of targets) { + facts.push(makeFact("relation", target, file, lineNumber, rawLine, "EXTRACTED")); + } +} + +/** The files a wildcard import of `packagePath` brings in, minus hidden names. */ +function expandWildcard(packagePath: string, files: CodeCollectedFile[], hidden: ReadonlySet): string[] { + return files + .filter((candidate) => candidate.relativePath.includes(`/${packagePath}/`)) + .filter((candidate) => !hidden.has(fileName(candidate.relativePath))) + .map((candidate) => candidate.relativePath); +} + +/** `src/main/scala/core/Invoice.scala` → `Invoice`. */ +function fileName(relativePath: string): string { + return (relativePath.split("/").pop() ?? relativePath).replace(/\.\w+$/u, ""); } function makeFact( From 13cbdf6857aeebece9a5072749d627d2eb0572f1 Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:33:25 +0800 Subject: [PATCH 06/13] fix(wiki): honor Scala 3 `*`, cross-language wildcards, and declaring files MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Four import gaps: - `*` never expanded: the clause regex dropped it before the wildcard check, and an in-brace `*` entry was not recognized as a wildcard at all — both treated as a plain package import whose substring match re-added names the selector hides. - Wildcard expansion saw only the Scala batch, so a package provided by Java files fell back to the package path (linking a hidden name), and Scala candidates masked Java ones. extractForLanguage now passes every collected file to the extractor. - A named import assumed a same-named file (`Invoice` in `Invoice.scala`) though Scala freely puts many types in one `Models.scala`. The symbol now resolves to the package file that declares it, falling back to the conventional path. - Wildcard membership matched descendant directories too, so a subpackage's files rode along on the parent package's import; direct package members only. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 50 +++++ .../code-knowledge/code-extractors.ts | 2 +- .../code-knowledge/extractors/index.ts | 8 +- .../code-knowledge/extractors/scala.ts | 201 +++++++++++++----- 4 files changed, 198 insertions(+), 63 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 7903cd29a..9038ea10a 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -170,6 +170,56 @@ describe('Scala heuristic extractor', () => { expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); }); + it('expands a Scala 3 `*` wildcard the same way as `_`', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core.*\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core\n\nobject InternalAdmin\n', 'src/main/scala/com/demo/core/InternalAdmin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + }); + + it('expands a wildcard over Java files of the package, minus hidden names', () => { + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const invoiceJava: CodeCollectedFile = { + path: '/virtual/src/main/java/com/demo/core/Invoice.java', + relativePath: 'src/main/java/com/demo/core/Invoice.java', + language: 'java', + sha256: 'test', + content: 'package com.demo.core;\n\npublic class Invoice {}\n', + }; + const orderJava: CodeCollectedFile = { + ...invoiceJava, + relativePath: 'src/main/java/com/demo/core/Order.java', + content: 'package com.demo.core;\n\npublic class Order {}\n', + }; + const collected = [gateway, invoiceJava, orderJava]; + + const names = extractScala([gateway], collected).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/java/com/demo/core/Order.java']); + }); + + it('resolves a named import to the file that declares the symbol', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Models.scala']); + }); + + it('excludes a subpackage from a wildcard import', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core.internal\n\nobject Admin\n', 'src/main/scala/com/demo/core/internal/Admin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala']); + }); + it('produces a dependency edge for an internal import', () => { const gateway = scalaFile( ['package com.demo.payments', '', 'import com.demo.core.Invoice', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), diff --git a/src/wiki-engine/code-knowledge/code-extractors.ts b/src/wiki-engine/code-knowledge/code-extractors.ts index 7624014e4..cca721433 100644 --- a/src/wiki-engine/code-knowledge/code-extractors.ts +++ b/src/wiki-engine/code-knowledge/code-extractors.ts @@ -44,7 +44,7 @@ export function extractCodeFacts(files: CodeCollectedFile[]): CodeFact[] { const byLanguage = groupByLanguage(files); const allFacts: CodeFact[] = []; for (const [language, langFiles] of byLanguage) { - allFacts.push(...extractForLanguage(language, langFiles)); + allFacts.push(...extractForLanguage(language, langFiles, files)); } // Deduplicate facts by kind:name:file (same symbol in same file only kept once) const seen = new Set(); diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index 7fd7324ff..ca700a490 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -9,7 +9,7 @@ import { extractScala } from "./scala.js"; import { extractSwift } from "./swift.js"; import { extractTypescript } from "./typescript.js"; -type LanguageExtractor = (files: CodeCollectedFile[]) => CodeFact[]; +type LanguageExtractor = (files: CodeCollectedFile[], allFiles?: CodeCollectedFile[]) => CodeFact[]; /** * Registry mapping language identifiers to their specialized extractors. @@ -29,14 +29,16 @@ const EXTRACTOR_REGISTRY: Record = { /** * Dispatch extraction to the appropriate language-specific extractor. + * `allFiles` — every collected file, all languages — lets an extractor resolve + * cross-language constructs (a Scala wildcard importing Java files). * Falls back to an empty array for unsupported languages (json, yaml, text, etc.). */ -export function extractForLanguage(language: string, files: CodeCollectedFile[]): CodeFact[] { +export function extractForLanguage(language: string, files: CodeCollectedFile[], allFiles?: CodeCollectedFile[]): CodeFact[] { const extractor = EXTRACTOR_REGISTRY[language]; if (!extractor) { return []; } - return extractor(files); + return extractor(files, allFiles); } /** diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 4361757eb..dadaf0c33 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -1,12 +1,25 @@ import { type CodeCollectedFile } from "../code-collector.js"; import { type CodeFact, type CodeFactKind, mapKindToEvidenceType } from "../code-extractors.js"; +const TYPE_DECL_PATTERN = /^(class|object|enum)\s+([A-Z]\w*)/u; +const DEF_DECL_PATTERN = /^def\s+([a-z_]\w*)/u; +const TRAIT_DECL_PATTERN = /^trait\s+([A-Z]\w*)/u; + /** * Scala extractor. * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. + * + * `allFiles` — every collected file, all languages — resolves wildcard imports + * in a mixed project, where a Scala wildcard imports Java files just as freely + * as Scala ones. */ -export function extractScala(files: CodeCollectedFile[]): CodeFact[] { +export function extractScala(files: CodeCollectedFile[], allFiles: CodeCollectedFile[] = files): CodeFact[] { const facts: CodeFact[] = []; + const declarations = new Map>(); + for (const file of files) { + declarations.set(file.relativePath, declarationNames(file.content)); + } + const context: ExtractContext = { files, allFiles, declarations }; for (const file of files) { const lines = file.content.split(/\r?\n/); @@ -22,7 +35,7 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { if (openImport.selector.includes("}")) { const { line: start, prefix, selector } = openImport; openImport = undefined; - pushRelations(facts, files, prefix, selector.split("}")[0], file.relativePath, start, line); + pushRelations(facts, context, prefix, selector.split("}")[0], file.relativePath, start, line); } continue; } @@ -32,18 +45,18 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { const decl = stripLeadingModifiers(line); // --- Components --- - const typeDecl = /^(class|object|enum)\s+([A-Z]\w*)/u.exec(decl); + const typeDecl = TYPE_DECL_PATTERN.exec(decl); if (typeDecl) { facts.push(makeFact("component", typeDecl[2], file.relativePath, lineNumber, line, "EXTRACTED")); } - const defDecl = /^def\s+([a-z_]\w*)/u.exec(decl); + const defDecl = DEF_DECL_PATTERN.exec(decl); if (defDecl) { facts.push(makeFact("component", defDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); } // --- Interfaces --- - const traitDecl = /^trait\s+([A-Z]\w*)/u.exec(decl); + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); if (traitDecl) { facts.push(makeFact("interface", traitDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); } @@ -70,7 +83,7 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { if (importHead) { let rest = importHead[1]; while (rest.length > 0) { - const clause = /^([A-Za-z_]\w*(?:\.\w+)*)(\s*\.\s*\{([^}]*)\})?/u.exec(rest); + const clause = /^([A-Za-z_]\w*(?:\.\w+)*(?:\.\*)?)(\s*\.\s*\{([^}]*)\})?/u.exec(rest); if (!clause) { break; } @@ -79,7 +92,7 @@ export function extractScala(files: CodeCollectedFile[]): CodeFact[] { openImport = { line: lineNumber, prefix: clause[1], selector: after.replace(/^\s*\.\s*\{/u, "") }; break; } - pushRelations(facts, files, clause[1], clause[3], file.relativePath, lineNumber, line); + pushRelations(facts, context, clause[1], clause[3], file.relativePath, lineNumber, line); rest = after.replace(/^\s*,\s*/u, ""); } } @@ -103,29 +116,66 @@ function stripLeadingModifiers(line: string): string { return line.replace(MODIFIER_PATTERN, ""); } -/** `com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ -function toPath(dotted: string): string { - return dotted.replace(/[._*]+$/u, "").replace(/\./gu, "/"); +interface ExtractContext { + /** The Scala batch — its declarations resolve imported symbols to files. */ + files: CodeCollectedFile[]; + /** Every collected file — a wildcard imports Java files as freely as Scala ones. */ + allFiles: CodeCollectedFile[]; + /** relativePath → the type and trait names the file declares. */ + declarations: Map>; } /** - * `com/demo/core/Invoice/apply` → `com/demo/core/Invoice`: a lowercase tail - * after a type names a member of it, and the file that defines the type is - * the file the consumers can match. + * Emit one relation per imported symbol, as a path both relation consumers + * match — a bare package path would tie the importer to every file in the + * directory. A named symbol resolves to the file in the package that declares + * it (Scala freely puts many types in one `Models.scala`), or to the + * conventional path when no collected file declares it. A wildcard (a + * trailing `._`/`.*` or a `_`/`*` entry) names the direct members of the + * package directory, minus the names the selector hides; with no collected + * file to name (an external package) the package path is all that is left to + * say. */ -function toModulePath(path: string): string { - const segments = path.split("/"); - let lastType = -1; - for (let i = 0; i < segments.length; i++) { - if (/^[A-Z]/u.test(segments[i])) { - lastType = i; - } +function pushRelations( + facts: CodeFact[], + context: ExtractContext, + dottedPrefix: string, + selector: string | undefined, + file: string, + lineNumber: number, + rawLine: string +): void { + const packagePath = toPath(dottedPrefix); + const selection = selector !== undefined ? parseSelectors(selector) : undefined; + const wildcard = /[._*]$/u.test(dottedPrefix) || (selection?.wildcard ?? false); + let targets: string[]; + if (wildcard) { + const hidden = new Set(selection?.hidden ?? []); + const expanded = expandWildcard(packagePath, context.allFiles, hidden); + targets = expanded.length > 0 ? expanded : [packagePath]; + } else if (selection && selection.symbols.length > 0) { + targets = [...new Set(selection.symbols.map((symbol) => symbolTarget(symbol, packagePath, file, context)))]; + } else if (selection) { + // only hidden names — the package minus those names + targets = [packagePath]; + } else { + // A plain import's last type names a symbol (`com.foo.Bar`), the rest its + // package; all-lowercase is a package import with nothing to resolve. + const modulePath = toModulePath(packagePath); + const segments = modulePath.split("/"); + const symbol = segments[segments.length - 1]; + targets = + segments.length > 1 && /^[A-Z]/u.test(symbol) + ? [symbolTarget(symbol, segments.slice(0, -1).join("/"), file, context)] + : [modulePath]; + } + for (const target of targets) { + facts.push(makeFact("relation", target, file, lineNumber, rawLine, "EXTRACTED")); } - return lastType === -1 ? path : segments.slice(0, lastType + 1).join("/"); } interface ImportSelection { - /** A `_` entry imports everything in the package. */ + /** A `_` or `*` entry imports everything in the package. */ wildcard: boolean; /** The symbols the selector names (`Invoice as Inv` imports `Invoice`). */ symbols: string[]; @@ -136,9 +186,9 @@ interface ImportSelection { /** * `{Invoice as Inv, Order => O}` → symbols `["Invoice", "Order"]`: the name an * entry imports is the part before the rename (`=>` in Scala 2, `as` in - * Scala 3). An alias of `_` hides its name, and `_` alone imports the whole - * package. Anything that is not a plain identifier (nested selectors) is - * dropped; the package covers it. + * Scala 3). An alias of `_` hides its name, and `_` or `*` alone imports the + * whole package. Anything that is not a plain identifier (nested selectors) + * is dropped; the package covers it. */ function parseSelectors(selector: string): ImportSelection { const selection: ImportSelection = { wildcard: false, symbols: [], hidden: [] }; @@ -146,7 +196,7 @@ function parseSelectors(selector: string): ImportSelection { const item = entry.trim(); const renamed = /^([A-Za-z_]\w*)\s*(?:=>|\bas\b)\s*(.+)$/u.exec(item); const alias = renamed?.[2].trim(); - if (item === "_") { + if (item === "_" || item === "*") { selection.wildcard = true; } else if (renamed && alias === "_") { selection.hidden.push(renamed[1]); @@ -160,49 +210,82 @@ function parseSelectors(selector: string): ImportSelection { return selection; } +/** `com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ +function toPath(dotted: string): string { + return dotted.replace(/[._*]+$/u, "").replace(/\./gu, "/"); +} + /** - * Emit one relation per imported symbol, as a slash-separated path - * (`com.foo.{Bar, Baz => B}` → `com/foo/Bar`, `com/foo/Baz`) — a bare package - * path would tie the importer to every file in the directory. A wildcard - * (a trailing `._`/`.*` or a `_` entry) names the collected files of the - * package itself, minus the names the selector hides, so the excluded file - * gets no edge; with no collected file to name (an external package) the - * package path is all that is left to say. + * `com/demo/core/Invoice/apply` → `com/demo/core/Invoice`: a lowercase tail + * after a type names a member of it, and the file that defines the type is + * the file the consumers can match. */ -function pushRelations( - facts: CodeFact[], - files: CodeCollectedFile[], - dottedPrefix: string, - selector: string | undefined, - file: string, - lineNumber: number, - rawLine: string -): void { - const packagePath = toPath(dottedPrefix); - const selection = selector !== undefined ? parseSelectors(selector) : undefined; - const wildcard = /[._*]$/u.test(dottedPrefix) || (selection?.wildcard ?? false); - let targets: string[]; - if (wildcard) { - const hidden = new Set(selection?.hidden ?? []); - const expanded = expandWildcard(packagePath, files, hidden); - targets = expanded.length > 0 ? expanded : [packagePath]; - } else { - const symbols = selection?.symbols ?? []; - targets = [...new Set((symbols.length > 0 ? symbols.map((symbol) => `${packagePath}/${symbol}`) : [packagePath]).map(toModulePath))]; - } - for (const target of targets) { - facts.push(makeFact("relation", target, file, lineNumber, rawLine, "EXTRACTED")); +function toModulePath(path: string): string { + const segments = path.split("/"); + let lastType = -1; + for (let i = 0; i < segments.length; i++) { + if (/^[A-Z]/u.test(segments[i])) { + lastType = i; + } } + return lastType === -1 ? path : segments.slice(0, lastType + 1).join("/"); } /** The files a wildcard import of `packagePath` brings in, minus hidden names. */ -function expandWildcard(packagePath: string, files: CodeCollectedFile[], hidden: ReadonlySet): string[] { - return files - .filter((candidate) => candidate.relativePath.includes(`/${packagePath}/`)) +function expandWildcard(packagePath: string, allFiles: CodeCollectedFile[], hidden: ReadonlySet): string[] { + return packageMembers(packagePath, allFiles) .filter((candidate) => !hidden.has(fileName(candidate.relativePath))) .map((candidate) => candidate.relativePath); } +/** + * The file in the package that declares `symbol`, or the conventional path if + * none does — the symbol's own name is not a file name to rely on. + */ +function symbolTarget(symbol: string, packagePath: string, importer: string, context: ExtractContext): string { + for (const candidate of packageMembers(packagePath, context.files)) { + if (candidate.relativePath !== importer && context.declarations.get(candidate.relativePath)?.has(symbol)) { + return candidate.relativePath; + } + } + return toModulePath(`${packagePath}/${symbol}`); +} + +/** The direct members of the package directory — a subpackage's files are not among them. */ +function packageMembers(packagePath: string, files: CodeCollectedFile[]): CodeCollectedFile[] { + return files.filter((candidate) => { + const tail = afterPackage(candidate.relativePath, packagePath); + return tail !== undefined && !tail.includes("/"); + }); +} + +/** The part of `relativePath` below `com/demo/core`, or undefined when the file is elsewhere. */ +function afterPackage(relativePath: string, packagePath: string): string | undefined { + if (relativePath.startsWith(`${packagePath}/`)) { + return relativePath.slice(packagePath.length + 1); + } + const marker = `/${packagePath}/`; + const at = relativePath.indexOf(marker); + return at === -1 ? undefined : relativePath.slice(at + marker.length); +} + +/** The type and trait names a file declares — the names another file can import. */ +function declarationNames(content: string): Set { + const names = new Set(); + for (const rawLine of content.split(/\r?\n/)) { + const decl = stripLeadingModifiers(rawLine); + const typeDecl = TYPE_DECL_PATTERN.exec(decl); + if (typeDecl) { + names.add(typeDecl[2]); + } + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); + if (traitDecl) { + names.add(traitDecl[1]); + } + } + return names; +} + /** `src/main/scala/core/Invoice.scala` → `Invoice`. */ function fileName(relativePath: string): string { return (relativePath.split("/").pop() ?? relativePath).replace(/\.\w+$/u, ""); From bb480c147b5aa1783d216769adeb405a56f5a77a Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 21:54:05 +0800 Subject: [PATCH 07/13] fix(wiki): keep Scala import resolution whole across incremental runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An incremental run re-extracts only changed files, so the extractor's resolution context went blind: wildcard expansion fell back to the package path (fuzzy-matching hidden names and subpackages back in) and named imports lost the file that declares the symbol. extractCodeFacts now takes the run's full file list plus the previous run's declarations (rebuilt from the cached facts — component facts carry name and file), so unchanged files resolve without being re-read. Also: hidden-name exclusion now consults the declarations, not just the file name — Models.scala declaring only a hidden Invoice is skipped too; wildcard membership is JVM-importable sources only, so a schema.sql beside the package does not become a dependency; a plain import always tries its last segment as a symbol, which resolves Scala 3 top-level defs (import com.demo.core.validate → Helpers.scala); and a file's several imports of the same target collapse to one relation. The MR diff filter in ci/extract-mr.ts now collects swift and scala as well. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 62 +++++++++++- src/ci/extract-mr.ts | 2 +- src/codebase-extract.ts | 35 ++++++- .../code-knowledge/code-extractors.ts | 9 +- .../code-knowledge/extractors/index.ts | 22 ++++- .../code-knowledge/extractors/scala.ts | 96 +++++++++++++------ 6 files changed, 182 insertions(+), 44 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 9038ea10a..39ad965f3 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -134,8 +134,9 @@ describe('Scala heuristic extractor', () => { // The fixture file sits outside com/demo/core, so the wildcard has no // collected file to name; a selector of only hidden names likewise leaves - // nothing but the package. Neither may produce a relation for `Invoice`. - expect(facts).toEqual(['relation:com/demo/core', 'relation:com/demo/core']); + // nothing but the package. Neither may produce a relation for `Invoice`, + // and both imports name the same dependency, so one relation suffices. + expect(facts).toEqual(['relation:com/demo/core']); }); it('reads a brace import scalafmt wraps across lines', () => { @@ -196,11 +197,66 @@ describe('Scala heuristic extractor', () => { }; const collected = [gateway, invoiceJava, orderJava]; - const names = extractScala([gateway], collected).filter((f) => f.kind === 'relation').map((f) => f.name); + const names = extractScala([gateway], { allFiles: collected, priorDeclarations: new Map() }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); expect(names).toEqual(['src/main/java/com/demo/core/Order.java']); }); + it('resolves against a previous run\'s declarations when only the importer changed', () => { + // Incremental run: the batch holds only the changed importer; the rest of + // the project is known by path (content-less stubs) and cached facts. + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const stub = (relativePath: string): CodeCollectedFile => ({ path: `/virtual/${relativePath}`, relativePath, language: 'scala', sha256: '', content: '' }); + const priorDeclarations = new Map>([ + ['src/main/scala/com/demo/core/Invoice.scala', new Set(['Invoice'])], + ['src/main/scala/com/demo/core/Main.scala', new Set(['Main'])], + ['src/main/scala/com/demo/core/Models.scala', new Set(['Order'])], + ]); + + const names = extractScala([gateway], { + allFiles: [gateway, stub('src/main/scala/com/demo/core/Invoice.scala'), stub('src/main/scala/com/demo/core/Main.scala'), stub('src/main/scala/com/demo/core/Models.scala')], + priorDeclarations, + }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('keeps a declared-hidden symbol out of the wildcard even in a differently named file', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Models.scala declares only the hidden Invoice, so the wildcard skips it too. + expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala']); + }); + + it('does not import package-adjacent resources', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const schema = scalaFile('-- schema', 'src/main/resources/com/demo/core/schema.sql'); + + const names = extractScala([main, invoice], { allFiles: [main, invoice, schema], priorDeclarations: new Map() }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala']); + }); + + it('resolves a top-level def import to its file', () => { + const helpers = scalaFile('package com.demo.core\n\ndef validate(s: String): Boolean = true\n', 'src/main/scala/com/demo/core/Helpers.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.core.validate\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([helpers, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['src/main/scala/com/demo/core/Helpers.scala']); + }); + it('resolves a named import to the file that declares the symbol', () => { const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); diff --git a/src/ci/extract-mr.ts b/src/ci/extract-mr.ts index 44185ddde..9f53edc84 100644 --- a/src/ci/extract-mr.ts +++ b/src/ci/extract-mr.ts @@ -256,7 +256,7 @@ export async function ciExtractMr(opts: CiExtractMrOptions): Promise { { cwd: businessRepo, encoding: 'utf-8', timeout: 10_000 }, ); changedFiles = diffOutput.trim().split('\n') - .filter(f => f && /\.(ts|tsx|js|jsx|py|go|rs|java)$/.test(f)); + .filter(f => f && /\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/.test(f)); if (changedFiles.length > 0) break; } catch { continue; diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 41a2092df..064db6d5f 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -28,6 +28,8 @@ import { formatAstStatsSummary, } from './wiki-engine/adapters/index.js'; import type { CodeFact, InterfaceInventory, CallChain } from './wiki-engine/adapters/index.js'; +import type { CodeCollectedFile } from './wiki-engine/code-knowledge/code-collector.js'; +import type { ExtractorContext } from './wiki-engine/code-knowledge/extractors/index.js'; import { loadFactsCache, saveFactsCache, @@ -597,17 +599,42 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise 0 ? extractCodeFacts(files) : []; + const indicesDir = path.join(wikiRoot, '.indices'); + + // 增量模式下，跨文件解析（通配展开、符号定位到声明文件）需要未变更文件的 + // 路径与声明；两者都能从上一轮 facts 缓存得到，未变更文件无需重新读取 + let cachedFacts: CodeFact[] | undefined; + let extractionContext: ExtractorContext | undefined; + if (changedFiles !== undefined) { + cachedFacts = await loadFactsCache(indicesDir); + const priorDeclarations = new Map>(); + const stubs = new Map(); + const removed = new Set([...changedFiles, ...deletedFiles]); + for (const fact of cachedFacts) { + if (!removed.has(fact.file)) { + stubs.set(fact.file, { path: fact.file, relativePath: fact.file, language: 'text', sha256: '', content: '' }); + } + if (fact.kind !== 'relation') { + const names = priorDeclarations.get(fact.file) ?? new Set(); + names.add(fact.name); + priorDeclarations.set(fact.file, names); + } + } + for (const file of files) { + stubs.delete(file.relativePath); // 变更文件以本批为准 + } + extractionContext = { allFiles: [...files, ...stubs.values()], priorDeclarations }; + } + + const newFacts = files.length > 0 ? extractCodeFacts(files, extractionContext) : []; // 增量模式：加载缓存 → 剪除 → 合并 let facts: CodeFact[]; let interfaceInventory: InterfaceInventory; - const indicesDir = path.join(wikiRoot, '.indices'); if (changedFiles !== undefined) { // 增量模式（含 changedFiles=[] 即仅删除场景） - const oldFacts = await loadFactsCache(indicesDir); + const oldFacts = cachedFacts ?? (await loadFactsCache(indicesDir)); const oldInterfaces = await loadInterfacesCache(indicesDir); // 剪除已变更/删除的旧数据 diff --git a/src/wiki-engine/code-knowledge/code-extractors.ts b/src/wiki-engine/code-knowledge/code-extractors.ts index cca721433..553e6b2f5 100644 --- a/src/wiki-engine/code-knowledge/code-extractors.ts +++ b/src/wiki-engine/code-knowledge/code-extractors.ts @@ -1,5 +1,5 @@ import { type CodeCollectedFile } from "./code-collector.js"; -import { extractForLanguage } from "./extractors/index.js"; +import { extractForLanguage, type ExtractorContext } from "./extractors/index.js"; export type CodeFactKind = "component" | "interface" | "config" | "error" | "data" | "style" | "relation"; @@ -39,12 +39,15 @@ export interface CodeFact { /** * Extract code facts from collected files. * Groups files by language, then dispatches to language-specific extractors. + * `context` — the run's full file list and the previous run's declarations — + * lets extractors resolve cross-file constructs (see ExtractorContext). */ -export function extractCodeFacts(files: CodeCollectedFile[]): CodeFact[] { +export function extractCodeFacts(files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { const byLanguage = groupByLanguage(files); const allFacts: CodeFact[] = []; + const effectiveContext: ExtractorContext = context ?? { allFiles: files, priorDeclarations: new Map() }; for (const [language, langFiles] of byLanguage) { - allFacts.push(...extractForLanguage(language, langFiles, files)); + allFacts.push(...extractForLanguage(language, langFiles, effectiveContext)); } // Deduplicate facts by kind:name:file (same symbol in same file only kept once) const seen = new Set(); diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index ca700a490..c97191fd6 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -9,7 +9,19 @@ import { extractScala } from "./scala.js"; import { extractSwift } from "./swift.js"; import { extractTypescript } from "./typescript.js"; -type LanguageExtractor = (files: CodeCollectedFile[], allFiles?: CodeCollectedFile[]) => CodeFact[]; +/** What an extractor may need beyond its own language batch. */ +export interface ExtractorContext { + /** Every collected file of the run, all languages — a Scala wildcard imports Java files just as freely. */ + allFiles: CodeCollectedFile[]; + /** + * The symbol names each file declares, from the previous run's facts. An + * incremental run re-extracts only changed files, so unchanged files are + * known by their cached declarations alone. + */ + priorDeclarations: Map>; +} + +type LanguageExtractor = (files: CodeCollectedFile[], context?: ExtractorContext) => CodeFact[]; /** * Registry mapping language identifiers to their specialized extractors. @@ -29,16 +41,16 @@ const EXTRACTOR_REGISTRY: Record = { /** * Dispatch extraction to the appropriate language-specific extractor. - * `allFiles` — every collected file, all languages — lets an extractor resolve - * cross-language constructs (a Scala wildcard importing Java files). + * `context` — the run's full file list and the previous run's declarations — + * lets an extractor resolve cross-language and cross-file constructs. * Falls back to an empty array for unsupported languages (json, yaml, text, etc.). */ -export function extractForLanguage(language: string, files: CodeCollectedFile[], allFiles?: CodeCollectedFile[]): CodeFact[] { +export function extractForLanguage(language: string, files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { const extractor = EXTRACTOR_REGISTRY[language]; if (!extractor) { return []; } - return extractor(files, allFiles); + return extractor(files, context); } /** diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index dadaf0c33..33000da69 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -1,5 +1,6 @@ import { type CodeCollectedFile } from "../code-collector.js"; import { type CodeFact, type CodeFactKind, mapKindToEvidenceType } from "../code-extractors.js"; +import { type ExtractorContext } from "./index.js"; const TYPE_DECL_PATTERN = /^(class|object|enum)\s+([A-Z]\w*)/u; const DEF_DECL_PATTERN = /^def\s+([a-z_]\w*)/u; @@ -9,17 +10,21 @@ const TRAIT_DECL_PATTERN = /^trait\s+([A-Z]\w*)/u; * Scala extractor. * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. * - * `allFiles` — every collected file, all languages — resolves wildcard imports - * in a mixed project, where a Scala wildcard imports Java files just as freely - * as Scala ones. + * `context.allFiles` — every collected file, all languages — resolves wildcard + * imports in a mixed project, where a Scala wildcard imports Java files just as + * freely as Scala ones. `context.priorDeclarations` — the previous run's + * declarations — keeps symbol resolution working in an incremental run, which + * re-extracts only changed files. */ -export function extractScala(files: CodeCollectedFile[], allFiles: CodeCollectedFile[] = files): CodeFact[] { +export function extractScala(files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { const facts: CodeFact[] = []; - const declarations = new Map>(); + // An incremental run re-parses only changed files; everything else is known + // by its cached declarations, which the fresh parse then overrides. + const declarations = new Map(context?.priorDeclarations ?? []); for (const file of files) { declarations.set(file.relativePath, declarationNames(file.content)); } - const context: ExtractContext = { files, allFiles, declarations }; + const extractContext: ExtractContext = { allFiles: context?.allFiles ?? files, declarations }; for (const file of files) { const lines = file.content.split(/\r?\n/); @@ -35,7 +40,7 @@ export function extractScala(files: CodeCollectedFile[], allFiles: CodeCollected if (openImport.selector.includes("}")) { const { line: start, prefix, selector } = openImport; openImport = undefined; - pushRelations(facts, context, prefix, selector.split("}")[0], file.relativePath, start, line); + pushRelations(facts, extractContext, prefix, selector.split("}")[0], file.relativePath, start, line); } continue; } @@ -92,14 +97,30 @@ export function extractScala(files: CodeCollectedFile[], allFiles: CodeCollected openImport = { line: lineNumber, prefix: clause[1], selector: after.replace(/^\s*\.\s*\{/u, "") }; break; } - pushRelations(facts, context, clause[1], clause[3], file.relativePath, lineNumber, line); + pushRelations(facts, extractContext, clause[1], clause[3], file.relativePath, lineNumber, line); rest = after.replace(/^\s*,\s*/u, ""); } } } } - return facts; + // Two imports of one file (a wildcard plus a named symbol, say) name the + // same dependency twice — keep one relation per target per file. + const seenRelations = new Set(); + const uniqueFacts: CodeFact[] = []; + for (const fact of facts) { + if (fact.kind !== "relation") { + uniqueFacts.push(fact); + continue; + } + const key = `${fact.file}|${fact.name}`; + if (seenRelations.has(key)) { + continue; + } + seenRelations.add(key); + uniqueFacts.push(fact); + } + return uniqueFacts; } /** @@ -117,11 +138,9 @@ function stripLeadingModifiers(line: string): string { } interface ExtractContext { - /** The Scala batch — its declarations resolve imported symbols to files. */ - files: CodeCollectedFile[]; - /** Every collected file — a wildcard imports Java files as freely as Scala ones. */ + /** Every collected file of the run — a wildcard imports Java files as freely as Scala ones. */ allFiles: CodeCollectedFile[]; - /** relativePath → the type and trait names the file declares. */ + /** relativePath → the symbol names the file declares, this run or cached. */ declarations: Map>; } @@ -151,7 +170,7 @@ function pushRelations( let targets: string[]; if (wildcard) { const hidden = new Set(selection?.hidden ?? []); - const expanded = expandWildcard(packagePath, context.allFiles, hidden); + const expanded = expandWildcard(packagePath, context, hidden); targets = expanded.length > 0 ? expanded : [packagePath]; } else if (selection && selection.symbols.length > 0) { targets = [...new Set(selection.symbols.map((symbol) => symbolTarget(symbol, packagePath, file, context)))]; @@ -159,13 +178,14 @@ function pushRelations( // only hidden names — the package minus those names targets = [packagePath]; } else { - // A plain import's last type names a symbol (`com.foo.Bar`), the rest its - // package; all-lowercase is a package import with nothing to resolve. + // A plain import names a symbol (`com.foo.Bar` — or a Scala 3 top-level + // `def validate`) preceded by its package; resolving it against the + // package's declarations keeps the conventional same-name path as fallback. const modulePath = toModulePath(packagePath); const segments = modulePath.split("/"); const symbol = segments[segments.length - 1]; targets = - segments.length > 1 && /^[A-Z]/u.test(symbol) + segments.length > 1 ? [symbolTarget(symbol, segments.slice(0, -1).join("/"), file, context)] : [modulePath]; } @@ -231,10 +251,20 @@ function toModulePath(path: string): string { return lastType === -1 ? path : segments.slice(0, lastType + 1).join("/"); } -/** The files a wildcard import of `packagePath` brings in, minus hidden names. */ -function expandWildcard(packagePath: string, allFiles: CodeCollectedFile[], hidden: ReadonlySet): string[] { - return packageMembers(packagePath, allFiles) - .filter((candidate) => !hidden.has(fileName(candidate.relativePath))) +/** + * The files a wildcard import of `packagePath` brings in: the package's + * JVM-importable members, minus those hidden by file name and those whose + * every declared symbol is hidden. + */ +function expandWildcard(packagePath: string, context: ExtractContext, hidden: ReadonlySet): string[] { + return packageMembers(packagePath, context.allFiles) + .filter((candidate) => { + if (hidden.has(fileName(candidate.relativePath))) { + return false; + } + const declared = context.declarations.get(candidate.relativePath); + return !(declared && declared.size > 0 && [...declared].every((name) => hidden.has(name))); + }) .map((candidate) => candidate.relativePath); } @@ -243,7 +273,7 @@ function expandWildcard(packagePath: string, allFiles: CodeCollectedFile[], hidd * none does — the symbol's own name is not a file name to rely on. */ function symbolTarget(symbol: string, packagePath: string, importer: string, context: ExtractContext): string { - for (const candidate of packageMembers(packagePath, context.files)) { + for (const candidate of packageMembers(packagePath, context.allFiles)) { if (candidate.relativePath !== importer && context.declarations.get(candidate.relativePath)?.has(symbol)) { return candidate.relativePath; } @@ -251,12 +281,18 @@ function symbolTarget(symbol: string, packagePath: string, importer: string, con return toModulePath(`${packagePath}/${symbol}`); } -/** The direct members of the package directory — a subpackage's files are not among them. */ +/** + * The JVM-importable direct members of the package directory — a subpackage's + * files are not among them, and neither is a resource like a `.sql` schema + * that happens to sit beside the sources. + */ function packageMembers(packagePath: string, files: CodeCollectedFile[]): CodeCollectedFile[] { - return files.filter((candidate) => { - const tail = afterPackage(candidate.relativePath, packagePath); - return tail !== undefined && !tail.includes("/"); - }); + return files + .filter((candidate) => /\.(?:scala|java)$/u.test(candidate.relativePath)) + .filter((candidate) => { + const tail = afterPackage(candidate.relativePath, packagePath); + return tail !== undefined && !tail.includes("/"); + }); } /** The part of `relativePath` below `com/demo/core`, or undefined when the file is elsewhere. */ @@ -269,7 +305,7 @@ function afterPackage(relativePath: string, packagePath: string): string | undef return at === -1 ? undefined : relativePath.slice(at + marker.length); } -/** The type and trait names a file declares — the names another file can import. */ +/** The type, trait and def names a file declares — the names another file can import. */ function declarationNames(content: string): Set { const names = new Set(); for (const rawLine of content.split(/\r?\n/)) { @@ -282,6 +318,10 @@ function declarationNames(content: string): Set { if (traitDecl) { names.add(traitDecl[1]); } + const defDecl = DEF_DECL_PATTERN.exec(decl); + if (defDecl) { + names.add(defDecl[1]); + } } return names; } From b527575cf64e766126e666f0de517b27518a54d0 Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:08:31 +0800 Subject: [PATCH 08/13] fix(wiki): invalidate Scala wildcard importers, and three import gaps MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A wildcard materializes to the package files present at extraction time, so an incremental run that only adds a package member left the unchanged importer's cached relations stale — no edge to the new file. Each wildcard now also emits a symbolic `scala-wildcard:` relation (matched by no consumer) and the incremental pass re-extracts every importer whose wildcard package gained or lost a scala/java file. Nested defs are no longer package-level declarations: the declarations index tracks brace depth and column, so a class member `def total` does not rescue a file whose only top-level name the selector hides. Cached declarations rebuild from component/interface facts only — a config fact like `API_KEY` is not an importable symbol and used to restore dependencies on files the selector hid. A wildcard on an object (`import com.demo.Models.*`) now resolves to the file declaring the object when the package directory holds nothing, instead of falling back to a path no consumer matches. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 41 ++++++++++--- src/codebase-extract.ts | 33 ++++++++--- .../code-knowledge/extractors/index.ts | 2 +- .../code-knowledge/extractors/scala.ts | 57 ++++++++++++++----- 4 files changed, 102 insertions(+), 31 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 39ad965f3..02d314cdb 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -119,7 +119,9 @@ describe('Scala heuristic extractor', () => { 'relation:com/payments/gateway', 'relation:java/util/concurrent/TimeUnit', 'relation:scala/collection/mutable/Map', + 'relation:scala-wildcard:com/payments/events', 'relation:com/payments/events', + 'relation:scala-wildcard:com/payments/core', 'relation:com/payments/core', 'relation:com/payments/model/Invoice', 'relation:com/payments/model/Order', @@ -136,7 +138,7 @@ describe('Scala heuristic extractor', () => { // collected file to name; a selector of only hidden names likewise leaves // nothing but the package. Neither may produce a relation for `Invoice`, // and both imports name the same dependency, so one relation suffices. - expect(facts).toEqual(['relation:com/demo/core']); + expect(facts).toEqual(['relation:scala-wildcard:com/demo/core', 'relation:com/demo/core']); }); it('reads a brace import scalafmt wraps across lines', () => { @@ -158,7 +160,28 @@ describe('Scala heuristic extractor', () => { const names = extractScala([gateway, invoice, main]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('ignores nested defs when a hidden name decides wildcard exclusion', () => { + const models = scalaFile('package com.demo.core\n\nclass Invoice {\n def total: Int = 1\n}\n', 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Models.scala's only package-level name is the hidden Invoice — its + // nested `def total` is not something the wildcard could have imported. + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('resolves a wildcard on an object to the file declaring it', () => { + const domain = scalaFile('package com.demo\n\nobject Models {\n val all = List(1)\n}\n', 'src/main/scala/com/demo/Domain.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.Models.*\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([domain, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/Models', 'src/main/scala/com/demo/Domain.scala']); }); it('expands a plain wildcard over the package files', () => { @@ -168,7 +191,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); }); it('expands a Scala 3 `*` wildcard the same way as `_`', () => { @@ -178,7 +201,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); }); it('expands a wildcard over Java files of the package, minus hidden names', () => { @@ -201,7 +224,7 @@ describe('Scala heuristic extractor', () => { .filter((f) => f.kind === 'relation') .map((f) => f.name); - expect(names).toEqual(['src/main/java/com/demo/core/Order.java']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/java/com/demo/core/Order.java']); }); it('resolves against a previous run\'s declarations when only the importer changed', () => { @@ -222,7 +245,7 @@ describe('Scala heuristic extractor', () => { .filter((f) => f.kind === 'relation') .map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala', 'src/main/scala/com/demo/core/Models.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala', 'src/main/scala/com/demo/core/Models.scala']); }); it('keeps a declared-hidden symbol out of the wildcard even in a differently named file', () => { @@ -233,7 +256,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); // Models.scala declares only the hidden Invoice, so the wildcard skips it too. - expect(names).toEqual(['src/main/scala/com/demo/core/Main.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); }); it('does not import package-adjacent resources', () => { @@ -245,7 +268,7 @@ describe('Scala heuristic extractor', () => { .filter((f) => f.kind === 'relation') .map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala']); }); it('resolves a top-level def import to its file', () => { @@ -273,7 +296,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Invoice.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala']); }); it('produces a dependency edge for an internal import', () => { diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 064db6d5f..971affa94 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -30,6 +30,7 @@ import { import type { CodeFact, InterfaceInventory, CallChain } from './wiki-engine/adapters/index.js'; import type { CodeCollectedFile } from './wiki-engine/code-knowledge/code-collector.js'; import type { ExtractorContext } from './wiki-engine/code-knowledge/extractors/index.js'; +import { SCALA_WILDCARD_PREFIX } from './wiki-engine/code-knowledge/extractors/index.js'; import { loadFactsCache, saveFactsCache, @@ -588,6 +589,27 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise /\.(?:scala|java)$/.test(f)); + if (jvmChanged.length > 0) { + const staleImporters = new Set(); + for (const fact of cachedFacts) { + if (fact.kind !== 'relation' || !fact.name.startsWith(SCALA_WILDCARD_PREFIX)) continue; + const wildcardPackage = fact.name.slice(SCALA_WILDCARD_PREFIX.length); + const touched = jvmChanged.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)); + if (touched) staleImporters.add(fact.file); + } + changedFiles = [...new Set([...changedFiles, ...staleImporters])]; + } + } + const { files, manifest: collectionManifest } = await collectCode({ root, maxFiles, changedFiles }); if (files.length === 0 && !changedFiles) { // 全量模式下无文件 @@ -599,22 +621,19 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise>(); const stubs = new Map(); const removed = new Set([...changedFiles, ...deletedFiles]); - for (const fact of cachedFacts) { + for (const fact of cachedFacts ?? []) { if (!removed.has(fact.file)) { stubs.set(fact.file, { path: fact.file, relativePath: fact.file, language: 'text', sha256: '', content: '' }); } - if (fact.kind !== 'relation') { + if (fact.kind === 'component' || fact.kind === 'interface') { const names = priorDeclarations.get(fact.file) ?? new Set(); names.add(fact.name); priorDeclarations.set(fact.file, names); diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index c97191fd6..d6793fff3 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -64,6 +64,6 @@ export { extractGo } from "./go.js"; export { extractJava } from "./java.js"; export { extractPython } from "./python.js"; export { extractRust } from "./rust.js"; -export { extractScala } from "./scala.js"; +export { extractScala, SCALA_WILDCARD_PREFIX } from "./scala.js"; export { extractSwift } from "./swift.js"; export { extractTypescript } from "./typescript.js"; diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 33000da69..96ea45e5e 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -6,6 +6,13 @@ const TYPE_DECL_PATTERN = /^(class|object|enum)\s+([A-Z]\w*)/u; const DEF_DECL_PATTERN = /^def\s+([a-z_]\w*)/u; const TRAIT_DECL_PATTERN = /^trait\s+([A-Z]\w*)/u; +/** + * Marks a wildcard relation: `scala-wildcard:com/demo/core`. The name matches + * no file, so no consumer resolves it; the incremental layer reads these + * markers to re-extract importers whose package membership changed. + */ +export const SCALA_WILDCARD_PREFIX = "scala-wildcard:"; + /** * Scala extractor. * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. @@ -169,9 +176,22 @@ function pushRelations( const wildcard = /[._*]$/u.test(dottedPrefix) || (selection?.wildcard ?? false); let targets: string[]; if (wildcard) { + facts.push(makeFact("relation", `${SCALA_WILDCARD_PREFIX}${packagePath}`, file, lineNumber, rawLine, "EXTRACTED")); const hidden = new Set(selection?.hidden ?? []); const expanded = expandWildcard(packagePath, context, hidden); - targets = expanded.length > 0 ? expanded : [packagePath]; + if (expanded.length > 0) { + targets = expanded; + } else { + // `com.demo.Models.*` wildcards an object, not a package — resolve it to + // the file that declares the object, as a named import would. + const modulePath = toModulePath(packagePath); + const segments = modulePath.split("/"); + const symbol = segments[segments.length - 1]; + targets = + segments.length > 1 && /^[A-Z]/u.test(symbol) + ? [symbolTarget(symbol, segments.slice(0, -1).join("/"), file, context)] + : [packagePath]; + } } else if (selection && selection.symbols.length > 0) { targets = [...new Set(selection.symbols.map((symbol) => symbolTarget(symbol, packagePath, file, context)))]; } else if (selection) { @@ -305,23 +325,32 @@ function afterPackage(relativePath: string, packagePath: string): string | undef return at === -1 ? undefined : relativePath.slice(at + marker.length); } -/** The type, trait and def names a file declares — the names another file can import. */ +/** + * The top-level type, trait and def names a file declares — the names another + * file can import from the package. A member sits inside braces or, in + * significant-indentation Scala 3, at least one indent; either way it is not + * a package-level name. + */ function declarationNames(content: string): Set { const names = new Set(); + let depth = 0; for (const rawLine of content.split(/\r?\n/)) { - const decl = stripLeadingModifiers(rawLine); - const typeDecl = TYPE_DECL_PATTERN.exec(decl); - if (typeDecl) { - names.add(typeDecl[2]); - } - const traitDecl = TRAIT_DECL_PATTERN.exec(decl); - if (traitDecl) { - names.add(traitDecl[1]); - } - const defDecl = DEF_DECL_PATTERN.exec(decl); - if (defDecl) { - names.add(defDecl[1]); + if (depth === 0 && !/^\s/u.test(rawLine)) { + const decl = stripLeadingModifiers(rawLine); + const typeDecl = TYPE_DECL_PATTERN.exec(decl); + if (typeDecl) { + names.add(typeDecl[2]); + } + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); + if (traitDecl) { + names.add(traitDecl[1]); + } + const defDecl = DEF_DECL_PATTERN.exec(decl); + if (defDecl) { + names.add(defDecl[1]); + } } + depth += (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length; } return names; } From 379fcac3f9e4d3884f9ecd0b0ed1b3745222e91d Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Wed, 7 Oct 2026 22:24:31 +0800 Subject: [PATCH 09/13] fix(wiki): invalidate named-import targets, clean metadata relations A named import materializes to its current declaring file, so a declaration moving between files staled the unchanged importer's cached relation just like a package member change stales a wildcard. The incremental pass now re-extracts an importer whenever a cached materialized target (a relation name ending in .scala/.java) is among the changed or deleted files, alongside the existing wildcard-package rule. The declarations index no longer rebuilds from component facts, which cannot tell a nested member from a package-level name. Each Scala file now records its top-level names in a `scala-decl:` metadata relation, and the incremental context rebuilds from those markers alone. Top-level detection also learns the Scala 3 `package com.demo.core:` block: its members sit one indent in, and declarations deeper than the block's first indent level are members, not package names. The `scala-wildcard:`/`scala-decl:` metadata relations are now filtered from the relation evidence page, the gap detector, and the graph builder, so they no longer surface as undocumented dependencies. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 38 +++++++++---- src/codebase-extract.ts | 41 ++++++++------ .../code-knowledge/code-extractors.ts | 10 ++++ src/wiki-engine/code-knowledge/code-graph.ts | 4 +- .../code-knowledge/extractors/index.ts | 2 +- .../code-knowledge/extractors/scala.ts | 53 ++++++++++++++----- 6 files changed, 104 insertions(+), 44 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 02d314cdb..0b1e2f551 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -160,7 +160,25 @@ describe('Scala heuristic extractor', () => { const names = extractScala([gateway, invoice, main]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala', 'scala-decl:Invoice', 'scala-decl:Main']); + }); + + it('reads top-level declarations under a Scala 3 package block', () => { + const models = scalaFile( + ['package com.demo.core:', ' case class Invoice(id: Int)', ' object Registry:', ' def build: Int = 1'].join('\n'), + 'src/main/scala/com/demo/core/Models.scala', + ); + const main = scalaFile('package com.demo.core:\n object Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile( + 'package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\nimport com.demo.core.Registry\n', + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // `def build` sits one indent deeper than Registry — a member, not a name + // the package exports; Invoice is hidden, Registry keeps Models.scala in. + expect(names).toEqual(['scala-decl:Invoice,Registry', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Models.scala', 'src/main/scala/com/demo/core/Main.scala']); }); it('ignores nested defs when a hidden name decides wildcard exclusion', () => { @@ -172,7 +190,7 @@ describe('Scala heuristic extractor', () => { // Models.scala's only package-level name is the hidden Invoice — its // nested `def total` is not something the wildcard could have imported. - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); }); it('resolves a wildcard on an object to the file declaring it', () => { @@ -181,7 +199,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([domain, main]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/Models', 'src/main/scala/com/demo/Domain.scala']); + expect(names).toEqual(['scala-decl:Models', 'scala-wildcard:com/demo/Models', 'src/main/scala/com/demo/Domain.scala']); }); it('expands a plain wildcard over the package files', () => { @@ -191,7 +209,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala', 'scala-decl:Invoice', 'scala-decl:InternalAdmin']); }); it('expands a Scala 3 `*` wildcard the same way as `_`', () => { @@ -201,7 +219,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala', 'scala-decl:Invoice', 'scala-decl:InternalAdmin']); }); it('expands a wildcard over Java files of the package, minus hidden names', () => { @@ -256,7 +274,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); // Models.scala declares only the hidden Invoice, so the wildcard skips it too. - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); }); it('does not import package-adjacent resources', () => { @@ -268,7 +286,7 @@ describe('Scala heuristic extractor', () => { .filter((f) => f.kind === 'relation') .map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'scala-decl:Invoice']); }); it('resolves a top-level def import to its file', () => { @@ -277,7 +295,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([helpers, main]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Helpers.scala']); + expect(names).toEqual(['scala-decl:validate', 'src/main/scala/com/demo/core/Helpers.scala']); }); it('resolves a named import to the file that declares the symbol', () => { @@ -286,7 +304,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['src/main/scala/com/demo/core/Models.scala']); + expect(names).toEqual(['scala-decl:Invoice,Order', 'src/main/scala/com/demo/core/Models.scala']); }); it('excludes a subpackage from a wildcard import', () => { @@ -296,7 +314,7 @@ describe('Scala heuristic extractor', () => { const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); - expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala']); + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'scala-decl:Invoice', 'scala-decl:Admin']); }); it('produces a dependency edge for an internal import', () => { diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 971affa94..2e7f046b8 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -30,7 +30,8 @@ import { import type { CodeFact, InterfaceInventory, CallChain } from './wiki-engine/adapters/index.js'; import type { CodeCollectedFile } from './wiki-engine/code-knowledge/code-collector.js'; import type { ExtractorContext } from './wiki-engine/code-knowledge/extractors/index.js'; -import { SCALA_WILDCARD_PREFIX } from './wiki-engine/code-knowledge/extractors/index.js'; +import { SCALA_DECL_PREFIX, SCALA_WILDCARD_PREFIX } from './wiki-engine/code-knowledge/extractors/index.js'; +import { isMetadataRelation } from './wiki-engine/code-knowledge/code-extractors.js'; import { loadFactsCache, saveFactsCache, @@ -99,7 +100,7 @@ function detectKnowledgeGaps( } // 1. 未解析的外部依赖：import target 不在扫描范围内 - const relationFacts = facts.filter((f) => f.kind === 'relation'); + const relationFacts = facts.filter((f) => f.kind === 'relation' && !isMetadataRelation(f.name)); const unresolvedImports = new Set(); for (const rel of relationFacts) { const target = rel.name; @@ -219,7 +220,7 @@ function buildEvidencePages( pages.set(`${kind}.md`, lines.join('\n')); } - const relationFacts = facts.filter((f) => f.kind === 'relation'); + const relationFacts = facts.filter((f) => f.kind === 'relation' && !isMetadataRelation(f.name)); if (relationFacts.length > 0) { const byDir = new Map(); for (const fact of relationFacts) { @@ -589,23 +590,29 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise /\.(?:scala|java)$/.test(f)); - if (jvmChanged.length > 0) { - const staleImporters = new Set(); - for (const fact of cachedFacts) { - if (fact.kind !== 'relation' || !fact.name.startsWith(SCALA_WILDCARD_PREFIX)) continue; + const changedSet = new Set([...changedFiles, ...deletedFiles]); + const staleImporters = new Set(); + for (const fact of cachedFacts) { + if (fact.kind !== 'relation') continue; + if (fact.name.startsWith(SCALA_WILDCARD_PREFIX)) { const wildcardPackage = fact.name.slice(SCALA_WILDCARD_PREFIX.length); - const touched = jvmChanged.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)); + const touched = changedFiles.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)) + || deletedFiles.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)); if (touched) staleImporters.add(fact.file); + } else if (/\.(?:scala|java)$/.test(fact.name) && changedSet.has(fact.name)) { + // a materialized target changed — the importer must re-resolve it + staleImporters.add(fact.file); } + } + if (staleImporters.size > 0) { changedFiles = [...new Set([...changedFiles, ...staleImporters])]; } } @@ -623,7 +630,8 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise>(); @@ -633,9 +641,8 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise(); - names.add(fact.name); + if (fact.kind === 'relation' && fact.name.startsWith(SCALA_DECL_PREFIX)) { + const names = new Set(fact.name.slice(SCALA_DECL_PREFIX.length).split(',')); priorDeclarations.set(fact.file, names); } } diff --git a/src/wiki-engine/code-knowledge/code-extractors.ts b/src/wiki-engine/code-knowledge/code-extractors.ts index 553e6b2f5..ac550056a 100644 --- a/src/wiki-engine/code-knowledge/code-extractors.ts +++ b/src/wiki-engine/code-knowledge/code-extractors.ts @@ -36,6 +36,16 @@ export interface CodeFact { evidenceType?: CodeEvidenceType; } +/** + * Relations that carry extraction metadata for the incremental layer — a + * Scala wildcard's package, a file's top-level declarations — rather than a + * dependency. Evidence pages and gap detection skip them, and the graph + * builder resolves none of them. + */ +export function isMetadataRelation(name: string): boolean { + return /^scala-(?:wildcard|decl):/u.test(name); +} + /** * Extract code facts from collected files. * Groups files by language, then dispatches to language-specific extractors. diff --git a/src/wiki-engine/code-knowledge/code-graph.ts b/src/wiki-engine/code-knowledge/code-graph.ts index f91b7585c..a4339fefd 100644 --- a/src/wiki-engine/code-knowledge/code-graph.ts +++ b/src/wiki-engine/code-knowledge/code-graph.ts @@ -1,6 +1,6 @@ import path from "node:path"; -import { type CodeFact } from "./code-extractors.js"; +import { type CodeFact, isMetadataRelation } from "./code-extractors.js"; import { type GraphIndex, type GraphNode, @@ -27,7 +27,7 @@ export function buildCodeGraph(facts: CodeFact[]): GraphIndex { const nodeFiles = new Set(facts.filter(f => f.kind !== "relation").map(f => f.file)); const edges: GraphEdge[] = facts - .filter((fact) => fact.kind === "relation") + .filter((fact) => fact.kind === "relation" && !isMetadataRelation(fact.name)) .flatMap((fact) => { // AST-derived relation facts carry a resolved target file in fact.name and a // "(code-ast)" marker in detail — trust them directly instead of fuzzy matching. diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index d6793fff3..11ff8c8ac 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -64,6 +64,6 @@ export { extractGo } from "./go.js"; export { extractJava } from "./java.js"; export { extractPython } from "./python.js"; export { extractRust } from "./rust.js"; -export { extractScala, SCALA_WILDCARD_PREFIX } from "./scala.js"; +export { extractScala, SCALA_DECL_PREFIX, SCALA_WILDCARD_PREFIX } from "./scala.js"; export { extractSwift } from "./swift.js"; export { extractTypescript } from "./typescript.js"; diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 96ea45e5e..6f43ca0f5 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -13,6 +13,15 @@ const TRAIT_DECL_PATTERN = /^trait\s+([A-Z]\w*)/u; */ export const SCALA_WILDCARD_PREFIX = "scala-wildcard:"; +/** + * Marks a file's top-level declaration names, comma-joined: + * `scala-decl:Invoice,Order`. Like the wildcard marker it is metadata for the + * incremental layer — the next run rebuilds the declarations index from these + * instead of component facts, which cannot tell a nested member from a + * package-level name. + */ +export const SCALA_DECL_PREFIX = "scala-decl:"; + /** * Scala extractor. * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. @@ -35,6 +44,10 @@ export function extractScala(files: CodeCollectedFile[], context?: ExtractorCont for (const file of files) { const lines = file.content.split(/\r?\n/); + const declared = declarations.get(file.relativePath); + if (declared && declared.size > 0) { + facts.push(makeFact("relation", `${SCALA_DECL_PREFIX}${[...declared].join(",")}`, file.relativePath, 1, "", "EXTRACTED")); + } // A brace selector scalafmt wraps across lines stays open until its `}`. let openImport: { line: number; prefix: string; selector: string } | undefined; @@ -328,26 +341,38 @@ function afterPackage(relativePath: string, packagePath: string): string | undef /** * The top-level type, trait and def names a file declares — the names another * file can import from the package. A member sits inside braces or, in - * significant-indentation Scala 3, at least one indent; either way it is not - * a package-level name. + * significant-indentation Scala 3, deeper than the package block's first + * indent; either way it is not a package-level name. */ function declarationNames(content: string): Set { const names = new Set(); let depth = 0; + let inPackageBlock = false; // `package com.demo.core:` — members sit one indent in + let memberIndent = -1; for (const rawLine of content.split(/\r?\n/)) { - if (depth === 0 && !/^\s/u.test(rawLine)) { - const decl = stripLeadingModifiers(rawLine); - const typeDecl = TYPE_DECL_PATTERN.exec(decl); - if (typeDecl) { - names.add(typeDecl[2]); - } - const traitDecl = TRAIT_DECL_PATTERN.exec(decl); - if (traitDecl) { - names.add(traitDecl[1]); + const trimmed = rawLine.trim(); + const indent = rawLine.length - rawLine.trimStart().length; + if (!inPackageBlock && /^package\s+[\w.]+\s*:\s*$/u.test(trimmed)) { + inPackageBlock = true; + } else if (trimmed !== "") { + if (inPackageBlock && memberIndent === -1) { + memberIndent = indent; } - const defDecl = DEF_DECL_PATTERN.exec(decl); - if (defDecl) { - names.add(defDecl[1]); + const topLevel = depth === 0 && (indent === 0 || (inPackageBlock && indent <= memberIndent)); + if (topLevel) { + const decl = stripLeadingModifiers(rawLine); + const typeDecl = TYPE_DECL_PATTERN.exec(decl); + if (typeDecl) { + names.add(typeDecl[2]); + } + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); + if (traitDecl) { + names.add(traitDecl[1]); + } + const defDecl = DEF_DECL_PATTERN.exec(decl); + if (defDecl) { + names.add(defDecl[1]); + } } } depth += (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length; From f52c7cf6e0f9bf2d2542585b43ca6d603351c1fd Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:41:18 +0800 Subject: [PATCH 10/13] fix(wiki): re-resolve unresolved Scala imports when their package changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An unresolved named import caches as its conventional path (com/demo/core/Invoice), which the invalidation rules never matched: the wildcard rule looks at packages, the target rule at materialized .scala/.java names. A third rule now invalidates an importer whenever a file is added to or removed from the package directory its unresolved relation names — the target may have become resolvable. An exclusion-only selector ({Order => _}) imports nothing, so it now emits no relation at all; the package fallback tied the importer to every file in the directory. Metadata relations no longer inflate statistics: the evidence index, overview and router fact counts, and the CLI/JSON summary all count visible facts only, with markers still persisted in the cache where the incremental layer needs them. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 5 +-- src/codebase-extract.ts | 37 ++++++++++++------- .../code-knowledge/extractors/scala.ts | 4 +- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 0b1e2f551..38d91018f 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -135,9 +135,8 @@ describe('Scala heuristic extractor', () => { ); // The fixture file sits outside com/demo/core, so the wildcard has no - // collected file to name; a selector of only hidden names likewise leaves - // nothing but the package. Neither may produce a relation for `Invoice`, - // and both imports name the same dependency, so one relation suffices. + // collected file to name. A selector of only hidden names (`{Order => _}`) + // imports nothing and emits nothing — no relation for `Invoice` either. expect(facts).toEqual(['relation:scala-wildcard:com/demo/core', 'relation:com/demo/core']); }); diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 2e7f046b8..18d507308 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -86,6 +86,11 @@ interface KnowledgeGap { source: string; } +/** 用户可见的 facts：元数据 relation（通配包名、顶层声明标记）不进任何统计。 */ +function visibleFactsOf(facts: CodeFact[]): CodeFact[] { + return facts.filter((f) => f.kind !== 'relation' || !isMetadataRelation(f.name)); +} + function detectKnowledgeGaps( facts: CodeFact[], graph: GraphIndex, @@ -302,7 +307,7 @@ function buildEvidencePages( '', `# ${project}`, '', - `Facts: ${facts.length} | Pages: ${pages.size}`, + `Facts: ${visibleFactsOf(facts).length} | Pages: ${pages.size}`, '', ]; @@ -471,7 +476,7 @@ function buildOverview( '', `# ${project}`, '', - `**${facts.length} facts** extracted from ${new Set(facts.map(f => f.file)).size} files.`, + `**${visibleFactsOf(facts).length} facts** extracted from ${new Set(facts.map(f => f.file)).size} files.`, `Graph: ${graph.nodes.length} nodes, ${graph.edges.length} edges.`, '', '## Module Structure', @@ -599,17 +604,23 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise(); for (const fact of cachedFacts) { - if (fact.kind !== 'relation') continue; + if (fact.kind !== 'relation' || isMetadataRelation(fact.name)) continue; + const touchedPackage = (pkg: string): boolean => + originalChangedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)) || + deletedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)); if (fact.name.startsWith(SCALA_WILDCARD_PREFIX)) { - const wildcardPackage = fact.name.slice(SCALA_WILDCARD_PREFIX.length); - const touched = changedFiles.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)) - || deletedFiles.some((f) => f.includes(`/${wildcardPackage}/`) || f.startsWith(`${wildcardPackage}/`)); - if (touched) staleImporters.add(fact.file); - } else if (/\.(?:scala|java)$/.test(fact.name) && changedSet.has(fact.name)) { - // a materialized target changed — the importer must re-resolve it - staleImporters.add(fact.file); + if (touchedPackage(fact.name.slice(SCALA_WILDCARD_PREFIX.length))) staleImporters.add(fact.file); + } else if (/\.(?:scala|java)$/.test(fact.name)) { + // 物化目标本身变了——importer 必须重新解析 + if (changedSet.has(fact.name)) staleImporters.add(fact.file); + } else { + // 未物化的常规路径（com/demo/core/Invoice）：包目录里有文件增删时， + // 原本解析不到的目标可能已经可解析 + const pkg = fact.name.split('/').slice(0, -1).join('/'); + if (pkg && touchedPackage(pkg)) staleImporters.add(fact.file); } } if (staleImporters.size > 0) { @@ -876,7 +887,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise 0 ? ifByType : undefined, @@ -982,14 +993,14 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise = {}; - for (const fact of facts) { + for (const fact of visibleFactsOf(facts)) { byKind[fact.kind] = (byKind[fact.kind] ?? 0) + 1; } const result: ExtractResult = { project, filesScanned: files.length, - facts: { total: facts.length, byKind }, + facts: { total: visibleFactsOf(facts).length, byKind }, graph: { nodes: repoGraph.nodes.length, edges: repoGraph.edges.length }, incremental: !!opts.incremental && !!changedFiles, outputDir: wikiRoot, diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 6f43ca0f5..f1fc46f0d 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -208,8 +208,8 @@ function pushRelations( } else if (selection && selection.symbols.length > 0) { targets = [...new Set(selection.symbols.map((symbol) => symbolTarget(symbol, packagePath, file, context)))]; } else if (selection) { - // only hidden names — the package minus those names - targets = [packagePath]; + // only hidden names: the clause imports nothing, so no relation at all + targets = []; } else { // A plain import names a symbol (`com.foo.Bar` — or a Scala 3 top-level // `def validate`) preceded by its package; resolving it against the From 0d45ffebe2abc5a5539643c1cfbbf8af2fba4fad Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Thu, 8 Oct 2026 13:53:49 +0800 Subject: [PATCH 11/13] fix(wiki): restore wildcard invalidation, _root_ imports, nested packages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The R9 cleanup skipped every metadata relation before the wildcard-prefix branch, leaving the wildcard invalidation rule unreachable — a package member added incrementally no longer re-extracted the importer. Skip only scala-decl markers (they carry no dependency); regression-tested with the add-member incremental scenario. `import _root_.com.foo.Bar` kept the qualifier and matched no path; it is now stripped like any other package prefix. Scala 3 nests package blocks (`package com.demo:` then `package core:`); each block now carries its own member indent, so declarations under the inner block are package-level names of the inner package. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 21 +++++++++++++++ src/codebase-extract.ts | 2 +- .../code-knowledge/extractors/scala.ts | 27 +++++++++++-------- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 38d91018f..ae942cf55 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -297,6 +297,27 @@ describe('Scala heuristic extractor', () => { expect(names).toEqual(['scala-decl:validate', 'src/main/scala/com/demo/core/Helpers.scala']); }); + it('strips the _root_ qualifier from an import', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport _root_.com.demo.core.Invoice\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Invoice', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('reads nested Scala 3 package blocks', () => { + const models = scalaFile( + ['package com.demo:', ' package core:', ' case class Invoice(id: Int)', ' object Registry'].join('\n'), + 'src/main/scala/com/demo/core/Models.scala', + ); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Invoice\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Invoice,Registry', 'src/main/scala/com/demo/core/Models.scala']); + }); + it('resolves a named import to the file that declares the symbol', () => { const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 18d507308..2f9003733 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -607,7 +607,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise(); for (const fact of cachedFacts) { - if (fact.kind !== 'relation' || isMetadataRelation(fact.name)) continue; + if (fact.kind !== 'relation' || fact.name.startsWith(SCALA_DECL_PREFIX)) continue; const touchedPackage = (pkg: string): boolean => originalChangedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)) || deletedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)); diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index f1fc46f0d..2ce000253 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -263,9 +263,9 @@ function parseSelectors(selector: string): ImportSelection { return selection; } -/** `com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ +/** `_root_.com.foo.Bar`/`com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ function toPath(dotted: string): string { - return dotted.replace(/[._*]+$/u, "").replace(/\./gu, "/"); + return dotted.replace(/^_root_\./u, "").replace(/[._*]+$/u, "").replace(/\./gu, "/"); } /** @@ -341,24 +341,29 @@ function afterPackage(relativePath: string, packagePath: string): string | undef /** * The top-level type, trait and def names a file declares — the names another * file can import from the package. A member sits inside braces or, in - * significant-indentation Scala 3, deeper than the package block's first - * indent; either way it is not a package-level name. + * significant-indentation Scala 3, deeper than its package block's member + * indent (Scala 3 nests package blocks, each with its own level); either + * way it is not a package-level name. */ function declarationNames(content: string): Set { const names = new Set(); let depth = 0; - let inPackageBlock = false; // `package com.demo.core:` — members sit one indent in - let memberIndent = -1; + const packageMemberIndents: number[] = []; // open `package x:` blocks, innermost last; -1 until its member indent is seen for (const rawLine of content.split(/\r?\n/)) { const trimmed = rawLine.trim(); const indent = rawLine.length - rawLine.trimStart().length; - if (!inPackageBlock && /^package\s+[\w.]+\s*:\s*$/u.test(trimmed)) { - inPackageBlock = true; + const last = packageMemberIndents.length - 1; + if (trimmed !== "" && last >= 0 && packageMemberIndents[last] === -1) { + packageMemberIndents[last] = indent; // the first content line fixes the block's member indent + } + if (/^package\s+[\w.]+\s*:\s*$/u.test(trimmed)) { + packageMemberIndents.push(-1); } else if (trimmed !== "") { - if (inPackageBlock && memberIndent === -1) { - memberIndent = indent; + while (packageMemberIndents.length > 1 && packageMemberIndents[packageMemberIndents.length - 1] > indent) { + packageMemberIndents.pop(); // dedented out of the inner block } - const topLevel = depth === 0 && (indent === 0 || (inPackageBlock && indent <= memberIndent)); + const memberIndent = packageMemberIndents[packageMemberIndents.length - 1]; + const topLevel = depth === 0 && (memberIndent === undefined ? indent === 0 : indent === memberIndent); if (topLevel) { const decl = stripLeadingModifiers(rawLine); const typeDecl = TYPE_DECL_PATTERN.exec(decl); From 03fa7b8ebb13a915f7dbfb9b8cdc3d48236e8dcf Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:07:19 +0800 Subject: [PATCH 12/13] fix(wiki): survive collector upgrades and three Scala import forms An incremental run against a manifest written before .scala was collected saw an empty commit diff at the same clean commit and exited "no changes", leaving Scala files permanently absent. The source manifest now records CODE_COLLECTION_VERSION (bumped when the collected set changes), and a version mismatch skips the git fast path for the sha256 diff, which surfaces the newly supported files as added. A braced package (`package com.demo.core { ... }`) counted its brace as declaration nesting, so nothing inside produced a scala-decl marker. The package brace now opens a block like the colon form instead. `import com.demo.Models.Invoice`, where Invoice lives inside `object Models` in Domain.scala, now resolves: the symbol walk tries each trailing segment as a name in the remaining prefix's package, landing on Models' declaring file. Plain `as` renames in a comma-separated import (`import a.B as Alias, c.D`) are consumed so later clauses parse. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 27 ++++++++ src/codebase-extract.ts | 3 +- .../code-knowledge/code-collector.ts | 8 +++ .../code-knowledge/code-incremental.ts | 20 ++++-- .../code-knowledge/extractors/scala.ts | 66 +++++++++++-------- 5 files changed, 87 insertions(+), 37 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index ae942cf55..72dfad257 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -318,6 +318,33 @@ describe('Scala heuristic extractor', () => { expect(names).toEqual(['scala-decl:Invoice,Registry', 'src/main/scala/com/demo/core/Models.scala']); }); + it('reads declarations inside a braced package', () => { + const models = scalaFile(['package com.demo.core {', ' class Invoice {', ' def total: Int = 1', ' }', '}'].join('\n'), 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // The package brace does not nest the declarations away, and Models.scala + // declares only the hidden Invoice — the wildcard skips it. + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('resolves an import of a member declared inside an object', () => { + const domain = scalaFile('package com.demo\n\nobject Models {\n case class Invoice(id: Int)\n}\n', 'src/main/scala/com/demo/Domain.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.Models.Invoice\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([domain, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Models', 'src/main/scala/com/demo/Domain.scala']); + }); + + it('reads a comma-separated import with a plain `as` rename', () => { + const facts = extracted('import com.demo.A as Alias, com.demo.B'); + + expect(facts).toEqual(['relation:com/demo/A', 'relation:com/demo/B']); + }); + it('resolves a named import to the file that declares the symbol', () => { const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 2f9003733..53cdf6647 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -29,6 +29,7 @@ import { } from './wiki-engine/adapters/index.js'; import type { CodeFact, InterfaceInventory, CallChain } from './wiki-engine/adapters/index.js'; import type { CodeCollectedFile } from './wiki-engine/code-knowledge/code-collector.js'; +import { CODE_COLLECTION_VERSION } from './wiki-engine/code-knowledge/code-collector.js'; import type { ExtractorContext } from './wiki-engine/code-knowledge/extractors/index.js'; import { SCALA_DECL_PREFIX, SCALA_WILDCARD_PREFIX } from './wiki-engine/code-knowledge/extractors/index.js'; import { isMetadataRelation } from './wiki-engine/code-knowledge/code-extractors.js'; @@ -972,7 +973,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise = { - version: 1, + version: CODE_COLLECTION_VERSION, lastScan: new Date().toISOString(), files: allManifestFiles, }; diff --git a/src/wiki-engine/code-knowledge/code-collector.ts b/src/wiki-engine/code-knowledge/code-collector.ts index 39d174340..d3b6cec7c 100644 --- a/src/wiki-engine/code-knowledge/code-collector.ts +++ b/src/wiki-engine/code-knowledge/code-collector.ts @@ -16,6 +16,14 @@ const execFileAsync = promisify(execFile); */ const GIT_EXEC_OPTIONS = { windowsHide: true } as const; +/** + * Bumped whenever the set of files collectCode picks up changes (a new + * extension whitelisted, a skip rule added). An incremental run against a + * manifest written by an older version skips the git fast path and diffs by + * sha256 instead, so files the old run never collected surface as added. + */ +export const CODE_COLLECTION_VERSION = 2; + export interface CodeCollectedFile { path: string; relativePath: string; diff --git a/src/wiki-engine/code-knowledge/code-incremental.ts b/src/wiki-engine/code-knowledge/code-incremental.ts index 641f4847c..f3aa15154 100644 --- a/src/wiki-engine/code-knowledge/code-incremental.ts +++ b/src/wiki-engine/code-knowledge/code-incremental.ts @@ -1,7 +1,7 @@ import { readFile, writeFile, stat, mkdir } from "node:fs/promises"; import path from "node:path"; -import { collectCode, gitCommit, gitDiffNameStatus, isWorkingTreeClean } from "./code-collector.js"; +import { CODE_COLLECTION_VERSION, collectCode, gitCommit, gitDiffNameStatus, isWorkingTreeClean } from "./code-collector.js"; import type { CodeFact } from "./code-extractors.js"; import type { InterfaceInventory } from "../interface-scanner.js"; @@ -20,18 +20,24 @@ export async function detectCodeIncrementalChanges( const previous = (await exists(manifestPath)) ? (JSON.parse(await readFile(manifestPath, "utf8")) as { headSha?: string; + version?: number; files?: Array<{ relativePath: string; sha256: string }>; }) : { files: [] }; const oldSha = previous.headSha; const newSha = await gitCommit(root); - - // Git incremental path: only when both commits are known AND the working - // tree is clean. A dirty tree has uncommitted/untracked changes that a - // commit-to-commit diff cannot see, so we fall back to the full sha256 - // scan (which reads the working tree) to avoid silent staleness. - if (oldSha && newSha && (await isWorkingTreeClean(root))) { + // A manifest from an older collector may simply not know files this + // version collects (.scala, say) — a commit-to-commit diff would report + // no changes and keep them absent. The sha256 path below spots them. + const manifestIsCurrent = previous.version === CODE_COLLECTION_VERSION; + + // Git incremental path: only when both commits are known, the manifest + // comes from this collector version, AND the working tree is clean. A + // dirty tree has uncommitted/untracked changes that a commit-to-commit + // diff cannot see, so we fall back to the full sha256 scan (which reads + // the working tree) to avoid silent staleness. + if (oldSha && newSha && manifestIsCurrent && (await isWorkingTreeClean(root))) { const gitChanges = await gitDiffNameStatus(root, oldSha, newSha); if (gitChanges !== null) { const { added, changed, deleted } = gitChanges; diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index 2ce000253..d9db491aa 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -112,13 +112,17 @@ export function extractScala(files: CodeCollectedFile[], context?: ExtractorCont if (!clause) { break; } + // A plain clause may carry a Scala 3 rename (`a.B as Alias`) — the + // imported name is the part before it. const after = rest.slice(clause[0].length); - if (/^\s*\.\s*\{/u.test(after)) { - openImport = { line: lineNumber, prefix: clause[1], selector: after.replace(/^\s*\.\s*\{/u, "") }; + const plainRename = /^\s+as\s+[A-Za-z_]\w*/u.exec(after)?.[0].length ?? 0; + const consumed = after.slice(plainRename); + if (/^\s*\.\s*\{/u.test(consumed)) { + openImport = { line: lineNumber, prefix: clause[1], selector: consumed.replace(/^\s*\.\s*\{/u, "") }; break; } pushRelations(facts, extractContext, clause[1], clause[3], file.relativePath, lineNumber, line); - rest = after.replace(/^\s*,\s*/u, ""); + rest = consumed.replace(/^\s*,\s*/u, ""); } } } @@ -197,30 +201,24 @@ function pushRelations( } else { // `com.demo.Models.*` wildcards an object, not a package — resolve it to // the file that declares the object, as a named import would. - const modulePath = toModulePath(packagePath); - const segments = modulePath.split("/"); - const symbol = segments[segments.length - 1]; - targets = - segments.length > 1 && /^[A-Z]/u.test(symbol) - ? [symbolTarget(symbol, segments.slice(0, -1).join("/"), file, context)] - : [packagePath]; + const resolved = resolveSymbolPath(toModulePath(packagePath), file, context); + targets = resolved !== undefined ? [resolved] : [packagePath]; } } else if (selection && selection.symbols.length > 0) { - targets = [...new Set(selection.symbols.map((symbol) => symbolTarget(symbol, packagePath, file, context)))]; + targets = [ + ...new Set( + selection.symbols.map((symbol) => resolveSymbolPath(`${packagePath}/${symbol}`, file, context) ?? toModulePath(`${packagePath}/${symbol}`)), + ), + ]; } else if (selection) { // only hidden names: the clause imports nothing, so no relation at all targets = []; } else { // A plain import names a symbol (`com.foo.Bar` — or a Scala 3 top-level - // `def validate`) preceded by its package; resolving it against the - // package's declarations keeps the conventional same-name path as fallback. + // `def validate`) preceded by its package; the conventional path is the + // fallback when no collected file declares the symbol. const modulePath = toModulePath(packagePath); - const segments = modulePath.split("/"); - const symbol = segments[segments.length - 1]; - targets = - segments.length > 1 - ? [symbolTarget(symbol, segments.slice(0, -1).join("/"), file, context)] - : [modulePath]; + targets = [resolveSymbolPath(modulePath, file, context) ?? modulePath]; } for (const target of targets) { facts.push(makeFact("relation", target, file, lineNumber, rawLine, "EXTRACTED")); @@ -302,16 +300,23 @@ function expandWildcard(packagePath: string, context: ExtractContext, hidden: Re } /** - * The file in the package that declares `symbol`, or the conventional path if - * none does — the symbol's own name is not a file name to rely on. + * The file a dotted import path resolves to: the package member that declares + * the last type, or — failing that — an earlier one (`com/demo/Models/Invoice` + * names a member of `object Models`, which lives in Models' own declaring + * file). Undefined when no collected file declares any of them. */ -function symbolTarget(symbol: string, packagePath: string, importer: string, context: ExtractContext): string { - for (const candidate of packageMembers(packagePath, context.allFiles)) { - if (candidate.relativePath !== importer && context.declarations.get(candidate.relativePath)?.has(symbol)) { - return candidate.relativePath; +function resolveSymbolPath(modulePath: string, importer: string, context: ExtractContext): string | undefined { + const segments = modulePath.split("/"); + for (let i = segments.length - 1; i >= 1; i--) { + const symbol = segments[i]; + const packagePath = segments.slice(0, i).join("/"); + for (const candidate of packageMembers(packagePath, context.allFiles)) { + if (candidate.relativePath !== importer && context.declarations.get(candidate.relativePath)?.has(symbol)) { + return candidate.relativePath; + } } } - return toModulePath(`${packagePath}/${symbol}`); + return undefined; } /** @@ -348,7 +353,7 @@ function afterPackage(relativePath: string, packagePath: string): string | undef function declarationNames(content: string): Set { const names = new Set(); let depth = 0; - const packageMemberIndents: number[] = []; // open `package x:` blocks, innermost last; -1 until its member indent is seen + const packageMemberIndents: number[] = []; // open package blocks (`x:` or `x {`), innermost last; -1 until its member indent is seen for (const rawLine of content.split(/\r?\n/)) { const trimmed = rawLine.trim(); const indent = rawLine.length - rawLine.trimStart().length; @@ -356,7 +361,10 @@ function declarationNames(content: string): Set { if (trimmed !== "" && last >= 0 && packageMemberIndents[last] === -1) { packageMemberIndents[last] = indent; // the first content line fixes the block's member indent } - if (/^package\s+[\w.]+\s*:\s*$/u.test(trimmed)) { + // `package x {` opens a block, not a declaration nest: its brace does not + // count toward the depth that hides members. + const packageOpener = /^package\s+[\w.]*\s*[:{]/u.test(trimmed); + if (packageOpener) { packageMemberIndents.push(-1); } else if (trimmed !== "") { while (packageMemberIndents.length > 1 && packageMemberIndents[packageMemberIndents.length - 1] > indent) { @@ -379,8 +387,8 @@ function declarationNames(content: string): Set { names.add(defDecl[1]); } } + depth += (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length; } - depth += (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length; } return names; } From 56ba775ce7cd146c67618a1907556185e1b53a64 Mon Sep 17 00:00:00 2001 From: yudongyouqing <142670774+yudongyouqing@users.noreply.github.com> Date: Thu, 8 Oct 2026 14:21:04 +0800 Subject: [PATCH 13/13] fix(wiki): declaration-first wildcard exclusion and three review fixes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Hiding a name dropped its whole file from a wildcard even when the file declared other, live symbols — Invoice.scala declaring Invoice and Order lost the Order dependency under `{Invoice as _, *}`. Exclusion now goes by declarations when they are known (drop the file only when every declared name is hidden), falling back to the file name for files whose declarations are unknown. The forced v1→v2 migration scan in detectCodeIncrementalChanges ran with the default 200-file cap instead of the caller's --max-files, which could classify the tail of a larger prior extraction as deleted and prune its facts; the limit is now passed through. The call-chain tracer filters metadata relations before its bounded slice so they no longer crowd real imports out of traversal. And the brace depth clamps at zero so declarations in a package block following a braced package are still recorded. Co-Authored-By: Claude Code --- src/__tests__/scala-extractor.test.ts | 19 +++++++++++++++++++ src/codebase-extract.ts | 2 +- src/wiki-engine/call-chain-tracer.ts | 6 ++++-- .../code-knowledge/code-incremental.ts | 7 +++++-- .../code-knowledge/extractors/scala.ts | 16 +++++++++++----- 5 files changed, 40 insertions(+), 10 deletions(-) diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts index 72dfad257..716db1147 100644 --- a/src/__tests__/scala-extractor.test.ts +++ b/src/__tests__/scala-extractor.test.ts @@ -345,6 +345,25 @@ describe('Scala heuristic extractor', () => { expect(facts).toEqual(['relation:com/demo/A', 'relation:com/demo/B']); }); + it('keeps a file whose hidden name shares it with live symbols', () => { + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice as _, *}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([invoice, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Hiding Invoice does not hide Order, which lives in the same file. + expect(names).toEqual(['scala-decl:Invoice,Order', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('reads declarations after a braced package closes', () => { + const models = scalaFile(['package a.b {', ' class X', '}', '', 'package c.d {', ' class Y', '}'].join('\n'), 'src/main/scala/c/d/Models.scala'); + + const names = extractScala([models]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:X,Y']); + }); + it('resolves a named import to the file that declares the symbol', () => { const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 53cdf6647..8acf5a5b7 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -575,7 +575,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise { const map = new Map(); for (const fact of facts) { - if (fact.kind !== "relation") continue; + // Metadata relations (wildcard packages, declaration markers) name no + // target and would only crowd the bounded relation slice in traversal. + if (fact.kind !== "relation" || isMetadataRelation(fact.name)) continue; const group = map.get(fact.file) ?? []; group.push(fact); map.set(fact.file, group); diff --git a/src/wiki-engine/code-knowledge/code-incremental.ts b/src/wiki-engine/code-knowledge/code-incremental.ts index f3aa15154..1211938c4 100644 --- a/src/wiki-engine/code-knowledge/code-incremental.ts +++ b/src/wiki-engine/code-knowledge/code-incremental.ts @@ -16,6 +16,7 @@ export async function detectCodeIncrementalChanges( root: string, manifestPath: string, project: string, + maxFiles?: number, ): Promise { const previous = (await exists(manifestPath)) ? (JSON.parse(await readFile(manifestPath, "utf8")) as { @@ -50,8 +51,10 @@ export async function detectCodeIncrementalChanges( } } - // Fallback: full sha256 comparison when git is unavailable or no baseline - const current = await collectCode({ root }); + // Fallback: full sha256 comparison when git is unavailable or no baseline. + // Honors the caller's --max-files: a default-capped scan against a larger + // prior extraction would classify the tail as deleted and prune it. + const current = await collectCode({ root, maxFiles }); const previousByPath = new Map((previous.files ?? []).map((file) => [file.relativePath, file.sha256])); const currentByPath = new Map(current.manifest.files.map((file) => [file.relativePath, file.sha256])); const added = [...currentByPath.keys()].filter((file) => !previousByPath.has(file)).sort(); diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts index d9db491aa..0dc858800 100644 --- a/src/wiki-engine/code-knowledge/extractors/scala.ts +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -290,11 +290,14 @@ function toModulePath(path: string): string { function expandWildcard(packagePath: string, context: ExtractContext, hidden: ReadonlySet): string[] { return packageMembers(packagePath, context.allFiles) .filter((candidate) => { - if (hidden.has(fileName(candidate.relativePath))) { - return false; - } + // Exclusion is declaration-based when the names are known: a file with + // any live declaration still carries its other symbols. Only a file + // with no known declarations falls back to its file name. const declared = context.declarations.get(candidate.relativePath); - return !(declared && declared.size > 0 && [...declared].every((name) => hidden.has(name))); + if (declared && declared.size > 0) { + return ![...declared].every((name) => hidden.has(name)); + } + return !hidden.has(fileName(candidate.relativePath)); }) .map((candidate) => candidate.relativePath); } @@ -387,7 +390,10 @@ function declarationNames(content: string): Set { names.add(defDecl[1]); } } - depth += (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length; + // A package block's closing brace has no matching opener in the count + // (openers are deliberately skipped) — clamp so later blocks still sit + // at depth 0. + depth = Math.max(0, depth + (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length); } } return names;