diff --git a/skill-data/wiki/references/agents/kb-doc-generator.md b/skill-data/wiki/references/agents/kb-doc-generator.md index 448c7afe4..da308d242 100644 --- a/skill-data/wiki/references/agents/kb-doc-generator.md +++ b/skill-data/wiki/references/agents/kb-doc-generator.md @@ -131,6 +131,7 @@ Use the `Glob → Grep → Read` three-step method (**adapt to the language of t TypeScript: app.ts / index.ts / main.ts / server.ts Rust: main.rs / src/main.rs Swift: main.swift / App.swift + Scala: Main.scala / App.scala 2. Grep: locate the core Handlers/Routers (choose the pattern by language + framework) Go: grep -rn 'func.*Handler\|\.GET\|\.POST\|router\.\|@handler' diff --git a/skill-data/wiki/scripts/scan_repo.py b/skill-data/wiki/scripts/scan_repo.py index 08691ab2b..ca965ae18 100644 --- a/skill-data/wiki/scripts/scan_repo.py +++ b/skill-data/wiki/scripts/scan_repo.py @@ -53,7 +53,7 @@ # Language extension map LANG_MAP = { ".py": "Python", ".go": "Go", ".js": "JavaScript", ".ts": "TypeScript", - ".java": "Java", ".rs": "Rust", ".swift": "Swift", ".rb": "Ruby", ".php": "PHP", + ".java": "Java", ".rs": "Rust", ".swift": "Swift", ".scala": "Scala", ".rb": "Ruby", ".php": "PHP", ".c": "C", ".cpp": "C++", ".h": "C/C++ Header", ".proto": "Protobuf", ".thrift": "Thrift", ".graphql": "GraphQL", ".sql": "SQL", ".sh": "Shell", ".bash": "Shell", diff --git a/src/__tests__/scala-extractor.test.ts b/src/__tests__/scala-extractor.test.ts new file mode 100644 index 000000000..716db1147 --- /dev/null +++ b/src/__tests__/scala-extractor.test.ts @@ -0,0 +1,490 @@ +import { describe, it, expect } from 'vitest'; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { CodeCollectedFile } from '../wiki-engine/code-knowledge/code-collector.js'; +import { collectCode } from '../wiki-engine/code-knowledge/code-collector.js'; +import { buildCodeGraph } from '../wiki-engine/code-knowledge/code-graph.js'; +import { traceCallChains } from '../wiki-engine/call-chain-tracer.js'; +import { extractScala } from '../wiki-engine/code-knowledge/extractors/scala.js'; +import { extractForLanguage, supportedLanguages } from '../wiki-engine/code-knowledge/extractors/index.js'; + +function scalaFile(content: string, relativePath = 'src/main/scala/com/payments/App.scala'): CodeCollectedFile { + return { + path: `/virtual/${relativePath}`, + relativePath, + language: 'scala', + sha256: 'test', + content, + }; +} + +/** Collect the `kind:name` pairs the extractor produced. */ +function extracted(content: string): string[] { + return extractScala([scalaFile(content)]).map((f) => `${f.kind}:${f.name}`); +} + +describe('Scala heuristic extractor', () => { + it('is registered for the scala language', () => { + expect(supportedLanguages()).toContain('scala'); + expect(extractForLanguage('scala', [scalaFile('object App { }\n')])).not.toEqual([]); + }); + + it('extracts classes, objects, traits, enums and defs', () => { + const facts = extracted( + [ + 'package com.payments', + '', + 'trait PaymentGateway {', + ' def charge(order: Order): Receipt', + '}', + '', + 'final case class Invoice(id: Int, amount: BigDecimal)', + '', + 'case object Paid', + '', + 'class OrderRepository(cache: Cache)', + '', + 'enum Color {', + ' case Red, Blue', + '}', + '', + 'object Main extends App {', + ' def run(): Unit = ()', + '}', + ].join('\n'), + ); + + expect(facts).toContain('interface:PaymentGateway'); + expect(facts).toContain('component:charge'); + expect(facts).toContain('component:Invoice'); + expect(facts).toContain('component:Paid'); + expect(facts).toContain('component:OrderRepository'); + expect(facts).toContain('component:Color'); + expect(facts).toContain('component:Main'); + expect(facts).toContain('component:run'); + }); + + it('sees past annotations and modifiers, including a `case` match clause', () => { + const facts = extracted( + [ + '@Singleton', + 'final case class Invoice(id: Int)', + '', + 'sealed abstract class Payment(val id: String)', + '', + 'private[payments] object PaymentRegistry', + '', + 'implicit class RichString(s: String) {', + ' def quoted: String = s', + '}', + '', + 'transparent inline def show(x: Any): String = x.toString', + '', + 'val label = payload match {', + ' case Invoice(id) => id', + ' case other => 0', + '}', + ].join('\n'), + ); + + // `case Invoice(id) =>` is a match clause, not a declaration. + expect(facts.filter((f) => f === 'component:Invoice')).toHaveLength(1); + expect(facts).toContain('component:Payment'); + expect(facts).toContain('component:PaymentRegistry'); + expect(facts).toContain('component:RichString'); + expect(facts).toContain('component:quoted'); + expect(facts).toContain('component:show'); + }); + + it('extracts import relations regardless of the import form', () => { + const facts = extracted( + [ + 'import com.payments.gateway', + 'import java.util.concurrent.TimeUnit', + 'import scala.collection.mutable.{Map => MMap}', + 'import com.payments.events._', + 'import com.payments.core.*', + 'import com.payments.model.{Invoice as Inv, Order => O}', + 'import com.demo.core.Invoice.apply', + ].join('\n'), + ); + + // Relation names are slash-separated paths: buildCodeGraph and the + // call-chain tracer both match against file paths, not dotted packages. + // Brace selectors expand to one relation per imported symbol; a member + // import narrows to the type's path. + expect(facts).toEqual([ + 'relation:com/payments/gateway', + 'relation:java/util/concurrent/TimeUnit', + 'relation:scala/collection/mutable/Map', + 'relation:scala-wildcard:com/payments/events', + 'relation:com/payments/events', + 'relation:scala-wildcard:com/payments/core', + 'relation:com/payments/core', + 'relation:com/payments/model/Invoice', + 'relation:com/payments/model/Order', + 'relation:com/demo/core/Invoice', + ]); + }); + + it('falls back to the package path when no collected file matches the import', () => { + const facts = extracted( + ['import com.demo.core.{Invoice => _, _}', 'import com.demo.core.{Order => _}'].join('\n'), + ); + + // The fixture file sits outside com/demo/core, so the wildcard has no + // collected file to name. A selector of only hidden names (`{Order => _}`) + // imports nothing and emits nothing — no relation for `Invoice` either. + expect(facts).toEqual(['relation:scala-wildcard:com/demo/core', 'relation:com/demo/core']); + }); + + it('reads a brace import scalafmt wraps across lines', () => { + const facts = extracted(['import com.demo.core.{', ' Invoice,', ' Order => O', '}'].join('\n')); + + expect(facts).toEqual(['relation:com/demo/core/Invoice', 'relation:com/demo/core/Order']); + }); + + it('reads every clause of a comma-separated import', () => { + const facts = extracted('import com.demo.A, com.demo.B'); + + expect(facts).toEqual(['relation:com/demo/A', 'relation:com/demo/B']); + }); + + it('expands a wildcard over the package files and skips hidden names', () => { + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + + const names = extractScala([gateway, invoice, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala', 'scala-decl:Invoice', 'scala-decl:Main']); + }); + + it('reads top-level declarations under a Scala 3 package block', () => { + const models = scalaFile( + ['package com.demo.core:', ' case class Invoice(id: Int)', ' object Registry:', ' def build: Int = 1'].join('\n'), + 'src/main/scala/com/demo/core/Models.scala', + ); + const main = scalaFile('package com.demo.core:\n object Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile( + 'package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\nimport com.demo.core.Registry\n', + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // `def build` sits one indent deeper than Registry — a member, not a name + // the package exports; Invoice is hidden, Registry keeps Models.scala in. + expect(names).toEqual(['scala-decl:Invoice,Registry', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Models.scala', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('ignores nested defs when a hidden name decides wildcard exclusion', () => { + const models = scalaFile('package com.demo.core\n\nclass Invoice {\n def total: Int = 1\n}\n', 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Models.scala's only package-level name is the hidden Invoice — its + // nested `def total` is not something the wildcard could have imported. + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('resolves a wildcard on an object to the file declaring it', () => { + const domain = scalaFile('package com.demo\n\nobject Models {\n val all = List(1)\n}\n', 'src/main/scala/com/demo/Domain.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.Models.*\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([domain, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Models', 'scala-wildcard:com/demo/Models', 'src/main/scala/com/demo/Domain.scala']); + }); + + it('expands a plain wildcard over the package files', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core\n\nobject InternalAdmin\n', 'src/main/scala/com/demo/core/InternalAdmin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala', 'scala-decl:Invoice', 'scala-decl:InternalAdmin']); + }); + + it('expands a Scala 3 `*` wildcard the same way as `_`', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core.*\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core\n\nobject InternalAdmin\n', 'src/main/scala/com/demo/core/InternalAdmin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/InternalAdmin.scala', 'scala-decl:Invoice', 'scala-decl:InternalAdmin']); + }); + + it('expands a wildcard over Java files of the package, minus hidden names', () => { + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const invoiceJava: CodeCollectedFile = { + path: '/virtual/src/main/java/com/demo/core/Invoice.java', + relativePath: 'src/main/java/com/demo/core/Invoice.java', + language: 'java', + sha256: 'test', + content: 'package com.demo.core;\n\npublic class Invoice {}\n', + }; + const orderJava: CodeCollectedFile = { + ...invoiceJava, + relativePath: 'src/main/java/com/demo/core/Order.java', + content: 'package com.demo.core;\n\npublic class Order {}\n', + }; + const collected = [gateway, invoiceJava, orderJava]; + + const names = extractScala([gateway], { allFiles: collected, priorDeclarations: new Map() }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/java/com/demo/core/Order.java']); + }); + + it('resolves against a previous run\'s declarations when only the importer changed', () => { + // Incremental run: the batch holds only the changed importer; the rest of + // the project is known by path (content-less stubs) and cached facts. + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + const stub = (relativePath: string): CodeCollectedFile => ({ path: `/virtual/${relativePath}`, relativePath, language: 'scala', sha256: '', content: '' }); + const priorDeclarations = new Map>([ + ['src/main/scala/com/demo/core/Invoice.scala', new Set(['Invoice'])], + ['src/main/scala/com/demo/core/Main.scala', new Set(['Main'])], + ['src/main/scala/com/demo/core/Models.scala', new Set(['Order'])], + ]); + + const names = extractScala([gateway], { + allFiles: [gateway, stub('src/main/scala/com/demo/core/Invoice.scala'), stub('src/main/scala/com/demo/core/Main.scala'), stub('src/main/scala/com/demo/core/Models.scala')], + priorDeclarations, + }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('keeps a declared-hidden symbol out of the wildcard even in a differently named file', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Models.scala declares only the hidden Invoice, so the wildcard skips it too. + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('does not import package-adjacent resources', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const schema = scalaFile('-- schema', 'src/main/resources/com/demo/core/schema.sql'); + + const names = extractScala([main, invoice], { allFiles: [main, invoice, schema], priorDeclarations: new Map() }) + .filter((f) => f.kind === 'relation') + .map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'scala-decl:Invoice']); + }); + + it('resolves a top-level def import to its file', () => { + const helpers = scalaFile('package com.demo.core\n\ndef validate(s: String): Boolean = true\n', 'src/main/scala/com/demo/core/Helpers.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.core.validate\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([helpers, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:validate', 'src/main/scala/com/demo/core/Helpers.scala']); + }); + + it('strips the _root_ qualifier from an import', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport _root_.com.demo.core.Invoice\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Invoice', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('reads nested Scala 3 package blocks', () => { + const models = scalaFile( + ['package com.demo:', ' package core:', ' case class Invoice(id: Int)', ' object Registry'].join('\n'), + 'src/main/scala/com/demo/core/Models.scala', + ); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Invoice\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Invoice,Registry', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('reads declarations inside a braced package', () => { + const models = scalaFile(['package com.demo.core {', ' class Invoice {', ' def total: Int = 1', ' }', '}'].join('\n'), 'src/main/scala/com/demo/core/Models.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice => _, _}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // The package brace does not nest the declarations away, and Models.scala + // declares only the hidden Invoice — the wildcard skips it. + expect(names).toEqual(['scala-decl:Invoice', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('resolves an import of a member declared inside an object', () => { + const domain = scalaFile('package com.demo\n\nobject Models {\n case class Invoice(id: Int)\n}\n', 'src/main/scala/com/demo/Domain.scala'); + const main = scalaFile('package com.demo.app\n\nimport com.demo.Models.Invoice\n', 'src/main/scala/com/demo/app/Main.scala'); + + const names = extractScala([domain, main]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Models', 'src/main/scala/com/demo/Domain.scala']); + }); + + it('reads a comma-separated import with a plain `as` rename', () => { + const facts = extracted('import com.demo.A as Alias, com.demo.B'); + + expect(facts).toEqual(['relation:com/demo/A', 'relation:com/demo/B']); + }); + + it('keeps a file whose hidden name shares it with live symbols', () => { + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const main = scalaFile('package com.demo.core\n\nobject Main\n', 'src/main/scala/com/demo/core/Main.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.{Invoice as _, *}\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([invoice, main, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + // Hiding Invoice does not hide Order, which lives in the same file. + expect(names).toEqual(['scala-decl:Invoice,Order', 'scala-decl:Main', 'scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'src/main/scala/com/demo/core/Main.scala']); + }); + + it('reads declarations after a braced package closes', () => { + const models = scalaFile(['package a.b {', ' class X', '}', '', 'package c.d {', ' class Y', '}'].join('\n'), 'src/main/scala/c/d/Models.scala'); + + const names = extractScala([models]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:X,Y']); + }); + + it('resolves a named import to the file that declares the symbol', () => { + const models = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\ncase class Order(id: Int)\n', 'src/main/scala/com/demo/core/Models.scala'); + const gateway = scalaFile('package com.demo.payments\n\nimport com.demo.core.Order\n', 'src/main/scala/com/demo/payments/Gateway.scala'); + + const names = extractScala([models, gateway]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-decl:Invoice,Order', 'src/main/scala/com/demo/core/Models.scala']); + }); + + it('excludes a subpackage from a wildcard import', () => { + const main = scalaFile('package com.demo.app\n\nimport com.demo.core._\n', 'src/main/scala/com/demo/app/Main.scala'); + const invoice = scalaFile('package com.demo.core\n\ncase class Invoice(id: Int)\n', 'src/main/scala/com/demo/core/Invoice.scala'); + const admin = scalaFile('package com.demo.core.internal\n\nobject Admin\n', 'src/main/scala/com/demo/core/internal/Admin.scala'); + + const names = extractScala([main, invoice, admin]).filter((f) => f.kind === 'relation').map((f) => f.name); + + expect(names).toEqual(['scala-wildcard:com/demo/core', 'src/main/scala/com/demo/core/Invoice.scala', 'scala-decl:Invoice', 'scala-decl:Admin']); + }); + + it('produces a dependency edge for an internal import', () => { + const gateway = scalaFile( + ['package com.demo.payments', '', 'import com.demo.core.Invoice', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + + const graph = buildCodeGraph([...extractScala([gateway]), ...extractScala([invoice])]); + const edge = graph.edges.find((e) => e.relation === 'DEPENDS_ON'); + + expect(edge?.from).toBe('src/main/scala/com/demo/payments/Gateway.scala'); + expect(edge?.to).toBe('src/main/scala/com/demo/core/Invoice.scala'); + }); + + it('scopes a brace import to the selected symbol, not the whole package directory', () => { + const gateway = scalaFile( + ['package com.demo.payments', '', 'import com.demo.core.{Invoice}', '', 'object Gateway {', ' def charge(i: Invoice): Boolean = true', '}'].join('\n'), + 'src/main/scala/com/demo/payments/Gateway.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + const admin = scalaFile( + ['package com.demo.core', '', 'object InternalAdmin'].join('\n'), + 'src/main/scala/com/demo/core/InternalAdmin.scala', + ); + + const graph = buildCodeGraph([...extractScala([gateway]), ...extractScala([invoice]), ...extractScala([admin])]); + const deps = graph.edges.filter((e) => e.relation === 'DEPENDS_ON'); + + expect(deps).toHaveLength(1); + expect(deps[0].from).toBe('src/main/scala/com/demo/payments/Gateway.scala'); + expect(deps[0].to).toBe('src/main/scala/com/demo/core/Invoice.scala'); + }); + + it('resolves a brace import in the call-chain tracer', () => { + const main = scalaFile( + ['package com.demo.app', '', 'import com.demo.core.{Invoice}', '', 'object Main extends App {', ' val i: Invoice = null', '}'].join('\n'), + 'src/main/scala/com/demo/app/Main.scala', + ); + const invoice = scalaFile( + ['package com.demo.core', '', 'case class Invoice(id: Int)'].join('\n'), + 'src/main/scala/com/demo/core/Invoice.scala', + ); + const admin = scalaFile( + ['package com.demo.core', '', 'object InternalAdmin'].join('\n'), + 'src/main/scala/com/demo/core/InternalAdmin.scala', + ); + const files = [main, invoice, admin]; + const facts = files.flatMap((f) => extractScala([f])); + + const chains = traceCallChains(facts, files); + const mainChain = chains.find((c) => c.entryPoint.includes('Main.scala')); + + // The brace import resolves to Invoice.scala's component; the sibling + // InternalAdmin.scala in the same package directory is not pulled in. + expect(mainChain?.steps[0]?.callsTo).toContain('Invoice'); + expect(mainChain?.steps.some((s) => s.file.endsWith('InternalAdmin.scala'))).toBe(false); + }); + + it('infers error types from the Error/Exception suffix and reads environment config', () => { + const facts = extracted( + [ + 'case class PaymentError(msg: String) extends RuntimeException("boom")', + '', + 'sealed trait AppException extends Exception', + '', + 'object Config {', + ' val apiKey = sys.env("PAYMENTS_API_KEY")', + ' val region = System.getenv("AWS_REGION")', + '}', + ].join('\n'), + ); + + expect(facts).toContain('error:PaymentError'); + expect(facts).toContain('error:AppException'); + expect(facts).toContain('config:PAYMENTS_API_KEY'); + expect(facts).toContain('config:AWS_REGION'); + }); +}); + +describe('Scala source collection', () => { + it('collects .scala files with the scala language and key-file marking', async () => { + const root = mkdtempSync(path.join(os.tmpdir(), 'teamai-scala-collect-')); + try { + mkdirSync(path.join(root, 'src', 'main', 'scala'), { recursive: true }); + writeFileSync(path.join(root, 'src', 'main', 'scala', 'Main.scala'), 'object Main extends App\n'); + writeFileSync(path.join(root, 'src', 'main', 'scala', 'Helper.scala'), 'object Helper { }\n'); + + const { manifest } = await collectCode({ root }); + const byPath = new Map(manifest.files.map((f) => [f.relativePath, f])); + + expect(byPath.get('src/main/scala/Main.scala')?.language).toBe('scala'); + expect(byPath.get('src/main/scala/Main.scala')?.isKeyFile).toBe(true); + expect(byPath.get('src/main/scala/Helper.scala')?.language).toBe('scala'); + expect(byPath.get('src/main/scala/Helper.scala')?.isKeyFile).toBe(false); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/src/ci/extract-mr.ts b/src/ci/extract-mr.ts index 44185ddde..9f53edc84 100644 --- a/src/ci/extract-mr.ts +++ b/src/ci/extract-mr.ts @@ -256,7 +256,7 @@ export async function ciExtractMr(opts: CiExtractMrOptions): Promise { { cwd: businessRepo, encoding: 'utf-8', timeout: 10_000 }, ); changedFiles = diffOutput.trim().split('\n') - .filter(f => f && /\.(ts|tsx|js|jsx|py|go|rs|java)$/.test(f)); + .filter(f => f && /\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/.test(f)); if (changedFiles.length > 0) break; } catch { continue; diff --git a/src/codebase-extract.ts b/src/codebase-extract.ts index 41a2092df..8acf5a5b7 100644 --- a/src/codebase-extract.ts +++ b/src/codebase-extract.ts @@ -28,6 +28,11 @@ import { formatAstStatsSummary, } from './wiki-engine/adapters/index.js'; import type { CodeFact, InterfaceInventory, CallChain } from './wiki-engine/adapters/index.js'; +import type { CodeCollectedFile } from './wiki-engine/code-knowledge/code-collector.js'; +import { CODE_COLLECTION_VERSION } from './wiki-engine/code-knowledge/code-collector.js'; +import type { ExtractorContext } from './wiki-engine/code-knowledge/extractors/index.js'; +import { SCALA_DECL_PREFIX, SCALA_WILDCARD_PREFIX } from './wiki-engine/code-knowledge/extractors/index.js'; +import { isMetadataRelation } from './wiki-engine/code-knowledge/code-extractors.js'; import { loadFactsCache, saveFactsCache, @@ -82,6 +87,11 @@ interface KnowledgeGap { source: string; } +/** 用户可见的 facts：元数据 relation（通配包名、顶层声明标记）不进任何统计。 */ +function visibleFactsOf(facts: CodeFact[]): CodeFact[] { + return facts.filter((f) => f.kind !== 'relation' || !isMetadataRelation(f.name)); +} + function detectKnowledgeGaps( facts: CodeFact[], graph: GraphIndex, @@ -96,7 +106,7 @@ function detectKnowledgeGaps( } // 1. 未解析的外部依赖：import target 不在扫描范围内 - const relationFacts = facts.filter((f) => f.kind === 'relation'); + const relationFacts = facts.filter((f) => f.kind === 'relation' && !isMetadataRelation(f.name)); const unresolvedImports = new Set(); for (const rel of relationFacts) { const target = rel.name; @@ -216,7 +226,7 @@ function buildEvidencePages( pages.set(`${kind}.md`, lines.join('\n')); } - const relationFacts = facts.filter((f) => f.kind === 'relation'); + const relationFacts = facts.filter((f) => f.kind === 'relation' && !isMetadataRelation(f.name)); if (relationFacts.length > 0) { const byDir = new Map(); for (const fact of relationFacts) { @@ -298,7 +308,7 @@ function buildEvidencePages( '', `# ${project}`, '', - `Facts: ${facts.length} | Pages: ${pages.size}`, + `Facts: ${visibleFactsOf(facts).length} | Pages: ${pages.size}`, '', ]; @@ -467,7 +477,7 @@ function buildOverview( '', `# ${project}`, '', - `**${facts.length} facts** extracted from ${new Set(facts.map(f => f.file)).size} files.`, + `**${visibleFactsOf(facts).length} facts** extracted from ${new Set(facts.map(f => f.file)).size} files.`, `Graph: ${graph.nodes.length} nodes, ${graph.edges.length} edges.`, '', '## Module Structure', @@ -565,7 +575,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise(); + for (const fact of cachedFacts) { + if (fact.kind !== 'relation' || fact.name.startsWith(SCALA_DECL_PREFIX)) continue; + const touchedPackage = (pkg: string): boolean => + originalChangedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)) || + deletedFiles.some((f) => f.includes(`/${pkg}/`) || f.startsWith(`${pkg}/`)); + if (fact.name.startsWith(SCALA_WILDCARD_PREFIX)) { + if (touchedPackage(fact.name.slice(SCALA_WILDCARD_PREFIX.length))) staleImporters.add(fact.file); + } else if (/\.(?:scala|java)$/.test(fact.name)) { + // 物化目标本身变了——importer 必须重新解析 + if (changedSet.has(fact.name)) staleImporters.add(fact.file); + } else { + // 未物化的常规路径（com/demo/core/Invoice）：包目录里有文件增删时， + // 原本解析不到的目标可能已经可解析 + const pkg = fact.name.split('/').slice(0, -1).join('/'); + if (pkg && touchedPackage(pkg)) staleImporters.add(fact.file); + } + } + if (staleImporters.size > 0) { + changedFiles = [...new Set([...changedFiles, ...staleImporters])]; + } + } + const { files, manifest: collectionManifest } = await collectCode({ root, maxFiles, changedFiles }); if (files.length === 0 && !changedFiles) { // 全量模式下无文件 @@ -597,17 +640,39 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise 0 ? extractCodeFacts(files) : []; + // 增量模式下，跨文件解析（通配展开、符号定位到声明文件）需要未变更文件的 + // 路径与声明；两者都能从上一轮 facts 缓存得到，未变更文件无需重新读取。 + // 声明只从 scala-decl: 标记重建——component facts 分不出嵌套成员， + // 标记在提取时就只记包级名字 + let extractionContext: ExtractorContext | undefined; + if (changedFiles !== undefined) { + const priorDeclarations = new Map>(); + const stubs = new Map(); + const removed = new Set([...changedFiles, ...deletedFiles]); + for (const fact of cachedFacts ?? []) { + if (!removed.has(fact.file)) { + stubs.set(fact.file, { path: fact.file, relativePath: fact.file, language: 'text', sha256: '', content: '' }); + } + if (fact.kind === 'relation' && fact.name.startsWith(SCALA_DECL_PREFIX)) { + const names = new Set(fact.name.slice(SCALA_DECL_PREFIX.length).split(',')); + priorDeclarations.set(fact.file, names); + } + } + for (const file of files) { + stubs.delete(file.relativePath); // 变更文件以本批为准 + } + extractionContext = { allFiles: [...files, ...stubs.values()], priorDeclarations }; + } + + const newFacts = files.length > 0 ? extractCodeFacts(files, extractionContext) : []; // 增量模式：加载缓存 → 剪除 → 合并 let facts: CodeFact[]; let interfaceInventory: InterfaceInventory; - const indicesDir = path.join(wikiRoot, '.indices'); if (changedFiles !== undefined) { // 增量模式（含 changedFiles=[] 即仅删除场景） - const oldFacts = await loadFactsCache(indicesDir); + const oldFacts = cachedFacts ?? (await loadFactsCache(indicesDir)); const oldInterfaces = await loadInterfacesCache(indicesDir); // 剪除已变更/删除的旧数据 @@ -823,7 +888,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise 0 ? ifByType : undefined, @@ -908,7 +973,7 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise = { - version: 1, + version: CODE_COLLECTION_VERSION, lastScan: new Date().toISOString(), files: allManifestFiles, }; @@ -929,14 +994,14 @@ export async function extractCodebase(opts: ExtractCodebaseOptions): Promise = {}; - for (const fact of facts) { + for (const fact of visibleFactsOf(facts)) { byKind[fact.kind] = (byKind[fact.kind] ?? 0) + 1; } const result: ExtractResult = { project, filesScanned: files.length, - facts: { total: facts.length, byKind }, + facts: { total: visibleFactsOf(facts).length, byKind }, graph: { nodes: repoGraph.nodes.length, edges: repoGraph.edges.length }, incremental: !!opts.incremental && !!changedFiles, outputDir: wikiRoot, diff --git a/src/import-repo.ts b/src/import-repo.ts index f7b8fd360..4d559c479 100644 --- a/src/import-repo.ts +++ b/src/import-repo.ts @@ -119,7 +119,7 @@ export function detectCrossRepoEdges( for (const edge of overlay.edges) { if (edge.relation !== 'imports') continue; const segments = edge.to.split('/'); - const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift)$/, '') ?? ''; + const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/, '') ?? ''; const pascalName = fileName.split(/[-_]/).map(s => s.charAt(0).toUpperCase() + s.slice(1)).join(''); const match = existingIndex.get(pascalName.toLowerCase()); @@ -146,7 +146,7 @@ export function detectCrossRepoEdges( for (const edge of existing.edges) { if (edge.relation !== 'imports') continue; const segments = edge.to.split('/'); - const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift)$/, '') ?? ''; + const fileName = segments[segments.length - 1]?.replace(/\.(ts|tsx|js|jsx|py|go|rs|java|swift|scala)$/, '') ?? ''; const pascalName = fileName.split(/[-_]/).map(s => s.charAt(0).toUpperCase() + s.slice(1)).join(''); const match = overlayIndex.get(pascalName.toLowerCase()); diff --git a/src/wiki-engine/call-chain-tracer.ts b/src/wiki-engine/call-chain-tracer.ts index 0332dbe25..cd7b2773d 100644 --- a/src/wiki-engine/call-chain-tracer.ts +++ b/src/wiki-engine/call-chain-tracer.ts @@ -1,5 +1,5 @@ import type { CodeCollectedFile } from './code-knowledge/code-collector.js'; -import type { CodeFact } from './code-knowledge/code-extractors.js'; +import { type CodeFact, isMetadataRelation } from './code-knowledge/code-extractors.js'; export type CallChainLayer = "entry" | "orchestration" | "service" | "data"; @@ -24,9 +24,11 @@ const ENTRY_PATTERNS = [ /route/i, /controller/i, /endpoint/i, - /main\.(ts|go|py|rs|java|swift)$/, - /server\.(ts|go|py|rs|java|swift)$/, - /app\.(ts|go|py|rs|java|swift)$/, + // Case-insensitive: Scala's key files are Main.scala / App.scala, and a + // capitalized App.ts or Main.go is just as much an entry point. + /main\.(ts|go|py|rs|java|swift|scala)$/i, + /server\.(ts|go|py|rs|java|swift|scala)$/i, + /app\.(ts|go|py|rs|java|swift|scala)$/i, ]; const ORCHESTRATION_PATTERNS = [ @@ -195,7 +197,9 @@ function findEntryPoints(facts: CodeFact[], files: CodeCollectedFile[]): EntryPo function buildRelationsByFile(facts: CodeFact[]): Map { const map = new Map(); for (const fact of facts) { - if (fact.kind !== "relation") continue; + // Metadata relations (wildcard packages, declaration markers) name no + // target and would only crowd the bounded relation slice in traversal. + if (fact.kind !== "relation" || isMetadataRelation(fact.name)) continue; const group = map.get(fact.file) ?? []; group.push(fact); map.set(fact.file, group); @@ -237,7 +241,7 @@ function resolveRelationTarget(importPath: string, filesByModule: Map = { java: [/Application\.java$/, /Controller\.java$/, /Service\.java$/], typescript: [/index\.ts$/, /server\.ts$/, /app\.ts$/, /router\.ts$/], rust: [/main\.rs$/, /lib\.rs$/, /mod\.rs$/], - swift: [/main\.swift$/, /App\.swift$/, /Package\.swift$/] + swift: [/main\.swift$/, /App\.swift$/, /Package\.swift$/], + scala: [/Main\.scala$/, /App\.scala$/] }; export function isKeyFile(relativePath: string, language: string): boolean { @@ -132,7 +141,7 @@ async function walk(directory: string, results: string[], includeTests: boolean) } function isCodeFile(filePath: string): boolean { - return [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".swift", ".json", ".yaml", ".yml", ".toml", ".sql", ".conf", ".ini"].includes( + return [".ts", ".tsx", ".js", ".jsx", ".mjs", ".cjs", ".py", ".go", ".rs", ".java", ".swift", ".scala", ".json", ".yaml", ".yml", ".toml", ".sql", ".conf", ".ini"].includes( path.extname(filePath).toLowerCase() ); } @@ -145,7 +154,7 @@ function languageFor(filePath: string): string { const ext = path.extname(filePath).toLowerCase(); const map: Record = { ".ts": "typescript", ".tsx": "typescript", ".js": "javascript", ".jsx": "javascript", - ".py": "python", ".go": "go", ".rs": "rust", ".java": "java", ".swift": "swift", + ".py": "python", ".go": "go", ".rs": "rust", ".java": "java", ".swift": "swift", ".scala": "scala", ".json": "json", ".yaml": "yaml", ".yml": "yaml", ".toml": "toml", ".sql": "sql", ".conf": "toml", ".ini": "toml", }; diff --git a/src/wiki-engine/code-knowledge/code-extractors.ts b/src/wiki-engine/code-knowledge/code-extractors.ts index 7624014e4..ac550056a 100644 --- a/src/wiki-engine/code-knowledge/code-extractors.ts +++ b/src/wiki-engine/code-knowledge/code-extractors.ts @@ -1,5 +1,5 @@ import { type CodeCollectedFile } from "./code-collector.js"; -import { extractForLanguage } from "./extractors/index.js"; +import { extractForLanguage, type ExtractorContext } from "./extractors/index.js"; export type CodeFactKind = "component" | "interface" | "config" | "error" | "data" | "style" | "relation"; @@ -36,15 +36,28 @@ export interface CodeFact { evidenceType?: CodeEvidenceType; } +/** + * Relations that carry extraction metadata for the incremental layer — a + * Scala wildcard's package, a file's top-level declarations — rather than a + * dependency. Evidence pages and gap detection skip them, and the graph + * builder resolves none of them. + */ +export function isMetadataRelation(name: string): boolean { + return /^scala-(?:wildcard|decl):/u.test(name); +} + /** * Extract code facts from collected files. * Groups files by language, then dispatches to language-specific extractors. + * `context` — the run's full file list and the previous run's declarations — + * lets extractors resolve cross-file constructs (see ExtractorContext). */ -export function extractCodeFacts(files: CodeCollectedFile[]): CodeFact[] { +export function extractCodeFacts(files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { const byLanguage = groupByLanguage(files); const allFacts: CodeFact[] = []; + const effectiveContext: ExtractorContext = context ?? { allFiles: files, priorDeclarations: new Map() }; for (const [language, langFiles] of byLanguage) { - allFacts.push(...extractForLanguage(language, langFiles)); + allFacts.push(...extractForLanguage(language, langFiles, effectiveContext)); } // Deduplicate facts by kind:name:file (same symbol in same file only kept once) const seen = new Set(); diff --git a/src/wiki-engine/code-knowledge/code-graph.ts b/src/wiki-engine/code-knowledge/code-graph.ts index f91b7585c..a4339fefd 100644 --- a/src/wiki-engine/code-knowledge/code-graph.ts +++ b/src/wiki-engine/code-knowledge/code-graph.ts @@ -1,6 +1,6 @@ import path from "node:path"; -import { type CodeFact } from "./code-extractors.js"; +import { type CodeFact, isMetadataRelation } from "./code-extractors.js"; import { type GraphIndex, type GraphNode, @@ -27,7 +27,7 @@ export function buildCodeGraph(facts: CodeFact[]): GraphIndex { const nodeFiles = new Set(facts.filter(f => f.kind !== "relation").map(f => f.file)); const edges: GraphEdge[] = facts - .filter((fact) => fact.kind === "relation") + .filter((fact) => fact.kind === "relation" && !isMetadataRelation(fact.name)) .flatMap((fact) => { // AST-derived relation facts carry a resolved target file in fact.name and a // "(code-ast)" marker in detail — trust them directly instead of fuzzy matching. diff --git a/src/wiki-engine/code-knowledge/code-incremental.ts b/src/wiki-engine/code-knowledge/code-incremental.ts index 641f4847c..1211938c4 100644 --- a/src/wiki-engine/code-knowledge/code-incremental.ts +++ b/src/wiki-engine/code-knowledge/code-incremental.ts @@ -1,7 +1,7 @@ import { readFile, writeFile, stat, mkdir } from "node:fs/promises"; import path from "node:path"; -import { collectCode, gitCommit, gitDiffNameStatus, isWorkingTreeClean } from "./code-collector.js"; +import { CODE_COLLECTION_VERSION, collectCode, gitCommit, gitDiffNameStatus, isWorkingTreeClean } from "./code-collector.js"; import type { CodeFact } from "./code-extractors.js"; import type { InterfaceInventory } from "../interface-scanner.js"; @@ -16,22 +16,29 @@ export async function detectCodeIncrementalChanges( root: string, manifestPath: string, project: string, + maxFiles?: number, ): Promise { const previous = (await exists(manifestPath)) ? (JSON.parse(await readFile(manifestPath, "utf8")) as { headSha?: string; + version?: number; files?: Array<{ relativePath: string; sha256: string }>; }) : { files: [] }; const oldSha = previous.headSha; const newSha = await gitCommit(root); - - // Git incremental path: only when both commits are known AND the working - // tree is clean. A dirty tree has uncommitted/untracked changes that a - // commit-to-commit diff cannot see, so we fall back to the full sha256 - // scan (which reads the working tree) to avoid silent staleness. - if (oldSha && newSha && (await isWorkingTreeClean(root))) { + // A manifest from an older collector may simply not know files this + // version collects (.scala, say) — a commit-to-commit diff would report + // no changes and keep them absent. The sha256 path below spots them. + const manifestIsCurrent = previous.version === CODE_COLLECTION_VERSION; + + // Git incremental path: only when both commits are known, the manifest + // comes from this collector version, AND the working tree is clean. A + // dirty tree has uncommitted/untracked changes that a commit-to-commit + // diff cannot see, so we fall back to the full sha256 scan (which reads + // the working tree) to avoid silent staleness. + if (oldSha && newSha && manifestIsCurrent && (await isWorkingTreeClean(root))) { const gitChanges = await gitDiffNameStatus(root, oldSha, newSha); if (gitChanges !== null) { const { added, changed, deleted } = gitChanges; @@ -44,8 +51,10 @@ export async function detectCodeIncrementalChanges( } } - // Fallback: full sha256 comparison when git is unavailable or no baseline - const current = await collectCode({ root }); + // Fallback: full sha256 comparison when git is unavailable or no baseline. + // Honors the caller's --max-files: a default-capped scan against a larger + // prior extraction would classify the tail as deleted and prune it. + const current = await collectCode({ root, maxFiles }); const previousByPath = new Map((previous.files ?? []).map((file) => [file.relativePath, file.sha256])); const currentByPath = new Map(current.manifest.files.map((file) => [file.relativePath, file.sha256])); const added = [...currentByPath.keys()].filter((file) => !previousByPath.has(file)).sort(); diff --git a/src/wiki-engine/code-knowledge/extractors/index.ts b/src/wiki-engine/code-knowledge/extractors/index.ts index 913e9e6f1..11ff8c8ac 100644 --- a/src/wiki-engine/code-knowledge/extractors/index.ts +++ b/src/wiki-engine/code-knowledge/extractors/index.ts @@ -5,10 +5,23 @@ import { extractGo } from "./go.js"; import { extractJava } from "./java.js"; import { extractPython } from "./python.js"; import { extractRust } from "./rust.js"; +import { extractScala } from "./scala.js"; import { extractSwift } from "./swift.js"; import { extractTypescript } from "./typescript.js"; -type LanguageExtractor = (files: CodeCollectedFile[]) => CodeFact[]; +/** What an extractor may need beyond its own language batch. */ +export interface ExtractorContext { + /** Every collected file of the run, all languages — a Scala wildcard imports Java files just as freely. */ + allFiles: CodeCollectedFile[]; + /** + * The symbol names each file declares, from the previous run's facts. An + * incremental run re-extracts only changed files, so unchanged files are + * known by their cached declarations alone. + */ + priorDeclarations: Map>; +} + +type LanguageExtractor = (files: CodeCollectedFile[], context?: ExtractorContext) => CodeFact[]; /** * Registry mapping language identifiers to their specialized extractors. @@ -20,6 +33,7 @@ const EXTRACTOR_REGISTRY: Record = { python: extractPython, java: extractJava, rust: extractRust, + scala: extractScala, swift: extractSwift, toml: extractToml, sql: extractSql, @@ -27,14 +41,16 @@ const EXTRACTOR_REGISTRY: Record = { /** * Dispatch extraction to the appropriate language-specific extractor. + * `context` — the run's full file list and the previous run's declarations — + * lets an extractor resolve cross-language and cross-file constructs. * Falls back to an empty array for unsupported languages (json, yaml, text, etc.). */ -export function extractForLanguage(language: string, files: CodeCollectedFile[]): CodeFact[] { +export function extractForLanguage(language: string, files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { const extractor = EXTRACTOR_REGISTRY[language]; if (!extractor) { return []; } - return extractor(files); + return extractor(files, context); } /** @@ -48,5 +64,6 @@ export { extractGo } from "./go.js"; export { extractJava } from "./java.js"; export { extractPython } from "./python.js"; export { extractRust } from "./rust.js"; +export { extractScala, SCALA_DECL_PREFIX, SCALA_WILDCARD_PREFIX } from "./scala.js"; export { extractSwift } from "./swift.js"; export { extractTypescript } from "./typescript.js"; diff --git a/src/wiki-engine/code-knowledge/extractors/scala.ts b/src/wiki-engine/code-knowledge/extractors/scala.ts new file mode 100644 index 000000000..0dc858800 --- /dev/null +++ b/src/wiki-engine/code-knowledge/extractors/scala.ts @@ -0,0 +1,416 @@ +import { type CodeCollectedFile } from "../code-collector.js"; +import { type CodeFact, type CodeFactKind, mapKindToEvidenceType } from "../code-extractors.js"; +import { type ExtractorContext } from "./index.js"; + +const TYPE_DECL_PATTERN = /^(class|object|enum)\s+([A-Z]\w*)/u; +const DEF_DECL_PATTERN = /^def\s+([a-z_]\w*)/u; +const TRAIT_DECL_PATTERN = /^trait\s+([A-Z]\w*)/u; + +/** + * Marks a wildcard relation: `scala-wildcard:com/demo/core`. The name matches + * no file, so no consumer resolves it; the incremental layer reads these + * markers to re-extract importers whose package membership changed. + */ +export const SCALA_WILDCARD_PREFIX = "scala-wildcard:"; + +/** + * Marks a file's top-level declaration names, comma-joined: + * `scala-decl:Invoice,Order`. Like the wildcard marker it is metadata for the + * incremental layer — the next run rebuilds the declarations index from these + * instead of component facts, which cannot tell a nested member from a + * package-level name. + */ +export const SCALA_DECL_PREFIX = "scala-decl:"; + +/** + * Scala extractor. + * Extracts classes, objects, traits, enums, defs, configs, errors, and import relations. + * + * `context.allFiles` — every collected file, all languages — resolves wildcard + * imports in a mixed project, where a Scala wildcard imports Java files just as + * freely as Scala ones. `context.priorDeclarations` — the previous run's + * declarations — keeps symbol resolution working in an incremental run, which + * re-extracts only changed files. + */ +export function extractScala(files: CodeCollectedFile[], context?: ExtractorContext): CodeFact[] { + const facts: CodeFact[] = []; + // An incremental run re-parses only changed files; everything else is known + // by its cached declarations, which the fresh parse then overrides. + const declarations = new Map(context?.priorDeclarations ?? []); + for (const file of files) { + declarations.set(file.relativePath, declarationNames(file.content)); + } + const extractContext: ExtractContext = { allFiles: context?.allFiles ?? files, declarations }; + + for (const file of files) { + const lines = file.content.split(/\r?\n/); + const declared = declarations.get(file.relativePath); + if (declared && declared.size > 0) { + facts.push(makeFact("relation", `${SCALA_DECL_PREFIX}${[...declared].join(",")}`, file.relativePath, 1, "", "EXTRACTED")); + } + // A brace selector scalafmt wraps across lines stays open until its `}`. + let openImport: { line: number; prefix: string; selector: string } | undefined; + + for (let i = 0; i < lines.length; i++) { + const line = lines[i]; + const lineNumber = i + 1; + + if (openImport) { + openImport.selector += ` ${line}`; + if (openImport.selector.includes("}")) { + const { line: start, prefix, selector } = openImport; + openImport = undefined; + pushRelations(facts, extractContext, prefix, selector.split("}")[0], file.relativePath, start, line); + } + continue; + } + + // Annotations (`@deprecated`), modifiers (`final case`), and Scala 3 + // modifiers (`transparent inline`) precede the keyword on the same line. + const decl = stripLeadingModifiers(line); + + // --- Components --- + const typeDecl = TYPE_DECL_PATTERN.exec(decl); + if (typeDecl) { + facts.push(makeFact("component", typeDecl[2], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + const defDecl = DEF_DECL_PATTERN.exec(decl); + if (defDecl) { + facts.push(makeFact("component", defDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Interfaces --- + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); + if (traitDecl) { + facts.push(makeFact("interface", traitDecl[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Configs --- + const envRead = /\b(?:sys\.env|System\.getenv)\s*\(\s*"([A-Z][A-Z0-9_]+)"\s*\)/u.exec(line); + if (envRead) { + facts.push(makeFact("config", envRead[1], file.relativePath, lineNumber, line, "EXTRACTED")); + } + + // --- Errors --- + const typeName = typeDecl?.[2] ?? traitDecl?.[1]; + if (typeName && /(?:Error|Exception)$/u.test(typeName)) { + facts.push(makeFact("error", typeName, file.relativePath, lineNumber, line, "INFERRED")); + } + + // --- Relations --- + // One clause per imported symbol, as a path both relation consumers can + // match: buildCodeGraph compares file paths, and the call-chain tracer + // looks files up by path and basename. Several imports may share one + // line (`import a.B, c.D`); a brace selector may continue on the lines + // below, in which case it opens and waits for its `}`. + const importHead = /^import\s+(.+)$/u.exec(decl); + if (importHead) { + let rest = importHead[1]; + while (rest.length > 0) { + const clause = /^([A-Za-z_]\w*(?:\.\w+)*(?:\.\*)?)(\s*\.\s*\{([^}]*)\})?/u.exec(rest); + if (!clause) { + break; + } + // A plain clause may carry a Scala 3 rename (`a.B as Alias`) — the + // imported name is the part before it. + const after = rest.slice(clause[0].length); + const plainRename = /^\s+as\s+[A-Za-z_]\w*/u.exec(after)?.[0].length ?? 0; + const consumed = after.slice(plainRename); + if (/^\s*\.\s*\{/u.test(consumed)) { + openImport = { line: lineNumber, prefix: clause[1], selector: consumed.replace(/^\s*\.\s*\{/u, "") }; + break; + } + pushRelations(facts, extractContext, clause[1], clause[3], file.relativePath, lineNumber, line); + rest = consumed.replace(/^\s*,\s*/u, ""); + } + } + } + } + + // Two imports of one file (a wildcard plus a named symbol, say) name the + // same dependency twice — keep one relation per target per file. + const seenRelations = new Set(); + const uniqueFacts: CodeFact[] = []; + for (const fact of facts) { + if (fact.kind !== "relation") { + uniqueFacts.push(fact); + continue; + } + const key = `${fact.file}|${fact.name}`; + if (seenRelations.has(key)) { + continue; + } + seenRelations.add(key); + uniqueFacts.push(fact); + } + return uniqueFacts; +} + +/** + * `case` is the one token that is both a declaration modifier (`case class`) + * and a `match` clause (`case Invoice(id) =>`). It is stripped either way, but + * a clause line then starts with a capitalized name — no declaration keyword — + * so it cannot be mistaken for a declaration. + */ +const MODIFIER_PATTERN = + /^\s*(?:@\w+(?:\[[^\]]*\])?(?:\([^)]*\))?\s+)*(?:(?:private|protected)(?:\[[^\]]*\])?\s+|(?:final|sealed|abstract|case|implicit|lazy|override|open|transparent|inline|infix|erased)\s+)*/u; + +/** Anything after the leading annotations/modifiers is the declaration itself. */ +function stripLeadingModifiers(line: string): string { + return line.replace(MODIFIER_PATTERN, ""); +} + +interface ExtractContext { + /** Every collected file of the run — a wildcard imports Java files as freely as Scala ones. */ + allFiles: CodeCollectedFile[]; + /** relativePath → the symbol names the file declares, this run or cached. */ + declarations: Map>; +} + +/** + * Emit one relation per imported symbol, as a path both relation consumers + * match — a bare package path would tie the importer to every file in the + * directory. A named symbol resolves to the file in the package that declares + * it (Scala freely puts many types in one `Models.scala`), or to the + * conventional path when no collected file declares it. A wildcard (a + * trailing `._`/`.*` or a `_`/`*` entry) names the direct members of the + * package directory, minus the names the selector hides; with no collected + * file to name (an external package) the package path is all that is left to + * say. + */ +function pushRelations( + facts: CodeFact[], + context: ExtractContext, + dottedPrefix: string, + selector: string | undefined, + file: string, + lineNumber: number, + rawLine: string +): void { + const packagePath = toPath(dottedPrefix); + const selection = selector !== undefined ? parseSelectors(selector) : undefined; + const wildcard = /[._*]$/u.test(dottedPrefix) || (selection?.wildcard ?? false); + let targets: string[]; + if (wildcard) { + facts.push(makeFact("relation", `${SCALA_WILDCARD_PREFIX}${packagePath}`, file, lineNumber, rawLine, "EXTRACTED")); + const hidden = new Set(selection?.hidden ?? []); + const expanded = expandWildcard(packagePath, context, hidden); + if (expanded.length > 0) { + targets = expanded; + } else { + // `com.demo.Models.*` wildcards an object, not a package — resolve it to + // the file that declares the object, as a named import would. + const resolved = resolveSymbolPath(toModulePath(packagePath), file, context); + targets = resolved !== undefined ? [resolved] : [packagePath]; + } + } else if (selection && selection.symbols.length > 0) { + targets = [ + ...new Set( + selection.symbols.map((symbol) => resolveSymbolPath(`${packagePath}/${symbol}`, file, context) ?? toModulePath(`${packagePath}/${symbol}`)), + ), + ]; + } else if (selection) { + // only hidden names: the clause imports nothing, so no relation at all + targets = []; + } else { + // A plain import names a symbol (`com.foo.Bar` — or a Scala 3 top-level + // `def validate`) preceded by its package; the conventional path is the + // fallback when no collected file declares the symbol. + const modulePath = toModulePath(packagePath); + targets = [resolveSymbolPath(modulePath, file, context) ?? modulePath]; + } + for (const target of targets) { + facts.push(makeFact("relation", target, file, lineNumber, rawLine, "EXTRACTED")); + } +} + +interface ImportSelection { + /** A `_` or `*` entry imports everything in the package. */ + wildcard: boolean; + /** The symbols the selector names (`Invoice as Inv` imports `Invoice`). */ + symbols: string[]; + /** Names renamed to `_` — imported only to be hidden. */ + hidden: string[]; +} + +/** + * `{Invoice as Inv, Order => O}` → symbols `["Invoice", "Order"]`: the name an + * entry imports is the part before the rename (`=>` in Scala 2, `as` in + * Scala 3). An alias of `_` hides its name, and `_` or `*` alone imports the + * whole package. Anything that is not a plain identifier (nested selectors) + * is dropped; the package covers it. + */ +function parseSelectors(selector: string): ImportSelection { + const selection: ImportSelection = { wildcard: false, symbols: [], hidden: [] }; + for (const entry of selector.split(",")) { + const item = entry.trim(); + const renamed = /^([A-Za-z_]\w*)\s*(?:=>|\bas\b)\s*(.+)$/u.exec(item); + const alias = renamed?.[2].trim(); + if (item === "_" || item === "*") { + selection.wildcard = true; + } else if (renamed && alias === "_") { + selection.hidden.push(renamed[1]); + } else if (renamed && alias && /^[A-Za-z_]\w*$/u.test(alias)) { + selection.symbols.push(renamed[1]); + } else if (!renamed && /^[A-Za-z_]\w*$/u.test(item)) { + selection.symbols.push(item); + } + } + selection.symbols = [...new Set(selection.symbols)]; + return selection; +} + +/** `_root_.com.foo.Bar`/`com.foo.Bar` → `com/foo/Bar`; a trailing wildcard (`_`, `*`) names the whole package and drops off. */ +function toPath(dotted: string): string { + return dotted.replace(/^_root_\./u, "").replace(/[._*]+$/u, "").replace(/\./gu, "/"); +} + +/** + * `com/demo/core/Invoice/apply` → `com/demo/core/Invoice`: a lowercase tail + * after a type names a member of it, and the file that defines the type is + * the file the consumers can match. + */ +function toModulePath(path: string): string { + const segments = path.split("/"); + let lastType = -1; + for (let i = 0; i < segments.length; i++) { + if (/^[A-Z]/u.test(segments[i])) { + lastType = i; + } + } + return lastType === -1 ? path : segments.slice(0, lastType + 1).join("/"); +} + +/** + * The files a wildcard import of `packagePath` brings in: the package's + * JVM-importable members, minus those hidden by file name and those whose + * every declared symbol is hidden. + */ +function expandWildcard(packagePath: string, context: ExtractContext, hidden: ReadonlySet): string[] { + return packageMembers(packagePath, context.allFiles) + .filter((candidate) => { + // Exclusion is declaration-based when the names are known: a file with + // any live declaration still carries its other symbols. Only a file + // with no known declarations falls back to its file name. + const declared = context.declarations.get(candidate.relativePath); + if (declared && declared.size > 0) { + return ![...declared].every((name) => hidden.has(name)); + } + return !hidden.has(fileName(candidate.relativePath)); + }) + .map((candidate) => candidate.relativePath); +} + +/** + * The file a dotted import path resolves to: the package member that declares + * the last type, or — failing that — an earlier one (`com/demo/Models/Invoice` + * names a member of `object Models`, which lives in Models' own declaring + * file). Undefined when no collected file declares any of them. + */ +function resolveSymbolPath(modulePath: string, importer: string, context: ExtractContext): string | undefined { + const segments = modulePath.split("/"); + for (let i = segments.length - 1; i >= 1; i--) { + const symbol = segments[i]; + const packagePath = segments.slice(0, i).join("/"); + for (const candidate of packageMembers(packagePath, context.allFiles)) { + if (candidate.relativePath !== importer && context.declarations.get(candidate.relativePath)?.has(symbol)) { + return candidate.relativePath; + } + } + } + return undefined; +} + +/** + * The JVM-importable direct members of the package directory — a subpackage's + * files are not among them, and neither is a resource like a `.sql` schema + * that happens to sit beside the sources. + */ +function packageMembers(packagePath: string, files: CodeCollectedFile[]): CodeCollectedFile[] { + return files + .filter((candidate) => /\.(?:scala|java)$/u.test(candidate.relativePath)) + .filter((candidate) => { + const tail = afterPackage(candidate.relativePath, packagePath); + return tail !== undefined && !tail.includes("/"); + }); +} + +/** The part of `relativePath` below `com/demo/core`, or undefined when the file is elsewhere. */ +function afterPackage(relativePath: string, packagePath: string): string | undefined { + if (relativePath.startsWith(`${packagePath}/`)) { + return relativePath.slice(packagePath.length + 1); + } + const marker = `/${packagePath}/`; + const at = relativePath.indexOf(marker); + return at === -1 ? undefined : relativePath.slice(at + marker.length); +} + +/** + * The top-level type, trait and def names a file declares — the names another + * file can import from the package. A member sits inside braces or, in + * significant-indentation Scala 3, deeper than its package block's member + * indent (Scala 3 nests package blocks, each with its own level); either + * way it is not a package-level name. + */ +function declarationNames(content: string): Set { + const names = new Set(); + let depth = 0; + const packageMemberIndents: number[] = []; // open package blocks (`x:` or `x {`), innermost last; -1 until its member indent is seen + for (const rawLine of content.split(/\r?\n/)) { + const trimmed = rawLine.trim(); + const indent = rawLine.length - rawLine.trimStart().length; + const last = packageMemberIndents.length - 1; + if (trimmed !== "" && last >= 0 && packageMemberIndents[last] === -1) { + packageMemberIndents[last] = indent; // the first content line fixes the block's member indent + } + // `package x {` opens a block, not a declaration nest: its brace does not + // count toward the depth that hides members. + const packageOpener = /^package\s+[\w.]*\s*[:{]/u.test(trimmed); + if (packageOpener) { + packageMemberIndents.push(-1); + } else if (trimmed !== "") { + while (packageMemberIndents.length > 1 && packageMemberIndents[packageMemberIndents.length - 1] > indent) { + packageMemberIndents.pop(); // dedented out of the inner block + } + const memberIndent = packageMemberIndents[packageMemberIndents.length - 1]; + const topLevel = depth === 0 && (memberIndent === undefined ? indent === 0 : indent === memberIndent); + if (topLevel) { + const decl = stripLeadingModifiers(rawLine); + const typeDecl = TYPE_DECL_PATTERN.exec(decl); + if (typeDecl) { + names.add(typeDecl[2]); + } + const traitDecl = TRAIT_DECL_PATTERN.exec(decl); + if (traitDecl) { + names.add(traitDecl[1]); + } + const defDecl = DEF_DECL_PATTERN.exec(decl); + if (defDecl) { + names.add(defDecl[1]); + } + } + // A package block's closing brace has no matching opener in the count + // (openers are deliberately skipped) — clamp so later blocks still sit + // at depth 0. + depth = Math.max(0, depth + (rawLine.match(/\{/gu) ?? []).length - (rawLine.match(/\}/gu) ?? []).length); + } + } + return names; +} + +/** `src/main/scala/core/Invoice.scala` → `Invoice`. */ +function fileName(relativePath: string): string { + return (relativePath.split("/").pop() ?? relativePath).replace(/\.\w+$/u, ""); +} + +function makeFact( + kind: CodeFactKind, + name: string, + file: string, + lineStart: number, + rawLine: string, + confidence: CodeFact["confidence"] +): CodeFact { + return { kind, name, file, lineStart, detail: rawLine.trim(), confidence, evidenceType: mapKindToEvidenceType(kind) }; +}