From 78867d619d270b9465eaf8e8bf0a11cf4ae75a29 Mon Sep 17 00:00:00 2001 From: rochala Date: Thu, 3 Sep 2026 00:03:27 +0200 Subject: [PATCH] Resolve sources jars for the whole classpath, per entry get-source fetched only the named coordinate's sources jar, so any symbol defined in a dependency (inherited members, cats.Functor via cats-effect) could not be shown. Resolve the sources classifier in the same coursier fetch as the binaries and map each classpath entry to its sources jar; the symbol's own entry then says where its source lives. Co-Authored-By: Claude Fable 5.1 --- README.md | 2 +- lib/src/cellar/ContextResource.scala | 30 +++++++++---- lib/src/cellar/CoursierFetchClient.scala | 44 +++++++++++-------- lib/src/cellar/SourceFetcher.scala | 15 ------- lib/src/cellar/SourceJars.scala | 42 ++++++++++++++++++ lib/src/cellar/handlers/GetHandler.scala | 2 +- .../cellar/handlers/GetSourceHandler.scala | 13 +++--- lib/src/cellar/handlers/ListHandler.scala | 2 +- lib/src/cellar/handlers/SearchHandler.scala | 2 +- lib/test/src/cellar/ContextResourceTest.scala | 6 ++- 10 files changed, 106 insertions(+), 52 deletions(-) create mode 100644 lib/src/cellar/SourceJars.scala diff --git a/README.md b/README.md index 141a6be..69e6a7e 100644 --- a/README.md +++ b/README.md @@ -142,7 +142,7 @@ cellar deps |---|---| | `get` | Symbol info from the current project (signature, flags, members, docs) | | `get-external` | Symbol info from a Maven coordinate | -| `get-source` | Source code from a published `-sources.jar` | +| `get-source` | Source code from the `-sources.jar` of the artifact defining the symbol, including transitive dependencies | | `list` | List public symbols in a package/class from the current project | | `list-external` | List public symbols from a Maven coordinate | | `search` | Case-insensitive substring search in the current project | diff --git a/lib/src/cellar/ContextResource.scala b/lib/src/cellar/ContextResource.scala index ae63bc0..e4d00c7 100644 --- a/lib/src/cellar/ContextResource.scala +++ b/lib/src/cellar/ContextResource.scala @@ -2,6 +2,7 @@ package cellar import cats.effect.{IO, Resource} import cats.syntax.all.* +import cellar.CoursierFetchClient.ResolvedClasspath import coursierapi.Repository import fs2.io.file.Path import org.typelevel.log4cats.Logger @@ -15,6 +16,13 @@ object ContextResource: tracer: Tracer[IO], logger: Logger[IO] = StderrLogger.off ): Resource[IO, (Context, Classpath)] = + makeWithSources(ResolvedClasspath(jars, Map.empty), jreClasspath).map((ctx, cp, _) => (ctx, cp)) + + def makeWithSources(resolved: ResolvedClasspath, jreClasspath: Classpath)(using + tracer: Tracer[IO], + logger: Logger[IO] = StderrLogger.off + ): Resource[IO, (Context, Classpath, SourceJars)] = + val jars = resolved.jars Resource.eval { tracer.span("tasty.context.init").surround { for @@ -27,13 +35,17 @@ object ContextResource: e ) } - (jarClasspath, dropped) = loaded + (kept, jarClasspath, dropped) = loaded _ <- dropped.traverse_(p => logger.warn(s"dropped unreadable classpath entry (tasty-query MatchError): $p") ) classpath = jreClasspath ++ jarClasspath ctx <- IO.blocking(Context.initialize(classpath)) - yield (ctx, classpath) + sourceJars <- IO(SourceJars.pair(kept, jarClasspath, resolved.sourcesJars)).flatTap { + case Some(_) => IO.unit + case None => logger.warn("classpath entries do not line up with jars; sources unavailable") + } + yield (ctx, classpath, sourceJars.getOrElse(SourceJars.empty)) } } @@ -42,8 +54,8 @@ object ContextResource: * alongside the classpath so the caller can report them — dropping an entry silently can turn a * present symbol into a "not found". */ - private def readClasspathRobust(paths: List[Path], dropped: List[Path] = Nil): (Classpath, List[Path]) = - try (ClasspathLoaders.read(paths.map(_.toNioPath)), dropped) + private def readClasspathRobust(paths: List[Path], dropped: List[Path] = Nil): (List[Path], Classpath, List[Path]) = + try (paths, ClasspathLoaders.read(paths.map(_.toNioPath)), dropped) catch case e: MatchError => val bad = paths.find { p => @@ -61,18 +73,18 @@ object ContextResource: )(using tracer: Tracer[IO], logger: Logger[IO] = StderrLogger.off - ): Resource[IO, (Context, Classpath)] = - Resource.eval(CoursierFetchClient.fetchClasspath(coord, extraRepositories)).flatMap { jars => - make(jars, jreClasspath).evalMap { (ctx, classpath) => + ): Resource[IO, (Context, Classpath, SourceJars)] = + Resource.eval(CoursierFetchClient.fetchClasspathWithSources(coord, extraRepositories)).flatMap { resolved => + makeWithSources(resolved, jreClasspath).evalMap { (ctx, classpath, sourceJars) => IO.blocking { - if jars.nonEmpty then + if resolved.jars.nonEmpty then val jarEntries = classpath.filter(_.toString.endsWith(".jar")) val hasSymbols = jarEntries.exists { entry => try ctx.findSymbolsByClasspathEntry(entry).nonEmpty catch case _: Exception => false } if !hasSymbols then throw CellarError.EmptyArtifact(coord) - (ctx, classpath) + (ctx, classpath, sourceJars) } } } diff --git a/lib/src/cellar/CoursierFetchClient.scala b/lib/src/cellar/CoursierFetchClient.scala index bc36e01..d166376 100644 --- a/lib/src/cellar/CoursierFetchClient.scala +++ b/lib/src/cellar/CoursierFetchClient.scala @@ -15,22 +15,6 @@ object CoursierFetchClient: val repos = if extraRepositories.isEmpty then "default repositories" else extraRepositories.mkString(", ") logger.debug(s"$what ${coord.render} from $repos") - def fetchSourcesJar( - coord: MavenCoordinate, - extraRepositories: Seq[Repository] = Seq.empty - )(using logger: Logger[IO] = StderrLogger.off): IO[Option[Path]] = - logAttempt(coord, "fetching sources for", extraRepositories) *> - IO.blocking { - val dep = coord.toCoursierDependency.withTransitive(false) - val fetch = Fetch.create() - .addDependencies(dep) - .withCache(Cache.create()) - .addClassifiers("sources") - .withMainArtifacts(false) - if extraRepositories.nonEmpty then fetch.addRepositories(extraRepositories*): Unit - fetch.fetch().asScala.headOption.map(file => Path.fromNioPath(file.toPath)) - }.handleErrorWith(e => logger.debug(s"no sources jar for ${coord.render}: ${e.getMessage}").as(None)) - def fetchPom( coord: MavenCoordinate, extraRepositories: Seq[Repository] = Seq.empty @@ -55,20 +39,44 @@ object CoursierFetchClient: case e => IO.raiseError(e) } + /** The main jars of `coord`'s transitive closure, each paired with its `-sources.jar` when the + * publisher shipped one. Coursier treats classifier artifacts as optional, so a dependency + * without sources costs nothing but its absence from `sourcesJars`. + */ + case class ResolvedClasspath(jars: Seq[Path], sourcesJars: Map[Path, Path]) + def fetchClasspath( coord: MavenCoordinate, extraRepositories: Seq[Repository] = Seq.empty )(using tracer: Tracer[IO], logger: Logger[IO] = StderrLogger.off): IO[Seq[Path]] = + fetchClasspathWithSources(coord, extraRepositories).map(_.jars) + + def fetchClasspathWithSources( + coord: MavenCoordinate, + extraRepositories: Seq[Repository] = Seq.empty + )(using tracer: Tracer[IO], logger: Logger[IO] = StderrLogger.off): IO[ResolvedClasspath] = tracer.span("coursier.fetch").surround { logAttempt(coord, "resolving", extraRepositories) *> IO.blocking { val dep = coord.toCoursierDependency - val fetch = Fetch.create().addDependencies(dep).withCache(Cache.create()) + val fetch = Fetch.create() + .addDependencies(dep) + .withCache(Cache.create()) + .addClassifiers("sources") + .withMainArtifacts(true) if extraRepositories.nonEmpty then fetch.addRepositories(extraRepositories*): Unit - fetch.fetch().asScala.toSeq.map(file => Path.fromNioPath(file.toPath)) + val files = fetch.fetch().asScala.toSeq.map(file => Path.fromNioPath(file.toPath)) + val (sources, jars) = files.partition(_.fileName.toString.endsWith(SourcesSuffix)) + val sourcesByStem = sources.map(p => p.fileName.toString.stripSuffix(SourcesSuffix) -> p).toMap + val paired = jars.flatMap { jar => + sourcesByStem.get(jar.fileName.toString.stripSuffix(".jar")).map(jar -> _) + } + ResolvedClasspath(jars, paired.toMap) }.handleErrorWith { case e: coursierapi.error.CoursierError => CoordinateCompleter.suggest(coord, extraRepositories).flatMap { suggestions => IO.raiseError(CellarError.CoordinateNotFound(coord, e, suggestions)) } } } + + private val SourcesSuffix = "-sources.jar" diff --git a/lib/src/cellar/SourceFetcher.scala b/lib/src/cellar/SourceFetcher.scala index 60308c5..8456e75 100644 --- a/lib/src/cellar/SourceFetcher.scala +++ b/lib/src/cellar/SourceFetcher.scala @@ -1,7 +1,6 @@ package cellar import cats.effect.{IO, Resource} -import coursierapi.Repository import fs2.io.file.Path import fs2.io.readInputStream @@ -12,20 +11,6 @@ object SourceFetcher: case class SourceResult(entryPath: String, startLine: Int, endLine: Int, lines: IndexedSeq[String]) def fetch( - coord: MavenCoordinate, - sourceFilePath: String, - startLine: Int, - endLine: Int, - extraRepositories: Seq[Repository] = Seq.empty - ): IO[Either[String, SourceResult]] = - CoursierFetchClient.fetchSourcesJar(coord, extraRepositories).flatMap { - case None => - IO.pure(Left(s"No sources JAR published for '${coord.render}'.")) - case Some(jar) => - extractLines(jar, sourceFilePath, startLine, endLine) - } - - private def extractLines( jar: Path, sourceFilePath: String, startLine: Int, diff --git a/lib/src/cellar/SourceJars.scala b/lib/src/cellar/SourceJars.scala new file mode 100644 index 0000000..b353b14 --- /dev/null +++ b/lib/src/cellar/SourceJars.scala @@ -0,0 +1,42 @@ +package cellar + +import fs2.io.file.Path +import tastyquery.Classpaths.{Classpath, ClasspathEntry} +import tastyquery.Symbols.{PackageSymbol, Symbol, TermOrTypeSymbol} + +/** Which `-sources.jar` holds the source of a symbol: the one published next to the classpath + * entry the symbol was loaded from, which for an inherited or transitive member is not the jar + * the user named. + */ +final class SourceJars private (byEntry: Map[ClasspathEntry, Path]): + def forSymbol(sym: Symbol): Option[Path] = + for + top <- topLevelClass(sym) + pkg <- top.owner match + case p: PackageSymbol => Some(p.fullName.toString) + case _ => None + (entry, _) <- byEntry.find { (entry, _) => + entry.listAllPackages().exists { data => + data.dotSeparatedName == pkg && data.getClassDataByBinaryName(top.name.toString).isDefined + } + } + yield byEntry(entry) + + private def topLevelClass(sym: Symbol): Option[TermOrTypeSymbol] = + sym match + case s: TermOrTypeSymbol => + s.owner match + case _: PackageSymbol => Some(s) + case owner: TermOrTypeSymbol => topLevelClass(owner) + case _ => None + +object SourceJars: + val empty: SourceJars = new SourceJars(Map.empty) + + /** `ClasspathLoaders.read` yields one entry per path, in order; anything else means the pairing + * would be a guess, so the caller gets `None` rather than wrong sources. + */ + def pair(jars: List[Path], entries: Classpath, sourcesJars: Map[Path, Path]): Option[SourceJars] = + Option.when(jars.length == entries.length) { + new SourceJars(jars.zip(entries).flatMap((jar, entry) => sourcesJars.get(jar).map(entry -> _)).toMap) + } diff --git a/lib/src/cellar/handlers/GetHandler.scala b/lib/src/cellar/handlers/GetHandler.scala index 46aa8e8..2d610ea 100644 --- a/lib/src/cellar/handlers/GetHandler.scala +++ b/lib/src/cellar/handlers/GetHandler.scala @@ -26,7 +26,7 @@ object GetHandler: val program = for jreClasspath <- javaHome.fold(JreClasspath.jrtPath())(JreClasspath.jrtPath) - result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath) => + result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath, _) => given Context = ctx runCore(fqn, classpath, Some(coord), limit, hideInherited, groupInherited, logger) } diff --git a/lib/src/cellar/handlers/GetSourceHandler.scala b/lib/src/cellar/handlers/GetSourceHandler.scala index 55b7fac..dafd1eb 100644 --- a/lib/src/cellar/handlers/GetSourceHandler.scala +++ b/lib/src/cellar/handlers/GetSourceHandler.scala @@ -25,7 +25,7 @@ object GetSourceHandler: val program = for jreClasspath <- javaHome.fold(JreClasspath.jrtPath())(JreClasspath.jrtPath) - result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath) => + result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath, sourceJars) => given Context = ctx SymbolResolver.resolve(fqn).flatMap { case LookupResult.IsPackage => @@ -39,13 +39,16 @@ object GetSourceHandler: case LookupResult.LookupFailed(cause) => IO.raiseError(CellarError.SymbolLookupFailed(fqn, cause)) case LookupResult.Found(symbols) => - IO.blocking(combinedSourceRef(symbols.head)(using ctx)).flatMap { - case None => + val sym = symbols.head + IO.blocking((combinedSourceRef(sym)(using ctx), sourceJars.forSymbol(sym))).flatMap { + case (None, _) => Console[IO].errorln( s"No source position for '$fqn'. Only Scala 3 (TASTy) and Java symbols are supported." ).as(ExitCode.Error) - case Some(ref) => - SourceFetcher.fetch(coord, ref.filePath, ref.startLine, ref.endLine, extraRepositories).flatMap { + case (Some(_), None) => + Console[IO].errorln(s"No sources JAR published for the artifact defining '$fqn'.").as(ExitCode.Error) + case (Some(ref), Some(jar)) => + SourceFetcher.fetch(jar, ref.filePath, ref.startLine, ref.endLine).flatMap { case Left(err) => Console[IO].errorln(err).as(ExitCode.Error) case Right(result) => diff --git a/lib/src/cellar/handlers/ListHandler.scala b/lib/src/cellar/handlers/ListHandler.scala index 8cc893e..fcacb47 100644 --- a/lib/src/cellar/handlers/ListHandler.scala +++ b/lib/src/cellar/handlers/ListHandler.scala @@ -22,7 +22,7 @@ object ListHandler: val program = for jreClasspath <- javaHome.fold(JreClasspath.jrtPath())(JreClasspath.jrtPath) - result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, _) => + result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, _, _) => given tastyquery.Contexts.Context = ctx runCore(fqn, limit, Some(coord)) } diff --git a/lib/src/cellar/handlers/SearchHandler.scala b/lib/src/cellar/handlers/SearchHandler.scala index ac74fed..2b9cbe4 100644 --- a/lib/src/cellar/handlers/SearchHandler.scala +++ b/lib/src/cellar/handlers/SearchHandler.scala @@ -22,7 +22,7 @@ object SearchHandler: val program = for jreClasspath <- javaHome.fold(JreClasspath.jrtPath())(JreClasspath.jrtPath) - result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath) => + result <- ContextResource.makeFromCoord(coord, jreClasspath, extraRepositories).use { (ctx, classpath, _) => given tastyquery.Contexts.Context = ctx runCore(query, limit, classpath, jreClasspath) } diff --git a/lib/test/src/cellar/ContextResourceTest.scala b/lib/test/src/cellar/ContextResourceTest.scala index 8152cbd..fd8cfd2 100644 --- a/lib/test/src/cellar/ContextResourceTest.scala +++ b/lib/test/src/cellar/ContextResourceTest.scala @@ -35,9 +35,13 @@ class ContextResourceTest extends CatsEffectSuite: JreClasspath.jrtPath().flatMap { jrePaths => ContextResource .makeFromCoord(TestFixtures.scala3Coord, jrePaths, Seq(TestFixtures.localM2Repo)) - .use { (ctx, _) => + .use { (ctx, _, sourceJars) => IO.blocking(ctx.findStaticClass("cellar.fixture.scala3.CellarA")).map { cls => assertEquals(cls.name.toString, "CellarA") + val sources = sourceJars.forSymbol(cls) + assert(sources.exists(_.fileName.toString.endsWith("-sources.jar")), s"sources jar for CellarA: $sources") + val option = ctx.findStaticClass("scala.Option") + assert(sourceJars.forSymbol(option).exists(_.fileName.toString.startsWith("scala-library")), "transitive dependency sources") } } }