From fe2db2bb081080c959ebd11300f3b27453244122 Mon Sep 17 00:00:00 2001 From: NePreby Date: Mon, 9 Nov 2026 22:56:22 +0700 Subject: [PATCH] feat(manifest): parse Cargo.lock and package-lock.json with transitive deps (fixes #6) --- frontend/src/lib/__tests__/manifest.test.ts | 235 +++++++++++++- frontend/src/lib/manifest.ts | 319 +++++++++++++++++++- frontend/src/pages/Cascades.tsx | 13 +- 3 files changed, 553 insertions(+), 14 deletions(-) diff --git a/frontend/src/lib/__tests__/manifest.test.ts b/frontend/src/lib/__tests__/manifest.test.ts index bc26051..526079c 100644 --- a/frontend/src/lib/__tests__/manifest.test.ts +++ b/frontend/src/lib/__tests__/manifest.test.ts @@ -1,5 +1,10 @@ import { describe, expect, it } from "vitest"; -import { equalWeights, parseManifest } from "../manifest"; +import { + equalWeights, + parseCargoLock, + parseManifest, + parsePackageLock, +} from "../manifest"; describe("parseCargoToml", () => { it("reads all common dependency forms", () => { @@ -37,16 +42,242 @@ libc = "0.2" # comment describe("parsePackageJson", () => { it("reads deps, peer and dev deps without duplicates", () => { const m = parseManifest( - JSON.stringify({ name: "x", dependencies: { a: "^1" }, peerDependencies: { b: "2" }, devDependencies: { a: "1", c: "3" } }), + JSON.stringify({ + name: "x", + dependencies: { a: "^1" }, + peerDependencies: { b: "2" }, + devDependencies: { a: "1", c: "3" }, + }), ); expect(m.ecosystem).toBe("npm"); expect(m.deps.map((d) => `${d.name}:${d.kind}`)).toEqual(["a:normal", "b:peer", "c:dev"]); }); + it("errors on invalid json", () => { expect(() => parseManifest("{ nope", "package.json")).toThrow(/valid JSON/); }); }); +describe("parseCargoLock", () => { + const cargoLockFixture = ` +# This file is automatically @generated by Cargo. +version = 3 + +[[package]] +name = "flowpulse-contracts" +version = "0.1.0" +dependencies = [ + "soroban-sdk", + "tokio 1.35.0", +] + +[[package]] +name = "soroban-sdk" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "soroban-env-host", + "soroban-sdk-macros", +] + +[[package]] +name = "soroban-env-host" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "soroban-sdk-macros" +version = "29.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "tokio" +version = "1.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "pin-project-lite", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +`; + + it("extracts direct dependencies only by default", () => { + const m = parseCargoLock(cargoLockFixture); + expect(m.ecosystem).toBe("cargo"); + expect(m.name).toBe("flowpulse-contracts"); + expect(m.deps.map((d) => d.name)).toEqual(["soroban-sdk", "tokio"]); + expect(m.deps.find((d) => d.name === "tokio")?.version).toBe("1.35.0"); + }); + + it("includes transitive dependencies when includeTransitive is true", () => { + const m = parseCargoLock(cargoLockFixture, { includeTransitive: true }); + expect(m.name).toBe("flowpulse-contracts"); + // Direct deps first: soroban-sdk, tokio. Then transitive: soroban-env-host, soroban-sdk-macros, pin-project-lite + const names = m.deps.map((d) => d.name); + expect(names).toContain("soroban-sdk"); + expect(names).toContain("tokio"); + expect(names).toContain("soroban-env-host"); + expect(names).toContain("soroban-sdk-macros"); + expect(names).toContain("pin-project-lite"); + // Root package itself must not be included + expect(names).not.toContain("flowpulse-contracts"); + expect(names.length).toBe(5); + }); + + it("caps total dependencies at 32 rows (or custom maxDeps)", () => { + // Generate a Cargo.lock with 40 packages + let bigLock = `[[package]]\nname = "root"\nversion = "0.1.0"\ndependencies = [\n`; + for (let i = 0; i < 40; i++) { + bigLock += ` "pkg-${i}",\n`; + } + bigLock += `]\n\n`; + for (let i = 0; i < 40; i++) { + bigLock += `[[package]]\nname = "pkg-${i}"\nversion = "1.0.0"\nsource = "registry"\n\n`; + } + + const defaultCapped = parseCargoLock(bigLock); + expect(defaultCapped.deps.length).toBe(32); + + const customCapped = parseCargoLock(bigLock, { maxDeps: 10 }); + expect(customCapped.deps.length).toBe(10); + }); +}); + +describe("parsePackageLock", () => { + const packageLockV3Fixture = JSON.stringify({ + name: "flowpulse-frontend", + version: "0.1.0", + lockfileVersion: 3, + packages: { + "": { + name: "flowpulse-frontend", + version: "0.1.0", + dependencies: { + "@stellar/stellar-sdk": "^13.0.0", + react: "^19.0.0", + }, + devDependencies: { + vite: "^6.0.0", + }, + }, + "node_modules/@stellar/stellar-sdk": { + version: "13.1.2", + dependencies: { + "@stellar/stellar-base": "13.1.2", + axios: "^1.7.0", + }, + }, + "node_modules/@stellar/stellar-base": { + version: "13.1.2", + }, + "node_modules/axios": { + version: "1.7.4", + dependencies: { + "follow-redirects": "^1.15.6", + }, + }, + "node_modules/follow-redirects": { + version: "1.15.6", + }, + "node_modules/react": { + version: "19.0.0", + }, + "node_modules/vite": { + version: "6.0.7", + }, + }, + }); + + it("parses v3 direct dependencies only by default", () => { + const m = parsePackageLock(packageLockV3Fixture); + expect(m.ecosystem).toBe("npm"); + expect(m.name).toBe("flowpulse-frontend"); + const names = m.deps.map((d) => d.name); + expect(names).toEqual(["@stellar/stellar-sdk", "react", "vite"]); + expect(m.deps.find((d) => d.name === "@stellar/stellar-sdk")?.version).toBe("13.1.2"); + }); + + it("parses v3 transitive dependencies when includeTransitive is true", () => { + const m = parsePackageLock(packageLockV3Fixture, { includeTransitive: true }); + const names = m.deps.map((d) => d.name); + // Direct: @stellar/stellar-sdk, react, vite + // Transitive: @stellar/stellar-base, axios, follow-redirects + expect(names).toContain("@stellar/stellar-sdk"); + expect(names).toContain("react"); + expect(names).toContain("vite"); + expect(names).toContain("@stellar/stellar-base"); + expect(names).toContain("axios"); + expect(names).toContain("follow-redirects"); + expect(names).not.toContain("flowpulse-frontend"); + expect(names.length).toBe(6); + }); + + const packageLockV1Fixture = JSON.stringify({ + name: "legacy-app", + version: "1.0.0", + lockfileVersion: 1, + dependencies: { + lodash: { + version: "4.17.21", + }, + request: { + version: "2.88.2", + dependencies: { + qs: { + version: "6.5.3", + }, + }, + }, + }, + }); + + it("parses v1 lockfiles with direct and transitive dependencies", () => { + const directOnly = parsePackageLock(packageLockV1Fixture, { includeTransitive: false }); + expect(directOnly.deps.map((d) => d.name)).toEqual(["lodash", "request"]); + + const withTransitive = parsePackageLock(packageLockV1Fixture, { includeTransitive: true }); + expect(withTransitive.deps.map((d) => d.name)).toEqual(["lodash", "request", "qs"]); + }); + + it("errors on invalid json package-lock", () => { + expect(() => parsePackageLock("{ not json")).toThrow(/valid JSON/); + }); +}); + +describe("parseManifest routing & integration", () => { + it("detects and parses Cargo.lock by filename", () => { + const lock = `[[package]]\nname = "root"\nversion = "1.0"\ndependencies = ["depA"]\n\n[[package]]\nname = "depA"\nversion = "0.2"`; + const m = parseManifest(lock, "Cargo.lock"); + expect(m.ecosystem).toBe("cargo"); + expect(m.deps.map((d) => d.name)).toEqual(["depA"]); + }); + + it("detects and parses Cargo.lock by [[package]] content without filename", () => { + const lock = `[[package]]\nname = "root"\nversion = "1.0"\ndependencies = ["depA"]\n\n[[package]]\nname = "depA"\nversion = "0.2"`; + const m = parseManifest(lock); + expect(m.ecosystem).toBe("cargo"); + expect(m.deps.map((d) => d.name)).toEqual(["depA"]); + }); + + it("detects and parses package-lock.json by filename and content", () => { + const lock = JSON.stringify({ + name: "app", + lockfileVersion: 3, + packages: { + "": { dependencies: { chalk: "^5" } }, + "node_modules/chalk": { version: "5.3.0" }, + }, + }); + const m = parseManifest(lock, "package-lock.json"); + expect(m.ecosystem).toBe("npm"); + expect(m.deps[0].name).toBe("chalk"); + expect(m.deps[0].version).toBe("5.3.0"); + }); +}); + describe("equalWeights", () => { it("sums exactly to the target", () => { const w = equalWeights(7, 9000); diff --git a/frontend/src/lib/manifest.ts b/frontend/src/lib/manifest.ts index 306d01f..6b673f1 100644 --- a/frontend/src/lib/manifest.ts +++ b/frontend/src/lib/manifest.ts @@ -1,6 +1,6 @@ /** * Dependency manifest parsing for split tables. - * Supports Cargo.toml, package.json, and Cargo.lock / package-lock.json (top-level only). + * Supports Cargo.toml, package.json, and Cargo.lock / package-lock.json (with transitive dependencies option). */ export type Ecosystem = "cargo" | "npm"; @@ -18,9 +18,37 @@ export interface Manifest { deps: Dependency[]; } -export function parseManifest(text: string, filename = ""): Manifest { +export interface ParseOptions { + includeTransitive?: boolean; + maxDeps?: number; +} + +export const MAX_SPLIT_ROWS = 32; + +export function parseManifest(text: string, filename = "", options: ParseOptions = {}): Manifest { const t = text.trim(); - if (filename.endsWith(".json") || t.startsWith("{")) return parsePackageJson(t); + const lower = filename.toLowerCase(); + + if (lower.endsWith("cargo.lock") || (!lower.endsWith(".toml") && t.includes("[[package]]"))) { + return parseCargoLock(t, options); + } + + if (lower.endsWith("package-lock.json")) { + return parsePackageLock(t, options); + } + + if (lower.endsWith(".json") || t.startsWith("{")) { + try { + const parsed = JSON.parse(t); + if (parsed && typeof parsed === "object" && ("lockfileVersion" in parsed || "packages" in parsed)) { + return parsePackageLock(t, options); + } + } catch { + // Let parsePackageJson throw standard error + } + return parsePackageJson(t); + } + return parseCargoToml(t); } @@ -50,6 +78,275 @@ export function parsePackageJson(text: string): Manifest { return { ecosystem: "npm", name: typeof j.name === "string" ? j.name : undefined, deps }; } +interface CargoLockPackage { + name: string; + version: string; + source?: string; + dependencies: string[]; +} + +/** + * Parse Cargo.lock format. + * Extracts direct dependencies by default, or all transitive dependencies if includeTransitive is true. + * Deduplicates package names and caps output at maxDeps (default 32). + */ +export function parseCargoLock(text: string, options: ParseOptions = {}): Manifest { + const maxDeps = options.maxDeps ?? MAX_SPLIT_ROWS; + const includeTransitive = options.includeTransitive ?? false; + + const packages: CargoLockPackage[] = []; + let current: Partial | null = null; + let inDeps = false; + + for (const raw of text.split(/\r?\n/)) { + const line = raw.replace(/\s+#.*$/, "").trim(); + if (!line) continue; + + if (line === "[[package]]") { + if (current && current.name && current.version) { + packages.push({ + name: current.name, + version: current.version, + source: current.source, + dependencies: current.dependencies ?? [], + }); + } + current = { dependencies: [] }; + inDeps = false; + continue; + } + + if (!current) continue; + + if (line === "dependencies = [" || line.startsWith("dependencies = [")) { + inDeps = true; + const inline = line.match(/dependencies\s*=\s*\[(.*)\]/); + if (inline) { + for (const depStr of inline[1].split(",")) { + const s = depStr.trim().replace(/^["']|["']$/g, ""); + if (s) current.dependencies!.push(s); + } + inDeps = false; + } + continue; + } + + if (inDeps) { + if (line === "]") { + inDeps = false; + continue; + } + const depName = line.replace(/,$/, "").trim().replace(/^["']|["']$/g, ""); + if (depName) current.dependencies!.push(depName); + continue; + } + + const kv = line.match(/^("?)([A-Za-z0-9_\-]+)\1\s*=\s*(.+)$/); + if (!kv) continue; + const key = kv[2]; + const val = unquote(kv[3].trim()); + + if (key === "name") current.name = val; + else if (key === "version") current.version = val; + else if (key === "source") current.source = val; + } + + if (current && current.name && current.version) { + packages.push({ + name: current.name, + version: current.version, + source: current.source, + dependencies: current.dependencies ?? [], + }); + } + + if (packages.length === 0) { + return { ecosystem: "cargo", deps: [] }; + } + + // Root package is the member without source, or the first package + const rootPkg = packages.find((p) => !p.source) ?? packages[0]; + const rootName = rootPkg.name; + + const directNames = new Set( + rootPkg.dependencies.map((d) => d.split(" ")[0]), + ); + + const deps: Dependency[] = []; + const seen = new Set(); + seen.add(rootName); + + // 1. Direct dependencies from root package + for (const pkg of packages) { + if (directNames.has(pkg.name) && !seen.has(pkg.name)) { + seen.add(pkg.name); + deps.push({ + name: pkg.name, + version: pkg.version, + ecosystem: "cargo", + kind: "normal", + }); + if (deps.length >= maxDeps) break; + } + } + + // 2. Transitive dependencies if requested + if (includeTransitive && deps.length < maxDeps) { + for (const pkg of packages) { + if (!seen.has(pkg.name)) { + seen.add(pkg.name); + deps.push({ + name: pkg.name, + version: pkg.version, + ecosystem: "cargo", + kind: "normal", + }); + if (deps.length >= maxDeps) break; + } + } + } + + return { + ecosystem: "cargo", + name: rootName, + deps, + }; +} + +/** + * Parse package-lock.json format (v1, v2, v3). + * Extracts direct dependencies by default, or all transitive dependencies if includeTransitive is true. + * Deduplicates package names and caps output at maxDeps (default 32). + */ +export function parsePackageLock(text: string, options: ParseOptions = {}): Manifest { + const maxDeps = options.maxDeps ?? MAX_SPLIT_ROWS; + const includeTransitive = options.includeTransitive ?? false; + + let j: Record; + try { + j = JSON.parse(text); + } catch { + throw new Error("package-lock.json is not valid JSON"); + } + + const rootName = typeof j.name === "string" ? j.name : undefined; + const deps: Dependency[] = []; + const seen = new Set(); + if (rootName) seen.add(rootName); + + const cleanPkgName = (pkgPath: string): string => { + const idx = pkgPath.lastIndexOf("node_modules/"); + if (idx !== -1) { + return pkgPath.slice(idx + "node_modules/".length); + } + return pkgPath; + }; + + // v2 / v3 lockfile format with j.packages + if (j.packages && typeof j.packages === "object") { + const pkgs = j.packages as Record>; + const rootPkg = pkgs[""] ?? {}; + const directDeps = { + ...((rootPkg.dependencies as Record) ?? {}), + ...((rootPkg.devDependencies as Record) ?? {}), + ...((rootPkg.peerDependencies as Record) ?? {}), + }; + + // 1. Direct dependencies + for (const [depName, verRange] of Object.entries(directDeps)) { + if (seen.has(depName)) continue; + seen.add(depName); + + let exactVer = String(verRange); + const pkgEntry = pkgs[`node_modules/${depName}`]; + if (pkgEntry && typeof pkgEntry.version === "string") { + exactVer = pkgEntry.version; + } + + deps.push({ + name: depName, + version: exactVer, + ecosystem: "npm", + kind: "normal", + }); + if (deps.length >= maxDeps) break; + } + + // 2. Transitive dependencies + if (includeTransitive && deps.length < maxDeps) { + for (const [pathKey, pkg] of Object.entries(pkgs)) { + if (!pathKey || pathKey === "" || !pathKey.includes("node_modules/")) continue; + const name = cleanPkgName(pathKey); + if (seen.has(name)) continue; + seen.add(name); + + const ver = pkg && typeof pkg.version === "string" ? pkg.version : "*"; + deps.push({ + name, + version: ver, + ecosystem: "npm", + kind: "normal", + }); + if (deps.length >= maxDeps) break; + } + } + + return { ecosystem: "npm", name: rootName, deps }; + } + + // v1 lockfile format with j.dependencies + if (j.dependencies && typeof j.dependencies === "object") { + const depMap = j.dependencies as Record>; + + // 1. Direct dependencies (top level of dependencies) + for (const [depName, depObj] of Object.entries(depMap)) { + if (seen.has(depName)) continue; + seen.add(depName); + const ver = depObj?.version ? String(depObj.version) : "*"; + deps.push({ + name: depName, + version: ver, + ecosystem: "npm", + kind: "normal", + }); + if (deps.length >= maxDeps) break; + } + + // 2. Transitive dependencies + if (includeTransitive && deps.length < maxDeps) { + const collectTransitive = (obj: Record>) => { + for (const [name, val] of Object.entries(obj)) { + if (!seen.has(name)) { + seen.add(name); + deps.push({ + name, + version: val?.version ? String(val.version) : "*", + ecosystem: "npm", + kind: "normal", + }); + if (deps.length >= maxDeps) return; + } + if (val && typeof val.dependencies === "object") { + collectTransitive(val.dependencies as Record>); + if (deps.length >= maxDeps) return; + } + } + }; + + for (const val of Object.values(depMap)) { + if (val && typeof val.dependencies === "object") { + collectTransitive(val.dependencies as Record>); + if (deps.length >= maxDeps) break; + } + } + } + + return { ecosystem: "npm", name: rootName, deps }; + } + + return { ecosystem: "npm", name: rootName, deps: [] }; +} + /** * Minimal TOML reader covering the dependency tables used in practice: * [dependencies] foo = "1" | foo = { version = "1", ... } | foo.workspace = true @@ -145,15 +442,21 @@ export function registryUrl(d: Pick) { return d.ecosystem === "cargo" ? `https://crates.io/crates/${d.name}` : `https://www.npmjs.com/package/${d.name}`; } -/** Fetch a manifest from a public GitHub repo (tries Cargo.toml then package.json). */ -export async function fetchRepoManifest(repo: string): Promise<{ text: string; file: string }> { +/** Fetch a manifest from a public GitHub repo. */ +export async function fetchRepoManifest( + repo: string, + options: ParseOptions = {}, +): Promise<{ text: string; file: string }> { const slug = repo.replace(/^https?:\/\/github\.com\//, "").replace(/\.git$/, "").replace(/\/$/, ""); if (!/^[\w.-]+\/[\w.-]+$/.test(slug)) throw new Error("Use owner/repo or a github.com URL"); - for (const file of ["Cargo.toml", "package.json"]) { - for (const branch of ["HEAD"]) { + const files = options.includeTransitive + ? ["Cargo.lock", "package-lock.json", "Cargo.toml", "package.json"] + : ["Cargo.toml", "package.json", "Cargo.lock", "package-lock.json"]; + for (const file of files) { + for (const branch of ["HEAD", "main", "master"]) { const res = await fetch(`https://raw.githubusercontent.com/${slug}/${branch}/${file}`); if (res.ok) return { text: await res.text(), file }; } } - throw new Error(`No Cargo.toml or package.json at the root of ${slug}`); + throw new Error(`No manifest or lockfile found at the root of ${slug}`); } diff --git a/frontend/src/pages/Cascades.tsx b/frontend/src/pages/Cascades.tsx index f7ac376..5d0794f 100644 --- a/frontend/src/pages/Cascades.tsx +++ b/frontend/src/pages/Cascades.tsx @@ -43,6 +43,7 @@ export function CascadesPage() { const [onChain, setOnChain] = useState(null); const [loadingChain, setLoadingChain] = useState(false); const [includeDev, setIncludeDev] = useState(false); + const [includeTransitive, setIncludeTransitive] = useState(false); // Preview / distribute const [owner, setOwner] = useState(config.readSource); // demo table until a wallet connects @@ -89,7 +90,7 @@ export function CascadesPage() { function parse() { setParseError(null); try { - const m = parseManifest(text); + const m = parseManifest(text, "", { includeTransitive }); if (m.deps.length === 0) throw new Error("No dependencies found"); applyManifest(m); } catch (e) { @@ -101,9 +102,9 @@ export function CascadesPage() { setParseError(null); setFetching(true); try { - const { text: t } = await fetchRepoManifest(repo.trim()); + const { text: t, file } = await fetchRepoManifest(repo.trim(), { includeTransitive }); setText(t); - const m = parseManifest(t); + const m = parseManifest(t, file, { includeTransitive }); if (m.deps.length === 0) throw new Error("No dependencies found"); applyManifest(m); } catch (e) { @@ -239,7 +240,7 @@ export function CascadesPage() { value={text} onChange={(e) => setText(e.target.value)} spellCheck={false} - aria-label="Cargo.toml or package.json" + aria-label="Cargo.toml, package.json, Cargo.lock, or package-lock.json" />