diff --git a/.gitignore b/.gitignore index 660a48e..676d34a 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ dist/ build/ *.egg-info/ __pycache__/ +memory/ diff --git a/README.md b/README.md index f793bd2..98aa4f8 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,22 @@ talk to Redis, OpenAI, etc. Large repos (350+ files) are automatically aggregate directory-level nodes so the graph stays readable; override with `--graph-detail files` or `--graph-detail dirs`. +Test files (`tests/`, `test_*.py`, `conftest.py`, `*.spec.ts`, `*_test.go`, …) get their own +**Test Graph** section below the main one, showing which modules and services each test reaches. +Fixtures and test doubles name every service they exercise, so keeping them in the main graph +buries the real dependency structure under service edges. + +Both sections have a **Flow | Graph** toggle over the same nodes and edges. **Flow** (the default +up to 80 nodes) is a left-to-right layered diagram: columns are dependency depth from the sources +(HTTP entry and files nothing imports), every detected service is its own node in the last column, +and import cycles are drawn dashed. **Graph** is the force-directed view. Moving dots show data +going both ways: a filled dot is the call/request, a hollow dot the return/response. On graphs +with more than 60 edges, only the paths of the clicked or hovered node animate. Search, "hide +isolated" and the selected node are shared between the two views. + +The older layered **Full System Architecture** diagram is collapsed by default, since the +Architecture Map covers the same ground. + By default the graph only shows what the *code* says (import direction, static "this file calls Redis"), which is honest but not the same as real traffic. Pass `--access-log /path/to/access.log` (any combined/common log format) to overlay real observed request counts onto the HTTP-entry diff --git a/SKILL.md b/SKILL.md index 9c16782..3247c04 100644 --- a/SKILL.md +++ b/SKILL.md @@ -58,7 +58,7 @@ Scan the current project and produce a complete visual system workflow with conc 2. **Run the analyzer**: ```bash - python3 ~/.claude/skills/workflow-generator/scripts/analyze.py /WORKFLOW.html + python3 ~/.claude/skills/workflow-generator/scripts/analyze.py /WORKFLOW.html --enrich-labels ``` Optional flags, appended after the two positional args: @@ -66,6 +66,11 @@ Scan the current project and produce a complete visual system workflow with conc the codebase dependency graph's HTTP-entry edges, instead of import-direction only. - `--graph-detail auto|files|dirs` — force file-level or directory-level graph nodes. Default `auto` aggregates to directories once a project exceeds ~350 source files. + - `--enrich-labels` — always pass this when running via this skill (you're already an + authenticated `claude` session, so the cost is already accepted). It relabels + Architecture Map service nodes by having a local `claude -p` call actually read each + file, instead of the mechanical stem-dictionary label (see Architecture Map below). + It's a no-op everywhere else if `claude` isn't on PATH — never fails the scan. 3. **Open the output**: ```bash @@ -119,7 +124,27 @@ The generated `WORKFLOW.html` always contains all of the following: from the same min() comparison used for the "Practical Throughput" stat, so this section and that stat can never disagree with each other. -6. **Codebase Dependency Graph** — a force-directed graph of every source file (Python, JS/TS, +6. **Architecture Map** — a curated, high-level view of *how the system works*, distinct from the + raw dependency graph below it: one node per entry point (grouped by router/controller file, not + by URL prefix), internal service, datastore, external integration, scheduled job, and — when + detected — AI agent/model/tool. Built entirely from already-detected facts plus a few light, + deterministic file-pattern checks (composition-root file, `crud`/`service`/`repository`-named + modules, tool-binding/agent-framework signatures); by default no LLM judgment is involved, so + the node/edge set is identical across the CLI, MCP server, Copilot extension, and this skill. + Service-node **labels** are the one part of this that stays mechanical by default (a fixed + stem→label dictionary, so a generically-named file gets no node at all) — pass + `--enrich-labels` (always on when invoked via this skill; opt-in flag/param on CLI, MCP's + `generate_workflow` tool, and the Copilot extension's `enrichLabels` request field) to have a + local `claude -p` call read each service file and relabel it with what the code actually does. + It only ever renames existing nodes — never invents new ones — and silently no-ops if `claude` + isn't installed, so the deterministic output is always the floor. Entry→service edges are only + drawn when the entry file actually references the service module's name — never a blanket + fan-out. Click any node for a detail panel (what it is, its source file); click **Show import + details** to jump to the Codebase Dependency Graph below with that component's exact files + isolated and highlighted. Nodes sharing a feature (inferred from router/service naming) cluster + together with a matching accent color and group label. + +7. **Codebase Dependency Graph** — a force-directed graph of every source file (Python, JS/TS, Go, Java, Rust, Ruby) as a node and every real, parser-resolved import as an edge; files that match an already-detected component (an LLM call, a database client, a queue) also get an edge to that component. Click a node to isolate its neighbors; hover for file details. Large @@ -127,8 +152,15 @@ The generated `WORKFLOW.html` always contains all of the following: import-direction only (static, honest) unless `--access-log` was supplied, in which case HTTP-entry edges are weighted with real observed request counts — the graph is explicit about which kind of edge is which so it's never mistaken for a traced request path. - -7. **Guided tour** — a spotlight walkthrough that dims the page and steps through each section + Test files are split into a separate **Test Graph** section (tests plus the modules/services + they import or call), so they don't clutter the main graph. + Each has a **Flow | Graph** toggle over the same nodes/edges: Flow is a layered left-to-right + diagram (depth columns, every service its own node in the last column, cycles dashed), Graph + the force-directed view. Dots animate data both ways (filled = call, hollow = return); past 60 + edges only the clicked/hovered node's paths animate. The Full System Architecture section is + collapsed by default. + +8. **Guided tour** — a spotlight walkthrough that dims the page and steps through each section in turn (Back/Next/Skip), shown automatically the first time a generated report is opened. Steps for sections that don't apply to a given project (e.g. no detected routes, no graph) are skipped automatically. Replayable anytime via the `?` button next to the theme toggle. diff --git a/copilot/index.js b/copilot/index.js index 227579b..d739fe1 100644 --- a/copilot/index.js +++ b/copilot/index.js @@ -27,7 +27,9 @@ app.post("/agent", async (req, res) => { res.write(`data: {"choices":[{"delta":{"content":"Scanning project at \`${projectDir}\`...\\n\\n"}}]}\n\n`); try { - const out = execFileSync("python3", [ANALYZE_PY, projectDir, outputFile], { encoding: "utf8" }); + const args = [ANALYZE_PY, projectDir, outputFile]; + if (req.body.enrichLabels) args.push("--enrich-labels"); + const out = execFileSync("python3", args, { encoding: "utf8" }); const lines = out.trim().split("\n").map(l => `> ${l}`).join("\n"); res.write(`data: {"choices":[{"delta":{"content":"${lines.replace(/"/g,'\\"').replace(/\n/g,'\\n')}\\n\\nWorkflow diagram written to \`${outputFile}\`"}}]}\n\n`); } catch (err) { diff --git a/pyproject.toml b/pyproject.toml index 7ab1bce..9032eba 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "workflow-generator-mcp" -version = "0.3.1" +version = "0.5.0" description = "Generate a visual system architecture diagram (WORKFLOW.html) with concurrency capacity estimates and bottleneck analysis from any codebase — CLI + MCP server, stdlib-only scanner." readme = "README.md" license = { text = "MIT" } diff --git a/server.json b/server.json index 452563b..7bd7775 100644 --- a/server.json +++ b/server.json @@ -7,12 +7,12 @@ "url": "https://github.com/askuma/workflow-generator", "source": "github" }, - "version": "0.3.1", + "version": "0.4.0", "packages": [ { "registryType": "pypi", "identifier": "workflow-generator-mcp", - "version": "0.3.1", + "version": "0.4.0", "transport": { "type": "stdio" } diff --git a/tests/test_graph_tests_split.py b/tests/test_graph_tests_split.py new file mode 100644 index 0000000..0626c1c --- /dev/null +++ b/tests/test_graph_tests_split.py @@ -0,0 +1,53 @@ +from conftest import write_project + +ANALYSIS = { + 'llm': {'providers': ['OpenAI']}, 'storage': [], 'queues': [], 'external_sources': [], + 'framework': None, 'access_log': None, +} + + +def test_is_test_path(az_module): + az = az_module + for rel in ('tests/helpers.py', 'pkg/test_x.py', 'pkg/x_test.go', 'conftest.py', + 'web/a.spec.ts', 'web/a.test.jsx', 'src/test/java/FooTest.java', 'lib/a_spec.rb'): + assert az._is_test_path(rel), rel + for rel in ('app/main.py', 'pkg/testing_utils.py', 'src/latest.ts', 'Manifest.java'): + assert not az._is_test_path(rel), rel + + +def test_tests_split_into_their_own_graph(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "app.py": "import util\n", + "util.py": "", + "tests/test_app.py": "import app\nfrom openai import OpenAI\nOpenAI().chat.completions.create()\n", + }) + main, tests = az.build_code_graphs(root, ANALYSIS, detail='files') + main_ids = {n['id'] for n in main['nodes']} + test_ids = {n['id'] for n in tests['nodes']} + + assert main_ids == {'app.py', 'util.py'} + assert main['meta']['files'] == 2 + # A service only a test reaches must not leak into the main graph. + assert 'svc:OpenAI' not in main_ids + + assert test_ids == {'tests/test_app.py', 'app.py', 'svc:OpenAI'} + assert tests['meta']['files'] == 1 + assert all(e['s'] == 'tests/test_app.py' for e in tests['links']) + + +def test_render_codebase_sections_have_flow_and_graph_views(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "app.py": "import util\n", + "util.py": "", + "tests/test_app.py": "import app\n", + }) + html = az.render_html(az.collect(root), "p") + assert 'id="section-flow"' not in html # merged into the Codebase section + for P in ('graph-', 'test-graph-'): + for suffix in ('tab-flow', 'tab-force', 'flow-canvas', 'canvas', 'data', 'animate'): + assert f'id="{P}{suffix}"' in html, P + suffix + assert html.index('id="section-graph"') < html.index('id="section-test-graph"') + # Small graphs open on the Flow view. + assert 'id="section-graph" data-default-view="flow"' in html diff --git a/tests/test_semantic_graph.py b/tests/test_semantic_graph.py new file mode 100644 index 0000000..fd2d420 --- /dev/null +++ b/tests/test_semantic_graph.py @@ -0,0 +1,680 @@ +import json + +from conftest import write_project + + +# ── detect_crons ────────────────────────────────────────────────────────── + +def test_detect_crons_celery_beat(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "tasks.py": "from celery import Celery\napp = Celery('t')\nbeat_schedule = {}\n", + }) + crons = az.detect_crons(root) + assert any(c["type"] == "celery" for c in crons) + + +def test_detect_crons_apscheduler(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "sched.py": "from apscheduler.schedulers.background import BackgroundScheduler\n", + }) + crons = az.detect_crons(root) + assert any(c["type"] == "apscheduler" for c in crons) + + +def test_detect_crons_schedule_library(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "sched.py": "import schedule\nschedule.every(10).minutes.do(job)\n", + }) + crons = az.detect_crons(root) + assert any(c["type"] == "schedule" for c in crons) + + +def test_detect_crons_github_actions_schedule(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + ".github/workflows/nightly.yml": "on:\n schedule:\n - cron: '0 3 * * *'\n", + }) + crons = az.detect_crons(root) + assert any(c["type"] == "github_actions" for c in crons) + + +def test_detect_crons_empty_when_nothing_matches(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('hello')\n"}) + assert az.detect_crons(root) == [] + + +# ── detect_tools ────────────────────────────────────────────────────────── + +def test_detect_tools_langchain(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "tools.py": "from langchain.tools import tool\n\n@tool\ndef search(q: str): ...\n", + }) + tools = az.detect_tools(root) + assert any(t["type"] == "langchain_tools" for t in tools) + + +def test_detect_tools_openai_functions_requires_openai_hint(az_module, tmp_path): + az = az_module + # tools=[...] alone (no OpenAI SDK signal) should NOT be misattributed to OpenAI. + root = write_project(tmp_path, {"app.py": "tools = [1, 2, 3]\n"}) + tools = az.detect_tools(root) + assert not any(t["type"] == "openai_functions" for t in tools) + + +def test_detect_tools_openai_functions_with_hint(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "app.py": ( + "from openai import OpenAI\n" + "client = OpenAI()\n" + "tools = [{'type': 'function', 'function': {'name': 'get_weather'}}]\n" + ), + }) + tools = az.detect_tools(root) + assert any(t["type"] == "openai_functions" for t in tools) + + +def test_detect_tools_empty_when_nothing_matches(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('hello')\n"}) + assert az.detect_tools(root) == [] + + +# ── detect_agents ───────────────────────────────────────────────────────── + +def test_detect_agents_langchain_executor(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "agent.py": "from langchain.agents import AgentExecutor, create_react_agent\n", + }) + agents = az.detect_agents(root) + assert any(a["type"] == "langchain_agent" for a in agents) + + +def test_detect_agents_crewai_requires_crewai_hint(az_module, tmp_path): + az = az_module + # A bare "Agent(" call with no crewai import shouldn't be misattributed. + root = write_project(tmp_path, {"app.py": "x = Agent(name='x')\n"}) + agents = az.detect_agents(root) + assert not any(a["type"] == "crewai_agent" for a in agents) + + +def test_detect_agents_crewai_with_hint(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "agent.py": "from crewai import Agent\nresearcher = Agent(role='researcher')\n", + }) + agents = az.detect_agents(root) + assert any(a["type"] == "crewai_agent" for a in agents) + + +def test_detect_agents_empty_when_nothing_matches(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('hello')\n"}) + assert az.detect_agents(root) == [] + + +# ── detect_composition_root ─────────────────────────────────────────────── + +def test_detect_composition_root_fastapi(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "app/main.py": ( + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "app.add_middleware(CORSMiddleware)\n" + ), + }) + comp = az.detect_composition_root(root) + assert comp is not None + assert comp["framework"] == "FastAPI" + assert comp["file"] == "app/main.py" + assert comp["has_middleware"] is True + + +def test_detect_composition_root_flask(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "app.py": "from flask import Flask\napp = Flask(__name__)\n", + }) + comp = az.detect_composition_root(root) + assert comp is not None + assert comp["framework"] == "Flask" + + +def test_detect_composition_root_express(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, { + "server.js": "const express = require('express');\nconst app = express();\napp.use(cors());\n", + }) + comp = az.detect_composition_root(root) + assert comp is not None + assert comp["framework"] == "Express" + assert comp["has_middleware"] is True + + +def test_detect_composition_root_none_when_no_framework(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"script.py": "print('just a script')\n"}) + assert az.detect_composition_root(root) is None + + +# ── build_semantic_graph ────────────────────────────────────────────────── + +def _fastapi_project(tmp_path): + return write_project(tmp_path, { + "app/main.py": ( + "from fastapi import FastAPI\n" + "app = FastAPI()\n" + "app.add_middleware(CORSMiddleware)\n" + ), + "app/api/routes/users.py": ( + "from fastapi import APIRouter\n" + "from app import crud\n" + "router = APIRouter()\n" + "@router.get('/{user_id}')\n" + "def get_user(user_id: str):\n" + " return crud.get_user_by_email(email='x')\n" + ), + "app/api/routes/items.py": ( + "from fastapi import APIRouter\n" + "router = APIRouter()\n" + "@router.get('/')\n" + "def list_items():\n" + " return []\n" + ), + "app/crud.py": ( + "def get_user_by_email(email):\n" + " return None\n" + ), + }) + + +def test_build_semantic_graph_creates_composition_root_and_entries(az_module, tmp_path): + az = az_module + root = _fastapi_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': az.detect_composition_root(root), + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + kinds = {n['kind'] for n in sg['graph']['nodes']} + assert 'service' in kinds # composition root + crud service + assert 'entry' in kinds + labels = {n['label'] for n in sg['graph']['nodes']} + assert 'Users API' in labels + assert 'Items API' in labels + + +def test_build_semantic_graph_links_entry_to_service_only_when_referenced(az_module, tmp_path): + # users.py imports+calls crud.py -> should get a 'calls' edge to the CRUD + # service node. items.py does NOT reference crud.py -> must NOT get one. + # (Regression test for the N x M edge-fanout bug found during manual review.) + az = az_module + root = _fastapi_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': az.detect_composition_root(root), + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + edges = sg['graph']['edges'] + users_to_crud = [e for e in edges if e['from'] == 'entry-users' and e['to'] == 'service-crud'] + items_to_crud = [e for e in edges if e['from'] == 'entry-items' and e['to'] == 'service-crud'] + assert len(users_to_crud) == 1 + assert len(items_to_crud) == 0 + + +def test_build_semantic_graph_node_members_tracks_file_paths(az_module, tmp_path): + az = az_module + root = _fastapi_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': az.detect_composition_root(root), + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + members = sg['node_members'] + assert 'app/api/routes/users.py' in members['entry-users'] + assert 'app/crud.py' in members['service-crud'] + + +def test_build_semantic_graph_excludes_non_integration_external_sources(az_module, tmp_path): + # "Users / API Clients" and "Cron / Scheduler" are synthetic entries + # detect_external_sources() always/sometimes adds; neither is a real + # third-party integration and both must be filtered out of the semantic + # graph's external nodes (regression test for the bug found during review). + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [ + {'name': 'Users / API Clients'}, + {'name': 'Cron / Scheduler'}, + {'name': 'Stripe'}, + ], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + ext_labels = {n['label'] for n in sg['graph']['nodes'] if n['kind'] == 'external'} + assert ext_labels == {'Stripe'} + + +def test_build_semantic_graph_stats_reflect_zero_ai_for_non_ai_repo(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert sg['stats'] == {'agents': 0, 'models': 0, 'tools': 0, 'integrations': 0} + + +def test_build_semantic_graph_agent_only_created_when_tools_present(az_module, tmp_path): + # An agent framework match with no tool signal shouldn't fabricate an + # agent node in isolation from the rest of the "never guess" philosophy + # this function follows elsewhere. + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [{'type': 'langchain_agent', 'name': 'LangChain Agent', 'framework': 'LangChain'}], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert not any(n['kind'] == 'agent' for n in sg['graph']['nodes']) + + +def test_build_semantic_graph_agent_created_with_tools_present(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [{'type': 'langchain_tools', 'name': 'LangChain Tools'}], + 'agents': [{'type': 'langchain_agent', 'name': 'LangChain Agent', 'framework': 'LangChain'}], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert any(n['kind'] == 'agent' for n in sg['graph']['nodes']) + assert sg['stats']['agents'] == 1 + + +def test_build_semantic_graph_store_domain_from_known_brand(az_module, tmp_path): + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [{'name': 'PostgreSQL', 'type': 'relational'}], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + store_nodes = [n for n in sg['graph']['nodes'] if n['kind'] == 'store'] + assert len(store_nodes) == 1 + assert store_nodes[0]['domain'] == 'postgresql.org' + + +def test_build_semantic_graph_cron_links_to_composition_root_not_all_services(az_module, tmp_path): + # Regression test: a detected cron pattern must not be wired to every + # service found anywhere in the repo (an honest "never fabricate a + # connection we can't confirm" constraint) — it should only link to the + # composition root, if one exists. + az = az_module + root = _fastapi_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [{'type': 'celery', 'name': 'Celery Beat', 'detail': 'Periodic tasks'}], + 'tools': [], + 'agents': [], + 'composition_root': az.detect_composition_root(root), + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + cron_edges = [e for e in sg['graph']['edges'] if e['from'] == 'cron-celery'] + assert len(cron_edges) == 1 + assert cron_edges[0]['to'] == 'app-root' + + +def test_build_semantic_graph_caps_nodes_at_60(az_module, tmp_path): + az = az_module + files = {f"app/routes/r{i}.py": f"from fastapi import APIRouter\nrouter = APIRouter()\n@router.get('/{i}')\ndef h(): ...\n" for i in range(80)} + root = write_project(tmp_path, files) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert len(sg['graph']['nodes']) <= 60 + + +def test_build_semantic_graph_top_integrations_populated(az_module, tmp_path): + # Regression test: topIntegrations used an undefined `ext_id` variable + # in its list comprehension (UnboundLocalError), only surfaced when + # build_semantic_graph() actually ran end-to-end against a repo with a + # real external integration (caught while regenerating the FastAPI + # template demo, not by any prior unit test). + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [{'name': 'Stripe'}], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert sg['topIntegrations'] == [{'id': 'stripe', 'label': 'Stripe', 'domain': 'stripe.com'}] + + +def _orm_session_project(tmp_path): + # crud.py never spells out "postgresql" anywhere (the connection lives + # in a separate core/db.py this fixture doesn't even need) — it only + # talks to the store through a generic SQLModel/SQLAlchemy-style + # session, which is how most layered ORM apps actually look. + return write_project(tmp_path, { + "app/crud.py": ( + "from sqlmodel import Session\n" + "def create_user(session: Session, user):\n" + " session.add(user)\n" + " session.commit()\n" + " session.refresh(user)\n" + " return user\n" + ), + }) + + +def test_build_semantic_graph_links_orm_session_to_sole_store(az_module, tmp_path): + # Regression test: Postgres was showing up with zero edges in the real + # FastAPI template demo because the literal-name heuristic requires + # "postgres" to appear in crud.py's own text, which it never does for + # SQLModel/SQLAlchemy-style layered apps. When exactly one non-cache + # store is detected repo-wide, a generic session.add()/session.exec() + # call is enough to honestly infer the connection. + az = az_module + root = _orm_session_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [{'name': 'PostgreSQL', 'type': 'relational'}], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + edges = sg['graph']['edges'] + to_store = [e for e in edges if e['to'] == 'store-postgresql'] + assert len(to_store) == 1 + assert to_store[0]['from'] == 'service-crud' + + +def test_build_semantic_graph_orm_session_skips_ambiguous_multi_store(az_module, tmp_path): + # With two non-cache stores present, a bare session.add() doesn't say + # which one it means — never fabricate a connection when it's genuinely + # ambiguous which store is meant. + az = az_module + root = _orm_session_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [ + {'name': 'PostgreSQL', 'type': 'relational'}, + {'name': 'MongoDB', 'type': 'nosql'}, + ], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + edges = sg['graph']['edges'] + service_to_store = [e for e in edges if e['from'] == 'service-crud' and e['to'].startswith('store-')] + assert service_to_store == [] + + +# ── enrich_service_labels (local `claude` CLI shell-out, opt-in) ───────────── +# All of these mock the subprocess boundary — a real `claude` process is never +# invoked in tests: that would be non-deterministic, slow, and require auth. + +def _semantic_graph_with_one_service(az, tmp_path): + root = _fastapi_project(tmp_path) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': az.detect_composition_root(root), + 'llm': {'providers': []}, + } + return root, az.build_semantic_graph(root, analysis) + + +def test_enrich_service_labels_noop_when_claude_not_on_path(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: None) + + def _boom(*a, **k): + raise AssertionError("subprocess.run must not be called when claude isn't on PATH") + monkeypatch.setattr(az.subprocess, 'run', _boom) + + before = [dict(n) for n in sg['graph']['nodes']] + out = az.enrich_service_labels(root, sg) + assert out['graph']['nodes'] == before + + +def test_collect_never_shells_out_when_enrich_labels_false(az_module, tmp_path, monkeypatch): + # The default (enrich_labels=False) path must never touch the subprocess + # boundary at all, regardless of whether claude happens to be on PATH. + az = az_module + root = _fastapi_project(tmp_path) + + def _boom(*a, **k): + raise AssertionError("subprocess.run must not be called when enrich_labels=False") + monkeypatch.setattr(az.subprocess, 'run', _boom) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + + az.collect(root, enrich_labels=False) # must not raise + + +def test_enrich_service_labels_applies_valid_relabel(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + + payload = json.dumps([ + {'id': 'service-crud', 'label': 'User records', 'sub': 'reads/writes user rows', 'detail': 'x'}, + ]) + + class _FakeProc: + returncode = 0 + stdout = json.dumps({'result': payload}) + + monkeypatch.setattr(az.subprocess, 'run', lambda *a, **k: _FakeProc()) + out = az.enrich_service_labels(root, sg) + crud = next(n for n in out['graph']['nodes'] if n['id'] == 'service-crud') + assert crud['label'] == 'User records' + assert crud['sub'] == 'reads/writes user rows' + # untouched fields + assert crud['kind'] == 'service' + assert crud['sourceRef'] == 'app/crud.py' + + +def test_enrich_service_labels_truncates_oversized_fields(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + + long_label = 'X' * 200 + payload = json.dumps([{'id': 'service-crud', 'label': long_label}]) + + class _FakeProc: + returncode = 0 + stdout = json.dumps({'result': payload}) + + monkeypatch.setattr(az.subprocess, 'run', lambda *a, **k: _FakeProc()) + out = az.enrich_service_labels(root, sg) + crud = next(n for n in out['graph']['nodes'] if n['id'] == 'service-crud') + assert len(crud['label']) == az._ENRICH_LABEL_MAX + + +def test_enrich_service_labels_ignores_unknown_ids(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + before_label = next(n for n in sg['graph']['nodes'] if n['id'] == 'service-crud')['label'] + + payload = json.dumps([{'id': 'service-does-not-exist', 'label': 'Whatever'}]) + + class _FakeProc: + returncode = 0 + stdout = json.dumps({'result': payload}) + + monkeypatch.setattr(az.subprocess, 'run', lambda *a, **k: _FakeProc()) + out = az.enrich_service_labels(root, sg) + crud = next(n for n in out['graph']['nodes'] if n['id'] == 'service-crud') + assert crud['label'] == before_label + + +def test_enrich_service_labels_handles_malformed_json_response(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + before = [dict(n) for n in sg['graph']['nodes']] + + class _FakeProc: + returncode = 0 + stdout = 'not json at all' + + monkeypatch.setattr(az.subprocess, 'run', lambda *a, **k: _FakeProc()) + out = az.enrich_service_labels(root, sg) + assert out['graph']['nodes'] == before + + +def test_enrich_service_labels_handles_nonzero_exit(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + before = [dict(n) for n in sg['graph']['nodes']] + + class _FakeProc: + returncode = 1 + stdout = '' + + monkeypatch.setattr(az.subprocess, 'run', lambda *a, **k: _FakeProc()) + out = az.enrich_service_labels(root, sg) + assert out['graph']['nodes'] == before + + +def test_enrich_service_labels_handles_timeout(az_module, tmp_path, monkeypatch): + az = az_module + root, sg = _semantic_graph_with_one_service(az, tmp_path) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + before = [dict(n) for n in sg['graph']['nodes']] + + def _timeout(*a, **k): + raise az.subprocess.TimeoutExpired(cmd='claude', timeout=45) + monkeypatch.setattr(az.subprocess, 'run', _timeout) + + out = az.enrich_service_labels(root, sg) + assert out['graph']['nodes'] == before + + +def test_enrich_service_labels_noop_when_no_service_nodes(az_module, tmp_path, monkeypatch): + az = az_module + root = write_project(tmp_path, {"app.py": "print('x')\n"}) + analysis = { + 'project_name': 'Demo', + 'api_server': {'routes': []}, + 'storage': [], + 'external_sources': [], + 'crons': [], + 'tools': [], + 'agents': [], + 'composition_root': None, + 'llm': {'providers': []}, + } + sg = az.build_semantic_graph(root, analysis) + assert not any(n['kind'] == 'service' for n in sg['graph']['nodes']) + monkeypatch.setattr(az.shutil, 'which', lambda _: '/usr/bin/claude') + + def _boom(*a, **k): + raise AssertionError("subprocess.run must not be called when there are no service nodes") + monkeypatch.setattr(az.subprocess, 'run', _boom) + + az.enrich_service_labels(root, sg) # must not raise diff --git a/workflow_generator_mcp/__init__.py b/workflow_generator_mcp/__init__.py index 4dceb26..13f4979 100644 --- a/workflow_generator_mcp/__init__.py +++ b/workflow_generator_mcp/__init__.py @@ -1,3 +1,3 @@ """workflow-generator — architecture diagrams with capacity analysis from any codebase.""" -__version__ = "0.3.1" +__version__ = "0.4.0" diff --git a/workflow_generator_mcp/analyze.py b/workflow_generator_mcp/analyze.py index af1fb47..8015e86 100644 --- a/workflow_generator_mcp/analyze.py +++ b/workflow_generator_mcp/analyze.py @@ -8,7 +8,7 @@ python3 analyze.py [project_dir] [output_file] """ -import ast, datetime, json, re, sys +import ast, datetime, json, re, shutil, subprocess, sys from pathlib import Path # ── Design tokens (identical to generated WORKFLOW.html) ───────────────────── @@ -25,6 +25,26 @@ 'gray': '#94a3b8', 'gray_d': '#1a2535', } +# ── Known brand domains, for Architecture Map node branding ────────────────── +# Bundled/static (no network favicon lookups) — matches the names already +# produced by detect_storage()/detect_external_sources()/detect_llm() etc. +# Names outside this list simply render without a domain; never guessed. +_BRAND_DOMAINS = { + 'PostgreSQL': 'postgresql.org', 'Postgres': 'postgresql.org', + 'MySQL': 'mysql.com', 'MongoDB': 'mongodb.com', 'SQLite': 'sqlite.org', + 'Redis': 'redis.io', 'Qdrant': 'qdrant.tech', 'Pinecone': 'pinecone.io', + 'Weaviate': 'weaviate.io', 'ChromaDB': 'trychroma.com', 'Chroma': 'trychroma.com', + 'FAISS': 'github.com', 'Milvus': 'milvus.io', 'pgvector': 'postgresql.org', + 'Celery': 'docs.celeryq.dev', 'BullMQ': 'bullmq.io', 'Kafka': 'kafka.apache.org', + 'RabbitMQ': 'rabbitmq.com', 'RQ': 'python-rq.org', 'AWS SQS': 'aws.amazon.com', + 'Jira': 'atlassian.com', 'Azure DevOps': 'azure.microsoft.com', 'Slack': 'slack.com', + 'GitHub': 'github.com', 'Stripe': 'stripe.com', 'Salesforce': 'salesforce.com', + 'Twilio': 'twilio.com', 'Sentry': 'sentry.io', 'Traefik': 'traefik.io', + 'nginx': 'nginx.org', 'Caddy': 'caddyserver.com', 'OpenAI': 'openai.com', + 'Anthropic': 'claude.ai', 'Cohere': 'cohere.com', 'AWS Bedrock': 'aws.amazon.com', + 'TruLens': 'trulens.org', 'RAGAS': 'docs.ragas.io', 'LangSmith': 'smith.langchain.com', +} + # ── Pattern library ─────────────────────────────────────────────────────────── _RE = { # Workers & replicas @@ -753,6 +773,194 @@ def hit(re_key): return sources +# ── Semantic graph detectors (crons, tools, agents, composition-root) ──────── + +def detect_crons(root: Path) -> list: + """Detect scheduled jobs and cron-like tasks.""" + py_src = _read_all(root, '*.py') + yaml_src = _read_all(root, '*.yml') + _read_all(root, '*.yaml') + json_src = _read_all(root, '*.json') + all_src = py_src + yaml_src + json_src + all_src_lower = all_src.lower() + + crons = [] + + # Celery beat schedule + if re.search(r'@.*\.task|@shared_task|beat_schedule|periodic_task', all_src, re.I): + for m in _RE['celery'].finditer(all_src_lower): + crons.append({'type': 'celery', 'name': 'Celery Beat', 'detail': 'Periodic tasks'}) + break + + # APScheduler + if re.search(r'APScheduler|BackgroundScheduler|CronTrigger', all_src, re.I): + crons.append({'type': 'apscheduler', 'name': 'APScheduler', 'detail': 'Scheduled jobs'}) + + # schedule.every() + if re.search(r'schedule\.every\(|schedule\.run_', all_src, re.I): + crons.append({'type': 'schedule', 'name': 'schedule library', 'detail': 'Simple jobs'}) + + # GitHub Actions workflows + if 'on: schedule:' in all_src or 'schedule:' in yaml_src: + crons.append({'type': 'github_actions', 'name': 'GitHub Actions', 'detail': 'Scheduled workflows'}) + + # Docker-compose/K8s CronJob (look for naming patterns) + if re.search(r'kind:\s*CronJob|cron[_-]?job|scheduled[_-]?job', yaml_src, re.I): + crons.append({'type': 'cronjob', 'name': 'Scheduled Job', 'detail': 'Infrastructure-level scheduling'}) + + return crons + + +def detect_tools(root: Path) -> list: + """Detect tool/function-calling definitions (OpenAI, Anthropic, LangChain, etc.).""" + py_src = _read_all(root, '*.py') + js_src = _read_all(root, '*.js') + _read_all(root, '*.ts') + all_src = py_src + js_src + all_src_lower = all_src.lower() + + tools = [] + + # OpenAI function calling + if re.search(r'functions\s*=\s*\[|tools\s*=\s*\[|\{.*"type"\s*:\s*"function"', all_src, re.I): + if _hinted_search('openai_chat', all_src, all_src_lower): + tools.append({'type': 'openai_functions', 'name': 'OpenAI Functions'}) + + # Anthropic tool use + if re.search(r'tools\s*=\s*\[|"type"\s*:\s*"tool"', all_src, re.I): + if _hinted_search('anthropic_chat', all_src, all_src_lower): + tools.append({'type': 'anthropic_tools', 'name': 'Anthropic Tools'}) + + # LangChain tools + if re.search(r'@tool|from langchain.*tools|Tool\(|StructuredTool', all_src, re.I): + tools.append({'type': 'langchain_tools', 'name': 'LangChain Tools'}) + + # Vercel AI SDK + if re.search(r'tool\(\{|tools:\s*\[|createTool', all_src, re.I): + if 'vercel' in all_src_lower or '@ai-sdk' in all_src_lower: + tools.append({'type': 'vercel_ai_tools', 'name': 'Vercel AI Tools'}) + + # CrewAI tools + if re.search(r'@tool|from crewai', all_src, re.I): + if 'crewai' in all_src_lower: + tools.append({'type': 'crewai_tools', 'name': 'CrewAI Tools'}) + + return tools + + +def detect_agents(root: Path) -> list: + """Detect agent frameworks and agent-framework constructs.""" + py_src = _read_all(root, '*.py') + js_src = _read_all(root, '*.js') + _read_all(root, '*.ts') + all_src = py_src + js_src + all_src_lower = all_src.lower() + + agents = [] + + # LangChain AgentExecutor / agent pool + if re.search(r'AgentExecutor|create_.*_agent|agent=|from langchain.*agent', all_src, re.I): + agents.append({'type': 'langchain_agent', 'name': 'LangChain Agent', 'framework': 'LangChain'}) + + # CrewAI agents + if re.search(r'class.*Agent|Agent\(|from crewai', all_src, re.I): + if 'crewai' in all_src_lower: + agents.append({'type': 'crewai_agent', 'name': 'CrewAI Agent', 'framework': 'CrewAI'}) + + # AutoGen + if re.search(r'ConversableAgent|UserProxyAgent|from autogen', all_src, re.I): + agents.append({'type': 'autogen_agent', 'name': 'AutoGen Agent', 'framework': 'AutoGen'}) + + # LlamaIndex agent + if re.search(r'AgentRunner|OpenAIAgent|from llama_index.*agent', all_src, re.I): + agents.append({'type': 'llamaindex_agent', 'name': 'LlamaIndex Agent', 'framework': 'LlamaIndex'}) + + # OpenAI Assistants API + if re.search(r'client\.beta\.assistants|Assistant\(|thread', all_src, re.I): + if _hinted_search('openai_chat', all_src, all_src_lower): + agents.append({'type': 'openai_assistant', 'name': 'OpenAI Assistant', 'framework': 'OpenAI'}) + + return agents + + +def detect_composition_root(root: Path) -> dict | None: + """Detect where the framework app is constructed (FastAPI, Flask, Express, etc.).""" + py_files = list(root.rglob('*.py')) + js_files = list(root.rglob('*.js')) + list(root.rglob('*.ts')) + + # Python frameworks + for py_file in py_files: + try: + content = py_file.read_text(errors='ignore') + except: + continue + + # FastAPI + if re.search(r'app\s*=\s*FastAPI\(|from fastapi import FastAPI', content): + rel_path = py_file.relative_to(root).as_posix() + # Look for middleware/setup in same file + has_middleware = bool(re.search(r'app\.add_middleware|app\.middleware|@app\.on_event', content)) + return { + 'framework': 'FastAPI', + 'uses_sentry': bool(re.search(r'sentry_sdk|SENTRY_DSN|@sentry/', content, re.I)), + 'file': rel_path, + 'has_middleware': has_middleware, + 'detail': 'FastAPI application' + } + + # Flask + if re.search(r'app\s*=\s*Flask\(|from flask import Flask', content): + rel_path = py_file.relative_to(root).as_posix() + has_middleware = bool(re.search(r'app\.before_request|app\.after_request|@app\.before_|@app\.teardown_', content)) + return { + 'framework': 'Flask', + 'uses_sentry': bool(re.search(r'sentry_sdk|SENTRY_DSN|@sentry/', content, re.I)), + 'file': rel_path, + 'has_middleware': has_middleware, + 'detail': 'Flask application' + } + + # Django (settings.py or wsgi.py) + if 'settings.py' in py_file.name and re.search(r'INSTALLED_APPS|MIDDLEWARE', content): + rel_path = py_file.relative_to(root).as_posix() + return { + 'framework': 'Django', + 'uses_sentry': bool(re.search(r'sentry_sdk|SENTRY_DSN|@sentry/', content, re.I)), + 'file': rel_path, + 'has_middleware': True, + 'detail': 'Django settings' + } + + # JavaScript/Node frameworks + for js_file in js_files: + try: + content = js_file.read_text(errors='ignore') + except: + continue + + # Express + if re.search(r'const app\s*=\s*express\(\)|const app\s*=\s*require.*express', content): + rel_path = js_file.relative_to(root).as_posix() + has_middleware = bool(re.search(r'app\.use\(|app\.middleware', content)) + return { + 'framework': 'Express', + 'uses_sentry': bool(re.search(r'sentry_sdk|SENTRY_DSN|@sentry/', content, re.I)), + 'file': rel_path, + 'has_middleware': has_middleware, + 'detail': 'Express application' + } + + # NestJS + if re.search(r'@Module|NestFactory|INestApplication', content): + rel_path = js_file.relative_to(root).as_posix() + return { + 'framework': 'NestJS', + 'uses_sentry': bool(re.search(r'sentry_sdk|SENTRY_DSN|@sentry/', content, re.I)), + 'file': rel_path, + 'has_middleware': True, + 'detail': 'NestJS application' + } + + return None + + # ── Concurrency calculation ──────────────────────────────────────────────────── def _rate_str_to_per_min(rate_str: str) -> float | None: @@ -955,12 +1163,12 @@ def infer_flows(analysis: dict) -> list: :root{--bg:#0f172a;--bg2:#111827;--bg3:#161e2e;--bg4:#1a2535; --border:#1e293b;--border2:#263347;--text:#f1f5f9;--muted:#64748b;--dim:#334155; --sans:'IBM Plex Sans',system-ui,sans-serif;--mono:'JetBrains Mono',monospace} -:root[data-theme="light"]{--bg:#f8fafc;--bg2:#ffffff;--bg3:#ffffff;--bg4:#f1f5f9; +:root[data-theme="light"]{--bg:#eef2f7;--bg2:#ffffff;--bg3:#ffffff;--bg4:#f1f5f9; --border:#e2e8f0;--border2:#cbd5e1;--text:#0f172a;--muted:#64748b;--dim:#94a3b8} -:root[data-theme="light"] body{background-image:radial-gradient(circle,#cbd5e1 1.4px,transparent 1.4px);background-size:24px 24px} :root[data-theme="light"] h1{color:#0f172a} :root[data-theme="light"] code{color:#0891b2} -:root[data-theme="light"] .arch-wrap{box-shadow:0 1px 3px rgba(15,23,42,.06)} +:root[data-theme="light"] .arch-wrap,:root[data-theme="light"] .graph-wrap,:root[data-theme="light"] .flow-wrap, +:root[data-theme="light"] .semantic-scroll{box-shadow:0 1px 3px rgba(15,23,42,.08)} *{box-sizing:border-box;margin:0;padding:0} body{background:var(--bg);color:var(--text);font-family:var(--sans);font-size:14px;line-height:1.6;transition:background .25s,color .25s} .page{max-width:1440px;margin:0 auto;padding:40px 32px 80px} @@ -972,7 +1180,8 @@ def infer_flows(analysis: dict) -> list: .theme-toggle:hover{border-color:var(--border2)} .subtitle{color:var(--muted);font-size:14px;margin-bottom:40px} .section{margin-bottom:56px} -.section-title{font-size:13px;font-weight:600;letter-spacing:.08em;text-transform:uppercase; +.section-intro{font-size:13px;line-height:1.6;color:var(--muted);margin:-8px 0 16px;max-width:1100px} +.section-title{font-size:16px;font-weight:600;letter-spacing:.08em;text-transform:uppercase; color:var(--muted);border-bottom:1px solid var(--border);padding-bottom:10px;margin-bottom:24px} .stat-row{display:flex;gap:16px;flex-wrap:wrap;margin-bottom:40px} .stat-card{background:var(--bg3);border:1px solid var(--border);border-radius:12px; @@ -986,20 +1195,54 @@ def infer_flows(analysis: dict) -> list: .graph-wrap{background:var(--bg3);border:1px solid var(--border);border-radius:14px; padding:16px;position:relative;height:640px} .graph-wrap canvas{width:100%;height:100%;display:block;border-radius:10px;cursor:grab} -.graph-toolbar{position:absolute;top:26px;left:26px;right:26px;z-index:2;display:flex; -align-items:center;gap:10px;flex-wrap:wrap} -#graph-search{background:var(--bg2); +.cg-bar{display:flex;align-items:center;gap:10px;flex-wrap:wrap;margin-bottom:10px} +.cg-tabs{display:inline-flex;background:var(--bg2);border:1px solid var(--border);border-radius:8px;padding:2px} +.cg-tabs button{background:none;border:0;border-radius:6px;padding:5px 14px;font-size:12.5px; +color:var(--muted);font-family:var(--sans);cursor:pointer} +.cg-tabs button.on{background:var(--bg4);color:var(--text);font-weight:600} +.cg-view[hidden],.cg-view-text[hidden]{display:none} +.arch-details>summary{cursor:pointer;font-size:12px;color:var(--muted);margin-bottom:12px} +.arch-details>summary:hover{color:var(--text)} +.graph-search{background:var(--bg2); border:1px solid var(--border);border-radius:8px;padding:6px 10px;font-size:12px; -color:var(--text);width:180px;font-family:var(--sans)} -#graph-search:focus{outline:none;border-color:var(--border2)} -#graph-hide-isolated-wrap{display:flex;align-items:center;gap:5px;font-size:11px; +color:var(--text);width:200px;font-family:var(--sans)} +.graph-search:focus{outline:none;border-color:var(--border2)} +.graph-hide-isolated-wrap{display:flex;align-items:center;gap:5px;font-size:12px; color:var(--muted);white-space:nowrap;cursor:pointer;user-select:none} -#graph-isolated-count{color:var(--muted)} -#graph-reset-view{background:var(--bg2);border:1px solid var(--border);border-radius:8px; -padding:6px 10px;font-size:11px;color:var(--muted);font-family:var(--sans);cursor:pointer} -#graph-reset-view:hover{color:var(--text);border-color:var(--border2)} +.graph-isolated-count{color:var(--muted)} +.graph-reset-view{background:var(--bg2);border:1px solid var(--border);border-radius:8px; +padding:6px 10px;font-size:12px;color:var(--muted);font-family:var(--sans);cursor:pointer} +.graph-reset-view:hover{color:var(--text);border-color:var(--border2)} +.flow-wrap{background:var(--bg3);border:1px solid var(--border);border-radius:14px; +padding:16px;overflow:auto;max-height:860px} +.flow-wrap svg{display:block} +.flow-col-head{font-size:10px;font-weight:600;letter-spacing:.08em;fill:var(--muted);text-transform:uppercase} +.flow-node{cursor:pointer} +.flow-node rect.box{fill:var(--bg2);stroke:var(--border2);stroke-width:1} +.flow-node:hover rect.box{stroke:var(--text)} +.flow-node.sel rect.box{stroke:var(--text);stroke-width:2} +.flow-node .lbl{font-size:11.5px;font-weight:600;fill:var(--text);font-family:var(--mono)} +.flow-node .sub{font-size:9.5px;fill:var(--muted);font-family:var(--mono)} +.flow-edge{fill:none;stroke-width:1.3;opacity:.4} +.flow-edge.import{stroke:var(--muted)} +.flow-edge.service{stroke:#60a5fa} +.flow-edge.entry{stroke:#22c55e} +.flow-edge.back{stroke-dasharray:4 3} +.flow-dim{opacity:.08 !important} +.flow-edge.hl{opacity:.95;stroke-width:1.8} +.flow-arrow.import{fill:var(--muted)}.flow-arrow.service{fill:#60a5fa}.flow-arrow.entry{fill:#22c55e} +.flow-dots{pointer-events:none} +.flow-dot{display:none} +.flow-dot.import{fill:var(--text)}.flow-dot.service{fill:#60a5fa}.flow-dot.entry{fill:#22c55e} +.flow-dot.ret{fill:var(--bg3);stroke-width:1.3} +.flow-dot.ret.import{stroke:var(--text)}.flow-dot.ret.service{stroke:#60a5fa}.flow-dot.ret.entry{stroke:#22c55e} +.flow-legend .dot{width:7px;height:7px;border-radius:50%;background:var(--text);display:inline-block} +.flow-legend .dot.ret{background:none;border:1.3px solid var(--text)} +.flow-legend{display:flex;gap:14px 20px;flex-wrap:wrap;font-size:12px;color:var(--muted);margin-top:10px} +.flow-legend span{display:flex;align-items:center;gap:6px} +.flow-legend i{width:18px;height:0;border-top:2px solid;display:inline-block} .graph-legend{position:absolute;bottom:26px;left:26px;right:26px;z-index:2;display:flex;gap:14px 20px; -flex-wrap:wrap;font-size:10.5px;color:var(--muted);background:rgba(0,0,0,.0)} +flex-wrap:wrap;font-size:12px;color:var(--muted);background:rgba(0,0,0,.0)} .graph-legend span{display:flex;align-items:center;gap:5px} .graph-legend .lg-dot{width:9px;height:9px;border-radius:50%;background:#818cf8;display:inline-block} .graph-legend .lg-square{width:9px;height:9px;background:#eab308;display:inline-block;border-radius:2px} @@ -1009,6 +1252,64 @@ def infer_flows(analysis: dict) -> list: display:none;max-width:320px;line-height:1.5} .graph-tooltip .tt-path{font-family:var(--mono);font-weight:600;word-break:break-all} .graph-tooltip .tt-meta{color:var(--muted);font-size:10.5px;margin-top:3px} +.semantic-wrap{position:relative;display:flex;gap:16px;align-items:flex-start} +.semantic-scroll{flex:1;overflow:auto;position:relative;background:var(--bg3);border:1px solid var(--border); +border-radius:12px;padding:16px} +/* Integrations strip and legend sit in normal flow (strip above the map, legend + below) so they can never cover nodes, whatever the layout's size. */ +.semantic-scroll{display:flex;flex-direction:column;align-items:flex-start;gap:12px} +#semantic-canvas{display:block} +.sem-integrations-card{order:-1;position:sticky;left:0;display:flex;flex-wrap:wrap;align-items:center; +gap:6px 8px;background:var(--bg2);border:1px solid var(--border2);border-radius:10px;padding:8px 12px; +max-width:100%} +.sem-integrations-card .sic-title{font-size:9.5px;font-weight:700;letter-spacing:.08em; +text-transform:uppercase;color:var(--muted);margin-right:4px} +.sem-integrations-card .sic-item{display:flex;align-items:center;gap:6px;font-size:12px; +color:var(--text);padding:2px 8px;background:var(--bg4);border-radius:999px} +.sem-integrations-card .sic-item i{width:6px;height:6px;border-radius:50%;background:var(--muted); +flex-shrink:0;display:inline-block} +.sem-group-box{fill:var(--gcolor,var(--border2));fill-opacity:.045;stroke:var(--gcolor,var(--border2)); +stroke-opacity:.5;stroke-width:1.25} +.sem-node-rect{fill:var(--bg2);stroke:var(--border2);stroke-width:1.25;transition:stroke .15s,opacity .15s} +.sem-node{cursor:pointer} +.sem-node:hover .sem-node-rect{stroke:var(--muted)} +.sem-node.sem-dim{opacity:.25} +.sem-icon{stroke-linecap:round;stroke-linejoin:round} +.sem-node-label{font-family:var(--sans);font-size:12.5px;font-weight:600;fill:var(--text)} +.sem-node-sub{font-family:var(--mono);font-size:9.5px;fill:var(--muted)} +.sem-group-tag{font-family:var(--sans);font-size:9.5px;font-weight:700;letter-spacing:.06em} +.sem-edge{opacity:.7;transition:opacity .15s,stroke .15s;fill:none} +.sem-edge.sem-edge-labeled{opacity:.85} +.sem-edge.sem-edge-active{opacity:1;stroke:var(--text)!important} +.sem-edge.sem-dim{opacity:.1} +.sem-edge-flow{opacity:.9;stroke:#38bdf8;stroke-width:1.4;stroke-dasharray:5 9; +stroke-linecap:round;transition:opacity .15s;animation:semEdgeFlow linear infinite} +@media(prefers-reduced-motion:reduce){.sem-edge-flow{animation:none;opacity:0}} +.sem-edge-flow.sem-edge-active{opacity:1} +.sem-edge-flow.sem-dim{opacity:0} +@keyframes semEdgeFlow{to{stroke-dashoffset:-28}} +.sem-edge-label{font-family:var(--sans);font-size:9.5px;fill:var(--muted);text-anchor:middle;font-weight:600} +.semantic-detail-panel{width:280px;flex-shrink:0;background:var(--bg3);border:1px solid var(--border); +border-radius:12px;padding:16px;position:sticky;top:16px} +.semantic-detail-panel[hidden]{display:none} +.sem-detail-close{float:right;background:none;border:none;color:var(--muted);cursor:pointer;font-size:16px;line-height:1} +.sem-detail-kind{font-size:10px;font-weight:700;letter-spacing:.08em;margin-bottom:6px} +.sem-detail-label{font-size:15px;font-weight:700;color:var(--text);margin-bottom:2px} +.sem-detail-sub{font-family:var(--mono);font-size:11px;color:var(--muted);margin-bottom:10px} +.sem-detail-desc{font-size:12.5px;color:var(--text);line-height:1.5;margin:0 0 10px} +.sem-detail-ref{font-family:var(--mono);font-size:10.5px;color:var(--muted);word-break:break-all; +background:var(--bg4);border-radius:6px;padding:6px 8px;margin-bottom:8px} +.sem-detail-domain{font-family:var(--mono);font-size:10.5px;color:var(--muted)} +.sem-detail-btn{margin-top:10px;width:100%;background:var(--bg4);border:1px solid var(--border2); +color:var(--text);border-radius:8px;padding:9px 10px;font-size:12px;font-weight:600;cursor:pointer; +font-family:var(--sans)} +.sem-detail-btn:hover{border-color:var(--muted)} +.semantic-detail-placeholder{color:var(--muted);font-size:12.5px} +.sem-legend{position:sticky;left:0;display:flex;flex-wrap:wrap;gap:6px;max-width:100%} +.sem-legend-item{display:inline-flex;align-items:center;gap:6px;font-size:12px;color:var(--muted); +background:var(--bg2);border:1px solid var(--border2);border-radius:999px;padding:5px 12px; +box-shadow:0 2px 8px rgba(0,0,0,.2)} +.sem-legend-item i{width:7px;height:7px;border-radius:50%;display:inline-block} .derivation{background:var(--bg3);border:1px solid var(--border);border-radius:10px;padding:12px 18px;margin:0 0 12px} .derivation summary{cursor:pointer;font-size:12px;font-weight:600;color:var(--text);list-style:revert} .derivation .muted-inline{font-weight:400;color:var(--muted);font-size:11px} @@ -1247,22 +1548,30 @@ def infer_flows(analysis: dict) -> list: clearTimeout(resizeTimer); resizeTimer = setTimeout(drawEdges, 150); }); + // Collapsed by default; the overlay can only be measured once it's open. + var archDetails = document.getElementById('arch-details'); + if (archDetails) archDetails.addEventListener('toggle', drawEdges); drawEdges(); })(); """ GRAPH_JS = """ -(function(){ - var dataEl = document.getElementById('graph-data'); +// One instance per graph canvas; P is the element-id prefix (graph-, test-graph-). +// Returns {select, isolate, fit, onSelect} for the Codebase section controller. +function wfgenCodeGraph(P){ + var dataEl = document.getElementById(P + 'data'); if (!dataEl) return; var data = JSON.parse(dataEl.textContent); - var canvas = document.getElementById('code-graph'); + var canvas = document.getElementById(P + 'canvas'); if (!canvas || !data.nodes.length) return; var ctx = canvas.getContext('2d'); var wrap = canvas.parentElement; - var tooltip = document.getElementById('graph-tooltip'); + var tooltip = document.getElementById(P + 'tooltip'); var reduceMotion = !window.matchMedia('(prefers-reduced-motion: no-preference)').matches; + var animateEl = document.getElementById(P + 'animate'); + function animateOn(){ return animateEl ? animateEl.checked : !reduceMotion; } + var api = {onSelect: null}; function hashStr(s){ var h = 2166136261; @@ -1387,9 +1696,9 @@ def infer_flows(analysis: dict) -> list: selected = null; hovered = null; searchMatch = null; } - var hideIsolatedEl = document.getElementById('graph-hide-isolated'); - var isolatedCountEl = document.getElementById('graph-isolated-count'); - var hideIsolatedWrapEl = document.getElementById('graph-hide-isolated-wrap'); + var hideIsolatedEl = document.getElementById(P + 'hide-isolated'); + var isolatedCountEl = document.getElementById(P + 'isolated-count'); + var hideIsolatedWrapEl = document.getElementById(P + 'hide-isolated-wrap'); if (isolatedCountEl) isolatedCountEl.textContent = isolatedCount ? ' (' + isolatedCount + ')' : ''; if (!isolatedCount && hideIsolatedWrapEl) hideIsolatedWrapEl.style.display = 'none'; if (hideIsolatedEl) hideIsolatedEl.checked = isolatedCount > 0; @@ -1398,7 +1707,7 @@ def infer_flows(analysis: dict) -> list: dragging = null; panStart = null; moved = false; buildLayout(hideIsolatedEl.checked); }); - var resetViewEl = document.getElementById('graph-reset-view'); + var resetViewEl = document.getElementById(P + 'reset-view'); if (resetViewEl) resetViewEl.addEventListener('click', function(){ fitView(); }); function worldToScreen(x, y){ @@ -1461,7 +1770,7 @@ def infer_flows(analysis: dict) -> list: ctx.stroke(); ctx.globalAlpha = 1; - if (!reduceMotion && !dim){ + if (animateOn() && !dim){ if (!l._phase) l._phase = l.phase; // Real observed-traffic weight (from --access-log) speeds the // particle up (log-scaled — raw request counts span orders of @@ -1469,13 +1778,17 @@ def infer_flows(analysis: dict) -> list: var hasTraffic = l.kind === 'entry' && l.w; var speedMul = hasTraffic ? Math.min(5, 1 + Math.log2(l.w + 1) * 0.5) : 1; l._phase = (l._phase + dt * particleSpeed * speedMul) % 1; - [l._phase, (l._phase + 0.5) % 1].forEach(function(ph){ + // To and fro: a filled dot carries the call/request source -> target, + // a hollow one carries the return/response back, half a cycle apart. + var dotColor = hasTraffic ? '#f59e0b' : (l.kind === 'service' ? '#60a5fa' : colors.muted); + [[l._phase, false], [1 - (l._phase + 0.5) % 1, true]].forEach(function(pr){ + var ph = pr[0]; var px = p1[0] + (p2[0]-p1[0]) * ph, py = p1[1] + (p2[1]-p1[1]) * ph; ctx.beginPath(); - ctx.arc(px, py, 1.8, 0, 7); - ctx.fillStyle = hasTraffic ? '#f59e0b' : (l.kind === 'service' ? '#60a5fa' : colors.muted); + ctx.arc(px, py, 1.9, 0, 7); ctx.globalAlpha = 0.9; - ctx.fill(); + if (pr[1]){ ctx.fillStyle = colors.bg; ctx.fill(); ctx.strokeStyle = dotColor; ctx.lineWidth = 1.2; ctx.stroke(); } + else { ctx.fillStyle = dotColor; ctx.fill(); } ctx.globalAlpha = 1; }); } @@ -1618,9 +1931,10 @@ def infer_flows(analysis: dict) -> list: var r = canvas.getBoundingClientRect(); var hit = pick(e.clientX - r.left, e.clientY - r.top); selected = hit ? (selected === hit.id ? null : hit.id) : null; + if (api.onSelect) api.onSelect(selected); }); - var search = document.getElementById('graph-search'); + var search = document.getElementById(P + 'search'); if (search){ search.addEventListener('input', function(){ var q = search.value.trim().toLowerCase(); @@ -1631,6 +1945,17 @@ def infer_flows(analysis: dict) -> list: }); } + // Used by the Architecture Map's "Show import details" drill-down (via the + // Codebase controller): highlights a semantic component's member files the + // same way a search match does, without requiring the user to type anything. + api.isolate = function(ids, label){ + var set = {}; + (ids || []).forEach(function(id){ set[id] = true; }); + searchMatch = Object.keys(set).length ? set : null; + selected = null; + if (search) search.value = label || ''; + }; + var running = false; var pulseUntil = null; var graphVisible = false; @@ -1659,7 +1984,415 @@ def infer_flows(analysis: dict) -> list: document.addEventListener('wfgen-tour-end', armPulseIfReady); running = true; requestAnimationFrame(draw); -})(); + + api.select = function(id){ selected = id && byId[id] ? id : null; }; + // The canvas measures 0x0 while its view is hidden, so re-fit on show. + api.fit = function(){ fitView(); }; + return api; +} +""" + +# Flow view of the Codebase section: the force graph's exact node/edge set (it +# reads the same

data JSON), laid out as a left-to-right layered diagram. +# Columns are longest-path depth from the sources (HTTP entry / files nothing +# imports); detected services always form the last column (as compact +# one-line nodes); zero-degree nodes get their own "isolated" column so nothing +# in the graph is dropped. Import cycles are broken by DFS back-edge removal for +# layering only; the back edges are still drawn, dashed. +# +# Data movement: each animated edge carries a filled dot source -> target (the +# call / request) and a hollow dot back (the return / response). To keep it +# readable, idle animation only runs on graphs of <= _FLOW_IDLE_ANIM_EDGES edges; +# once a node is selected or hovered, only its traced paths animate. +_FLOW_IDLE_ANIM_EDGES = 60 +_FLOW_MAX_NODES = 80 +FLOW_JS = """ +function wfgenFlow(P){ + var dataEl = document.getElementById(P + 'data'); + var svg = document.getElementById(P + 'flow-canvas'); + if (!dataEl || !svg) return null; + var data = JSON.parse(dataEl.textContent); + if (!(data.nodes || []).length) return null; + var wrap = svg.parentElement; + var reduceMotion = !window.matchMedia('(prefers-reduced-motion: no-preference)').matches; + var animateEl = document.getElementById(P + 'animate'); + var IDLE_ANIM_EDGES = """ + str(_FLOW_IDLE_ANIM_EDGES) + """; + var api = {onSelect: null}; + + function hashStr(s){ + var h = 2166136261; + for (var i = 0; i < s.length; i++){ h ^= s.charCodeAt(i); h = Math.imul(h, 16777619); } + return (h >>> 0); + } + function nodeColor(n){ + if (n.kind === 'service' || n.kind === 'entry') return n.color; + return 'hsl(' + (hashStr(n.pkg) % 360) + ',62%,58%)'; + } + function isSink(n){ return n.kind === 'service'; } + function el(tag, attrs, text){ + var e = document.createElementNS('http://www.w3.org/2000/svg', tag); + for (var k in attrs) e.setAttribute(k, attrs[k]); + if (text !== undefined) e.textContent = text; + return e; + } + + var W = 210, H = 40, HS = 28, VGAP = 10, HGAP = 110, PAD = 20, HEAD = 26; + var byId, links, ids, out, inn, pos, nodeEls, edgeEls, dotG; + var selected = null, hovered = null, highlight = null; + + function build(hideIsolated){ + var all = data.nodes; + var degAll = {}; + var idsAll = {}; + all.forEach(function(n){ idsAll[n.id] = true; degAll[n.id] = 0; }); + (data.links || []).forEach(function(l){ + if (idsAll[l.s] && idsAll[l.t]){ degAll[l.s]++; degAll[l.t]++; } + }); + var nodes = hideIsolated ? all.filter(function(n){ return degAll[n.id]; }) : all; + byId = {}; + nodes.forEach(function(n){ byId[n.id] = n; }); + links = (data.links || []).filter(function(l){ return byId[l.s] && byId[l.t]; }); + ids = nodes.map(function(n){ return n.id; }).sort(); + out = {}; inn = {}; + var deg = {}; + ids.forEach(function(id){ out[id] = []; inn[id] = []; deg[id] = 0; }); + links.forEach(function(l){ out[l.s].push(l.t); inn[l.t].push(l.s); deg[l.s]++; deg[l.t]++; }); + + // ---- Break cycles (DFS, deterministic id order) so layering terminates ---- + var back = {}, state = {}; + ids.forEach(function(root){ + if (state[root]) return; + var stack = [[root, 0]]; + state[root] = 1; + while (stack.length){ + var top = stack[stack.length - 1], v = top[0], kids = out[v].slice().sort(); + if (top[1] < kids.length){ + var w = kids[top[1]++]; + if (state[w] === 1) back[v + '>' + w] = true; + else if (!state[w]){ state[w] = 1; stack.push([w, 0]); } + } else { state[v] = 2; stack.pop(); } + } + }); + + // ---- Longest-path layering over non-service nodes ---- + var layer = {}, indeg = {}; + ids.forEach(function(id){ indeg[id] = 0; }); + links.forEach(function(l){ + if (back[l.s + '>' + l.t] || isSink(byId[l.t]) || isSink(byId[l.s])) return; + indeg[l.t]++; + }); + var q = ids.filter(function(id){ return !isSink(byId[id]) && !indeg[id]; }); + q.forEach(function(id){ layer[id] = 0; }); + for (var qi = 0; qi < q.length; qi++){ + var v = q[qi]; + out[v].forEach(function(w){ + if (back[v + '>' + w] || isSink(byId[w])) return; + layer[w] = Math.max(layer[w] || 0, layer[v] + 1); + if (--indeg[w] === 0) q.push(w); + }); + } + var maxL = 0; + ids.forEach(function(id){ + if (isSink(byId[id]) || !deg[id]) return; + if (layer[id] === undefined) layer[id] = 0; + maxL = Math.max(maxL, layer[id]); + }); + var hasSinks = ids.some(function(id){ return isSink(byId[id]) && deg[id]; }); + var sinkCol = maxL + 1, isoCol = maxL + (hasSinks ? 2 : 1); + var hasIso = false; + ids.forEach(function(id){ + if (!deg[id]){ layer[id] = isoCol; hasIso = true; } + else if (isSink(byId[id])) layer[id] = sinkCol; + }); + var nCols = hasIso ? isoCol + 1 : (hasSinks ? sinkCol + 1 : maxL + 1); + var cols = []; + for (var c = 0; c < nCols; c++) cols.push([]); + ids.forEach(function(id){ cols[layer[id]].push(id); }); + cols.forEach(function(col){ + col.sort(function(a, b){ + var na = byId[a], nb = byId[b]; + return (na.kind === 'entry' ? -1 : 0) - (nb.kind === 'entry' ? -1 : 0) + || (na.pkg || '').localeCompare(nb.pkg || '') || na.label.localeCompare(nb.label); + }); + }); + + // ---- Crossing reduction: alternating barycenter sweeps ---- + var rank = {}; + function reRank(){ cols.forEach(function(col){ col.forEach(function(id, i){ rank[id] = i; }); }); } + reRank(); + function bary(id, nbrs, prevRank){ + var sum = 0, n = 0; + nbrs.forEach(function(w){ if (layer[w] !== layer[id]){ sum += rank[w]; n++; } }); + return n ? sum / n : prevRank; + } + for (var sweep = 0; sweep < 6; sweep++){ + var down = sweep % 2 === 0; + var order = cols.map(function(_, i){ return down ? i : cols.length - 1 - i; }); + order.forEach(function(ci){ + if (ci === isoCol && hasIso) return; + var col = cols[ci], key = {}; + col.forEach(function(id){ key[id] = bary(id, down ? inn[id] : out[id], rank[id]); }); + col.sort(function(a, b){ return key[a] - key[b] || rank[a] - rank[b]; }); + col.forEach(function(id, i){ rank[id] = i; }); + }); + } + reRank(); + + // ---- Geometry: services are one-line nodes so a long services column stays short ---- + function hOf(id){ return isSink(byId[id]) ? HS : H; } + var colH = cols.map(function(col){ + return col.reduce(function(acc, id){ return acc + hOf(id) + VGAP; }, 0) - VGAP; + }); + var maxColH = Math.max.apply(null, colH.concat([0])); + var totalH = PAD * 2 + HEAD + maxColH; + pos = {}; + cols.forEach(function(col, ci){ + var y = PAD + HEAD + Math.max(0, maxColH - colH[ci]) / 2 * 0.35; + col.forEach(function(id){ + pos[id] = {x: PAD + ci * (W + HGAP), y: y, h: hOf(id)}; + y += hOf(id) + VGAP; + }); + }); + var totalW = PAD * 2 + nCols * W + (nCols - 1) * HGAP; + while (svg.firstChild) svg.removeChild(svg.firstChild); + svg.setAttribute('width', totalW); + svg.setAttribute('height', totalH); + svg.setAttribute('viewBox', '0 0 ' + totalW + ' ' + totalH); + + var defs = el('defs', {}); + ['import', 'service', 'entry'].forEach(function(k){ + var m = el('marker', {id: P + 'arrow-' + k, viewBox: '0 0 10 10', refX: '9', refY: '5', + markerWidth: '6', markerHeight: '6', orient: 'auto'}); + m.appendChild(el('path', {d: 'M0,0 L10,5 L0,10 z', 'class': 'flow-arrow ' + k})); + defs.appendChild(m); + }); + svg.appendChild(defs); + + cols.forEach(function(col, ci){ + var head = ci === isoCol && hasIso ? 'Isolated (' + col.length + ')' + : ci === sinkCol && hasSinks ? 'External services (' + col.length + ')' + : ci === 0 ? 'Sources (' + col.length + ')' + : 'Depth ' + ci + ' (' + col.length + ')'; + svg.appendChild(el('text', {x: PAD + ci * (W + HGAP), y: PAD + 10, 'class': 'flow-col-head'}, head)); + }); + + // Spread a node's anchors along its side in the order of the far end's y, + // so parallel edges fan out instead of stacking on the midpoint. + var outE = {}, inE = {}; + links.forEach(function(l){ (outE[l.s] = outE[l.s] || []).push(l); (inE[l.t] = inE[l.t] || []).push(l); }); + var offS = new Map(), offT = new Map(); + function spread(map, list, far, near){ + list.sort(function(a, b){ return pos[a[far]].y - pos[b[far]].y; }); + var n = list.length, span = Math.min(pos[list[0][near]].h - 10, n * 4); + list.forEach(function(l, i){ map.set(l, n < 2 ? 0 : i / (n - 1) * span - span / 2); }); + } + Object.keys(outE).forEach(function(id){ spread(offS, outE[id], 't', 's'); }); + Object.keys(inE).forEach(function(id){ spread(offT, inE[id], 's', 't'); }); + + var edgeG = el('g', {}), nodeG = el('g', {}); + dotG = el('g', {'class': 'flow-dots'}); + svg.appendChild(edgeG); svg.appendChild(dotG); svg.appendChild(nodeG); + edgeEls = []; + links.forEach(function(l){ + var a = pos[l.s], b = pos[l.t]; + var x1 = a.x + W, y1 = a.y + a.h / 2 + offS.get(l); + var x2 = b.x, y2 = b.y + b.h / 2 + offT.get(l); + var isBack = x2 <= x1, d; + if (!isBack){ + var mx = (x1 + x2) / 2; + d = 'M' + x1 + ',' + y1 + ' C' + mx + ',' + y1 + ' ' + mx + ',' + y2 + ' ' + (x2 - 2) + ',' + y2; + } else { + // Backward / same-column edge: loop out right, come back in from the left. + var bend = Math.max(40, (x1 - x2) * 0.25); + d = 'M' + x1 + ',' + y1 + ' C' + (x1 + bend) + ',' + y1 + ' ' + (x2 - bend) + ',' + y2 + ' ' + (x2 - 2) + ',' + y2; + } + var kind = l.kind === 'service' || l.kind === 'entry' ? l.kind : 'import'; + var p = el('path', {d: d, 'class': 'flow-edge ' + kind + (isBack ? ' back' : ''), + 'marker-end': 'url(#' + P + 'arrow-' + kind + ')'}); + if (l.w && l.w > 1) p.style.strokeWidth = Math.min(4, 1.3 + Math.log2(l.w) * 0.4); + p.appendChild(el('title', {}, l.s + ' \\u2192 ' + byId[l.t].label + (l.w > 1 ? ' (' + l.w + ')' : ''))); + edgeG.appendChild(p); + var out1 = el('circle', {r: 2.6, 'class': 'flow-dot ' + kind}); + var ret1 = el('circle', {r: 2.4, 'class': 'flow-dot ret ' + kind}); + dotG.appendChild(out1); dotG.appendChild(ret1); + edgeEls.push({l: l, el: p, out: out1, ret: ret1, len: 0, phase: (hashStr(l.s + l.t) % 1000) / 1000}); + }); + + nodeEls = {}; + ids.forEach(function(id){ + var n = byId[id], p = pos[id]; + var g = el('g', {'class': 'flow-node', transform: 'translate(' + p.x + ',' + p.y + ')'}); + g.appendChild(el('rect', {'class': 'box', width: W, height: p.h, rx: 7})); + g.appendChild(el('rect', {width: 4, height: p.h - 8, x: 3, y: 4, rx: 2, fill: nodeColor(n)})); + var label = (n.icon ? n.icon + ' ' : '') + n.label; + if (isSink(n)){ + if (label.length > 22) label = label.slice(0, 21) + '\\u2026'; + g.appendChild(el('text', {'class': 'lbl', x: 14, y: p.h / 2 + 4}, label)); + g.appendChild(el('text', {'class': 'sub', x: W - 8, y: p.h / 2 + 3.5, 'text-anchor': 'end'}, n.cat || 'service')); + } else { + if (label.length > 27) label = label.slice(0, 26) + '\\u2026'; + g.appendChild(el('text', {'class': 'lbl', x: 14, y: 17}, label)); + var sub = n.kind === 'file' ? (n.id.indexOf('/') !== -1 ? n.id.slice(0, n.id.lastIndexOf('/')) : '(root)') + ' \\u00b7 ' + n.loc + ' loc' + : n.kind === 'dir' ? (n.file_count || 0) + ' files \\u00b7 ' + n.loc + ' loc' + : 'entry point'; + if (sub.length > 34) sub = '\\u2026' + sub.slice(sub.length - 33); + g.appendChild(el('text', {'class': 'sub', x: 14, y: 31}, sub)); + } + g.appendChild(el('title', {}, n.id + '\\n' + inn[id].length + ' in \\u00b7 ' + out[id].length + ' out')); + g.addEventListener('click', function(ev){ + ev.stopPropagation(); + api.select(selected === id ? null : id); + if (api.onSelect) api.onSelect(selected); + }); + g.addEventListener('mouseenter', function(){ hovered = id; paint(); }); + g.addEventListener('mouseleave', function(){ if (hovered === id){ hovered = null; paint(); } }); + nodeG.appendChild(g); + nodeEls[id] = g; + }); + if (selected && !byId[selected]) selected = null; + paint(); + } + + function reach(start, adj){ + var seen = {}, stack = [start]; + seen[start] = true; + while (stack.length){ + adj[stack.pop()].forEach(function(w){ if (!seen[w]){ seen[w] = true; stack.push(w); } }); + } + return seen; + } + + // Which nodes/edges are "on": a selection traces everything upstream and + // downstream; a hover (with nothing selected) shows direct neighbours only; + // a search / drill-down highlight shows the matched set and edges among it. + var activeEdges = null; + function paint(){ + var nodeOn = null, edgeOn = null; + if (selected){ + var up = reach(selected, inn), dn = reach(selected, out); + nodeOn = function(id){ return up[id] || dn[id]; }; + edgeOn = function(l){ return (up[l.t] && up[l.s]) || (dn[l.s] && dn[l.t]); }; + } else if (hovered){ + nodeOn = function(id){ return id === hovered || out[hovered].indexOf(id) !== -1 || inn[hovered].indexOf(id) !== -1; }; + edgeOn = function(l){ return l.s === hovered || l.t === hovered; }; + } else if (highlight){ + nodeOn = function(id){ return highlight[id]; }; + edgeOn = function(l){ return highlight[l.s] && highlight[l.t]; }; + } + ids.forEach(function(id){ + nodeEls[id].classList.toggle('flow-dim', !!nodeOn && !nodeOn(id)); + nodeEls[id].classList.toggle('sel', id === selected); + }); + activeEdges = []; + edgeEls.forEach(function(e){ + var on = !edgeOn || !!edgeOn(e.l); + e.el.classList.toggle('flow-dim', !on); + e.el.classList.toggle('hl', !!edgeOn && on); + if (on && (edgeOn || edgeEls.length <= IDLE_ANIM_EDGES)) activeEdges.push(e); + }); + edgeEls.forEach(function(e){ e.out.style.display = e.ret.style.display = 'none'; }); + } + + // ---- Data movement (paused while scrolled away or on the other tab) ---- + var running = false, dotsShown = false; + function animateOn(){ return animateEl ? animateEl.checked : !reduceMotion; } + function hideDots(){ + if (!dotsShown) return; + edgeEls.forEach(function(e){ e.out.style.display = e.ret.style.display = 'none'; }); + dotsShown = false; + } + function onScreen(){ + var r = wrap.getBoundingClientRect(); + return r.width > 0 && r.bottom > 0 && r.top < window.innerHeight; + } + function frame(t){ + if (!animateOn()){ running = false; hideDots(); return; } + requestAnimationFrame(frame); + if (!onScreen()){ hideDots(); return; } + dotsShown = true; + activeEdges.forEach(function(e){ + if (!e.len) e.len = e.el.getTotalLength(); + if (!e.len) return; + // Constant on-screen speed (~70px/s) so long edges don't look frantic. + var period = Math.max(2200, e.len / 0.07); + var ph = (t / period + e.phase) % 1; + var po = e.el.getPointAtLength(ph * e.len); + var pr = e.el.getPointAtLength((1 - (ph + 0.5) % 1) * e.len); + e.out.setAttribute('cx', po.x); e.out.setAttribute('cy', po.y); + e.ret.setAttribute('cx', pr.x); e.ret.setAttribute('cy', pr.y); + e.out.style.display = e.ret.style.display = 'inline'; + }); + } + function kick(){ if (!running && animateOn()){ running = true; requestAnimationFrame(frame); } } + if (animateEl) animateEl.addEventListener('change', kick); + + var hideEl = document.getElementById(P + 'hide-isolated'); + build(hideEl ? hideEl.checked : false); + kick(); + if (hideEl) hideEl.addEventListener('change', function(){ build(hideEl.checked); }); + var search = document.getElementById(P + 'search'); + if (search) search.addEventListener('input', function(){ + var q2 = search.value.trim().toLowerCase(); + highlight = null; + if (q2){ + highlight = {}; + ids.forEach(function(id){ if (byId[id].label.toLowerCase().indexOf(q2) !== -1) highlight[id] = true; }); + } + paint(); + }); + svg.addEventListener('click', function(){ + api.select(null); + if (api.onSelect) api.onSelect(null); + }); + + api.select = function(id){ selected = id && byId[id] ? id : null; paint(); }; + api.highlight = function(list){ + highlight = {}; + (list || []).forEach(function(id){ highlight[id] = true; }); + if (!Object.keys(highlight).length) highlight = null; + selected = null; + paint(); + }; + api.reset = function(){ wrap.scrollTo(0, 0); }; + return api; +} + +// Codebase section controller: Flow | Graph tabs over one data set, with a +// shared toolbar (search, hide isolated, animate, reset) and shared selection. +function wfgenCodebase(P, isMain){ + var dataEl = document.getElementById(P + 'data'); + if (!dataEl) return; + var section = dataEl.closest('.section'); + // Both views read this checkbox each frame; default it from the OS motion setting. + var animateEl = document.getElementById(P + 'animate'); + if (animateEl) animateEl.checked = window.matchMedia('(prefers-reduced-motion: no-preference)').matches; + var force = wfgenCodeGraph(P), flow = wfgenFlow(P); + var views = {flow: document.getElementById(P + 'flow-view'), force: document.getElementById(P + 'force-view')}; + var tabs = {flow: document.getElementById(P + 'tab-flow'), force: document.getElementById(P + 'tab-force')}; + var current = null; + function show(v){ + current = v; + Object.keys(views).forEach(function(k){ + if (views[k]) views[k].hidden = k !== v; + if (tabs[k]){ tabs[k].classList.toggle('on', k === v); tabs[k].setAttribute('aria-selected', k === v ? 'true' : 'false'); } + }); + if (section) section.querySelectorAll('[data-view-text]').forEach(function(t){ t.hidden = t.getAttribute('data-view-text') !== v; }); + if (v === 'force' && force) force.fit(); + } + Object.keys(tabs).forEach(function(k){ if (tabs[k]) tabs[k].addEventListener('click', function(){ show(k); }); }); + if (force) force.onSelect = function(id){ if (flow) flow.select(id); }; + if (flow) flow.onSelect = function(id){ if (force) force.select(id); }; + var resetEl = document.getElementById(P + 'reset-view'); + if (resetEl) resetEl.addEventListener('click', function(){ if (current === 'flow' && flow) flow.reset(); }); + show((section && section.getAttribute('data-default-view')) || 'flow'); + if (isMain) window.__wfgenIsolateGraphNodes = function(ids, label){ + show('flow'); + if (force) force.isolate(ids, label); + if (flow) flow.highlight(ids); + }; +} +wfgenCodebase('graph-', true); +wfgenCodebase('test-graph-', false); """ TOUR_JS = """ @@ -1668,7 +2401,8 @@ def infer_flows(analysis: dict) -> list: var steps = [ {sel: '.stat-row', title: 'Capacity stats', body: "These are modeled estimates, not measurements. Expand \\u201cHow was Modeled Throughput computed?\\u201d below for the exact math, or run the generated k6 script against a real deploy to check them."}, {sel: '#tour-derivation', title: 'Where the numbers come from', body: 'Every number above traces back to something concrete \\u2014 worker count \\u00d7 per-worker concurrency, ranked against whichever constraint is tightest. Click either panel for the full derivation or a ready-to-run load test.'}, - {sel: '#section-graph', title: 'Codebase graph', body: 'Every node is a real file; every edge is an actual import or a pattern-confirmed service call \\u2014 not a guess. Drag to pan, scroll to zoom, click a node to isolate its connections, or search by name.'}, + {sel: '#section-semantic', title: 'Architecture map', body: 'A curated view of how the system actually works \\u2014 entry points, services, datastores, and integrations, grouped by feature. Click any node for details, then \\u201cShow import details\\u201d to jump to the exact files behind it.'}, + {sel: '#section-graph', title: 'Codebase', body: 'Every node is a real file; every edge is an actual import or a pattern-confirmed service call \\u2014 not a guess. Flow lays them out left to right with dots showing calls going out and returns coming back; Graph is the same data, force-directed. Click a node to trace its paths.'}, {sel: '#section-architecture', title: 'Architecture diagram', body: 'Solid boxes are confirmed in your source code; dashed boxes are declared in a manifest but never matched to real usage.'}, {sel: '#section-flows', title: 'Data flow paths', body: 'Each card animates a real request path \\u2014 write, read, background jobs \\u2014 inferred from what was actually detected.'}, {sel: '#section-concurrency', title: 'Concurrency model', body: "Layer-by-layer: what model each component uses, its ceiling, and what's actually limiting it."}, @@ -1818,6 +2552,425 @@ def infer_flows(analysis: dict) -> list: """ +# Semantic Architecture Map renderer. Layout is a Sugiyama-style topological +# layering (longest path from source nodes, computed via Kahn's algorithm on +# a graph where each `group` collapses to one meta-node for layering +# purposes) rather than a fixed kind-based column grid — this reproduces the +# left-to-right "sources -> logic -> sinks" reading order a real dependency +# flow naturally has. Groups render as an actual bounding box containing +# their stacked members (with a short internal connector when a real edge +# exists between two members), matching foglamp.dev/scan's grouped-stack +# presentation instead of only a shared accent color. +SEMANTIC_GRAPH_JS = """ +(function(){ + var dataEl = document.getElementById('semantic-graph-data'); + var membersEl = document.getElementById('semantic-node-members'); + if (!dataEl) return; + var data = JSON.parse(dataEl.textContent || '{"nodes":[],"edges":[]}'); + var members = membersEl ? JSON.parse(membersEl.textContent || '{}') : {}; + var nodes = data.nodes || [], edges = data.edges || []; + if (!nodes.length) return; + + var svg = document.getElementById('semantic-canvas'); + var panel = document.getElementById('semantic-detail-panel'); + var panelBody = document.getElementById('semantic-detail-body'); + var panelClose = document.getElementById('semantic-detail-close'); + var legend = document.getElementById('semantic-legend'); + if (!svg) return; + + var KIND_COLOR = { + entry:'#eab308', cron:'#eab308', agent:'#ec4899', model:'#6366f1', + tool:'#06b6d4', service:'#ec4899', store:'#22c55e', external:'#3b82f6' + }; + var KIND_LABEL = { + entry:'Trigger', cron:'Trigger', agent:'Service', model:'Model', + tool:'Tool', service:'Service', store:'Store', external:'External' + }; + var LEGEND_KINDS = [['entry','Triggers'],['service','Services'],['store','Stores'],['external','External']]; + var GROUP_PALETTE = ['#f472b6','#60a5fa','#34d399','#fbbf24','#a78bfa','#fb923c','#4ade80','#f87171']; + var groupColor = {}; + var gi = 0; + nodes.forEach(function(n){ + if (n.group && !(n.group in groupColor)){ + groupColor[n.group] = GROUP_PALETTE[gi % GROUP_PALETTE.length]; + gi++; + } + }); + // Only draw a bounding box for groups with >=2 members — a "group" of one + // (e.g. a service with no matching entry file) is just a plain node with + // a colored accent, not a boxed cluster. + var groupSize = {}; + nodes.forEach(function(n){ if (n.group) groupSize[n.group] = (groupSize[n.group]||0) + 1; }); + function isBoxedGroup(g){ return g && groupSize[g] >= 2; } + + var nodeById = {}; + nodes.forEach(function(n){ nodeById[n.id] = n; }); + + // ---- Layering: collapse each boxed group to one meta-node, run Kahn's + // algorithm for a longest-path topological layer per meta-node, then + // expand back so every real node inherits its meta-node's layer. ---- + function metaOf(id){ + var n = nodeById[id]; + return (n && isBoxedGroup(n.group)) ? ('grp:' + n.group) : id; + } + var metaIds = {}; + nodes.forEach(function(n){ metaIds[metaOf(n.id)] = true; }); + var metaOut = {}, metaIn = {}; + Object.keys(metaIds).forEach(function(m){ metaOut[m] = {}; metaIn[m] = 0; }); + edges.forEach(function(e){ + var a = metaOf(e.from), b = metaOf(e.to); + if (a === b) return; // internal group edge, not used for layering + if (!metaOut[a][b]){ metaOut[a][b] = true; metaIn[b] = (metaIn[b]||0) + 1; } + }); + var layer = {}; + var queue = Object.keys(metaIds).filter(function(m){ return !metaIn[m]; }); + queue.forEach(function(m){ layer[m] = 0; }); + var qi = 0, guard = 0; + while (qi < queue.length && guard++ < 5000){ + var m = queue[qi++]; + Object.keys(metaOut[m]).forEach(function(b){ + if (layer[b] === undefined || layer[b] < layer[m] + 1) layer[b] = layer[m] + 1; + metaIn[b]--; + if (metaIn[b] <= 0 && queue.indexOf(b) === -1) queue.push(b); + }); + } + // Any meta-node untouched by the Kahn pass (a true cycle) gets layer 0. + Object.keys(metaIds).forEach(function(m){ if (layer[m] === undefined) layer[m] = 0; }); + + var maxLayer = 0; + Object.keys(layer).forEach(function(m){ if (layer[m] > maxLayer) maxLayer = layer[m]; }); + + // Fully isolated meta-nodes (no in AND no out edges — e.g. a detected + // store nothing was confidently linked to) default to layer 0 from the + // Kahn seeding, which visually reads as "a source feeding everything." + // Push them to the end instead — an orphan reads more honestly as "found, + // but nothing connects to it" than as a false source. + Object.keys(metaIds).forEach(function(m){ + var hasOut = Object.keys(metaOut[m] || {}).length > 0; + var hasIn = Object.keys(metaOut).some(function(o){ return metaOut[o][m]; }); + if (!hasOut && !hasIn && maxLayer > 0) layer[m] = maxLayer; + }); + var layerCols = []; + for (var i = 0; i <= maxLayer; i++) layerCols.push([]); + var placedMeta = {}; + nodes.forEach(function(n){ + var m = metaOf(n.id); + if (!placedMeta[m]){ placedMeta[m] = true; layerCols[layer[m]].push(m); } + }); + layerCols.forEach(function(col){ col.sort(); }); + + // ---- Geometry ---- + var NODE_W = 190, NODE_H = 50, NODE_GAP = 12, COL_GAP = 100, PAD = 34; + var BOX_PAD = 14, BOX_LABEL_H = 20; + + var pos = {}; // node id -> {x,y} + var colX = []; + var x = PAD; + layerCols.forEach(function(col, ci){ + colX.push(x); + var maxW = NODE_W; + col.forEach(function(m){ + if (m.indexOf('grp:') === 0) maxW = Math.max(maxW, NODE_W + BOX_PAD * 2); + }); + x += maxW + COL_GAP; + }); + var totalW = x - COL_GAP + PAD; + + var maxColHeight = 0; + layerCols.forEach(function(col, ci){ + var y = PAD; + col.forEach(function(m){ + if (m.indexOf('grp:') === 0){ + var groupName = m.slice(4); + var memberNodes = nodes.filter(function(n){ return n.group === groupName; }); + // Order members so a real internal edge points top -> bottom. + memberNodes.sort(function(a, b){ + var aHead = edges.some(function(e){ return e.to === a.id && e.from === b.id; }); + if (aHead) return 1; + var bHead = edges.some(function(e){ return e.to === b.id && e.from === a.id; }); + if (bHead) return -1; + return (a.label||'').localeCompare(b.label||''); + }); + var innerY = y + BOX_LABEL_H + BOX_PAD; + memberNodes.forEach(function(n){ + pos[n.id] = {x: colX[ci] + BOX_PAD, y: innerY, layer: ci, group: groupName}; + innerY += NODE_H + NODE_GAP; + }); + pos['grp:' + groupName] = { + x: colX[ci], y: y, w: NODE_W + BOX_PAD * 2, + h: (innerY - NODE_GAP) - y + BOX_PAD, label: groupName + }; + y = innerY - NODE_GAP + BOX_PAD + 22; + } else { + var n = nodeById[m]; + pos[m] = {x: colX[ci], y: y, layer: ci}; + y += NODE_H + NODE_GAP + (n && n.group ? 18 : 0); + } + }); + if (y > maxColHeight) maxColHeight = y; + }); + var totalH = Math.max(maxColHeight + PAD, 320); + + svg.setAttribute('viewBox', '0 0 ' + totalW + ' ' + totalH); + svg.setAttribute('width', totalW); + svg.setAttribute('height', totalH); + + function el(tag, attrs){ + var e = document.createElementNS('http://www.w3.org/2000/svg', tag); + for (var k in attrs) e.setAttribute(k, attrs[k]); + return e; + } + + var defs = el('defs', {}); + [['sem-arrow','var(--muted)'], ['sem-arrow-hl','currentColor']].forEach(function(pair){ + var marker = el('marker', {id:pair[0], viewBox:'0 0 10 10', refX:'8.5', refY:'5', + markerWidth:'6.5', markerHeight:'6.5', orient:'auto-start-reverse'}); + marker.appendChild(el('path', {d:'M0,0 L10,5 L0,10 z', fill:pair[1]})); + defs.appendChild(marker); + }); + svg.appendChild(defs); + + // ---- Group bounding boxes (drawn first, behind everything) ---- + var boxLayer = el('g', {'class':'sem-boxes'}); + var edgeLayer = el('g', {'class':'sem-edges'}); + var nodeLayer = el('g', {'class':'sem-nodes'}); + svg.appendChild(boxLayer); + svg.appendChild(edgeLayer); + svg.appendChild(nodeLayer); + + Object.keys(pos).forEach(function(id){ + if (id.indexOf('grp:') !== 0) return; + var b = pos[id]; + var color = groupColor[b.label] || 'var(--border2)'; + boxLayer.appendChild(el('rect', { + x:b.x, y:b.y, width:b.w, height:b.h, rx:10, class:'sem-group-box', style:'--gcolor:' + color + })); + var tag = el('text', {x:b.x + BOX_PAD, y:b.y + 15, class:'sem-group-tag', fill:color}); + tag.textContent = b.label.toUpperCase(); + boxLayer.appendChild(tag); + }); + + // ---- Internal (within-group) connectors: short vertical arrow ---- + var groupInternalEdges = {}; + Object.keys(nodeById).length && edges.forEach(function(e){ + var s = nodeById[e.from], t = nodeById[e.to]; + if (s && t && s.group && s.group === t.group && isBoxedGroup(s.group)){ + groupInternalEdges[e.from + '>' + e.to] = e; + } + }); + + // Fan-out staggering: when several edges leave one node (or arrive at + // one node), spread their anchor points along the node's edge instead of + // stacking them all on the vertical center — the curves separate cleanly + // the way the reference renderer's do. + var outSlots = {}, inSlots = {}, outSeen = {}, inSeen = {}; + edges.forEach(function(e){ + var s = nodeById[e.from], t = nodeById[e.to]; + if (!s || !t) return; + if (s.group && s.group === t.group && isBoxedGroup(s.group)) return; + outSlots[e.from] = (outSlots[e.from] || 0) + 1; + inSlots[e.to] = (inSlots[e.to] || 0) + 1; + }); + function slotOffset(id, slots, seen){ + var n = slots[id] || 1; + var i = seen[id] = (seen[id] || 0) + 1; + if (n < 2) return 0; + var span = Math.min(NODE_H - 16, n * 9); + return (i - 1) / (n - 1) * span - span / 2; + } + + function anchor(id, side, dy){ + var p = pos[id]; + var y = p.y + NODE_H / 2 + (dy || 0); + return side === 'r' ? {x: p.x + NODE_W, y: y} : {x: p.x, y: y}; + } + + function drawEdge(e, internal){ + var s = nodeById[e.from], t = nodeById[e.to]; + if (!s || !t || !pos[s.id] || !pos[t.id]) return; + var ps = pos[s.id], pt = pos[t.id]; + var a, b, d, pathId = 'sem-edge-' + edgeIdx++; + if (internal){ + a = {x: ps.x + NODE_W/2, y: ps.y + NODE_H}; + b = {x: pt.x + NODE_W/2, y: pt.y}; + d = 'M' + a.x + ',' + a.y + ' L' + b.x + ',' + b.y; + } else { + var sameCol = ps.layer === pt.layer; + var oOff = slotOffset(e.from, outSlots, outSeen); + var iOff = slotOffset(e.to, inSlots, inSeen); + a = sameCol ? {x: ps.x + NODE_W/2, y: ps.y + NODE_H} : anchor(s.id, ps.layer <= pt.layer ? 'r' : 'l', oOff); + b = sameCol ? {x: pt.x + NODE_W/2, y: pt.y} : anchor(t.id, ps.layer <= pt.layer ? 'l' : 'r', iOff); + if (sameCol){ + d = 'M' + a.x + ',' + a.y + ' L' + b.x + ',' + b.y; + } else { + var dx = Math.max(36, Math.abs(b.x - a.x) * 0.42); + var sign = ps.layer <= pt.layer ? 1 : -1; + d = 'M' + a.x + ',' + a.y + ' C ' + (a.x + sign*dx) + ',' + a.y + ' ' + + (b.x - sign*dx) + ',' + b.y + ' ' + b.x + ',' + b.y; + } + } + var path = el('path', { + d:d, fill:'none', stroke:'var(--muted)', 'stroke-width': e.label ? '1.5' : '1.2', + 'marker-end':'url(#sem-arrow)', 'data-from':s.id, 'data-to':t.id, + 'class':'sem-edge' + (e.label ? ' sem-edge-labeled' : ''), id: pathId + }); + edgeLayer.appendChild(path); + // Flowing-current animation: a dashed overlay whose dash offset is + // animated by CSS — continuous movement along every path, like the + // reference renderer, with no per-frame JS. + var flow = el('path', { + d:d, fill:'none', 'data-from':s.id, 'data-to':t.id, 'class':'sem-edge-flow' + }); + flow.style.animationDuration = (1.1 + (edgeIdx % 4) * 0.3) + 's'; + flow.style.animationDelay = ((edgeIdx % 5) * 0.18) + 's'; + edgeLayer.appendChild(flow); + if (e.label){ + var mx = (a.x + b.x) / 2, my = (a.y + b.y) / 2; + var txt = el('text', {x:mx, y: my - 7, 'text-anchor':'middle', class:'sem-edge-label'}); + txt.textContent = e.label; + edgeLayer.appendChild(txt); + } + } + + var edgeIdx = 0; + Object.keys(groupInternalEdges).forEach(function(key){ drawEdge(groupInternalEdges[key], true); }); + edges.forEach(function(e){ + var key = e.from + '>' + e.to; + if (groupInternalEdges[key]) return; + drawEdge(e, false); + }); + + // ---- Icon glyphs (inline paths, no external font/icon set) ---- + function icon(kind){ + var g = el('g', {'class':'sem-icon'}); + if (kind === 'entry' || kind === 'cron'){ + g.appendChild(el('path', {d:'M6.5 0.5 L1 7.5h3.2L3 11.5l5.5-7.5H5.3z'})); + } else if (kind === 'store'){ + [0,3.2,6.4].forEach(function(dy){ + g.appendChild(el('ellipse', {cx:5, cy:2 + dy, rx:5, ry:1.8})); + }); + g.appendChild(el('path', {d:'M0,2 L0,8.4 A5 1.8 0 0 0 10 8.4 L10 2', fill:'none', 'stroke-width':'1'})); + } else if (kind === 'external'){ + g.appendChild(el('circle', {cx:5, cy:5, r:5, fill:'none', 'stroke-width':'1.1'})); + g.appendChild(el('ellipse', {cx:5, cy:5, rx:2.1, ry:5, fill:'none', 'stroke-width':'1'})); + g.appendChild(el('line', {x1:0, y1:5, x2:10, y2:5, 'stroke-width':'1'})); + } else if (kind === 'model' || kind === 'tool'){ + g.appendChild(el('rect', {x:1.5, y:1.5, width:7, height:7, rx:1.5, fill:'none', 'stroke-width':'1.3'})); + } else { + g.appendChild(el('circle', {cx:5, cy:5, r:4})); + } + return g; + } + + Object.keys(pos).forEach(function(id){ + if (id.indexOf('grp:') === 0) return; + var n = nodeById[id]; + if (!n) return; + var p = pos[id]; + var g = el('g', {'class':'sem-node', 'data-id':n.id, transform:'translate(' + p.x + ',' + p.y + ')'}); + if (n.group && !isBoxedGroup(n.group)){ + var tag = el('text', {x:0, y:-6, class:'sem-group-tag', fill: groupColor[n.group] || (KIND_COLOR[n.kind])}); + tag.textContent = n.group.toUpperCase(); + g.appendChild(tag); + } + var kcolor = KIND_COLOR[n.kind] || '#64748b'; + var rect = el('rect', {width:NODE_W, height:NODE_H, rx:9, class:'sem-node-rect'}); + g.appendChild(rect); + var chip = el('rect', {x:9, y:(NODE_H-20)/2, width:20, height:20, rx:5, fill:kcolor, opacity:'0.16'}); + g.appendChild(chip); + var iconG = icon(n.kind); + iconG.setAttribute('transform', 'translate(' + (9 + 5) + ',' + ((NODE_H-20)/2 + 5) + ')'); + iconG.setAttribute('fill', kcolor); + iconG.setAttribute('stroke', kcolor); + g.appendChild(iconG); + var label = el('text', {x:38, y: n.sub ? 21 : 30, class:'sem-node-label'}); + label.textContent = n.label || n.id; + g.appendChild(label); + if (n.sub){ + var sub = el('text', {x:38, y:36, class:'sem-node-sub'}); + sub.textContent = n.sub; + g.appendChild(sub); + } + g.addEventListener('click', function(){ selectNode(n.id); }); + g.addEventListener('mouseenter', function(){ hoverNode(n.id); }); + g.addEventListener('mouseleave', function(){ hoverNode(null); }); + nodeLayer.appendChild(g); + }); + + function hoverNode(id){ + var connected = {}; + if (id){ + connected[id] = true; + edges.forEach(function(e){ + if (e.from === id) connected[e.to] = true; + if (e.to === id) connected[e.from] = true; + }); + } + nodeLayer.querySelectorAll('.sem-node').forEach(function(g){ + var nid = g.getAttribute('data-id'); + g.classList.toggle('sem-dim', !!id && !connected[nid]); + }); + edgeLayer.querySelectorAll('.sem-edge, .sem-edge-flow').forEach(function(p){ + var isRelated = !id || (connected[p.getAttribute('data-from')] && connected[p.getAttribute('data-to')] + && (p.getAttribute('data-from') === id || p.getAttribute('data-to') === id)); + p.classList.toggle('sem-edge-active', !!id && isRelated); + p.classList.toggle('sem-dim', !!id && !isRelated); + }); + } + + function selectNode(id){ + var n = nodeById[id]; + if (!n || !panel) return; + var memberIds = members[id] || []; + var html = '

' + + (KIND_LABEL[n.kind] || n.kind).toUpperCase() + '
' + + '
' + escapeHtml(n.label || n.id) + '
' + + (n.sub ? '
' + escapeHtml(n.sub) + '
' : '') + + (n.detail ? '

' + escapeHtml(n.detail) + '

' : '') + + (n.sourceRef ? '
' + escapeHtml(n.sourceRef) + '
' : '') + + (n.domain ? '
' + escapeHtml(n.domain) + '
' : ''); + panelBody.innerHTML = html; + var btn = document.getElementById('semantic-detail-import-btn'); + if (btn) btn.remove(); + if (memberIds.length && window.__wfgenIsolateGraphNodes){ + var b = document.createElement('button'); + b.id = 'semantic-detail-import-btn'; + b.className = 'sem-detail-btn'; + b.textContent = 'Show import details (' + memberIds.length + ' file' + (memberIds.length===1?'':'s') + ')'; + b.addEventListener('click', function(){ + window.__wfgenIsolateGraphNodes(memberIds, n.label || n.id); + var target = document.getElementById('section-graph'); + if (target) target.scrollIntoView({behavior:'smooth', block:'start'}); + }); + panelBody.appendChild(b); + } + panel.hidden = false; + } + + function escapeHtml(s){ + return String(s).replace(/[&<>"']/g, function(c){ + return {'&':'&','<':'<','>':'>','"':'"',"'":'''}[c]; + }); + } + + if (panelClose) panelClose.addEventListener('click', function(){ panel.hidden = true; }); + + if (legend){ + var LEGEND_MAP = {entry:['entry','cron'], service:['service','agent'], store:['store'], external:['external','model','tool']}; + LEGEND_KINDS.forEach(function(pair){ + var repKind = pair[0], label = pair[1]; + var kindsForThis = LEGEND_MAP[repKind] || [repKind]; + if (!nodes.some(function(n){ return kindsForThis.indexOf(n.kind) !== -1; })) return; + var item = document.createElement('span'); + item.className = 'sem-legend-item'; + item.innerHTML = '' + label; + legend.appendChild(item); + }); + } +})(); +""" + + def _box(name: str, desc: str, color: str, code: str = '', extra_desc: str = '', declared: bool = False, icon: str = '') -> str: bg = C.get(f'{color}_d', '#1a2535') border = C.get(color, '#475569') @@ -2130,6 +3283,528 @@ def compute_edges(flows: list, box_names: set) -> list: return edges +def build_semantic_graph(root: Path, analysis: dict) -> dict: + """Build semantic architecture graph from detected components. + + Generates a curated, high-level semantic graph following the foglamp.dev/scan + schema: entry points, services, stores, external integrations, agents, models, tools. + + Returns dict with keys: 'project', 'stats', 'topModels', 'topTools', 'topIntegrations', 'graph', 'node_members' + node_members maps semantic node IDs to file paths for drill-down filtering. + """ + + # Project metadata + project_name = analysis.get('project_name', 'Project') + project_slug = project_name.lower().replace(' ', '-') + from datetime import datetime + today = datetime.now().strftime('%Y-%m-%d') + + nodes = [] + edges = [] + node_members = {} # Tracks which files belong to each semantic node + + # 1. COMPOSITION ROOT node (if detected) + comp_root = analysis.get('composition_root') + if comp_root: + comp_root_id = 'app-root' + nodes.append({ + 'id': comp_root_id, + 'label': f"{comp_root['framework']} app", + 'kind': 'service', + 'sub': 'routing + middleware', + 'sourceRef': comp_root['file'], + 'detail': f"Wires all routes and global middleware" + }) + node_members[comp_root_id] = {comp_root['file']} + else: + comp_root_id = None + + # 1b. GATEWAY node (nginx/Caddy/Traefik) — the true left edge of the + # request path when present: gateway -> app root. + gateway = analysis.get('gateway') + if gateway and gateway.get('type'): + gw_type = gateway['type'] + nodes.append({ + 'id': 'gateway', + 'label': f"{gw_type} proxy"[:28], + 'kind': 'service', + 'sub': 'reverse proxy' + (' + TLS' if gateway.get('tls') else ''), + 'domain': _BRAND_DOMAINS.get(gw_type) or _BRAND_DOMAINS.get(gw_type.title()), + }) + if comp_root_id: + edges.append({'from': 'gateway', 'to': comp_root_id, 'kind': 'triggers'}) + + # 1c. FRONTEND node — the served SPA/dashboard, an entry the app hands + # to browsers (mirrors how a reader thinks of "the web app"). + frontend = analysis.get('frontend') or [] + if frontend: + fe_names = ', '.join(f['name'] for f in frontend[:3]) + nodes.append({ + 'id': 'frontend', + 'label': 'Web app (SPA)', + 'kind': 'entry', + 'sub': fe_names[:40], + }) + if comp_root_id: + edges.append({'from': comp_root_id, 'to': 'frontend', 'kind': 'triggers', 'label': 'serves SPA'}) + + # 2. ENTRY nodes — one per file that declares routes, grouped by the + # file's own name (routers/controllers are near-universally named after + # the resource they own: users.py, items.py, login.py). This is more + # reliable than grouping by URL path prefix: detect_api_server()'s route + # list is a flat, deduplicated, capped-at-20 set of "METHOD path" + # strings with no per-file provenance, and many frameworks (e.g. this + # project's own FastAPI routers) mount each router's own paths without + # repeating a shared prefix in the decorator string itself, so grouping + # by path segment silently produces empty/meaningless groups. + _GENERIC_STEMS = {'routes', 'router', 'routers', 'views', 'controllers', + 'controller', 'api', 'endpoints', 'handlers'} + entry_nodes = {} + route_patterns = [_RE['fastapi_route'], _RE['flask_route'], _RE['django_path'], _RE['express_route']] + for f in _collect_graph_files(root): + if f.suffix not in ('.py', '.js', '.ts'): + continue + try: + text = f.read_text(encoding='utf-8', errors='ignore') + except OSError: + continue + if not any(pat.search(text) for pat in route_patterns): + continue + stem = f.stem.lower() + if stem in _GENERIC_STEMS or stem == '__init__': + group_name = f.parent.name.replace('_', ' ').title() + else: + group_name = stem.replace('_', ' ').title() + group_id = re.sub(r'[^a-z0-9]+', '-', group_name.lower()).strip('-') or 'entry' + node_id = f"entry-{group_id}" + rel_path = f.relative_to(root).as_posix() + if node_id not in entry_nodes: + entry_nodes[node_id] = { + 'id': node_id, + 'label': f"{group_name} API"[:28], + 'kind': 'entry', + 'sub': rel_path, + 'sourceRef': rel_path, + 'group': group_name, + } + node_members[node_id] = set() + node_members[node_id].add(rel_path) + + # Cap entry nodes: merge the smallest (fewest member files) beyond 20 + # rather than truncating arbitrarily, so the biggest routers survive. + if len(entry_nodes) > 20: + ordered = sorted(entry_nodes.items(), key=lambda kv: len(node_members[kv[0]]), reverse=True) + kept, overflow = ordered[:19], ordered[19:] + entry_nodes = dict(kept) + merged_members = set() + for node_id, _ in overflow: + merged_members |= node_members.pop(node_id) + if merged_members: + entry_nodes['entry-other'] = { + 'id': 'entry-other', 'label': 'Other routes', 'kind': 'entry', + 'sub': f'{len(overflow)} smaller routers', + } + node_members['entry-other'] = merged_members + + # Add entry nodes to graph + for ent_id, ent_node in entry_nodes.items(): + nodes.append(ent_node) + # Link composition root to entries + if comp_root_id: + edges.append({ + 'from': comp_root_id, + 'to': ent_id, + 'kind': 'triggers' + }) + + # 3. SERVICE nodes — files matching common business-logic naming + # conventions (crud/services/repository layers, plus any file with a + # send_email-style function, which is common enough across frameworks to + # call out explicitly). Generalizes beyond a single project's layout; + # still name-pattern-based, not import-graph-based, consistent with this + # function's other detectors. + _SERVICE_STEM_LABELS = { + 'crud': ('CRUD service', 'Create/Read/Update/Delete operations', 'Data'), + 'services': ('Service layer', 'Business logic', 'Core'), + 'service': ('Service layer', 'Business logic', 'Core'), + 'repository': ('Repository', 'Data access layer', 'Data'), + 'dao': ('Data access', 'Data access layer', 'Data'), + } + service_nodes = {} + for f in _collect_graph_files(root): + if f.suffix != '.py': + continue + rel = f.relative_to(root).as_posix() + stem = f.stem.lower() + try: + text = f.read_text(encoding='utf-8', errors='ignore') + except OSError: + text = '' + label = sub = group = None + if stem in _SERVICE_STEM_LABELS: + label, sub, group = _SERVICE_STEM_LABELS[stem] + elif re.search(r'def send_email\b', text, re.I): + label, sub, group = 'Email service', 'Email templates + SMTP', 'Communication' + if label: + svc_id = f"service-{re.sub(r'[^a-z0-9]+', '-', stem)}" + service_nodes[svc_id] = { + 'id': svc_id, 'label': label, 'kind': 'service', 'sub': sub, + 'sourceRef': rel, 'group': group, + } + node_members[svc_id] = {rel} + + # Add service nodes, linking only entries whose files actually reference + # this service's module (by stem, e.g. `crud.create_user(` or + # `from app import crud`) — avoids an N-entries x M-services edge blowup. + for svc_id, svc_node in service_nodes.items(): + nodes.append(svc_node) + svc_stem = Path(svc_node['sourceRef']).stem + for ent_id, ent_node in entry_nodes.items(): + ent_files = node_members.get(ent_id, ()) + referenced = False + for rel in ent_files: + try: + ent_text = (root / rel).read_text(encoding='utf-8', errors='ignore') + except OSError: + continue + if re.search(r'\b' + re.escape(svc_stem) + r'\b', ent_text): + referenced = True + break + if referenced: + edges.append({'from': ent_id, 'to': svc_id, 'kind': 'calls'}) + + # 4. STORE nodes (from detected storage). detect_storage() returns + # {'name','type','desc','color'} — no domain, so branding comes from a + # small static lookup (mirrors the bundled-icon set planned for the + # renderer; brands outside this list simply render without one). + store_nodes = {} + store_types = {} + for store_type in analysis.get('storage', []): + store_name = store_type.get('name', 'Store') + store_id = f"store-{re.sub(r'[^a-z0-9]+', '-', store_name.lower())}" + if store_id not in store_nodes: + store_nodes[store_id] = { + 'id': store_id, + 'label': store_name, + 'kind': 'store', + 'domain': _BRAND_DOMAINS.get(store_name), + } + store_types[store_id] = store_type.get('type') + nodes.append(store_nodes[store_id]) + + # Generic ORM/session usage — a bare `session.add()`/`.objects.create()` + # talks to the app's primary datastore even when the file never spells + # the store's name out (common for layered apps where the connection + # lives in a separate db/core module, e.g. SQLModel/SQLAlchemy/Django + # ORM/Mongoose). Used only as a fallback when the literal-name match + # above finds nothing, and only when exactly one non-cache store is + # detected repo-wide — with more than one candidate, which store a bare + # `session.add()` means is genuinely ambiguous, so no edge is fabricated. + _GENERIC_DB_SESSION_RE = re.compile( + r'\bsession\.(exec|query|add|commit|delete|refresh)\(|' + r'\bdb\.(query|add|commit)\(|' + r'\.objects\.(create|get|filter|all)\(|' + r'\bawait\s+\w+\.save\(\)', + re.I, + ) + non_cache_store_ids = [sid for sid in store_nodes if store_types.get(sid) != 'cache'] + + # Link services to stores only when the service's file actually + # references the store by name (e.g. "postgres"/"redis" appearing in + # crud.py) — cheap, honest correlation rather than an N x M fan-out. + for svc_id, svc_node in service_nodes.items(): + try: + svc_text = (root / svc_node['sourceRef']).read_text(encoding='utf-8', errors='ignore').lower() + except (OSError, KeyError): + svc_text = '' + matched = False + for store_id, store_node in store_nodes.items(): + store_hint = store_node['label'].split(' ')[0].split('/')[0].lower() + if store_hint and store_hint in svc_text: + edges.append({'from': svc_node['id'], 'to': store_id, 'kind': 'writes'}) + matched = True + if not matched and len(non_cache_store_ids) == 1 and _GENERIC_DB_SESSION_RE.search(svc_text): + edges.append({'from': svc_node['id'], 'to': non_cache_store_ids[0], 'kind': 'writes'}) + + # 5. EXTERNAL nodes (from detected integrations). detect_external_sources() + # always appends "Users / API Clients" (an inbound source, not an + # outbound integration) and sometimes "Cron / Scheduler" (redundant with + # the dedicated detect_crons() pass above) — both excluded here since + # neither is a real third-party integration the app calls out to. + _NON_INTEGRATION_SOURCES = {'Users / API Clients', 'Cron / Scheduler'} + external_nodes = {} + for ext in analysis.get('external_sources', []): + ext_name = ext.get('name', 'service') + if ext_name in _NON_INTEGRATION_SOURCES: + continue + ext_id = f"external-{re.sub(r'[^a-z0-9]+', '-', ext_name.lower())}" + if ext_id not in external_nodes: + external_nodes[ext_id] = { + 'id': ext_id, + 'label': ext_name, + 'kind': 'external', + 'domain': _BRAND_DOMAINS.get(ext_name), + } + nodes.append(external_nodes[ext_id]) + + # 5b. Sentry — synthesized from the composition root's own file content + # (sentry_sdk / SENTRY_DSN / @sentry/ seen where the app is constructed), + # since detect_external_sources() has no Sentry pattern. The edge comes + # from the app root because that's literally where the init call lives. + if comp_root and comp_root.get('uses_sentry') and 'external-sentry' not in external_nodes: + external_nodes['external-sentry'] = { + 'id': 'external-sentry', 'label': 'Sentry', 'kind': 'external', + 'domain': 'sentry.io', 'sub': 'error tracking', + } + nodes.append(external_nodes['external-sentry']) + if comp_root_id: + edges.append({'from': comp_root_id, 'to': 'external-sentry', + 'kind': 'calls', 'label': 'reports errors'}) + + # 5c. SMTP — an email service (detected via a real `def send_email`) + # implies an outbound SMTP dependency; give it an explicit sink node so + # the mail path reads end-to-end instead of stopping at the service. + email_svc_ids = [sid for sid, sn in service_nodes.items() if sn['label'] == 'Email service'] + if email_svc_ids and 'external-smtp' not in external_nodes: + external_nodes['external-smtp'] = { + 'id': 'external-smtp', 'label': 'SMTP email', 'kind': 'external', + 'sub': 'outbound mail', + } + nodes.append(external_nodes['external-smtp']) + for sid in email_svc_ids: + edges.append({'from': sid, 'to': 'external-smtp', 'kind': 'calls'}) + + # 6. CRON nodes (if detected). No reliable signal ties a specific cron + # to a specific service without deeper analysis than this pass does, so + # — consistent with "never fabricate a connection" — cron nodes only + # link to the composition root (if one exists) rather than to every + # service found anywhere in the repo. + cron_nodes = {} + for cron in analysis.get('crons', []): + cron_id = f"cron-{cron.get('type', 'job')}" + if cron_id not in cron_nodes: + cron_nodes[cron_id] = { + 'id': cron_id, + 'label': cron.get('name', 'Scheduled Job'), + 'kind': 'cron', + 'sub': cron.get('detail', 'Periodic task') + } + nodes.append(cron_nodes[cron_id]) + if comp_root_id: + edges.append({'from': cron_id, 'to': comp_root_id, 'kind': 'triggers'}) + + # 7. MODEL nodes (from detected LLMs) + model_nodes = {} + top_models = [] + for llm in analysis.get('llm', {}).get('providers', []): + model_id = f"model-{llm.lower()}" + if model_id not in model_nodes: + label_map = { + 'openai': 'GPT-4', + 'anthropic': 'Claude', + 'google': 'Gemini', + 'cohere': 'Cohere', + } + label = label_map.get(llm, llm.title()) + domain_map = { + 'openai': 'openai.com', + 'anthropic': 'claude.ai', + 'google': 'gemini.google.com', + 'cohere': 'cohere.ai', + } + model_nodes[model_id] = { + 'id': model_id, + 'label': label, + 'kind': 'model', + 'domain': domain_map.get(llm) + } + nodes.append(model_nodes[model_id]) + if len(top_models) < 3: + top_models.append({'id': llm, 'label': label, 'domain': domain_map.get(llm)}) + + # 8. TOOL nodes (from detected tools) + tool_nodes = {} + top_tools = [] + for tool in analysis.get('tools', []): + tool_id = f"tool-{tool.get('type', 'tool')}" + if tool_id not in tool_nodes: + tool_nodes[tool_id] = { + 'id': tool_id, + 'label': tool.get('name', 'Tool'), + 'kind': 'tool' + } + nodes.append(tool_nodes[tool_id]) + if len(top_tools) < 10: + top_tools.append({'id': tool.get('type'), 'label': tool.get('name')}) + + # 9. AGENT nodes (if detected and have tools) + agent_nodes = {} + for agent in analysis.get('agents', []): + agent_id = f"agent-{agent.get('type', 'agent')}" + if agent_id not in agent_nodes and tool_nodes: # Only create agent if there are tools + agent_nodes[agent_id] = { + 'id': agent_id, + 'label': agent.get('name', 'Agent'), + 'kind': 'agent', + 'sub': agent.get('framework', 'Custom') + } + nodes.append(agent_nodes[agent_id]) + # Link agents to models and tools + for model_id in model_nodes: + edges.append({'from': agent_id, 'to': model_id, 'kind': 'calls'}) + for tool_id in tool_nodes: + edges.append({'from': agent_id, 'to': tool_id, 'kind': 'calls'}) + + # 10. Cap enforcement (nodes<=60, edges<=120) + if len(nodes) > 60: + # Merge smallest components + nodes = nodes[:60] + if len(edges) > 120: + edges = edges[:120] + + # 11. Stats and top items + stats = { + 'agents': len(agent_nodes), + 'models': len(model_nodes), + 'tools': len(tool_nodes), + 'integrations': len(external_nodes) + } + + top_integrations = [ + {'id': nodes_el['id'].replace('external-', ''), 'label': nodes_el['label'], 'domain': nodes_el.get('domain')} + for nodes_el in nodes + if nodes_el['kind'] == 'external' + ][:10] + + return { + 'version': 1, + 'project': { + 'name': project_name, + 'slug': project_slug, + 'date': today + }, + 'stats': stats, + 'topModels': top_models, + 'topTools': top_tools, + 'topIntegrations': top_integrations, + 'graph': { + 'nodes': nodes, + 'edges': edges + }, + 'node_members': node_members # For drill-down filtering + } + + +# Caps mirrored from build_semantic_graph's own node fields, so enriched +# labels can never exceed what the deterministic path already guarantees. +_ENRICH_LABEL_MAX = 28 +_ENRICH_SUB_MAX = 40 +_ENRICH_DETAIL_MAX = 200 +_ENRICH_FILE_EXCERPT_CHARS = 2000 + + +def enrich_service_labels(root: Path, semantic_graph: dict, *, timeout: int = 45) -> dict: + """Optional post-process: relabel `service` nodes using the user's local + `claude` CLI (headless/print mode) instead of the fixed stem dictionary + in build_semantic_graph(). Opt-in only (callers gate this on a flag), + and a no-op whenever `claude` isn't on PATH or anything about the call + fails — this must never turn a working deterministic scan into a broken + one. Only relabels nodes that already exist; never adds/removes nodes + or edges, consistent with this codebase's "never fabricate" policy. + """ + if shutil.which('claude') is None: + return semantic_graph + + service_nodes = [n for n in semantic_graph['graph']['nodes'] if n.get('kind') == 'service'] + if not service_nodes: + return semantic_graph + + entries = [] + for n in service_nodes: + src = n.get('sourceRef') + excerpt = '' + if src: + try: + excerpt = (root / src).read_text(encoding='utf-8', errors='ignore')[:_ENRICH_FILE_EXCERPT_CHARS] + except OSError: + excerpt = '' + entries.append({ + 'id': n['id'], + 'current_label': n.get('label'), + 'current_sub': n.get('sub'), + 'sourceRef': src, + 'excerpt': excerpt, + }) + + prompt = ( + "You are relabeling nodes in a software architecture diagram. For each " + "entry below, read the file excerpt and write a short label that reflects " + "what the code actually does, not the filename. Keep the same style as " + "the current labels (a short noun phrase ending in a generic-role word " + "like \"service\" or \"access\", e.g. \"Billing service\", \"User access\") " + "so the new labels stay visually consistent with the rest of the diagram. " + f"Hard limits: label <= {_ENRICH_LABEL_MAX} chars, sub <= {_ENRICH_SUB_MAX} chars, " + f"detail <= {_ENRICH_DETAIL_MAX} chars. " + "Respond with ONLY a JSON array, no prose, no markdown code fences, one " + "object per entry: [{\"id\": ..., \"label\": ..., \"sub\": ..., \"detail\": ...}]. " + "If an entry's excerpt doesn't tell you enough to improve on the current " + "label, just repeat the current label.\n\n" + + json.dumps(entries, indent=2) + ) + + try: + proc = subprocess.run( + ['claude', '-p', prompt, '--output-format', 'json'], + capture_output=True, text=True, timeout=timeout, + ) + except (OSError, subprocess.TimeoutExpired) as e: + print(f"Note: label enrichment skipped ({e}).", file=sys.stderr) + return semantic_graph + + if proc.returncode != 0 or not proc.stdout.strip(): + print(f"Note: label enrichment skipped (claude exited {proc.returncode}).", file=sys.stderr) + return semantic_graph + + try: + wrapper = json.loads(proc.stdout) + result_text = wrapper['result'] if isinstance(wrapper, dict) else proc.stdout + except (json.JSONDecodeError, KeyError, TypeError): + result_text = proc.stdout + + result_text = result_text.strip() + if result_text.startswith('```'): + result_text = re.sub(r'^```[a-zA-Z]*\n?', '', result_text) + result_text = re.sub(r'\n?```$', '', result_text) + + try: + updates = json.loads(result_text) + except json.JSONDecodeError: + print("Note: label enrichment skipped (unparseable response).", file=sys.stderr) + return semantic_graph + + if not isinstance(updates, list): + return semantic_graph + + by_id = {n['id']: n for n in service_nodes} + for u in updates: + if not isinstance(u, dict): + continue + node = by_id.get(u.get('id')) + if node is None: + continue + label = u.get('label') + if isinstance(label, str) and label.strip(): + node['label'] = label.strip()[:_ENRICH_LABEL_MAX] + sub = u.get('sub') + if isinstance(sub, str) and sub.strip(): + node['sub'] = sub.strip()[:_ENRICH_SUB_MAX] + detail = u.get('detail') + if isinstance(detail, str) and detail.strip(): + node['detail'] = detail.strip()[:_ENRICH_DETAIL_MAX] + + return semantic_graph + + def render_html(analysis: dict, project_name: str) -> str: today = datetime.date.today().strftime('%Y-%m-%d') cap = analysis['capacity'] @@ -2258,45 +3933,128 @@ def stat(val: str, color: str, label: str, sub: str = '') -> str: table_html = render_concurrency_table(analysis, cap) bn_html = render_bottlenecks(analysis, cap) - graph = analysis.get('graph') or {'nodes': [], 'links': [], 'meta': {'files': 0, 'links': 0, 'aggregated': False, 'shown_nodes': 0, 'has_access_log': False}} - graph_json = json.dumps(graph) - gm = graph['meta'] - graph_note = ( - f"{gm['files']} files aggregated into {gm['shown_nodes']} directories" if gm['aggregated'] - else f"{gm['shown_nodes']} modules" - ) - graph_section = '' - if graph['nodes']: - access_log = analysis.get('access_log') - traffic_note = ( - f" Amber particles on entry edges are real traffic, weighted by request count from {access_log['source']} ({access_log['total']:,} requests parsed)." - if gm.get('has_access_log') and access_log else - " This is static analysis, not live monitoring — particle motion shows dependency direction (importer → imported), not request traffic. Pass --access-log <path> to overlay real per-route request counts instead." - ) - graph_section = f""" -
-
Codebase Graph — {graph_note} · {gm['links']} real connections
-

Every node is a real file{' or directory' if gm['aggregated'] else ''} in this repo; every line is an actual import statement, or a pattern-confirmed service call scanned per-file (the same signal behind the summary below, localized to its source).{traffic_note} Drag a node, scroll to zoom, click to isolate its connections, or search below.

-
- -
-
- - - + sg = analysis.get('semantic_graph') or {} + sg_graph = sg.get('graph') or {'nodes': [], 'edges': []} + semantic_section = '' + if sg_graph.get('nodes'): + sg_nodes_json = json.dumps(sg_graph['nodes']) + sg_edges_json = json.dumps(sg_graph.get('edges', [])) + sg_members = sg.get('node_members') or {} + sg_members_json = json.dumps({k: sorted(v) if isinstance(v, set) else list(v) for k, v in sg_members.items()}) + sg_stats = sg.get('stats') or {} + stat_bits = ', '.join(f"{v} {k}" for k, v in sg_stats.items() if v) + top_integrations = sg.get('topIntegrations') or [] + integrations_card = '' + if top_integrations: + integ_items = ''.join( + f'
{i.get("label", i.get("id", ""))}
' + for i in top_integrations[:10] + ) + integrations_card = ( + f'
Integrations
{integ_items}
' + ) + semantic_section = f""" +
+
Architecture Map (components) — {len(sg_graph['nodes'])} components · {len(sg_graph.get('edges', []))} flows
+

A curated view of how the system works: entry points, services, datastores, and external integrations — grouped by feature, not by file layout.{' Detected: ' + stat_bits + '.' if stat_bits else ''} Click any node for details; use Show import details to jump to the exact files behind it in the Codebase section below.

+
+
+ +{integrations_card} +
+
+ +
+ +
+""" + + empty_graph = {'nodes': [], 'links': [], 'meta': {'files': 0, 'links': 0, 'aggregated': False, 'shown_nodes': 0, 'has_access_log': False}} + + def codebase_section(g: dict, P: str, section_id: str, title: str, intro: str) -> str: + gm = g['meta'] + # A layered diagram gets too tall to read past ~80 nodes; start big + # (or directory-aggregated) graphs on the force view instead. + default_view = 'force' if gm['aggregated'] or gm['shown_nodes'] > _FLOW_MAX_NODES else 'flow' + flow_anim_note = ('' if gm['links'] <= _FLOW_IDLE_ANIM_EDGES else + f' With {gm["links"]} edges, animating everything at once would be noise, so only the hovered or clicked node\'s paths move.') + return f""" +
+
{title}
+

{intro}

+

Flow view: files are laid out left → right by dependency depth, starting from the sources (the HTTP entry and files nothing imports); every detected service is its own node in the last column, and dashed edges close an import cycle. Moving dots show data going both ways: a filled dot is a call or request going out, a hollow dot the return or response coming back.{flow_anim_note} Hover a node to see its direct connections; click it to trace everything upstream and downstream.

+ +
+
+ + + + +
+
+
+
+entry → route file +import +service call (pattern-confirmed) +import cycle +call / request → +← return / response +click a node to trace everything upstream and downstream of it +
+
+ - +
""" + def graph_note(gm: dict, unit: str) -> str: + return (f"{gm['files']} {unit} aggregated into {gm['shown_nodes']} directories" if gm['aggregated'] + else f"{gm['shown_nodes']} modules") + + empty_graph = {'nodes': [], 'links': [], 'meta': {'files': 0, 'links': 0, 'aggregated': False, 'shown_nodes': 0, 'has_access_log': False}} + graph = analysis.get('graph') or empty_graph + gm = graph['meta'] + graph_section = '' + if graph['nodes']: + access_log = analysis.get('access_log') + traffic_note = ( + f" Amber particles on entry edges are real traffic, weighted by request count from {access_log['source']} ({access_log['total']:,} requests parsed)." + if gm.get('has_access_log') and access_log else + " This is static analysis, not live monitoring: the moving dots show direction, not measured traffic. Pass --access-log <path> to overlay real per-route request counts." + ) + graph_section = codebase_section( + graph, 'graph-', 'section-graph', + f"Codebase (files) — {graph_note(gm, 'files')} · {gm['links']} real connections", + f"Every node is a real source file{' or directory' if gm['aggregated'] else ''} in this repo (tests have their own section below); every line is an actual import statement, or a pattern-confirmed service call scanned per-file.{traffic_note}", + ) + + test_graph = analysis.get('test_graph') or empty_graph + tgm = test_graph['meta'] + test_graph_section = '' + if test_graph['nodes']: + test_graph_section = codebase_section( + test_graph, 'test-graph-', 'section-test-graph', + f"Tests (files) — {tgm['files']} test files · {graph_note(tgm, 'test files')} · {tgm['links']} connections", + "Test files and the source modules and services they import or call, kept out of the Codebase section above so fixtures and test doubles don't drown out the real dependency structure.", + ) + ext_count = len(analysis['external_sources']) storage_count = len(analysis['storage']) component_count = (1 + ext_count + storage_count + @@ -2321,15 +4079,19 @@ def stat(val: str, color: str, label: str, sub: str = '') -> str:

Component communication map · concurrent request capacity · bottleneck analysis · {today}

{stats}
-

Capacity figures are static-analysis estimates (heuristic: ~100 concurrent tasks per async worker), not load-test results.

+

Capacity figures are static-analysis estimates (heuristic: ~100 concurrent tasks per async worker), not load-test results.

{derivation_html}
+{semantic_section} {graph_section} +{test_graph_section}
Full System Architecture — {component_count} Components
-

Detection is pattern-based static analysis. Dashed boxes are dependencies declared in your manifest but not matched to a usage pattern in source — may be wired through a custom abstraction. Solid boxes are pattern-confirmed in source. A component not shown here isn't confirmed absent from the project — only undetected.

+

Detection is pattern-based static analysis. Dashed boxes are dependencies declared in your manifest but not matched to a usage pattern in source — may be wired through a custom abstraction. Solid boxes are pattern-confirmed in source. A component not shown here isn't confirmed absent from the project — only undetected.

+
Show the layered component diagram (overlaps the Architecture Map above)
{arch_html}
-{f'

{len(edges)} edge{"s" if len(edges) != 1 else ""} shown — drawn only where a data-flow step names two rendered components consecutively; not a claim about every real connection.

' if edges else ''} +{f'

{len(edges)} edge{"s" if len(edges) != 1 else ""} shown — drawn only where a data-flow step names two rendered components consecutively; not a claim about every real connection.

' if edges else ''} {route_html} +
@@ -2350,7 +4112,9 @@ def stat(val: str, color: str, label: str, sub: str = '') -> str:
{project_name} · System Workflow · {component_count} components · {today} · workflow-generator
- + + """ @@ -2390,6 +4154,10 @@ def stat(val: str, color: str, label: str, sub: str = '') -> str: 'Stripe': 'stripe', 'Salesforce': 'salesforce', 'Twilio': 'twilio', } _GRAPH_ENTRY_ID = 'entry:http' +_TEST_DIRS = frozenset({'tests', 'test', '__tests__', 'spec', 'specs', 'e2e'}) +_TEST_FILE_RE = re.compile( + r'^test_.*\.py$|_test\.(?:py|go|rb)$|^conftest\.py$|\.(?:test|spec)\.[jt]sx?$|_spec\.rb$|Tests?\.java$' +) def _py_module_name(root: Path, p: Path) -> str: @@ -2468,12 +4236,13 @@ def _resolve_js_spec(spec: str, from_file: Path, js_paths: set, alias_map: list[ return None -def build_code_graph(root: Path, analysis: dict, detail: str = 'auto') -> dict: +def build_code_graphs(root: Path, analysis: dict, detail: str = 'auto') -> tuple[dict, dict]: """Full module-dependency graph: every source file is a node, every real import is an edge, every file matching a detected component's own _RE pattern gets an edge to that component. Best-effort only — anything that can't be confidently resolved (dynamic imports, unusual layouts, Go - without a parseable go.mod) is silently skipped rather than guessed.""" + without a parseable go.mod) is silently skipped rather than guessed. + Returns (main, tests): the same scan split into source and test graphs.""" files = _collect_graph_files(root) total_files = len(files) file_ids = {p: p.relative_to(root).as_posix() for p in files} @@ -2691,7 +4460,7 @@ def add_edge(s, t, kind, w=None): # --- Service edges: same _RE pattern that already backs the summary, # scanned per-file so the graph shows *which* files actually call it. --- - def add_service_group(items, name_key, re_map, icon_fn): + def add_service_group(items, name_key, re_map, icon_fn, cat): for item in items: name = item[name_key] if isinstance(item, dict) else item re_key = re_map.get(name) @@ -2708,14 +4477,14 @@ def add_service_group(items, name_key, re_map, icon_fn): nodes[sid] = { 'id': sid, 'label': name, 'pkg': '(service)', 'loc': 0, 'kind': 'service', 'color': C.get(color_key, C['gray']), - 'icon': icon_fn(item), + 'icon': icon_fn(item), 'cat': cat, } - add_service_group(analysis['llm'].get('providers', []), None, _LLM_RE_KEYS, lambda i: '🧠') - add_service_group(analysis['storage'], 'name', _STORAGE_RE_KEYS, lambda i: _STORE_ICONS.get(i['type'], '📦')) - add_service_group(analysis['queues'], 'name', _QUEUE_RE_KEYS, lambda i: '📨') + add_service_group(analysis['llm'].get('providers', []), None, _LLM_RE_KEYS, lambda i: '🧠', 'LLM') + add_service_group(analysis['storage'], 'name', _STORAGE_RE_KEYS, lambda i: _STORE_ICONS.get(i['type'], '📦'), 'store') + add_service_group(analysis['queues'], 'name', _QUEUE_RE_KEYS, lambda i: '📨', 'queue') add_service_group([s for s in analysis['external_sources'] if s['name'] in _EXTSRC_RE_KEYS], - 'name', _EXTSRC_RE_KEYS, lambda i: '🌐') + 'name', _EXTSRC_RE_KEYS, lambda i: '🌐', 'external') # --- Entry node: HTTP clients -> every file with a route decorator. # When an --access-log was supplied, weight each edge with the real @@ -2743,6 +4512,41 @@ def add_service_group(items, name_key, re_map, icon_fn): 'loc': 0, 'kind': 'entry', 'color': C['green'], 'icon': '🌐', } + test_ids = {fid for fid in file_ids.values() if _is_test_path(fid)} + main_nodes, main_edges = _split_test_graph(nodes, edges, test_ids, 'main') + test_nodes, test_edges = _split_test_graph(nodes, edges, test_ids, 'tests') + main = _finalize_graph(main_nodes, main_edges, total_files - len(test_ids), detail, access_log) + tests = _finalize_graph(test_nodes, test_edges, len(test_ids), detail, access_log) + return main, tests + + +def build_code_graph(root: Path, analysis: dict, detail: str = 'auto') -> dict: + """Source-only module graph (test files are split out — see build_code_graphs).""" + return build_code_graphs(root, analysis, detail)[0] + + +def _is_test_path(rel: str) -> bool: + parts = Path(rel).parts + return any(d in _TEST_DIRS for d in parts[:-1]) or bool(_TEST_FILE_RE.search(parts[-1])) + + +def _split_test_graph(nodes: dict, edges: list, test_ids: set, scope: str) -> tuple[dict, list]: + """Tests otherwise swamp the main graph: fixtures and detector tests name + every service they exercise, so each one draws a fan of service edges. + 'main' drops test files (and any service/entry node only a test reached); + 'tests' keeps test files plus whatever they import or call.""" + if scope == 'tests': + edges = [e for e in edges if e['s'] in test_ids] + keep = test_ids | {e['t'] for e in edges} + else: + edges = [e for e in edges if e['s'] not in test_ids and e['t'] not in test_ids] + linked = {e['s'] for e in edges} | {e['t'] for e in edges} + keep = {nid for nid, n in nodes.items() + if nid not in test_ids and (n['kind'] == 'file' or nid in linked)} + return {nid: n for nid, n in nodes.items() if nid in keep}, edges + + +def _finalize_graph(nodes: dict, edges: list, total_files: int, detail: str, access_log) -> dict: file_node_count = sum(1 for n in nodes.values() if n['kind'] == 'file') should_aggregate = (detail == 'dirs') or (detail == 'auto' and file_node_count > _GRAPH_FILE_CAP) aggregated = False @@ -2808,7 +4612,8 @@ def dir_id(fid: str) -> str: # ── collect + main ───────────────────────────────────────────────────────────── -def collect(root: Path, graph_detail: str = 'auto', access_log_path: str | None = None) -> dict: +def collect(root: Path, graph_detail: str = 'auto', access_log_path: str | None = None, + enrich_labels: bool = False) -> dict: # Resolve once, up front: the graph builder compares Path objects for set # membership (js_paths, byId), and an unresolved relative root (the default — # `analyze.py .` is the most common invocation) makes every file path relative @@ -2824,6 +4629,11 @@ def collect(root: Path, graph_detail: str = 'auto', access_log_path: str | None storage = detect_storage(root) queues = detect_queues(root) external_sources = detect_external_sources(root) + # New detectors for semantic graph + crons = detect_crons(root) + tools = detect_tools(root) + agents = detect_agents(root) + composition_root = detect_composition_root(root) capacity = compute_concurrency(workers, gateway, concur, llm, api_server) flows = infer_flows({ 'external_sources': external_sources, @@ -2837,10 +4647,26 @@ def collect(root: Path, graph_detail: str = 'auto', access_log_path: str | None 'workers': workers, }) access_log = parse_access_log(access_log_path, api_server.get('routes', [])) if access_log_path else None - graph = build_code_graph(root, { + graph, test_graph = build_code_graphs(root, { 'llm': llm, 'storage': storage, 'queues': queues, 'external_sources': external_sources, 'framework': api_server.get('framework'), 'access_log': access_log, }, detail=graph_detail) + # Build semantic graph for architecture map + semantic_graph = build_semantic_graph(root, { + 'project_name': detect_project_name(root), + 'api_server': api_server, + 'storage': storage, + 'external_sources': external_sources, + 'crons': crons, + 'tools': tools, + 'agents': agents, + 'composition_root': composition_root, + 'gateway': gateway, + 'frontend': frontend, + 'llm': llm, + }) + if enrich_labels: + semantic_graph = enrich_service_labels(root, semantic_graph) return { 'workers': workers, 'gateway': gateway, @@ -2851,9 +4677,15 @@ def collect(root: Path, graph_detail: str = 'auto', access_log_path: str | None 'storage': storage, 'queues': queues, 'external_sources': external_sources, + 'crons': crons, + 'tools': tools, + 'agents': agents, + 'composition_root': composition_root, 'capacity': capacity, 'flows': flows, 'graph': graph, + 'test_graph': test_graph, + 'semantic_graph': semantic_graph, 'access_log': access_log, } @@ -2864,6 +4696,7 @@ def main(): argv = sys.argv[1:] graph_detail = 'auto' access_log_path = None + enrich_labels = False positional = [] i = 0 while i < len(argv): @@ -2876,6 +4709,8 @@ def main(): access_log_path = argv[i + 1]; i += 2; continue if a.startswith('--access-log='): access_log_path = a.split('=', 1)[1]; i += 1; continue + if a == '--enrich-labels': + enrich_labels = True; i += 1; continue positional.append(a); i += 1 if graph_detail not in ('auto', 'files', 'dirs'): @@ -2886,7 +4721,8 @@ def main(): output = Path(positional[1]) if len(positional) > 1 else root / 'WORKFLOW.html' project_name = detect_project_name(root) or root.resolve().name.replace('-', ' ').replace('_', ' ').title() - analysis = collect(root, graph_detail=graph_detail, access_log_path=access_log_path) + analysis = collect(root, graph_detail=graph_detail, access_log_path=access_log_path, + enrich_labels=enrich_labels) output.write_text(render_html(analysis, project_name)) diff --git a/workflow_generator_mcp/server.py b/workflow_generator_mcp/server.py index 50f2bff..3ab49df 100644 --- a/workflow_generator_mcp/server.py +++ b/workflow_generator_mcp/server.py @@ -58,6 +58,16 @@ async def handle_list_tools() -> list[types.Tool]: "description": "Open the generated file in the default browser.", "default": True, }, + "enrich_labels": { + "type": "boolean", + "description": ( + "Use the local `claude` CLI (if installed and on PATH) to " + "replace mechanical service-node labels with ones derived " + "from actually reading the code. No-op if `claude` isn't " + "found." + ), + "default": False, + }, }, "required": [], }, @@ -92,7 +102,11 @@ async def handle_call_tool(name: str, arguments: dict) -> list[types.TextContent return [types.TextContent(type="text", text=f"ERROR: Directory not found: {project_dir}")] project_name = _analyze.detect_project_name(project_dir) or project_dir.name.replace("-", " ").replace("_", " ").title() - analysis = _analyze.collect(project_dir) + # enrich_labels only affects generate_workflow's rendered HTML (analyze_workflow's + # trimmed JSON doesn't include semantic-graph labels at all), but it's harmless to + # thread through here unconditionally — analyze_workflow's schema doesn't declare + # the param, so well-behaved clients never set it for that tool. + analysis = _analyze.collect(project_dir, enrich_labels=bool(arguments.get("enrich_labels", False))) cap = analysis["capacity"] if name == "analyze_workflow":