diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 35c551c..3583d68 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "langfuse-observability", "description": "The Langfuse x Claude Code Observability Plugin", - "version": "1.0.0", + "version": "1.2.1", "author": { "name": "Langfuse" }, @@ -13,20 +13,18 @@ "type": "string", "title": "Langfuse Secret Key", "description": "Your Langfuse project secret key (sk-lf-...)", - "sensitive": true, - "required": true + "sensitive": true }, "LANGFUSE_PUBLIC_KEY": { "type": "string", "title": "Langfuse Public Key", - "description": "Your Langfuse project public key (pk-lf-...)", - "required": true + "description": "Your Langfuse project public key (pk-lf-...)" }, "LANGFUSE_BASE_URL": { "type": "string", "title": "Langfuse Base URL", "description": "Langfuse host. Use https://us.cloud.langfuse.com for US, https://cloud.langfuse.com for EU, or your self-hosted URL.", - "default": "https://us.cloud.langfuse.com" + "default": "https://cloud.langfuse.com" }, "LANGFUSE_USER_ID": { "type": "string", @@ -57,6 +55,17 @@ "description": "Include the injected skill instruction text in the Skill tool span output", "default": false }, + "CC_LANGFUSE_CAPTURE_IMAGES": { + "type": "boolean", + "title": "Capture images", + "description": "Upload images to Langfuse and show them in the trace. This includes pasted images and screenshots from tool results. Your Langfuse deployment must support media upload. Set to false to show short text markers instead.", + "default": true + }, + "CC_LANGFUSE_STATE_DIR": { + "type": "string", + "title": "State directory", + "description": "Optional. Absolute directory for the hook's state, lock and log files (default ~/.claude/state). Set a distinct value per CLAUDE_CONFIG_DIR installation to keep their state separate. Leave empty for the default." + }, "CC_LANGFUSE_TRACE_SEED": { "type": "string", "title": "Trace ID seed", diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..27f0db1 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,71 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: CI + +on: + pull_request: + push: + branches: + - main + +permissions: {} + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + test: + name: Test on Python ${{ matrix.python-version }} + runs-on: ubuntu-latest + permissions: + contents: read + strategy: + fail-fast: false + matrix: + python-version: ["3.10", "3.13"] + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Test + run: uv run --python ${{ matrix.python-version }} --group dev pytest + + lint: + name: Lint + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Ruff + run: uvx ruff check --select F,E9 . + + hook-smoke: + name: Hook starts from its own script header + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + with: + enable-cache: true + - name: Run the hook with an empty payload + run: echo '{}' | uv run --script hooks/langfuse_hook.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..8770363 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,44 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: Release + +on: + push: + tags: + - "v*" + +permissions: {} + +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: false + +jobs: + release: + name: Verify the tag and draft a GitHub release + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Validate release version + run: | + manifest_version="$(python3 -c 'import json; print(json.load(open(".claude-plugin/plugin.json"))["version"])')" + if [[ "$GITHUB_REF_NAME" != "v$manifest_version" ]]; then + echo "Tag $GITHUB_REF_NAME does not match plugin.json version $manifest_version" >&2 + exit 1 + fi + - name: Setup uv + uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 + - name: Test + run: uv run --group dev pytest + - name: Run the hook with an empty payload + run: echo '{}' | uv run --script hooks/langfuse_hook.py + # Draft, so the notes get a human pass before anyone sees them. + - name: Create draft GitHub release + run: gh release create "$GITHUB_REF_NAME" --draft --generate-notes --verify-tag + env: + GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml new file mode 100644 index 0000000..1f3aad2 --- /dev/null +++ b/.github/workflows/zizmor.yml @@ -0,0 +1,36 @@ +--- +# yaml-language-server: $schema=https://json.schemastore.org/github-workflow.json +name: Check GitHub Actions + +on: + workflow_dispatch: + push: + branches: + - "main" + merge_group: + pull_request: + branches: + - "main" + +permissions: {} + +jobs: + zizmor: + name: Check GitHub Actions security + runs-on: ubuntu-latest + permissions: + security-events: write + contents: read + actions: read + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Run zizmor + uses: zizmorcore/zizmor-action@70fb788f84895a7701f5643d103d587e460b5c99 # v0.6.3 + with: + # Findings upload to code scanning only on push; a pull request from a + # fork has no write token for it. + advanced-security: ${{ github.event_name == 'push' && 'true' || 'false' }} + min-severity: low diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..90a25fa --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,64 @@ +# Development and Maintenance + +See the [main CONTRIBUTING.md](https://github.com/langfuse/langfuse/blob/main/CONTRIBUTING.md) to learn how to contribute to Langfuse and its integrations. + +## Local Development + +The hook is a single uv script with inline dependency metadata, and Claude Code +runs `hooks/langfuse_hook.py` directly. There is no build step and no package to +install. + +Run the tests: + +```bash +uv run --group dev pytest +``` + +Lint with the same rules as CI: + +```bash +uvx ruff check --select F,E9 . +``` + +CI also starts the hook once with an empty payload, which catches a broken +script header before it reaches a user: + +```bash +echo '{}' | uv run --script hooks/langfuse_hook.py +``` + +Tests run against Python 3.10 and 3.13 in CI. Add `--python 3.10` to `uv run` to +reproduce the older matrix entry. + +To try your checkout as a real plugin, register the repository itself as a +marketplace instead of the GitHub one: + +```bash +claude plugin marketplace add /path/to/Claude-Observability-Plugin +claude plugin install langfuse-observability@langfuse-observability +``` + +Restart Claude Code afterwards, so that it loads the hook. Run +`claude plugin marketplace remove langfuse-observability` when you are done, and +add the GitHub marketplace again to go back to the released plugin. + +## Releasing + +1. From a clean, up-to-date `main` branch, raise `version` in + `.claude-plugin/plugin.json` and commit that change. + +2. Tag the release commit and push both: + + ```bash + git tag v1.2.3 + git push origin main --follow-tags + ``` + + The release workflow refuses a tag that does not match the manifest version. + It then runs the tests and the hook smoke check, and creates a draft GitHub + release with generated notes. + +3. Review the draft release, give the notes a human pass, and publish it. + +Installed plugins do not update themselves, so users reach the new version only +after `claude plugin marketplace update` and `claude plugin update`. diff --git a/README.md b/README.md index 278121b..04e0b6a 100644 --- a/README.md +++ b/README.md @@ -1,250 +1,91 @@ # Langfuse Observability Plugin for Claude Code -Trace hook-backed Claude Code sessions to [Langfuse](https://langfuse.com) — turns, generations, tool calls, and token usage — with zero code changes. +Claude Code plugin that sends [Claude Code](https://claude.com/claude-code) +session telemetry to [Langfuse](https://langfuse.com). It traces user prompts, +agent turns, model generations with their token usage and cost, thinking blocks, +tool calls, skills, subagents, and pasted or returned images. -## Install +Langfuse also documents this integration on the +[Claude Code integration page](https://langfuse.com/integrations/developer-tools/claude-code). -```bash -claude plugin marketplace add langfuse/Claude-Observability-Plugin -claude plugin install langfuse-observability@langfuse-observability -``` - -The marketplace command registers the plugin marketplace and refreshes its local cache. The install command enables the plugin for your Claude Code user scope. - -Restart Claude Code after install so the hook configuration is loaded. - -Then configure the plugin from within a Claude Code session. This is a Claude Code slash command, not a shell command. Run `claude` first, then type it at the session prompt: - -```text -$ claude -> /plugin configure langfuse-observability@langfuse-observability -``` +## Quick Start -Alternatively, pass configuration values during install: +Add the marketplace and install the plugin: ```bash -claude plugin install langfuse-observability@langfuse-observability \ - --config LANGFUSE_PUBLIC_KEY=pk-lf-... \ - --config LANGFUSE_SECRET_KEY=sk-lf-... \ - --config LANGFUSE_BASE_URL=https://cloud.langfuse.com -``` - -The plugin requires or accepts: - -| Field | Description | -| --------------------- | ---------------------------------------------------------------------------------------------------- | -| `LANGFUSE_SECRET_KEY` | Your Langfuse secret key (`sk-lf-...`). Stored in your OS keychain. | -| `LANGFUSE_PUBLIC_KEY` | Your Langfuse public key (`pk-lf-...`). | -| `LANGFUSE_BASE_URL` | https://us.cloud.langfuse.com (default), https://cloud.langfuse.com for EU, or your self-hosted URL. | -| `LANGFUSE_USER_ID` | Optional. User identifier attached to every trace (shown as the user in Langfuse). | -| `CC_LANGFUSE_DEBUG` | Verbose logging to `~/.claude/state/langfuse_hook.log`. | -| `CC_LANGFUSE_MAX_CHARS` | Truncate captured inputs/outputs to this many characters (default 20000). | -| `CC_LANGFUSE_SKILL_TAGS` | Tag traces with `skill:` for every skill invoked in the turn (default true). | -| `CC_LANGFUSE_CAPTURE_SKILL_CONTENT` | Include injected skill instruction text in the Skill tool span output (default false). | -| `CC_LANGFUSE_TRACE_SEED` | Optional. Opt-in seed for deterministic trace IDs — see [Deterministic trace IDs](#deterministic-trace-ids). | -| `CC_LANGFUSE_TRACEPARENT` | Optional, per run (env var, not plugin config). W3C traceparent of an existing trace to attach to — see [Attach runs to an existing trace](#attach-runs-to-an-existing-trace). | -| `CC_LANGFUSE_PARENT_TRACE_ID` / `CC_LANGFUSE_PARENT_SPAN_ID` | Optional, per run. Explicit alternative to `CC_LANGFUSE_TRACEPARENT` (32-hex trace id + 16-hex span id). | - -Get keys from your Langfuse project settings → API Keys. - -## Requirements - -One of: - -- [uv](https://docs.astral.sh/uv/) (recommended) on `PATH`. The hook uses `uv run --script` and installs the Langfuse SDK from the script metadata automatically. -- Python 3.10+ as `python3` with `langfuse>=4.0,<5` installed in that Python environment. This is only used as a fallback when `uv` is not on `PATH`. - -On the first hook run, uv downloads the Langfuse SDK from PyPI (declared in the script header) and caches it. Offline or proxied machines fail this download and retry on every turn until the cache is warm; you can pre-warm it from a networked terminal: - -```bash -echo '{}' | uv run --quiet --script /hooks/langfuse_hook.py -``` - -If no usable runtime is set up, the hook exits without tracing, and Claude Code is never blocked or slowed. Failure reasons are written to `~/.claude/state/langfuse_hook.log` (see Troubleshooting). - -## How it works - -A hook reads the session transcript incrementally on every turn (Stop) and at session end (SessionEnd), and emits a Langfuse trace with one span per turn, nested generations per assistant message, and child tool spans for every tool call. Token usage is captured when present. - -The plugin observes only data that Claude Code exposes through hooks and transcript files. - -Captured: - -- Claude Code CLI sessions -- Claude Code GUI Code mode sessions - -Not captured: - -- Regular Claude Desktop Chat mode conversations - -If you use the desktop app, switch to Code mode for hook-backed tracing. Regular Chat mode does not invoke Claude Code Stop or SessionEnd hooks and does not write the transcript file this plugin reads. - -State is kept in `~/.claude/state/langfuse_state.json` so re-runs only emit new turns. - -## Deterministic trace IDs - -By default, trace IDs are auto-generated, so external systems (CI harnesses, benchmark -runners, dataset-experiment services) that run Claude Code headlessly have to poll the -Langfuse API to discover the trace ID of a run. Setting `CC_LANGFUSE_TRACE_SEED` makes -trace IDs predictable: any system that knows the seed can compute the trace ID of each -turn **before** the trace exists. - -With the seed set, the trace ID for turn `N` (1-based, counted per session across hook -runs) is derived as: - -```python -from langfuse import Langfuse - -trace_id = Langfuse.create_trace_id(seed=f"{CC_LANGFUSE_TRACE_SEED}:{turn_number}") -# equivalent to: hashlib.sha256(f"{seed}:{turn_number}".encode()).hexdigest()[:32] -``` - -Use a **unique seed per session** — otherwise two sessions collide on the same trace -IDs. The recommended seed is the Claude session UUID you already pass via -`claude -p --session-id`. - -Example: a harness runs a single-turn headless prompt and links the resulting trace to -a Langfuse dataset run item without ever fetching the trace: - -```python -import subprocess, uuid, os -from langfuse import Langfuse - -langfuse = Langfuse() -session_id = str(uuid.uuid4()) - -# Precompute the trace ID for turn 1 — no polling needed. -trace_id = Langfuse.create_trace_id(seed=f"{session_id}:1") - -# Link it to a dataset run item BEFORE the run. -langfuse.api.dataset_run_items.create( - dataset_item_id=item.id, - run_name="claude-code-benchmark-v1", - trace_id=trace_id, -) - -subprocess.run( - ["claude", "-p", "Refactor utils.py", "--session-id", session_id], - env={**os.environ, "CC_LANGFUSE_TRACE_SEED": session_id}, -) -# When the hook uploads the turn, its trace appears in Langfuse under trace_id. -``` - -If derivation or wiring fails for any reason, the hook falls back to an auto-generated -trace ID (fail-open, like the rest of the hook). Unset or empty seed = unchanged behavior. - -## Attach runs to an existing trace - -When your application launches Claude Code headlessly (`claude -p`) as one step of a -larger, already-instrumented workflow, the agent run can join your application's -existing Langfuse trace instead of creating separate root traces. Create a span for -the agent run in your application (this becomes the run-level node), then pass its -trace context as environment variables when launching Claude Code: - -```python -from langfuse import get_client - -langfuse = get_client() - -with langfuse.start_as_current_span(name="Claude Code run") as run_span: - traceparent = f"00-{run_span.trace_id}-{run_span.id}-01" - subprocess.run( - ["claude", "-p", "Refactor utils.py"], - env={**os.environ, "CC_LANGFUSE_TRACEPARENT": traceparent}, - ) +claude plugin marketplace add langfuse/Claude-Observability-Plugin +claude plugin install langfuse-observability@langfuse-observability ``` -Every conversational turn of the session (with its LLM calls, tool calls, and -subagents) then appears as a child of your `Claude Code run` span, in your trace. -Instead of `CC_LANGFUSE_TRACEPARENT` you can pass `CC_LANGFUSE_PARENT_TRACE_ID` and -`CC_LANGFUSE_PARENT_SPAN_ID` explicitly. +Restart Claude Code after installing, so that it loads the hook. +Tracing covers the `claude` CLI and the desktop app in **Code** mode. Claude +Desktop **Chat** mode runs no Claude Code hooks and is not traced. -## Privacy +## Prerequisites -This plugin transmits your Claude Code session data — conversation turns, assistant -generations, tool calls, and token-usage statistics — to the Langfuse endpoint you -configure (`LANGFUSE_BASE_URL`, default `https://us.cloud.langfuse.com`; EU and -self-hosted endpoints are supported). Data is sent at the end of each turn (the -`Stop` hook) and at session end (`SessionEnd`) using the Langfuse API keys you -provide, which are stored in your OS keychain. No data is sent anywhere other than -the endpoint you configure. +The plugin needs [uv](https://docs.astral.sh/uv/) on `PATH`. The hook runs as a +uv script and installs the Langfuse SDK from its own inline script metadata. +Machines without uv fall back to Python 3.10+ as `python3` with +`langfuse>=4.7,<5` installed. -For how Langfuse Cloud handles data it receives, see the Langfuse privacy policy: -https://langfuse.com/privacy . When using a self-hosted Langfuse instance, your data -stays within your own infrastructure. +Without a usable runtime the hook writes the reason to its log and exits. It +never blocks or slows Claude Code. -## Reconfigure +## Langfuse Credentials -In Claude Code, run: +Configure the plugin from inside a Claude Code session. This is a Claude Code +slash command, not a shell command: ```text /plugin configure langfuse-observability@langfuse-observability ``` -## Disable tracing - -To stop tracing for new Claude Code CLI sessions and new Claude Code GUI Code mode sessions, disable the plugin: +You can also pass the values during install: ```bash -claude plugin disable langfuse-observability@langfuse-observability --scope user -claude plugin list +claude plugin install langfuse-observability@langfuse-observability \ + --config LANGFUSE_PUBLIC_KEY=pk-lf-... \ + --config LANGFUSE_SECRET_KEY=sk-lf-... \ + --config LANGFUSE_BASE_URL=https://cloud.langfuse.com ``` -`claude plugin list` should show the plugin as disabled. This keeps the plugin installed and does not delete your configuration. +Only `LANGFUSE_PUBLIC_KEY` and `LANGFUSE_SECRET_KEY` are required. Without +`LANGFUSE_BASE_URL` the plugin uses `https://cloud.langfuse.com` (EU region). +Get the keys from the API Keys page in your Langfuse project settings. The +secret key is held in your OS keychain, not in a file. -To enable tracing again: - -```bash -claude plugin enable langfuse-observability@langfuse-observability --scope user -``` +`/plugin configure` also lists the optional settings, such as the user ID, +debug logging, and image capture, with their defaults. -If a Claude Code session is already running, restart it after disabling the plugin. Running sessions may have loaded their hooks at startup, so disabling the plugin is intended for new sessions. +## Updating -## Uninstall +Automatic updates are off for marketplaces outside Anthropic's, so you run the +update yourself. ```bash -claude plugin uninstall langfuse-observability +claude plugin marketplace update langfuse-observability +claude plugin update langfuse-observability@langfuse-observability ``` -## Troubleshooting - -Nearly every failure explains itself in `~/.claude/state/langfuse_hook.log`. Send one message, then match the newest lines against this table: - -| What the log shows | Meaning | What to do | -| --- | --- | --- | -| No new lines at all | The hook never launched. Either the plugin is disabled, no usable runtime was found (no `uv`, no suitable `python3`), or uv could not download the SDK (offline/proxy) | Run `claude plugin list` **from the directory you use in the app**, since enable/disable can be project-scoped. Install uv and make sure it is on the PATH of the app that launches Claude Code. Pre-warm the SDK download (see Requirements) | -| `langfuse import failed (…) python=… PATH=…` | The Python that ran the hook cannot import the SDK; the line names the interpreter, version, and PATH | Install uv and make sure it is on the PATH the hook inherits (the log line shows that PATH), or make sure `python3` is 3.10+ with `langfuse>=4.0,<5` installed | -| `Langfuse config incomplete: missing …` | The named key(s) did not reach the hook. Either not configured yet, or configured but not delivered | Configure via `/plugin configure`. If the line also says `loaded under plugin identity '@inline'`, see the desktop section below | -| `Hook started` plus a skip reason (e.g. `Transcript path does not exist`) | The hook ran and skipped intentionally; these often come from background/utility sessions | Usually harmless. File an issue with the log line if actual turns are missing | -| `Processed N turns …` but nothing in Langfuse | Turns were handed to the SDK but delivery failed afterwards; export errors go to stderr, not this log | Check `LANGFUSE_BASE_URL` (EU `https://cloud.langfuse.com` vs US `https://us.cloud.langfuse.com`), key validity, and network/proxy reachability | - -`Hook started` and other `[DEBUG]` lines require debug logging (`CC_LANGFUSE_DEBUG=true` via `/plugin configure`); the failure lines above are written at `[INFO]` and appear without it. Regular Claude Desktop **Chat** mode is not hook-backed. Tracing covers the `claude` CLI and desktop **Code** mode. +Restart Claude Code afterwards. `claude plugin list` shows the installed +version. -### Desktop app (GUI) sessions - -GUI apps do not read your shell profile. `export`ed variables in `~/.zshrc` never reach GUI-spawned hooks, and the app resolves its `PATH` once at launch. After installing uv, fully quit and relaunch the desktop app. Keys must be configured via `/plugin configure` (or `--config` at install), not via shell exports. - -### Plugin options not delivered in GUI sessions +## Troubleshooting -Recent Claude Desktop builds load user-installed plugins under a different plugin identity (`langfuse-observability@inline`), so keys configured via `/plugin configure` are not delivered to the hook. Terminal sessions work, desktop sessions stay silent. When this happens, the log line contains `loaded under plugin identity '@inline'`. +The hook explains nearly every failure in +`~/.claude/state/langfuse_hook.log`. Send one message, then read the newest +lines. Turn on `CC_LANGFUSE_DEBUG` for verbose logging. -As a **temporary workaround**, duplicate your full options under the inline identity in `~/.claude/settings.json`: +A log with no new lines means that the hook never started. Either the plugin is +disabled, or no usable runtime was found. Run `claude plugin list` from the +directory you work in, and put uv on the `PATH` of the app that starts Claude +Code. -```json -"pluginConfigs": { - "langfuse-observability@inline": { - "options": { - "LANGFUSE_PUBLIC_KEY": "pk-lf-...", - "LANGFUSE_SECRET_KEY": "sk-lf-...", - "LANGFUSE_BASE_URL": "https://cloud.langfuse.com" - } - } -} -``` +## Contributing -The entry must include `LANGFUSE_SECRET_KEY` because the OS-keychain secret only applies to the installed identity. This stores the secret in plaintext, so consider a dedicated API key pair for the workaround period and remove the entry once it is no longer needed. No restart is needed; settings are picked up on the next message. +See the [contributing guide](./CONTRIBUTING.md). ## License -MIT +[MIT](./LICENSE) diff --git a/hooks/langfuse_hook.py b/hooks/langfuse_hook.py index 9df7ba9..a69b9af 100644 --- a/hooks/langfuse_hook.py +++ b/hooks/langfuse_hook.py @@ -2,7 +2,7 @@ # /// script # requires-python = ">=3.10" # dependencies = [ -# "langfuse>=4.0,<5", +# "langfuse>=4.7,<5", # ] # /// """ @@ -10,6 +10,7 @@ """ +import base64 import contextlib import json import logging @@ -26,29 +27,132 @@ from typing import Any, Dict, List, Optional, Tuple -# ----------------- Paths ----------------- -STATE_DIR = Path.home() / ".claude" / "state" -LOG_FILE = STATE_DIR / "langfuse_hook.log" -STATE_FILE = STATE_DIR / "langfuse_state.json" -LOCK_FILE = STATE_DIR / "langfuse_state.lock" - - # ----------------- Configuration ----------------- def _opt(name: str) -> str: - """Read a plugin userConfig value (CLAUDE_PLUGIN_OPTION_) with a fallback to a plain env var.""" - return os.environ.get(f"CLAUDE_PLUGIN_OPTION_{name}") or os.environ.get(name) or "" + """Read a config value: plain env var first, plugin userConfig (CLAUDE_PLUGIN_OPTION_) as fallback. + + Claude Code stores plugin userConfig at machine scope only, while `env` blocks + are per-repo — so the repo-level env var must win, wizard as fallback. + """ + return os.environ.get(name) or os.environ.get(f"CLAUDE_PLUGIN_OPTION_{name}") or "" DEBUG = _opt("CC_LANGFUSE_DEBUG").lower() == "true" SKILL_TAGS = (_opt("CC_LANGFUSE_SKILL_TAGS") or "true").lower() == "true" AGENT_TYPE_TAGS = (_opt("CC_LANGFUSE_AGENT_TYPE_TAGS") or "true").lower() == "true" CAPTURE_SKILL_CONTENT = _opt("CC_LANGFUSE_CAPTURE_SKILL_CONTENT").lower() == "true" +CAPTURE_IMAGES = (_opt("CC_LANGFUSE_CAPTURE_IMAGES") or "true").lower() == "true" +OPERATOR_TAGS_VAR = "CC_LANGFUSE_TRACE_TAGS" try: MAX_CHARS = int(_opt("CC_LANGFUSE_MAX_CHARS") or "20000") except ValueError: MAX_CHARS = 20000 +MAX_OPERATOR_TAGS = 20 +MAX_OPERATOR_TAG_CHARS = 200 + # Bound for unresolved task notifications kept in the state file between runs. MAX_PENDING_TASK_NOTIFICATIONS = 50 +INTERRUPTED_TURN_MARKER = "[Request interrupted by user]" + +def parse_operator_tags(raw: str) -> Tuple[List[str], str]: + """Parse caller tags from a JSON array or a comma-separated list. + + Returns the tags and a warning naming what was dropped. Bad input yields no + tags and a warning, never an exception: telemetry stays best-effort. + """ + raw = (raw or "").strip() + if not raw: + return [], "" + notes: List[str] = [] + values: List[Any] + if raw.startswith("{"): + return [], f"{OPERATOR_TAGS_VAR}: ignored, JSON is not an array" + if raw.startswith("["): + try: + parsed = json.loads(raw) + except ValueError: + return [], f"{OPERATOR_TAGS_VAR}: ignored, not valid JSON" + if not isinstance(parsed, list): + return [], f"{OPERATOR_TAGS_VAR}: ignored, JSON is not an array" + values = parsed + else: + values = raw.split(",") + + tags: List[str] = [] + for value in values: + if not isinstance(value, (str, int, float, bool)): + notes.append("non-scalar entries") + continue + tag = str(value).strip() + if not tag: + continue + if len(tag) > MAX_OPERATOR_TAG_CHARS: + notes.append(f"entries over {MAX_OPERATOR_TAG_CHARS} characters") + continue + if tag in tags: + continue + tags.append(tag) + + if len(tags) > MAX_OPERATOR_TAGS: + notes.append(f"everything past the first {MAX_OPERATOR_TAGS}") + tags = tags[:MAX_OPERATOR_TAGS] + warning = f"{OPERATOR_TAGS_VAR}: dropped {', '.join(dict.fromkeys(notes))}" if notes else "" + return tags, warning + +OPERATOR_TAGS, _OPERATOR_TAGS_WARNING = parse_operator_tags(_opt(OPERATOR_TAGS_VAR)) + + +# ----------------- Paths ----------------- +def _resolve_state_dir() -> Tuple[Path, str]: + """Resolve the state directory, taking CC_LANGFUSE_STATE_DIR into account. + + Multi-installation setups (CLAUDE_CONFIG_DIR) point each installation at + its own directory so installations stop sharing one log, state file and + lock. The historical default is ~/.claude/state, which is used when the + override is unset or invalid. + + Returns (directory, warning). A non-empty warning means the override was + rejected and the default is in use. + """ + default = Path.home() / ".claude" / "state" + override = _opt("CC_LANGFUSE_STATE_DIR") + if not override: + return default, "" + try: + # expanduser raises RuntimeError for '~unknownuser/...' paths + candidate = Path(override).expanduser() + except Exception as e: + return default, ( + f"CC_LANGFUSE_STATE_DIR {override!r} is unusable ({type(e).__name__}: {e}); " + f"falling back to {default}" + ) + if not candidate.is_absolute(): + return default, ( + f"CC_LANGFUSE_STATE_DIR {override!r} is not an absolute path; " + f"falling back to {default}" + ) + try: + candidate.mkdir(parents=True, exist_ok=True) + except Exception as e: + return default, ( + f"CC_LANGFUSE_STATE_DIR {override!r} is unusable ({type(e).__name__}: {e}); " + f"falling back to {default}" + ) + # mkdir(exist_ok=True) passes for a pre-existing dir the user cannot write + # to (root-owned, read-only volume); accepting it would kill log, lock and + # state at once — the one failure mode with no channel left to report itself. + if not os.access(candidate, os.W_OK | os.X_OK): + return default, ( + f"CC_LANGFUSE_STATE_DIR {override!r} is unusable (directory exists but is not writable); " + f"falling back to {default}" + ) + return candidate, "" + +STATE_DIR, _STATE_DIR_WARNING = _resolve_state_dir() +LOG_FILE = STATE_DIR / "langfuse_hook.log" +STATE_FILE = STATE_DIR / "langfuse_state.json" +LOCK_FILE = STATE_DIR / "langfuse_state.lock" + # How long the final flush+shutdown may take before the hook gives up on it. try: @@ -130,11 +234,26 @@ def get_parent_trace_context_from_env() -> Tuple[Optional[str], Optional[str]]: return None, None return parent_trace_id, parent_span_id +def _core_opt(name: str) -> Tuple[str, str]: + """Resolve NAME/CC_NAME source-first — any env spelling beats any wizard + spelling — returning (value, source) for mixed-source detection.""" + for source, key in ( + ("env", name), + ("env", f"CC_{name}"), + ("wizard", f"CLAUDE_PLUGIN_OPTION_{name}"), + ("wizard", f"CLAUDE_PLUGIN_OPTION_CC_{name}"), + ): + value = os.environ.get(key) + if value: + return value, source + return "", "" + def get_langfuse_config() -> Optional[LangfuseConfig]: - public_key = _opt("LANGFUSE_PUBLIC_KEY") or _opt("CC_LANGFUSE_PUBLIC_KEY") - secret_key = _opt("LANGFUSE_SECRET_KEY") or _opt("CC_LANGFUSE_SECRET_KEY") - host = _opt("LANGFUSE_BASE_URL") or _opt("CC_LANGFUSE_BASE_URL") or "https://us.cloud.langfuse.com" - user_id = _opt("LANGFUSE_USER_ID") or _opt("CC_LANGFUSE_USER_ID") or None + public_key, public_key_source = _core_opt("LANGFUSE_PUBLIC_KEY") + secret_key, secret_key_source = _core_opt("LANGFUSE_SECRET_KEY") + host, host_source = _core_opt("LANGFUSE_BASE_URL") + host = host or "https://cloud.langfuse.com" + user_id = _core_opt("LANGFUSE_USER_ID")[0] or None trace_seed = _opt("CC_LANGFUSE_TRACE_SEED") or None parent_trace_id, parent_span_id = get_parent_trace_context_from_env() if parent_trace_id is not None and trace_seed is not None: @@ -146,6 +265,17 @@ def get_langfuse_config() -> Optional[LangfuseConfig]: if not public_key or not secret_key: return None + # Export auth failures are swallowed downstream (capped background flush), + # so this is the only place a torn key/host pair is still diagnosable. + sources = {s for s in (public_key_source, secret_key_source, host_source) if s} + if len(sources) > 1: + info( + "Langfuse config is mixed-source: public_key from " + f"{public_key_source}, secret_key from {secret_key_source}, " + f"base_url from {host_source or 'default'} — mismatched key/host " + "pairs fail with 401 and traces are dropped." + ) + return LangfuseConfig( public_key=public_key, secret_key=secret_key, @@ -158,9 +288,9 @@ def get_langfuse_config() -> Optional[LangfuseConfig]: def _missing_langfuse_keys() -> List[str]: missing = [] - if not (_opt("LANGFUSE_PUBLIC_KEY") or _opt("CC_LANGFUSE_PUBLIC_KEY")): + if not _core_opt("LANGFUSE_PUBLIC_KEY")[0]: missing.append("LANGFUSE_PUBLIC_KEY") - if not (_opt("LANGFUSE_SECRET_KEY") or _opt("CC_LANGFUSE_SECRET_KEY")): + if not _core_opt("LANGFUSE_SECRET_KEY")[0]: missing.append("LANGFUSE_SECRET_KEY") return missing @@ -245,7 +375,17 @@ def info(msg: str) -> None: ) sys.exit(0) +# If this import fails, image capture falls back to text markers. +try: + from langfuse.media import LangfuseMedia +except Exception: + LangfuseMedia = None + def create_langfuse_client(config: LangfuseConfig) -> Optional[Langfuse]: + # With capture off, stop the SDK from uploading base64 images it finds + # in span payloads on its own. The SDK reads an empty value as enabled. + if not CAPTURE_IMAGES and not os.environ.get("LANGFUSE_MEDIA_UPLOAD_ENABLED"): + os.environ["LANGFUSE_MEDIA_UPLOAD_ENABLED"] = "false" # Read by the BatchSpanProcessor at construction, so it must be set before Langfuse(). os.environ.setdefault("OTEL_BSP_MAX_QUEUE_SIZE", DEFAULT_OTEL_BSP_MAX_QUEUE_SIZE) try: @@ -327,6 +467,10 @@ def is_session_end_hook_payload(payload: Dict[str, Any]) -> bool: # ----------------- State file concurrency control ----------------- +def get_session_lock_path(key: str) -> Path: + return STATE_DIR / f"langfuse_state.{key[:16]}.lock" + + class FileLock: def __init__(self, path: Path, timeout_s: float = 2.0): self.path = path @@ -452,6 +596,23 @@ def update_session_state(global_state: Dict[str, Any], key: str, session_state: "updated": datetime.now(timezone.utc).isoformat(), } +def sweep_stale_session_locks(state: Dict[str, Any], cutoff: datetime) -> None: + live = {k[:16] for k in state if isinstance(k, str)} + try: + locks = list(STATE_DIR.glob("langfuse_state.*.lock")) + except OSError: + return + for lock in locks: + stem = lock.name[len("langfuse_state."):-len(".lock")] + if len(stem) != 16 or stem in live: + continue + try: + if lock.stat().st_mtime < cutoff.timestamp(): + lock.unlink() + except OSError: + pass + + def save_hook_state(state: Dict[str, Any]) -> None: try: # Drop session entries older than 30 days to keep the file bounded. @@ -469,6 +630,7 @@ def save_hook_state(state: Dict[str, Any]) -> None: continue if ts < cutoff: del state[k] + sweep_stale_session_locks(state, cutoff) STATE_DIR.mkdir(parents=True, exist_ok=True) tmp = STATE_FILE.with_suffix(".tmp") tmp.write_text(json.dumps(state, indent=2, sort_keys=True), encoding="utf-8") @@ -518,6 +680,31 @@ def get_model(row: Dict[str, Any]) -> str: return m.get("model") or "claude" return "claude" +CACHE_WRITE_TTL_KEYS = ( + ("ephemeral_5m_input_tokens", "input_cache_creation_5m"), + ("ephemeral_1h_input_tokens", "input_cache_creation_1h"), +) + +def get_cache_write_details(usage: Dict[str, Any]) -> Dict[str, int]: + """Map cache-write tokens onto the price key for each lifetime. + + The flat total says nothing about the lifetime, so it is priced at the + cheaper rate. Use it only when the message omits the split. + """ + details: Dict[str, int] = {} + split = usage.get("cache_creation") + if isinstance(split, dict): + for src, dst in CACHE_WRITE_TTL_KEYS: + v = split.get(src) + if isinstance(v, int) and v > 0: + details[dst] = v + if details: + return details + v = usage.get("cache_creation_input_tokens") + if isinstance(v, int) and v > 0: + return {"cache_creation_input_tokens": v} + return {} + def get_usage_details_from_row(row: Dict[str, Any]) -> Optional[Dict[str, int]]: """Extract Anthropic token usage from an assistant message, if present.""" m = row.get("message") @@ -531,11 +718,11 @@ def get_usage_details_from_row(row: Dict[str, Any]) -> Optional[Dict[str, int]]: ("input_tokens", "input"), ("output_tokens", "output"), ("cache_read_input_tokens", "cache_read_input_tokens"), - ("cache_creation_input_tokens", "cache_creation_input_tokens"), ): v = u.get(src) if isinstance(v, int) and v > 0: details[dst] = v + details.update(get_cache_write_details(u)) return details or None def get_speed_from_row(row: Dict[str, Any]) -> Optional[str]: @@ -562,6 +749,43 @@ def parse_timestamp(value: Any) -> Optional[datetime]: except Exception: return None +def describe_image_block(block: Dict[str, Any]) -> str: + """Make a short text marker for an image block's base64 payload.""" + source = block.get("source") if isinstance(block.get("source"), dict) else {} + media_type = source.get("media_type") or "unknown type" + data = source.get("data") + if isinstance(data, str) and data: + return f"[image {media_type} ~{len(data) * 3 // 4 // 1024}KB]" + return f"[image {media_type}]" + +def media_from_image_block(block: Dict[str, Any]) -> Optional[Any]: + """Make a LangfuseMedia for a base64 image block. Return None when + capture is off, media support is missing, or the block is malformed. + The caller then keeps the text marker.""" + if not CAPTURE_IMAGES or LangfuseMedia is None: + return None + source = block.get("source") if isinstance(block.get("source"), dict) else {} + media_type = source.get("media_type") + data = source.get("data") + if source.get("type") != "base64" or not isinstance(media_type, str) or not media_type or not isinstance(data, str) or not data: + return None + try: + # The SDK does not raise on bad base64 and returns a hollow object + # that serializes as garbage, so decode here and pass bytes directly. + decoded = base64.b64decode("".join(data.split()), validate=True) + return LangfuseMedia(content_bytes=decoded, content_type=media_type) + except Exception as e: + debug(f"LangfuseMedia creation failed ({type(e).__name__}: {e}); keeping text marker only") + return None + +def get_image_blocks(content: Any) -> List[Dict[str, Any]]: + out: List[Dict[str, Any]] = [] + if isinstance(content, list): + for x in content: + if isinstance(x, dict) and x.get("type") == "image": + out.append(x) + return out + def extract_text_from_content(content: Any) -> str: if isinstance(content, str): return content @@ -570,6 +794,8 @@ def extract_text_from_content(content: Any) -> str: for x in content: if isinstance(x, dict) and x.get("type") == "text": parts.append(x.get("text", "")) + elif isinstance(x, dict) and x.get("type") == "image": + parts.append(describe_image_block(x)) elif isinstance(x, str): parts.append(x) return "\n".join([p for p in parts if p]) @@ -584,6 +810,40 @@ def truncate_text(s: str, max_chars: int = MAX_CHARS) -> Tuple[str, Dict[str, An head = s[:max_chars] return head, {"truncated": True, "orig_len": orig_len, "kept_len": len(head), "sha256": hashlib.sha256(s.encode("utf-8")).hexdigest()} +def build_status_message(value: Any, fallback: str) -> str: + """Render content as an observation status message.""" + text = extract_text_from_content(value).strip() + if not text and value: + try: + text = json.dumps(value, ensure_ascii=False) + except (TypeError, ValueError): + text = str(value) + text = text.strip() + return text or fallback + +def is_interrupted_turn_row(row: Dict[str, Any]) -> bool: + """Report the marker row that ends a turn the user stopped. + + The longer "for tool use" marker denies one tool and does not match, so a + turn that continues after a denied tool keeps its default level. + """ + if get_user_or_assistant_role_from_row(row) != "user": + return False + text = extract_text_from_content(get_content_from_row(row)).lstrip() + return text.startswith(INTERRUPTED_TURN_MARKER) + +def get_api_error_status(row: Dict[str, Any]) -> Optional[Tuple[str, str]]: + """Report the level for a synthetic row that carries an API failure.""" + if row.get("isApiErrorMessage") is not True: + return None + return "ERROR", build_status_message(get_content_from_row(row), "Claude API request failed") + +def get_status_kwargs(status: Optional[Tuple[str, str]]) -> Dict[str, Any]: + if status is None: + return {} + level, status_message = status + return {"level": level, "status_message": status_message} + def get_tool_use_blocks(content: Any) -> List[Dict[str, Any]]: out: List[Dict[str, Any]] = [] if isinstance(content, list): @@ -861,7 +1121,32 @@ def get_async_launch_flag_from_row(row: Dict[str, Any]) -> Optional[bool]: tool_use_result = row.get("toolUseResult") if not isinstance(tool_use_result, dict): return None - return tool_use_result.get("status") == "async_launched" or tool_use_result.get("isAsync") is True + # A teammate launch is async too: its result never carries the final output. + if tool_use_result.get("status") in ("async_launched", "teammate_spawned"): + return True + return tool_use_result.get("isAsync") is True + +def get_workflow_launch_marker_from_row(row: Dict[str, Any]) -> Optional[Dict[str, Any]]: + """Read the structured Workflow launch marker from a tool_result row. + + Workflow launches carry toolUseResult.taskType == "local_workflow" plus + the runId that names the run's transcript directory + (/subagents/workflows//). Only the structured + marker is trusted, the launch text is never parsed. + """ + tool_use_result = row.get("toolUseResult") + if not isinstance(tool_use_result, dict): + return None + if tool_use_result.get("taskType") != "local_workflow": + return None + run_id = tool_use_result.get("runId") + if not isinstance(run_id, str) or not run_id: + return None + workflow_launch_marker: Dict[str, Any] = {"run_id": run_id} + workflow_name = tool_use_result.get("workflowName") + if isinstance(workflow_name, str) and workflow_name: + workflow_launch_marker["workflow_name"] = workflow_name + return workflow_launch_marker def is_async_agent_launch_result(tool_result_entry: Any) -> bool: if not isinstance(tool_result_entry, dict): @@ -885,7 +1170,8 @@ def get_pending_agent_tool_use_ids(turn: Turn) -> List[str]: tool_use_ids: List[str] = [] for assistant_message in turn.assistant_msgs: for tool_use_block in get_tool_use_blocks(get_content_from_row(assistant_message)): - if tool_use_block.get("name") not in ("Agent", "Task"): + # Workflows resolve via task notifications too, so they hold the turn open. + if tool_use_block.get("name") not in ("Agent", "Task", "Workflow"): continue tool_use_id = str(tool_use_block.get("id") or "") if not tool_use_id: @@ -979,6 +1265,13 @@ def add_injected_context_row(row: Dict[str, Any], state: TurnAssemblyState) -> b state.current_rows.append(row) return True +def add_interrupted_turn_row(row: Dict[str, Any], state: TurnAssemblyState) -> bool: + """Keep the interrupt marker in the turn it ended.""" + if state.current_turn_user_row is None or not is_interrupted_turn_row(row): + return False + state.current_rows.append(row) + return True + def add_tool_result_row(row: Dict[str, Any], state: TurnAssemblyState) -> bool: # tool_result rows show up as role=user with content blocks of type tool_result. if not is_tool_result(row): @@ -987,6 +1280,7 @@ def add_tool_result_row(row: Dict[str, Any], state: TurnAssemblyState) -> bool: state.current_rows.append(row) row_timestamp = row.get("timestamp") is_async_launch = get_async_launch_flag_from_row(row) + workflow_launch_marker = get_workflow_launch_marker_from_row(row) for tool_result_block in get_tool_result_blocks(get_content_from_row(row)): tool_use_id = tool_result_block.get("tool_use_id") if tool_use_id: @@ -994,8 +1288,17 @@ def add_tool_result_row(row: Dict[str, Any], state: TurnAssemblyState) -> bool: "content": tool_result_block.get("content"), "timestamp": row_timestamp, } + if isinstance(tool_result_block.get("is_error"), bool): + tool_result_entry["is_error"] = tool_result_block["is_error"] if is_async_launch is not None: tool_result_entry["is_async_launch"] = is_async_launch + if workflow_launch_marker is not None: + # Links the launching tool_use to its workflow run so emission + # can attach the run's agent transcripts (which have no + # toolUseId of their own) under this tool's span. + tool_result_entry["workflow_run_id"] = workflow_launch_marker["run_id"] + if "workflow_name" in workflow_launch_marker: + tool_result_entry["workflow_name"] = workflow_launch_marker["workflow_name"] state.tool_results_by_id[str(tool_use_id)] = tool_result_entry return True @@ -1150,6 +1453,9 @@ def assemble_turns( if add_task_notification_row(row, state, task_id_to_tool_use_id, closed_turns=turns): continue + if add_interrupted_turn_row(row, state): + continue + role = get_user_or_assistant_role_from_row(row) if role == "user": @@ -1219,16 +1525,72 @@ def assign_turn_numbers(turns: List[Turn], trailing_turn: Optional[Turn], next_turn_number += 1 +def is_row_from_another_session(row: Dict[str, Any], session_id: str) -> bool: + """True if the row is not from the current session.""" + row_session_id = row.get("sessionId") + return ( + isinstance(row_session_id, str) + and bool(row_session_id) + and row_session_id != session_id + ) + + +def transcript_file_belongs_to_session(transcript_path: Path, session_id: str) -> bool: + """True if the transcript file has the name of the current session. + + Claude Code gives each transcript file the name of its session id. A + different name shows that the payload and the rows do not agree. + """ + return transcript_path.stem == session_id + + +def drop_rows_copied_from_another_session( + rows: List[Dict[str, Any]], + session_id: str, + session_state: SessionState, + task_id_to_tool_use_id: Optional[Dict[str, str]] = None, +) -> List[Dict[str, Any]]: + """Remove the rows that a fork copied from its original session. + + The original session exported these turns before. The number of the removed + turns increases the turn count. Then the later turns keep unique numbers. + """ + copied = [row for row in rows if is_row_from_another_session(row, session_id)] + if not copied: + return rows + + copied_turns = len(build_turns(copied, task_id_to_tool_use_id)) + session_state.turn_count = max(session_state.turn_count, copied_turns) + info( + f"Skipped {len(copied)} row(s) of history copied from another session " + f"({copied_turns} turn(s) already exported by it)" + ) + return [row for row in rows if not is_row_from_another_session(row, session_id)] + + def get_new_turns_from_transcript( transcript_path: Path, session_state: SessionState, subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]] = None, *, flush_deferred_agent_turns: bool = False, + session_id: Optional[str] = None, ) -> Tuple[List[Turn], SessionState]: rows, session_state = read_new_jsonl(transcript_path, session_state) task_id_to_tool_use_id = get_task_id_to_tool_use_id(subagent_transcripts_by_tool_use_id) + if session_id and rows: + if transcript_file_belongs_to_session(transcript_path, session_id): + rows = drop_rows_copied_from_another_session( + rows, session_id, session_state, task_id_to_tool_use_id + ) + else: + # Keeping duplicates as fallback. + debug( + f"transcript {transcript_path.name} is not named after session " + f"{session_id}; not checking for copied history" + ) + # Re-attach the trailing open turn from the previous run. Stop fires # multiple times within one logical turn, so a batch can begin with # user-less continuation rows (task notifications, assistant rows); with @@ -1278,31 +1640,91 @@ def get_new_turns_from_transcript( assign_turn_numbers(turns, trailing_turn, session_state) return turns, session_state +def resolve_agent_jsonl_and_id(meta_path: Path) -> Optional[Tuple[Path, str]]: + """Derive an agent's transcript path and agent id from its meta.json path. + + Returns None when the sibling .jsonl is missing (metas without a + transcript identify nothing worth emitting).""" + jsonl_path = meta_path.with_name(meta_path.name[: -len(".meta.json")] + ".jsonl") + if not jsonl_path.exists(): + return None + agent_id = meta_path.name[: -len(".meta.json")] + if agent_id.startswith("agent-"): + agent_id = agent_id[len("agent-"):] + return jsonl_path, agent_id + +def get_agent_launch_tool_use_ids_by_name(transcript_path: Path) -> Dict[str, str]: + """Map each agent name to the tool_use id of its launch. + The function streams the full transcript. It removes ambiguous names.""" + mapping: Dict[str, str] = {} + ambiguous: set = set() + try: + with transcript_path.open(encoding="utf-8") as lines: + for line in lines: + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except Exception: + continue + if not isinstance(row, dict): + continue + for tool_use in get_tool_use_blocks(get_content_from_row(row)): + tool_use_id = str(tool_use.get("id") or "") + tool_input = tool_use.get("input") + if tool_use.get("name") not in ("Agent", "Task") or not tool_use_id: + continue + name = tool_input.get("name") if isinstance(tool_input, dict) else None + if not isinstance(name, str) or not name: + continue + if mapping.get(name, tool_use_id) != tool_use_id: + ambiguous.add(name) + mapping[name] = tool_use_id + except Exception: + return {} + return {name: tool_use_id for name, tool_use_id in mapping.items() if name not in ambiguous} + def get_subagent_transcripts_by_tool_use_id(transcript_path: Path) -> Dict[str, Dict[str, Any]]: """Map launching Agent/Task tool_use ids to their subagent transcripts.""" subagent_dir = transcript_path.with_suffix("") / "subagents" if not subagent_dir.is_dir(): return {} + # Build the fallback map only when a meta has no toolUseId. + fallback_tool_use_ids: Optional[Dict[str, str]] = None + claimed_twice: set = set() subagent_transcripts_by_tool_use_id: Dict[str, Dict[str, Any]] = {} - for meta_path in subagent_dir.glob("*.meta.json"): + for meta_path in sorted(subagent_dir.glob("*.meta.json")): try: metadata = json.loads(meta_path.read_text(encoding="utf-8")) except Exception: continue + if not isinstance(metadata, dict): + continue + + resolved = resolve_agent_jsonl_and_id(meta_path) + if resolved is None: + continue + jsonl_path, agent_id = resolved tool_use_id = metadata.get("toolUseId") if not isinstance(tool_use_id, str) or not tool_use_id: + if fallback_tool_use_ids is None: + fallback_tool_use_ids = get_agent_launch_tool_use_ids_by_name(transcript_path) + # A teammate meta carries the launch input.name. + name = metadata.get("name") + tool_use_id = fallback_tool_use_ids.get(name) if isinstance(name, str) else None + if not tool_use_id: + info(f"subagent transcript not attributable to a launching tool_use, skipping: {meta_path}") continue - - jsonl_path = meta_path.with_name(meta_path.name[: -len(".meta.json")] + ".jsonl") - if not jsonl_path.exists(): + if tool_use_id in claimed_twice or tool_use_id in subagent_transcripts_by_tool_use_id: + # Two metas claim one launch -> Drop both. + claimed_twice.add(tool_use_id) + subagent_transcripts_by_tool_use_id.pop(tool_use_id, None) + info(f"multiple subagent metas claim tool_use {tool_use_id}, skipping: {meta_path}") continue - agent_id = meta_path.name[: -len(".meta.json")] - if agent_id.startswith("agent-"): - agent_id = agent_id[len("agent-"):] - subagent_transcripts_by_tool_use_id[tool_use_id] = { "path": jsonl_path, "agent_id": agent_id, @@ -1311,6 +1733,103 @@ def get_subagent_transcripts_by_tool_use_id(transcript_path: Path) -> Dict[str, } return subagent_transcripts_by_tool_use_id +def get_workflow_journal_results(run_dir: Path) -> Dict[str, Any]: + """Per-agent return values from a workflow run's journal.jsonl. + + The journal carries {"type":"result","agentId",...,"result":{...}} rows, + one per completed agent; unparseable lines are skipped.""" + journal_path = run_dir / "journal.jsonl" + try: + lines = journal_path.read_text(encoding="utf-8").splitlines() + except Exception: + return {} + results_by_agent_id: Dict[str, Any] = {} + for line in lines: + line = line.strip() + if not line: + continue + try: + journal_row = json.loads(line) + except Exception: + continue + if not isinstance(journal_row, dict) or journal_row.get("type") != "result": + continue + agent_id = journal_row.get("agentId") + if isinstance(agent_id, str) and agent_id: + results_by_agent_id[agent_id] = journal_row.get("result") + return results_by_agent_id + +def get_workflow_agents_in_run_dir(run_dir: Path) -> List[Dict[str, Any]]: + """The workflow-spawned agent transcripts of one Workflow run directory. + + Workflow-tool agents live under /subagents/workflows//; + their meta.json carries agentType=="workflow-subagent" and — unlike + classic subagents — no toolUseId, so a run is identified by its directory + name instead. The launching tool_use is linked via toolUseResult.runId on + the parent transcript's tool_result row (see + get_workflow_launch_marker_from_row). + """ + if not run_dir.is_dir(): + return [] + + journal_results = get_workflow_journal_results(run_dir) + agents: List[Dict[str, Any]] = [] + for meta_path in sorted(run_dir.glob("agent-*.meta.json")): + try: + metadata = json.loads(meta_path.read_text(encoding="utf-8")) + except Exception: + continue + if not isinstance(metadata, dict) or metadata.get("agentType") != "workflow-subagent": + continue + + resolved = resolve_agent_jsonl_and_id(meta_path) + if resolved is None: + continue + jsonl_path, agent_id = resolved + + agents.append({ + "path": jsonl_path, + "agent_id": agent_id, + "agent_type": metadata.get("agentType"), + "result": journal_results.get(agent_id), + }) + return agents + + +class WorkflowAgentTranscriptsByRunId: + """Per-run lookup of workflow-spawned agent transcripts. It reads on demand. + + The first lookup of a run id reads that run directory. The lookup then keeps + the result for its own life, which is one hook firing. Thus only a firing + that emits a reference to a run reads that run, and it always reads the + current files.""" + + def __init__(self, transcript_path: Path) -> None: + self._workflows_dir = transcript_path.with_suffix("") / "subagents" / "workflows" + self._agents_by_run_id: Dict[str, List[Dict[str, Any]]] = {} + + def get(self, run_id: Optional[str]) -> Optional[List[Dict[str, Any]]]: + if not isinstance(run_id, str) or not run_id: + return None + # A run id comes from transcript data, and this method makes a path from + # it. Thus the method must accept only a single plain directory name. + # The name test rejects each other shape, but not "..", whose Path.name + # is ".." again. + if run_id == ".." or run_id != Path(run_id).name: + return None + if run_id not in self._agents_by_run_id: + try: + agents = get_workflow_agents_in_run_dir(self._workflows_dir / run_id) + except OSError as e: + # The file system can refuse a run path, for example when the + # name is too long. Such a failure must not stop the turn. + debug(f"Workflow run {run_id} not readable: {e}") + agents = [] + if agents: + debug(f"Discovered {len(agents)} workflow agent transcript(s) for run {run_id}") + self._agents_by_run_id[run_id] = agents + return self._agents_by_run_id[run_id] or None + def get_task_id_to_tool_use_id( subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]], ) -> Dict[str, str]: @@ -1443,7 +1962,7 @@ def _start_backdated(langfuse: Langfuse, *, name: str, as_type: str, raise RuntimeError( f"Langfuse SDK {sdk_version} is missing _otel_tracer or " f"_create_observation_from_otel_span. This hook targets SDK 4.x; " - f"pin with `pip install \"langfuse>=4.0,<5\"` or update the hook script." + f"pin with `pip install \"langfuse>=4.7,<5\"` or update the hook script." ) start_ns = to_otel_nanoseconds(start_time) if parent_otel_span is not None: @@ -1527,6 +2046,22 @@ def collect_subagent_skill_tags( add_skill_tags_from_rows(rows, names, "subagent-skill:") return names +# Constant on purpose: a shared trace name keeps Langfuse name-based grouping usable. +TRACE_NAME = "Claude Code Turn" + +def get_trace_tags( + turn: Turn, + subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]] = None, +) -> List[str]: + tags = ["claude-code"] + if SKILL_TAGS: + tags += collect_skill_tags(turn) + tags += collect_subagent_skill_tags(turn, subagent_transcripts_by_tool_use_id) + if AGENT_TYPE_TAGS: + tags += collect_subagent_agent_type_tags(turn, subagent_transcripts_by_tool_use_id) + tags += [tag for tag in OPERATOR_TAGS if tag not in tags] + return tags + def collect_subagent_agent_type_tags( turn: Turn, @@ -1557,32 +2092,6 @@ def collect_subagent_agent_type_tags( names.append(tag) return names - -def short_session_label(session_id: str, max_len: int = 12) -> str: - """Return a compact session label for trace names.""" - sid = session_id.strip() - if not sid: - return "unknown" - parts = sid.split("-") - if len(parts) == 5 and len(parts[0]) == 8: - return parts[0] - return sid if len(sid) <= max_len else sid[:max_len].rstrip("-") - -def trace_display_name(session_id: str, turn_num: int) -> str: - return f"Claude Code - Turn {turn_num} ({short_session_label(session_id)})" - -def get_trace_tags( - turn: Turn, - subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]] = None, -) -> List[str]: - tags = ["claude-code"] - if SKILL_TAGS: - tags += collect_skill_tags(turn) - tags += collect_subagent_skill_tags(turn, subagent_transcripts_by_tool_use_id) - if AGENT_TYPE_TAGS: - tags += collect_subagent_agent_type_tags(turn, subagent_transcripts_by_tool_use_id) - return tags - # ---- Generation payloads ---- def build_generation_input( assistant_index: int, @@ -1600,20 +2109,134 @@ def build_generation_input( return {"role": "tool", "tool_results": tool_results} return None -def build_generation_output(assistant_text: str, tool_uses: List[Dict[str, Any]]) -> Dict[str, Any]: +def build_thinking_parts(content: Any) -> Tuple[List[Dict[str, str]], List[Dict[str, Any]]]: + """Make ChatML thinking parts from the thinking blocks of an assistant message. + + Langfuse renders these parts as thinking blocks. A block without text + holds no reasoning, so this function skips it. + """ + parts: List[Dict[str, str]] = [] + metas: List[Dict[str, Any]] = [] + if not isinstance(content, list): + return parts, metas + for block in content: + if not isinstance(block, dict) or block.get("type") != "thinking": + continue + thinking_raw = block.get("thinking") + if not isinstance(thinking_raw, str) or not thinking_raw.strip(): + continue + thinking_text, thinking_meta = truncate_text(thinking_raw) + parts.append({"type": "thinking", "content": thinking_text}) + metas.append(thinking_meta) + return parts, metas + +def build_generation_output(assistant_text: str, tool_uses: List[Dict[str, Any]], + thinking_parts: Optional[List[Dict[str, str]]] = None) -> Dict[str, Any]: output: Dict[str, Any] = {"role": "assistant"} if assistant_text: output["content"] = assistant_text + if thinking_parts: + output["thinking"] = thinking_parts if tool_uses: + # Langfuse renders tool calls only in this nested shape. A flat + # id and name pair stays raw JSON in the trace UI. output["tool_calls"] = [ { "id": tool_use.get("id"), - "name": tool_use.get("name"), + "type": "function", + "function": {"name": tool_use.get("name")}, } for tool_use in tool_uses ] return output + +# ---- Generation input history ---- +def build_user_history_content(user_row: Dict[str, Any]) -> Any: + """The user message for the history: text, plus the images of that turn. + + Each image stays at the position where it entered the conversation, so + the history shows it the way the model received it. + """ + user_content = get_content_from_row(user_row) + user_text, _ = truncate_text(extract_text_from_content(user_content)) + user_media = [ + media for media in (media_from_image_block(block) for block in get_image_blocks(user_content)) + if media is not None + ] + return [user_text, *user_media] if user_media else user_text + + +def build_turn_history_messages(turn: Turn) -> List[Dict[str, Any]]: + """The ChatML view of one turn, in the shape of the live generation payloads. + + Order: user message, then for each assistant step the generation output + and one tool message with the step results. An async result appears at + its launch step with the final output, not at its arrival time. + """ + messages: List[Dict[str, Any]] = [ + {"role": "user", "content": build_user_history_content(turn.user_msg)} + ] + for assistant_message in turn.assistant_msgs: + assistant_text, _ = truncate_text( + extract_text_from_content(get_content_from_row(assistant_message)) + ) + tool_uses = get_tool_use_blocks(get_content_from_row(assistant_message)) + messages.append(build_generation_output(assistant_text, tool_uses)) + tool_results: List[Dict[str, Any]] = [] + for tool_use in tool_uses: + entry = turn.tool_results_by_id.get(str(tool_use.get("id") or "")) + if entry is None: + continue + result = get_tool_result_for_observation(entry) + output = result.final_output if result.final_output is not None else result.output + tool_results.append({ + "tool_use_id": tool_use.get("id"), + "tool_name": tool_use.get("name"), + "output": output, + }) + if tool_results: + messages.append({"role": "tool", "tool_results": tool_results}) + return messages + + +@dataclass +class SessionHistory: + """Flat ChatML view of the whole transcript, with one index per turn.""" + messages: List[Dict[str, Any]] = field(default_factory=list) + start_index_by_user_row_uuid: Dict[str, int] = field(default_factory=dict) + + def prefix_for_turn(self, user_row_uuid: Any) -> Optional[List[Dict[str, Any]]]: + """All messages before the given turn, or None when the turn is unknown.""" + if not isinstance(user_row_uuid, str) or not user_row_uuid: + return None + start_index = self.start_index_by_user_row_uuid.get(user_row_uuid) + if start_index is None: + return None + return self.messages[:start_index] + + +def build_session_history( + transcript_path: Path, + task_id_to_tool_use_id: Optional[Dict[str, str]] = None, +) -> Optional[SessionHistory]: + """Rebuild the conversation of the whole transcript file as ChatML. + + Reads from byte 0 on purpose: the emission offset only tracks what was + exported, while a generation input needs every earlier message. + """ + rows = read_subagent_jsonl(transcript_path) + if not rows: + return None + history = SessionHistory() + for turn in build_turns(rows, task_id_to_tool_use_id): + user_row_uuid = turn.user_msg.get("uuid") + if isinstance(user_row_uuid, str) and user_row_uuid: + history.start_index_by_user_row_uuid.setdefault(user_row_uuid, len(history.messages)) + history.messages.extend(build_turn_history_messages(turn)) + return history + + # ---- Tool observations ---- @dataclass class ToolResultForObservation: @@ -1622,6 +2245,7 @@ class ToolResultForObservation: result_timestamp: Optional[datetime] = None final_output: Any = None final_result_timestamp: Optional[datetime] = None + status: Optional[Tuple[str, str]] = None @dataclass class EmittedSingleToolObservation: @@ -1645,14 +2269,40 @@ def get_tool_input_for_observation(tool_use: Dict[str, Any]) -> Tuple[Any, Optio return truncate_text(tool_input_raw) return tool_input_raw, None +def render_tool_result_content(raw: Any) -> Tuple[Any, Optional[Dict[str, Any]]]: + """Serialize a tool_result content value for an observation payload. + + Image blocks become text markers before json.dumps, so base64 cannot + use up the truncation budget or hide the text blocks around it. With + capture on, the media follows the truncated text in a list.""" + if isinstance(raw, str): + return truncate_text(raw) + media: List[Any] = [] + if isinstance(raw, list): + rendered_blocks: List[Any] = [] + for block in raw: + if isinstance(block, dict) and block.get("type") == "image": + media_object = media_from_image_block(block) + if media_object is not None: + media.append(media_object) + rendered_blocks.append({"type": "text", "text": describe_image_block(block)}) + else: + rendered_blocks.append(block) + raw = rendered_blocks + text, meta = truncate_text(json.dumps(raw, ensure_ascii=False)) + if media: + return [text, *media], meta + return text, meta + def get_tool_result_for_observation(tool_result_entry: Any) -> ToolResultForObservation: if not isinstance(tool_result_entry, dict): return ToolResultForObservation() - output_raw = tool_result_entry.get("content") - output_str = output_raw if isinstance(output_raw, str) else json.dumps(output_raw, ensure_ascii=False) - output, output_meta = truncate_text(output_str) + output, output_meta = render_tool_result_content(tool_result_entry.get("content")) result_timestamp = parse_timestamp(tool_result_entry.get("timestamp")) + status: Optional[Tuple[str, str]] = None + if tool_result_entry.get("is_error") is True: + status = "ERROR", build_status_message(tool_result_entry.get("content"), "Tool call failed") final_output_raw = tool_result_entry.get("final_content") if final_output_raw is None: @@ -1660,14 +2310,10 @@ def get_tool_result_for_observation(tool_result_entry: Any) -> ToolResultForObse output=output, output_meta=output_meta, result_timestamp=result_timestamp, + status=status, ) - final_output_str = ( - final_output_raw - if isinstance(final_output_raw, str) - else json.dumps(final_output_raw, ensure_ascii=False) - ) - final_output, _ = truncate_text(final_output_str) + final_output, _ = render_tool_result_content(final_output_raw) final_result_timestamp = parse_timestamp(tool_result_entry.get("final_timestamp")) return ToolResultForObservation( output=output, @@ -1675,6 +2321,7 @@ def get_tool_result_for_observation(tool_result_entry: Any) -> ToolResultForObse result_timestamp=result_timestamp, final_output=final_output, final_result_timestamp=final_result_timestamp, + status=status, ) def get_short_transcript_path_for_metadata(path: Any) -> Optional[str]: @@ -1708,7 +2355,8 @@ def build_tool_metadata( @dataclass class EmissionCursor: """Tracks which of a turn's observations were already emitted (namespaced - keys: gen:/tool:/subagent:) so continuation firings only emit what is new. + keys: gen:/tool:/subagent:/workflow-agent:) so continuation firings only + emit what is new. completed_only skips observations whose emitted form could still change (generation awaiting tool results, running async subagent); at turn close @@ -1750,6 +2398,7 @@ def emit_single_tool_observation( pending_async_tool_results: List[Dict[str, Any]], cursor: EmissionCursor, tool_key: str, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, ) -> EmittedSingleToolObservation: tool_use_id = str(tool_use.get("id") or "") tool_name = tool_use.get("name") or "unknown" @@ -1772,6 +2421,24 @@ def emit_single_tool_observation( ) tool_metadata = build_tool_metadata(tool_name, tool_use_id, tool_input_meta, tool_result, subagent) + workflow_run_id = ( + tool_result_entry.get("workflow_run_id") if isinstance(tool_result_entry, dict) else None + ) + workflow_name = ( + tool_result_entry.get("workflow_name") if isinstance(tool_result_entry, dict) else None + ) + workflow_agents = ( + workflow_agent_transcripts_by_run_id.get(workflow_run_id) or [] + if workflow_agent_transcripts_by_run_id and workflow_run_id + else [] + ) + if workflow_run_id: + tool_metadata["workflow_run_id"] = workflow_run_id + if workflow_name: + tool_metadata["workflow_name"] = workflow_name + if workflow_agents: + tool_metadata["workflow_agent_count"] = len(workflow_agents) + tool_use_timestamp = parse_timestamp(turn.tool_use_timestamps_by_id.get(tool_use_id)) or assistant_timestamp # A tool span's end time comes from its result row, so it only counts as # complete once that row exists. @@ -1785,6 +2452,7 @@ def emit_single_tool_observation( parent_otel_span=parent_otel_span, input=tool_input, metadata=tool_metadata, + **get_status_kwargs(tool_result.status), ) tool_span.update(output=tool_output) @@ -1815,9 +2483,35 @@ def emit_single_tool_observation( parent_otel_span, subagent, tool_use_timestamp, + subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, ) - tool_end_timestamp = _get_latest_timestamp(tool_result.result_timestamp, tool_use_timestamp) + workflow_end_timestamp = None + if workflow_agents: + # Agent transcripts may still grow until the completion + # notification (final_content) lands. + workflow_resolved = ( + isinstance(tool_result_entry, dict) + and tool_result_entry.get("final_content") is not None + ) + workflow_end_timestamp = emit_workflow_agent_observations( + langfuse, + # tool_span is None only when an earlier firing already exported + # it; late-discovered agents then nest under the turn root instead. + tool_span._otel_span if tool_span is not None else parent_otel_span, + workflow_agents, + workflow_run_id=workflow_run_id, + workflow_name=workflow_name, + workflow_resolved=workflow_resolved, + cursor=cursor, + emission_scope=tool_use_id or tool_key, + ) + + # Exported spans are immutable, so the end time set here must already + # cover the workflow agents nested under this span. + tool_end_timestamp = _get_latest_timestamp( + tool_result.result_timestamp, tool_use_timestamp, workflow_end_timestamp + ) handoff_timestamp = ( tool_result.result_timestamp or tool_result.final_result_timestamp @@ -1844,7 +2538,9 @@ def emit_single_tool_observation( "tool_name": tool_name, "output": tool_result.output, }, - latest_end_timestamp=_get_latest_timestamp(tool_end_timestamp, subagent_end_timestamp), + latest_end_timestamp=_get_latest_timestamp( + tool_end_timestamp, subagent_end_timestamp, workflow_end_timestamp + ), ) def emit_tool_observation_batch( @@ -1858,6 +2554,7 @@ def emit_tool_observation_batch( pending_subagents: List[Dict[str, Any]], pending_async_tool_results: List[Dict[str, Any]], cursor: EmissionCursor, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, ) -> EmittedToolObservationBatch: assistant_timestamp = parse_timestamp(assistant_message) generation_key = generation_emission_key(assistant_index, assistant_message) @@ -1879,6 +2576,7 @@ def emit_tool_observation_batch( pending_async_tool_results, cursor, tool_key, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, ) if emitted_tool.handoff_timestamp is not None: tool_result_timestamps.append(emitted_tool.handoff_timestamp) @@ -1964,9 +2662,10 @@ def build_generation_kwargs( previous_tool_results: List[Dict[str, Any]], ready_async_tool_results: List[Dict[str, Any]], ) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]: - assistant_text_raw = extract_text_from_content(get_content_from_row(assistant_message)) - assistant_text, assistant_text_meta = truncate_text(assistant_text_raw) - tool_uses = get_tool_use_blocks(get_content_from_row(assistant_message)) + assistant_content = get_content_from_row(assistant_message) + assistant_text, assistant_text_meta = truncate_text(extract_text_from_content(assistant_content)) + tool_uses = get_tool_use_blocks(assistant_content) + thinking_parts, thinking_metas = build_thinking_parts(assistant_content) speed = get_speed_from_row(assistant_message) @@ -1978,31 +2677,33 @@ def build_generation_kwargs( previous_tool_results, ready_async_tool_results, ), - output=build_generation_output(assistant_text, tool_uses), + output=build_generation_output(assistant_text, tool_uses, thinking_parts), metadata={ "assistant_index": assistant_index, "assistant_text": assistant_text_meta, "tool_count": len(tool_uses), }, ) + if thinking_metas: + generation_kwargs["metadata"]["thinking"] = thinking_metas if speed is not None: generation_kwargs["metadata"]["speed"] = speed usage_details = get_usage_details_from_row(assistant_message) if usage_details is not None: generation_kwargs["usage_details"] = usage_details + generation_kwargs.update(get_status_kwargs(get_api_error_status(assistant_message))) return generation_kwargs, tool_uses def emit_generation_observation( langfuse: Langfuse, parent_otel_span: Any, - generation_prefix: str, - assistant_index: int, + generation_name: str, start_timestamp: Optional[datetime], generation_kwargs: Dict[str, Any], ) -> Any: return _start_backdated( langfuse, - name=f"{generation_prefix} {assistant_index + 1}", + name=generation_name, as_type="generation", start_time=start_timestamp, parent_otel_span=parent_otel_span, @@ -2011,18 +2712,33 @@ def emit_generation_observation( def emit_turn_observations(langfuse: Langfuse, parent_otel_span: Any, turn: Turn, start_timestamp: Optional[datetime], - generation_prefix: str = "LLM Call", + generation_name: str = "LLM Call", subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]] = None, - cursor: Optional[EmissionCursor] = None) -> Optional[datetime]: + cursor: Optional[EmissionCursor] = None, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, + history_prefix: Optional[List[Dict[str, Any]]] = None) -> Optional[datetime]: """Emit a turn's generations and tool observations under an existing span. The full turn is always walked so cross-observation context (generation inputs from previous tool results, timestamps) stays correct; the cursor only gates which spans are actually created. Without a cursor everything is emitted (one-shot behavior). + + With history_prefix set, each generation input is the conversation up to + that point: the prefix, this turn's user message, and the earlier steps + of this turn. Callers pass None when the history is not available, + and the inputs then keep the delta form. """ cursor = cursor if cursor is not None else fresh_cursor() user_text, _ = truncate_text(extract_text_from_content(get_content_from_row(turn.user_msg))) + history_messages: Optional[List[Dict[str, Any]]] = None + if history_prefix is not None: + history_messages = list(history_prefix) + # Same shape as in build_turn_history_messages, so this turn sees its + # own images the way later turns see them in the history. + history_messages.append( + {"role": "user", "content": build_user_history_content(turn.user_msg)} + ) previous_timestamp = start_timestamp previous_tool_results: List[Dict[str, Any]] = [] pending_async_tool_results: List[Dict[str, Any]] = [] @@ -2042,6 +2758,7 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: parent_otel_span, pending_subagent["subagent"], pending_subagent.get("display_start_timestamp") or pending_subagent.get("start_timestamp"), + subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, ) latest_end_timestamp = _get_latest_timestamp(latest_end_timestamp, subagent_end_timestamp) @@ -2062,6 +2779,11 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: ) previous_timestamp = _get_latest_timestamp(previous_timestamp, ready_async_result_timestamp) + if history_messages is not None and ready_async_tool_results: + history_messages.append({ + "role": "tool", + "tool_results": [result["tool_result"] for result in ready_async_tool_results], + }) generation_kwargs, tool_uses = build_generation_kwargs( assistant_index, assistant_message, @@ -2069,11 +2791,15 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: previous_tool_results, ready_async_tool_results, ) + if history_messages is not None: + generation_kwargs["input"] = list(history_messages) + generation_kwargs["metadata"]["history"] = {"messages": len(history_messages)} generation_start_timestamp = previous_timestamp or assistant_timestamp # A generation is only complete when its emitted form cannot change - # anymore: (a) every tool_use of this message has its result (end - # time), (b) no earlier async launch is unresolved (a late - # notification would retroactively join this generation's input). + # anymore and its tool spans can ship with it: (a) every tool_use of + # this message has its result (the tool span end), (b) no earlier + # async launch is unresolved (a late notification would retroactively + # join this generation's input). # The trailing message needs no extra guard: Stop only fires after a # response is fully written, and no message.id ever grows across a # Stop boundary (0 cases across all local transcripts). @@ -2092,8 +2818,7 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: generation_span = emit_generation_observation( langfuse, parent_otel_span=parent_otel_span, - generation_prefix=generation_prefix, - assistant_index=assistant_index, + generation_name=generation_name, start_timestamp=generation_start_timestamp, generation_kwargs=generation_kwargs, ) @@ -2114,24 +2839,24 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: pending_subagents, pending_async_tool_results, cursor, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, ) latest_end_timestamp = _get_latest_timestamp( latest_end_timestamp, emitted_tools.latest_end_timestamp, ) - generation_end_timestamp = ( - max(emitted_tools.result_timestamps) - if emitted_tools.result_timestamps - else assistant_timestamp + generation_end_timestamp = _get_latest_timestamp( + assistant_timestamp or previous_timestamp, + generation_start_timestamp, ) if generation_span is not None: - generation_span.end( - end_time=to_otel_nanoseconds( - generation_end_timestamp or assistant_timestamp or previous_timestamp - ) - ) - latest_end_timestamp = _get_latest_timestamp(latest_end_timestamp, generation_end_timestamp) + generation_span.end(end_time=to_otel_nanoseconds(generation_end_timestamp)) + latest_end_timestamp = _get_latest_timestamp( + latest_end_timestamp, + generation_end_timestamp, + *emitted_tools.result_timestamps, + ) for tool_use in tool_uses: entry = turn.tool_results_by_id.get(str(tool_use.get("id") or "")) @@ -2140,6 +2865,14 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: ): unresolved_async_launch_seen = True + if history_messages is not None: + history_messages.append(generation_kwargs["output"]) + if emitted_tools.tool_results: + history_messages.append({ + "role": "tool", + "tool_results": emitted_tools.tool_results, + }) + previous_tool_results = emitted_tools.tool_results if emitted_tools.result_timestamps: previous_timestamp = max(emitted_tools.result_timestamps) @@ -2151,9 +2884,72 @@ def emit_pending_subagent(pending_subagent: Dict[str, Any]) -> None: return latest_end_timestamp +def emit_workflow_agent_observations( + langfuse: Langfuse, + parent_otel_span: Any, + workflow_agents: List[Dict[str, Any]], + *, + workflow_run_id: str, + workflow_name: Optional[str], + workflow_resolved: bool, + cursor: EmissionCursor, + emission_scope: str, +) -> Optional[datetime]: + """Emit each workflow-spawned agent transcript under the launching + "Tool: Workflow" span. + + Workflow agents carry no toolUseId of their own, so their emission keys + are derived from the launching tool_use (emission_scope) plus the agent + id — unique per agent and stable across Stop firings, so the incremental + EmissionCursor neither duplicates nor starves them. + """ + workflow_label = workflow_name or workflow_run_id + latest_end_timestamp: Optional[datetime] = None + for workflow_agent in workflow_agents: + agent_id = workflow_agent.get("agent_id") + if not isinstance(agent_id, str) or not agent_id: + continue + if not cursor.should_emit( + f"workflow-agent:{emission_scope}:{agent_id}", complete=workflow_resolved + ): + continue + extra_metadata: Dict[str, Any] = { + "workflow_run_id": workflow_run_id, + "workflow_agent_id": agent_id, + } + if workflow_name: + extra_metadata["workflow_name"] = workflow_name + agent_result = workflow_agent.get("result") + agent_result_json: Optional[str] = None + if agent_result is not None: + agent_result_json, _ = truncate_text(json.dumps(agent_result, ensure_ascii=False)) + extra_metadata["workflow_agent_result"] = agent_result_json + agent_end_timestamp = emit_subagent_observations( + langfuse, + parent_otel_span, + workflow_agent, + # No explicit start: the agent transcript's first row is the real start. + None, + span_name=f"Workflow agent: {workflow_label}/{agent_id}", + extra_metadata=extra_metadata, + generation_name="LLM Call", + # Workflow agents typically end on a bare StructuredOutput tool_use + # (no text in the final message); the journal result is the agent's + # actual return value, so it becomes the span output instead of "". + empty_output_fallback=agent_result_json, + ) + latest_end_timestamp = _get_latest_timestamp(latest_end_timestamp, agent_end_timestamp) + return latest_end_timestamp + def emit_subagent_observations(langfuse: Langfuse, parent_otel_span: Any, subagent: Dict[str, Any], - start_timestamp: Optional[datetime]) -> Optional[datetime]: + start_timestamp: Optional[datetime], + span_name: Optional[str] = None, + extra_metadata: Optional[Dict[str, Any]] = None, + generation_name: str = "Subagent LLM Call", + empty_output_fallback: Optional[str] = None, + subagent_transcripts_by_tool_use_id: Optional[Dict[str, Dict[str, Any]]] = None, + ) -> Optional[datetime]: path = subagent.get("path") if not isinstance(path, Path): return start_timestamp @@ -2172,35 +2968,51 @@ def emit_subagent_observations(langfuse: Langfuse, parent_otel_span: Any, last_turn = turns[-1] last_assistant = last_turn.assistant_msgs[-1] subagent_output_text, _ = truncate_text(extract_text_from_content(get_content_from_row(last_assistant))) + if not subagent_output_text and empty_output_fallback is not None: + # A final message of only thinking/tool_use blocks extracts no text; + # callers with a better source (workflow journal result) supply it here. + # Classic subagents pass no fallback, keeping their behavior unchanged. + subagent_output_text = empty_output_fallback description = subagent.get("description") - subagent_name = f"Subagent: {description}" if isinstance(description, str) and description else "Subagent" + if span_name is None: + span_name = f"Subagent: {description}" if isinstance(description, str) and description else "Subagent" + subagent_metadata: Dict[str, Any] = { + "agent_type": subagent.get("agent_type"), + "description": description, + "transcript_path": get_short_transcript_path_for_metadata(path), + "user_text": subagent_input_meta, + } + if extra_metadata: + subagent_metadata.update(extra_metadata) subagent_span = _start_backdated( langfuse, - name=subagent_name, + name=span_name, as_type="span", start_time=subagent_start_timestamp, parent_otel_span=parent_otel_span, input={"role": "user", "content": subagent_input_text}, - metadata={ - "agent_type": subagent.get("agent_type"), - "description": description, - "transcript_path": get_short_transcript_path_for_metadata(path), - "user_text": subagent_input_meta, - }, + metadata=subagent_metadata, + **get_status_kwargs(get_worst_turn_status(turns)), ) latest_end_timestamp = subagent_start_timestamp previous_start_timestamp = subagent_start_timestamp + # The agent transcript is complete on disk, so history accumulates + # across its turns the same way as in the main conversation. + subagent_history: List[Dict[str, Any]] = [] for turn in turns: latest_turn_timestamp = emit_turn_observations( langfuse, subagent_span._otel_span, turn, previous_start_timestamp, - generation_prefix="Subagent LLM Call", - subagent_transcripts_by_tool_use_id=None, + generation_name=generation_name, + # Agents launched by this agent are listed in the same map. + subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, + history_prefix=list(subagent_history), ) + subagent_history.extend(build_turn_history_messages(turn)) latest_end_timestamp = _get_latest_timestamp(latest_end_timestamp, latest_turn_timestamp) if latest_turn_timestamp is not None: previous_start_timestamp = latest_turn_timestamp @@ -2218,7 +3030,7 @@ def read_subagent_jsonl(path: Path) -> Optional[List[Dict[str, Any]]]: try: lines = path.read_text(encoding="utf-8").splitlines() except Exception as e: - info(f"subagent transcript read failed ({path}): {type(e).__name__}: {e}") + info(f"transcript read failed ({path}): {type(e).__name__}: {e}") return None rows: List[Dict[str, Any]] = [] @@ -2229,19 +3041,45 @@ def read_subagent_jsonl(path: Path) -> Optional[List[Dict[str, Any]]]: try: row = json.loads(line) except Exception as e: - info(f"subagent transcript line skipped ({path}:{line_number}): {type(e).__name__}: {e}") + info(f"transcript line skipped ({path}:{line_number}): {type(e).__name__}: {e}") continue if not isinstance(row, dict): - info(f"subagent transcript line skipped ({path}:{line_number}): expected JSON object") + info(f"transcript line skipped ({path}:{line_number}): expected JSON object") continue rows.append(row) return rows +def get_turn_status(turn: Turn) -> Optional[Tuple[str, str]]: + """Report the level for the turn as a whole. + + A failure that ends the turn outranks an interrupt. An API failure the turn + recovered from stays on its own generation and leaves the turn at default. + """ + if turn.assistant_msgs: + terminal_api_error = get_api_error_status(turn.assistant_msgs[-1]) + if terminal_api_error is not None: + return terminal_api_error + if any(is_interrupted_turn_row(row) for row in turn.rows): + return "WARNING", "Turn interrupted by user" + return None + +def get_worst_turn_status(turns: List[Turn]) -> Optional[Tuple[str, str]]: + """Report the most severe level across turns, ERROR before WARNING.""" + statuses = [status for status in map(get_turn_status, turns) if status is not None] + for level in ("ERROR", "WARNING"): + for status in statuses: + if status[0] == level: + return status + return None + def get_turn_end_timestamp(turn: Turn) -> Optional[datetime]: last_assistant_timestamp = parse_timestamp(turn.assistant_msgs[-1]) if turn.assistant_msgs else None + interrupt_timestamps = [ + parse_timestamp(row) for row in turn.rows if is_interrupted_turn_row(row) + ] candidate_end_timestamps = [ timestamp - for timestamp in [last_assistant_timestamp] + for timestamp in [last_assistant_timestamp, *interrupt_timestamps] if timestamp is not None ] for tool_result_entry in turn.tool_results_by_id.values(): @@ -2271,6 +3109,8 @@ def build_trace_metadata( value = turn.user_msg.get(src_key) if isinstance(value, str) and value: trace_metadata[dst_key] = value + if src_key == "cwd": + trace_metadata["project"] = Path(value).name return trace_metadata def is_valid_span_id_hex(span_id: Any) -> bool: @@ -2340,8 +3180,10 @@ def open_turn_root_span(langfuse: Langfuse, session_id: str, turn_num: int, turn trace id derived from seed and turn number; otherwise the session:user-row-uuid carrier pins the trace id. """ - user_text_raw = extract_text_from_content(get_content_from_row(turn.user_msg)) - user_text, user_text_meta = truncate_text(user_text_raw) + _, user_text_meta = truncate_text(extract_text_from_content(get_content_from_row(turn.user_msg))) + # The root keeps this turn's question only, so the trace list stays + # scannable; the conversation history lives on the LLM call inputs. + root_input = {"role": "user", "content": build_user_history_content(turn.user_msg)} trace_metadata = build_trace_metadata(session_id, turn_num, turn, transcript_path, user_text_meta) if parent_context is not None: parent_trace_id, parent_span_id = parent_context @@ -2355,8 +3197,9 @@ def open_turn_root_span(langfuse: Langfuse, session_id: str, turn_num: int, turn forced_trace_id=parent_trace_id, forced_parent_span_id=parent_span_id, as_root=False, - input={"role": "user", "content": user_text}, + input=root_input, metadata=trace_metadata, + **get_status_kwargs(get_turn_status(turn)), ) # Opt-in deterministic trace ids: fail open to the carrier-derived id. forced_trace_id: Optional[str] = None @@ -2373,8 +3216,9 @@ def open_turn_root_span(langfuse: Langfuse, session_id: str, turn_num: int, turn parent_otel_span=None if forced_trace_id else remote_parent(langfuse, session_id, turn.user_msg.get("uuid")), forced_trace_id=forced_trace_id, as_root=True, - input={"role": "user", "content": user_text}, + input=root_input, metadata=trace_metadata, + **get_status_kwargs(get_turn_status(turn)), ) @@ -2401,7 +3245,9 @@ def emit_turn(langfuse: Langfuse, session_id: str, turn_num: int, progress: Optional[Dict[str, Any]] = None, close: bool = True, trace_seed: Optional[str] = None, - parent_context: Optional[Tuple[str, str]] = None) -> Dict[str, Any]: + parent_context: Optional[Tuple[str, str]] = None, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, + history_prefix: Optional[List[Dict[str, Any]]] = None) -> Dict[str, Any]: """Emit a turn, resuming from prior firings' progress. With no progress and close=True this is the classic one-shot emission. @@ -2423,7 +3269,7 @@ def emit_turn(langfuse: Langfuse, session_id: str, turn_num: int, attribute_propagation = propagate_attributes( session_id=session_id, user_id=user_id, - trace_name=trace_display_name(session_id, turn_num), + trace_name=TRACE_NAME, tags=get_trace_tags(turn, subagent_transcripts_by_tool_use_id), ) else: @@ -2459,6 +3305,8 @@ def emit_turn(langfuse: Langfuse, session_id: str, turn_num: int, parse_timestamp(turn.user_msg), subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, cursor=cursor, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, + history_prefix=history_prefix, ) if trace_span is not None: # The root exports exactly once: end time and output are the @@ -2491,6 +3339,8 @@ def emit_and_close_ready_turns( subagent_transcripts_by_tool_use_id: Dict[str, Dict[str, Any]], trace_seed: Optional[str] = None, parent_context: Optional[Tuple[str, str]] = None, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, + session_history: Optional[SessionHistory] = None, ) -> int: emitted = 0 # Turns without a user-row uuid bypass assign_turn_numbers; seed their @@ -2521,6 +3371,12 @@ def emit_and_close_ready_turns( close=True, trace_seed=trace_seed, parent_context=parent_context, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, + history_prefix=( + session_history.prefix_for_turn(turn.user_msg.get("uuid")) + if session_history is not None + else None + ), ) except Exception as e: # Log at INFO so SDK incompatibilities (and other emit failures) @@ -2554,6 +3410,8 @@ def emit_ready_observations_of_open_turn( subagent_transcripts_by_tool_use_id: Dict[str, Dict[str, Any]], trace_seed: Optional[str] = None, parent_context: Optional[Tuple[str, str]] = None, + workflow_agent_transcripts_by_run_id: Optional[WorkflowAgentTranscriptsByRunId] = None, + session_history: Optional[SessionHistory] = None, ) -> None: """Emit the held open turn once its async activity is provably resolved. @@ -2597,6 +3455,12 @@ def emit_ready_observations_of_open_turn( close=False, trace_seed=trace_seed, parent_context=parent_context, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, + history_prefix=( + session_history.prefix_for_turn(user_row_uuid) + if session_history is not None + else None + ), ) session_state.turn_progress[user_row_uuid] = progress except Exception as e: @@ -2610,22 +3474,35 @@ def emit_new_turns_from_transcript( *, flush_deferred_agent_turns: bool = False, ) -> int: - with FileLock(LOCK_FILE): - state = load_hook_state() - key = get_session_state_key(session_id, str(transcript_path)) + key = get_session_state_key(session_id, str(transcript_path)) + + with FileLock(get_session_lock_path(key)): + with FileLock(LOCK_FILE): + state = load_hook_state() session_state = get_session_state(state, key) subagent_transcripts_by_tool_use_id = get_subagent_transcripts_by_tool_use_id(transcript_path) if subagent_transcripts_by_tool_use_id: debug(f"Discovered {len(subagent_transcripts_by_tool_use_id)} subagent transcript(s)") + workflow_agent_transcripts_by_run_id = WorkflowAgentTranscriptsByRunId(transcript_path) + turns, session_state = get_new_turns_from_transcript( transcript_path, session_state, subagent_transcripts_by_tool_use_id, flush_deferred_agent_turns=flush_deferred_agent_turns, + session_id=session_id, ) + # One full-file pass per firing serves every turn emitted below. + session_history = None + if turns or session_state.open_turn: + session_history = build_session_history( + transcript_path, + get_task_id_to_tool_use_id(subagent_transcripts_by_tool_use_id), + ) + emitted = 0 if turns: turns_to_emit = get_turns_to_emit( @@ -2643,6 +3520,8 @@ def emit_new_turns_from_transcript( subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, trace_seed=config.trace_seed, parent_context=config.parent_context, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, + session_history=session_history, ) session_state.turn_count += emitted @@ -2660,13 +3539,17 @@ def emit_new_turns_from_transcript( subagent_transcripts_by_tool_use_id=subagent_transcripts_by_tool_use_id, trace_seed=config.trace_seed, parent_context=config.parent_context, + workflow_agent_transcripts_by_run_id=workflow_agent_transcripts_by_run_id, + session_history=session_history, ) # Known limitation (accepted, like the crash-between-emit-and-save # duplicate window): progress is persisted before the SDK flush in # main(); a dropped flush leaves emitted_keys pointing at spans that # never reached the server. - save_session_state(state, key, session_state) + with FileLock(LOCK_FILE): + state = load_hook_state() + save_session_state(state, key, session_state) return emitted @@ -2697,9 +3580,14 @@ def main() -> int: start = time.time() debug("Hook started") + if _STATE_DIR_WARNING: + info(_STATE_DIR_WARNING) + if _OPERATOR_TAGS_WARNING: + info(_OPERATOR_TAGS_WARNING) + config = get_langfuse_config() if config is None: - debug("No LANGFUSE_PUBLIC_KEY/SECRET_KEY in environment; nothing to do") + log_missing_langfuse_config() return 0 payload = read_hook_payload() @@ -2728,7 +3616,7 @@ def main() -> int: return 0 except TimeoutError as e: - debug(f"lock timeout, skipping: {e}") + info(f"lock timeout, skipping: {e}") return 0 except Exception as e: diff --git a/tests/conftest.py b/tests/conftest.py index b6420f0..c6ceff1 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,6 +6,7 @@ import importlib.util import json import logging +import os import sys import types from pathlib import Path @@ -17,6 +18,18 @@ REPO_ROOT = Path(__file__).resolve().parents[1] FIXTURE_ROOT = REPO_ROOT / "tests" / "fixtures" / "transcripts" +# Remove developer environment values that would leak into the hook's +# import-time config. Tests that need these variables set them per test. +os.environ.pop("CC_LANGFUSE_STATE_DIR", None) +os.environ.pop("CLAUDE_PLUGIN_OPTION_CC_LANGFUSE_STATE_DIR", None) +os.environ.pop("CC_LANGFUSE_CAPTURE_IMAGES", None) +os.environ.pop("CLAUDE_PLUGIN_OPTION_CC_LANGFUSE_CAPTURE_IMAGES", None) + +# Mirrors OTel's active-span context: the stubbed use_span pushes the parent +# span here so FakeOtelSpan can record which span it was started under, +# letting tests assert observation nesting. +_ACTIVE_SPAN_STACK: list[Any] = [] + def _install_langfuse_stubs() -> None: langfuse_module = types.ModuleType("langfuse") @@ -37,12 +50,38 @@ def propagate_attributes(**_: Any) -> Iterator[None]: langfuse_module.propagate_attributes = propagate_attributes sys.modules["langfuse"] = langfuse_module + media_module = types.ModuleType("langfuse.media") + + class LangfuseMedia: + # Keyword-only with the real SDK constructor's parameters, so + # signature drift in the hook fails the tests, not production. + def __init__( + self, + *, + obj: Any = None, + base64_data_uri: str | None = None, + content_type: str | None = None, + content_bytes: bytes | None = None, + file_path: str | None = None, + ) -> None: + self.base64_data_uri = base64_data_uri + self.content_type = content_type + self.content_bytes = content_bytes + + media_module.LangfuseMedia = LangfuseMedia + langfuse_module.media = media_module + sys.modules["langfuse.media"] = media_module + opentelemetry_module = types.ModuleType("opentelemetry") trace_module = types.ModuleType("opentelemetry.trace") @contextlib.contextmanager - def use_span(*_: Any, **__: Any) -> Iterator[None]: - yield + def use_span(span: Any = None, *_: Any, **__: Any) -> Iterator[None]: + _ACTIVE_SPAN_STACK.append(span) + try: + yield + finally: + _ACTIVE_SPAN_STACK.pop() class TraceFlags(int): SAMPLED = 0x01 @@ -110,6 +149,9 @@ def __init__(self, name: str, start_time: int | None, context: Any = None) -> No self.start_time = start_time self.context = context self.attributes: dict[str, Any] = {} + # The span this one was started under (via the stubbed use_span); + # None for roots and carrier-context spans. + self.parent = _ACTIVE_SPAN_STACK[-1] if _ACTIVE_SPAN_STACK else None def set_attribute(self, key: str, value: Any) -> None: self.attributes[key] = value diff --git a/tests/fixtures/transcripts/human_wait_question/transcript.jsonl b/tests/fixtures/transcripts/human_wait_question/transcript.jsonl new file mode 100644 index 0000000..ccb5d71 --- /dev/null +++ b/tests/fixtures/transcripts/human_wait_question/transcript.jsonl @@ -0,0 +1,4 @@ +{"type": "user", "timestamp": "2026-01-01T00:01:00.000Z", "sessionId": "session-wait", "uuid": "user-wait-1", "cwd": "/repo", "gitBranch": "feature/test", "message": {"role": "user", "content": "Which backend should the cache use?"}} +{"type": "assistant", "timestamp": "2026-01-01T00:01:08.000Z", "sessionId": "session-wait", "uuid": "assistant-wait-1", "requestId": "req-wait-1", "message": {"id": "msg-wait-1", "role": "assistant", "model": "claude-test", "usage": {"input_tokens": 1840, "output_tokens": 1742}, "content": [{"type": "text", "text": "Both options work; the answer depends on durability."}, {"type": "tool_use", "id": "toolu_ask_1", "name": "AskUserQuestion", "input": {"questions": [{"question": "Which backend?", "header": "Backend", "multiSelect": false, "options": [{"label": "Redis", "description": "No durability."}, {"label": "ClickHouse", "description": "Durable."}]}]}}]}} +{"type": "user", "timestamp": "2026-01-01T03:01:08.000Z", "sessionId": "session-wait", "uuid": "tool-result-ask-1", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_ask_1", "content": "Backend: ClickHouse"}]}} +{"type": "assistant", "timestamp": "2026-01-01T03:01:14.000Z", "sessionId": "session-wait", "uuid": "assistant-wait-2", "requestId": "req-wait-2", "message": {"id": "msg-wait-2", "role": "assistant", "model": "claude-test", "usage": {"input_tokens": 2105, "output_tokens": 240}, "content": [{"type": "text", "text": "ClickHouse it is."}]}} diff --git a/tests/fixtures/transcripts/observation_levels/transcript.jsonl b/tests/fixtures/transcripts/observation_levels/transcript.jsonl new file mode 100644 index 0000000..d053316 --- /dev/null +++ b/tests/fixtures/transcripts/observation_levels/transcript.jsonl @@ -0,0 +1,12 @@ +{"type": "user", "timestamp": "2026-01-01T00:01:00.000Z", "sessionId": "session-levels", "uuid": "u1", "cwd": "/repo", "gitBranch": "main", "message": {"role": "user", "content": "Run three commands."}} +{"type": "assistant", "timestamp": "2026-01-01T00:01:01.000Z", "sessionId": "session-levels", "uuid": "a1", "requestId": "r1", "message": {"id": "m1", "role": "assistant", "model": "claude-test", "content": [{"type": "tool_use", "id": "toolu_fail", "name": "Bash", "input": {"command": "cat /nope"}}, {"type": "tool_use", "id": "toolu_false", "name": "Bash", "input": {"command": "echo ok"}}, {"type": "tool_use", "id": "toolu_absent", "name": "Read", "input": {"file_path": "README.md"}}]}} +{"type": "user", "timestamp": "2026-01-01T00:01:02.000Z", "sessionId": "session-levels", "uuid": "tr1", "message": {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_fail", "content": "cat: /nope: No such file or directory", "is_error": true}, {"type": "tool_result", "tool_use_id": "toolu_false", "content": "ok", "is_error": false}, {"type": "tool_result", "tool_use_id": "toolu_absent", "content": "# Example"}]}} +{"type": "assistant", "timestamp": "2026-01-01T00:01:03.000Z", "sessionId": "session-levels", "uuid": "a2", "requestId": "r2", "message": {"id": "m2", "role": "assistant", "model": "claude-test", "content": [{"type": "text", "text": "One failed."}]}} +{"type": "user", "timestamp": "2026-01-01T00:02:00.000Z", "sessionId": "session-levels", "uuid": "u2", "cwd": "/repo", "gitBranch": "main", "message": {"role": "user", "content": "Start something slow."}} +{"type": "assistant", "timestamp": "2026-01-01T00:02:01.000Z", "sessionId": "session-levels", "uuid": "a3", "requestId": "r3", "message": {"id": "m3", "role": "assistant", "model": "claude-test", "content": [{"type": "text", "text": "Starting."}]}} +{"type": "user", "timestamp": "2026-01-01T00:02:02.000Z", "sessionId": "session-levels", "uuid": "int1", "cwd": "/repo", "gitBranch": "main", "userType": "external", "entrypoint": "cli", "message": {"role": "user", "content": [{"type": "text", "text": "[Request interrupted by user]"}]}} +{"type": "user", "timestamp": "2026-01-01T00:03:00.000Z", "sessionId": "session-levels", "uuid": "u3", "cwd": "/repo", "gitBranch": "main", "message": {"role": "user", "content": "Try again."}} +{"type": "assistant", "timestamp": "2026-01-01T00:03:01.000Z", "sessionId": "session-levels", "uuid": "a4", "requestId": "r4", "isApiErrorMessage": true, "message": {"id": "m4", "role": "assistant", "model": "", "content": [{"type": "text", "text": "API Error: 500 Internal Server Error"}]}} +{"type": "user", "timestamp": "2026-01-01T00:04:00.000Z", "sessionId": "session-levels", "uuid": "u4", "cwd": "/repo", "gitBranch": "main", "message": {"role": "user", "content": "And again."}} +{"type": "assistant", "timestamp": "2026-01-01T00:04:01.000Z", "sessionId": "session-levels", "uuid": "a5", "requestId": "r5", "isApiErrorMessage": true, "message": {"id": "m5", "role": "assistant", "model": "", "content": [{"type": "text", "text": "API Error: 529 Overloaded"}]}} +{"type": "assistant", "timestamp": "2026-01-01T00:04:02.000Z", "sessionId": "session-levels", "uuid": "a6", "requestId": "r6", "message": {"id": "m6", "role": "assistant", "model": "claude-test", "content": [{"type": "text", "text": "Recovered and done."}]}} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript.jsonl b/tests/fixtures/transcripts/workflow_subagents/transcript.jsonl new file mode 100644 index 0000000..5e5b8c0 --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript.jsonl @@ -0,0 +1,8 @@ +{"type":"user","timestamp":"2026-07-23T10:00:00.000Z","sessionId":"session-workflow","uuid":"user-workflow-1","cwd":"/repo","gitBranch":"main","origin":{"kind":"human"},"parentUuid":null,"permissionMode":"default","promptId":"prompt-workflow-1","promptSource":"sdk","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":false,"message":{"role":"user","content":"Run the two-agent verification workflow."}} +{"type":"assistant","timestamp":"2026-07-23T10:00:05.000Z","sessionId":"session-workflow","uuid":"assistant-workflow-1","parentUuid":"user-workflow-1","requestId":"req-workflow-1","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":false,"message":{"id":"msg-workflow-1","type":"message","role":"assistant","model":"claude-test","content":[{"type":"text","text":"Launching the verification workflow."},{"type":"tool_use","id":"toolu_workflow_launch","name":"Workflow","input":{"script":"phase('Verify')\nawait parallel([() => agent('Verify claim A'), () => agent('Verify claim B')])\n"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","usage":{"input_tokens":2,"cache_creation_input_tokens":450,"cache_read_input_tokens":12000,"output_tokens":320}}} +{"type":"user","timestamp":"2026-07-23T10:00:08.000Z","sessionId":"session-workflow","uuid":"tool-result-workflow-1","parentUuid":"assistant-workflow-1","sourceToolAssistantUUID":"assistant-workflow-1","promptId":"prompt-workflow-1","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":false,"message":{"role":"user","content":[{"tool_use_id":"toolu_workflow_launch","type":"tool_result","content":"Workflow launched in background. Task ID: wtest001\nSummary: Two-agent verification workflow\nRun ID: wf_test001\n\nYou will be notified when it completes. Use /workflows to watch live progress.","is_error":false}]},"toolUseResult":{"status":"async_launched","taskId":"wtest001","taskType":"local_workflow","workflowName":"verify-claims","runId":"wf_test001","summary":"Two-agent verification workflow","transcriptDir":"/repo/transcript/subagents/workflows/wf_test001","scriptPath":"/repo/transcript/workflows/scripts/verify-claims-wf_test001.js"}} +{"type":"assistant","timestamp":"2026-07-23T10:00:15.000Z","sessionId":"session-workflow","uuid":"assistant-workflow-1b","parentUuid":"tool-result-workflow-1","requestId":"req-workflow-1b","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":false,"message":{"id":"msg-workflow-1b","type":"message","role":"assistant","model":"claude-test","content":[{"type":"text","text":"The verification workflow is running in the background; I'll summarize the findings when it completes."}],"stop_reason":"end_turn","usage":{"input_tokens":2,"cache_creation_input_tokens":120,"cache_read_input_tokens":12450,"output_tokens":180}}} +{"type":"queue-operation","operation":"enqueue","timestamp":"2026-07-23T10:02:20.000Z","sessionId":"session-workflow","content":"\nwtest001\ntoolu_workflow_launch\n/repo/tasks/wtest001.output\ncompleted\nDynamic workflow \"Two-agent verification workflow\" completed\n{\"findings\":[{\"verdict\":\"claim A holds\"},{\"verdict\":\"claim B holds\"}]}\nPer-agent results: /repo/transcript/subagents/workflows/wf_test001/journal.jsonl — one {\"type\":\"result\",...} line per completed agent with its full return value.\nIf the result above is empty or unexpected, Read this file BEFORE diagnosing — do not assume agents returned non-empty results.\nTo re-run with edited post-processing: Workflow({scriptPath: '/repo/transcript/workflows/scripts/verify-claims-wf_test001.js', resumeFromRunId: 'wf_test001'}) — agents whose (prompt, opts) are unchanged replay from cache.\n2200011392132000\n"} +{"type":"queue-operation","operation":"dequeue","timestamp":"2026-07-23T10:02:20.050Z","sessionId":"session-workflow"} +{"parentUuid":"assistant-workflow-1b","isSidechain":false,"promptId":"prompt-workflow-2","type":"user","message":{"role":"user","content":"\nwtest001\ntoolu_workflow_launch\n/repo/tasks/wtest001.output\ncompleted\nDynamic workflow \"Two-agent verification workflow\" completed\n{\"findings\":[{\"verdict\":\"claim A holds\"},{\"verdict\":\"claim B holds\"}]}\nPer-agent results: /repo/transcript/subagents/workflows/wf_test001/journal.jsonl — one {\"type\":\"result\",...} line per completed agent with its full return value.\nIf the result above is empty or unexpected, Read this file BEFORE diagnosing — do not assume agents returned non-empty results.\nTo re-run with edited post-processing: Workflow({scriptPath: '/repo/transcript/workflows/scripts/verify-claims-wf_test001.js', resumeFromRunId: 'wf_test001'}) — agents whose (prompt, opts) are unchanged replay from cache.\n2200011392132000\n"},"uuid":"task-notification-workflow-1","timestamp":"2026-07-23T10:02:20.100Z","permissionMode":"default","origin":{"kind":"task-notification"},"promptSource":"sdk","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"type":"assistant","timestamp":"2026-07-23T10:02:35.000Z","sessionId":"session-workflow","uuid":"assistant-workflow-2","parentUuid":"task-notification-workflow-1","requestId":"req-workflow-2","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":false,"message":{"id":"msg-workflow-2","type":"message","role":"assistant","model":"claude-test","content":[{"type":"text","text":"Both workflow agents completed; claims A and B hold."}],"stop_reason":"end_turn","usage":{"input_tokens":2,"cache_creation_input_tokens":510,"cache_read_input_tokens":13500,"output_tokens":240,"speed":"standard"}}} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.jsonl b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.jsonl new file mode 100644 index 0000000..fa87a26 --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.jsonl @@ -0,0 +1,5 @@ +{"type":"user","timestamp":"2026-07-23T10:00:12.000Z","sessionId":"session-workflow","agentId":"r1","uuid":"wf-r1-user-1","parentUuid":null,"promptId":"prompt-wf-r1-1","cwd":"/repo","gitBranch":"main","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":true,"message":{"role":"user","content":"Verify claim A."}} +{"parentUuid":"wf-r1-user-1","isSidechain":true,"agentId":"r1","attachment":{"type":"deferred_tools_delta","addedNames":["TaskCreate","TaskGet","WebFetch","WebSearch"],"addedLines":["TaskCreate","TaskGet","WebFetch","WebSearch"],"removedNames":[],"readdedNames":[]},"type":"attachment","uuid":"wf-r1-attachment-1","timestamp":"2026-07-23T10:00:12.001Z","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"parentUuid":"wf-r1-attachment-1","isSidechain":true,"agentId":"r1","attachment":{"type":"skill_listing","content":"- review: Review a GitHub pull request\n- init: Initialize a new CLAUDE.md file with codebase documentation","skillCount":2,"isInitial":true,"names":["review","init"]},"type":"attachment","uuid":"wf-r1-attachment-2","timestamp":"2026-07-23T10:00:12.002Z","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"type":"assistant","timestamp":"2026-07-23T10:00:45.000Z","sessionId":"session-workflow","agentId":"r1","uuid":"wf-r1-assistant-1","parentUuid":"wf-r1-attachment-2","requestId":"req-wf-r1-1","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":true,"message":{"id":"msg-wf-r1-1","type":"message","role":"assistant","model":"claude-test","content":[{"type":"text","text":"Examining claim A against the sources."}],"stop_reason":"end_turn","usage":{"input_tokens":2,"cache_creation_input_tokens":260,"output_tokens":210}}} +{"type":"assistant","timestamp":"2026-07-23T10:01:30.000Z","sessionId":"session-workflow","agentId":"r1","uuid":"wf-r1-assistant-2","parentUuid":"wf-r1-assistant-1","requestId":"req-wf-r1-2","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":true,"message":{"id":"msg-wf-r1-2","type":"message","role":"assistant","model":"claude-test","content":[{"type":"text","text":"Claim A holds."}],"stop_reason":"end_turn","usage":{"input_tokens":2,"cache_creation_input_tokens":180,"cache_read_input_tokens":3200,"output_tokens":340}}} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.meta.json b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.meta.json new file mode 100644 index 0000000..1ba006a --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r1.meta.json @@ -0,0 +1 @@ +{"agentType":"workflow-subagent","spawnDepth":1} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.jsonl b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.jsonl new file mode 100644 index 0000000..755cc60 --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.jsonl @@ -0,0 +1,6 @@ +{"type":"user","timestamp":"2026-07-23T10:00:13.000Z","sessionId":"session-workflow","agentId":"r2","uuid":"wf-r2-user-1","parentUuid":null,"promptId":"prompt-wf-r2-1","cwd":"/repo","gitBranch":"main","entrypoint":"cli","userType":"external","version":"2.1.215","isSidechain":true,"message":{"role":"user","content":"Verify claim B."}} +{"parentUuid":"wf-r2-user-1","isSidechain":true,"agentId":"r2","message":{"model":"claude-test","id":"msg-wf-r2-1","type":"message","role":"assistant","content":[{"type":"text","text":"Claim B checks out against the sources; recording the structured verdict."}],"stop_reason":null,"stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":300,"cache_read_input_tokens":2800,"cache_creation":{"ephemeral_5m_input_tokens":300,"ephemeral_1h_input_tokens":0},"output_tokens":2,"service_tier":"standard","inference_geo":"global"},"diagnostics":null},"requestId":"req-wf-r2-1","attributionAgent":"workflow-subagent","type":"assistant","uuid":"wf-r2-assistant-1","timestamp":"2026-07-23T10:01:40.000Z","effort":"high","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"parentUuid":"wf-r2-assistant-1","isSidechain":true,"agentId":"r2","message":{"model":"claude-test","id":"msg-wf-r2-1","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_wf_r2_so_1","name":"StructuredOutput","input":{"input":"{\"claim\": \"B\", \"verdict\": \"holds\"}"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":300,"cache_read_input_tokens":2800,"output_tokens":316,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":300},"inference_geo":"global","iterations":[{"input_tokens":2,"output_tokens":316,"cache_read_input_tokens":2800,"cache_creation_input_tokens":300,"cache_creation":{"ephemeral_5m_input_tokens":300,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req-wf-r2-1","attributionAgent":"workflow-subagent","type":"assistant","uuid":"wf-r2-assistant-2","timestamp":"2026-07-23T10:01:45.000Z","effort":"high","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"parentUuid":"wf-r2-assistant-2","isSidechain":true,"promptId":"prompt-wf-r2-1","agentId":"r2","type":"user","message":{"role":"user","content":[{"type":"tool_result","content":"Output does not match required schema: root: must have required property 'verdict'","is_error":true,"tool_use_id":"toolu_wf_r2_so_1"}]},"uuid":"wf-r2-tool-result-1","timestamp":"2026-07-23T10:01:47.000Z","toolUseResult":"Error: Output does not match required schema: root: must have required property 'verdict'","sourceToolAssistantUUID":"wf-r2-assistant-2","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"parentUuid":"wf-r2-tool-result-1","isSidechain":true,"agentId":"r2","message":{"model":"claude-test","id":"msg-wf-r2-2","type":"message","role":"assistant","content":[{"type":"tool_use","id":"toolu_wf_r2_so_2","name":"StructuredOutput","input":{"verdict":"claim B holds","confidence":"high"},"caller":{"type":"direct"}}],"stop_reason":"tool_use","stop_sequence":null,"stop_details":null,"usage":{"input_tokens":2,"cache_creation_input_tokens":180,"cache_read_input_tokens":3100,"output_tokens":273,"server_tool_use":{"web_search_requests":0,"web_fetch_requests":0},"service_tier":"standard","cache_creation":{"ephemeral_1h_input_tokens":0,"ephemeral_5m_input_tokens":180},"inference_geo":"global","iterations":[{"input_tokens":2,"output_tokens":273,"cache_read_input_tokens":3100,"cache_creation_input_tokens":180,"cache_creation":{"ephemeral_5m_input_tokens":180,"ephemeral_1h_input_tokens":0},"type":"message"}],"speed":"standard"},"diagnostics":null},"requestId":"req-wf-r2-2","attributionAgent":"workflow-subagent","type":"assistant","uuid":"wf-r2-assistant-3","timestamp":"2026-07-23T10:01:52.000Z","effort":"high","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} +{"parentUuid":"wf-r2-assistant-3","isSidechain":true,"promptId":"prompt-wf-r2-1","agentId":"r2","type":"user","message":{"role":"user","content":[{"tool_use_id":"toolu_wf_r2_so_2","type":"tool_result","content":"Structured output provided successfully"}]},"uuid":"wf-r2-tool-result-2","timestamp":"2026-07-23T10:01:55.000Z","toolEndsTurn":true,"sourceToolAssistantUUID":"wf-r2-assistant-3","userType":"external","entrypoint":"cli","cwd":"/repo","sessionId":"session-workflow","version":"2.1.215","gitBranch":"main"} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.meta.json b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.meta.json new file mode 100644 index 0000000..1ba006a --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/agent-r2.meta.json @@ -0,0 +1 @@ +{"agentType":"workflow-subagent","spawnDepth":1} diff --git a/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/journal.jsonl b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/journal.jsonl new file mode 100644 index 0000000..7c0a7c9 --- /dev/null +++ b/tests/fixtures/transcripts/workflow_subagents/transcript/subagents/workflows/wf_test001/journal.jsonl @@ -0,0 +1,5 @@ +{"type":"started","key":"v2:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","agentId":"r1"} +{"type":"started","key":"v2:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","agentId":"r2"} +{"type":"started","key":"v2:c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3c3","agentId":"r3"} +{"type":"result","key":"v2:a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1a1","agentId":"r1","result":{"verdict":"claim A holds","confidence":"high"}} +{"type":"result","key":"v2:b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2b2","agentId":"r2","result":{"verdict":"claim B holds","confidence":"high"}} diff --git a/tests/integration/test_emit_new_turns_from_transcript.py b/tests/integration/test_emit_new_turns_from_transcript.py index 49b1617..3ae7f6a 100644 --- a/tests/integration/test_emit_new_turns_from_transcript.py +++ b/tests/integration/test_emit_new_turns_from_transcript.py @@ -140,6 +140,91 @@ def roots(): assert len(roots()) == 1 +def test_named_teammate_transcript_attaches_to_its_launch_at_session_end_flush( + hook_module, + fake_langfuse, + isolated_hook_state, + tmp_path, +): + """A teammate meta has no toolUseId. The launch input.name + attributes its transcript when the deferred turn flushes at SessionEnd.""" + transcript = tmp_path / "transcript.jsonl" + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + # Meta and launch shapes below are harvested from a real 2.1.238 team session. + (subagent_dir / "agent-abuilder-issue-4-03b96faf.meta.json").write_text( + json.dumps({ + "agentType": "builder-issue-4", + "description": "Build issue 4", + "name": "builder-issue-4", + "spawnDepth": 0, + "model": "sonnet", + "taskKind": "in_process_teammate", + "teamName": "session-test", + "color": "blue", + "planModeRequired": False, + "permissionMode": "default", + }), + encoding="utf-8", + ) + (subagent_dir / "agent-abuilder-issue-4-03b96faf.jsonl").write_text( + "\n".join(json.dumps(row) for row in [ + {"type": "user", "uuid": "builder-user-1", "timestamp": "2026-01-01T00:00:03.000Z", + "message": {"role": "user", "content": "Build the fix."}}, + {"type": "assistant", "uuid": "builder-assistant-1", "timestamp": "2026-01-01T00:00:06.000Z", + "message": {"id": "msg-builder-1", "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": "Patch ready."}], + "usage": {"input_tokens": 12, "output_tokens": 34}}}, + ]) + "\n", + encoding="utf-8", + ) + launch_rows = [ + {"type": "user", "uuid": "user-1", "timestamp": "2026-01-01T00:00:00.000Z", + "sessionId": "session-teammate", "message": {"role": "user", "content": "Start a teammate."}}, + {"type": "assistant", "uuid": "assistant-1", "timestamp": "2026-01-01T00:00:01.000Z", + "message": {"id": "msg-1", "role": "assistant", "model": "claude-test", + "content": [{"type": "tool_use", "id": "toolu_builder", "name": "Agent", + "input": {"name": "builder-issue-4", + "taskKind": "in_process_teammate", + "prompt": "Build the fix."}}]}}, + {"type": "user", "uuid": "tool-result-1", "timestamp": "2026-01-01T00:00:02.000Z", + "toolUseResult": {"status": "teammate_spawned", "name": "builder-issue-4", + "agent_id": "builder-issue-4@session-test", + "teammate_id": "builder-issue-4@session-test", + "agent_type": "general-purpose", "team_name": "session-test"}, + "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_builder", + "content": [{"type": "text", "text": "Spawned successfully. agent_id: builder-issue-4@session-test"}]}]}}, + {"type": "assistant", "uuid": "assistant-2", "timestamp": "2026-01-01T00:00:02.500Z", + "message": {"id": "msg-2", "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": "The teammate is running."}]}}, + ] + transcript.write_text("\n".join(json.dumps(row) for row in launch_rows) + "\n", encoding="utf-8") + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + # First Stop: a teammate sends no task-notification. The hook emits nothing. + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-teammate", transcript) + assert fake_langfuse.observations == [] + + # SessionEnd: the hook flushes the deferred turn with the teammate transcript. + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-teammate", transcript, flush_deferred_agent_turns=True, + ) + + names = [observation.name for observation in fake_langfuse.observations] + assert "Conversational Turn" in names + assert "Tool: Agent" in names + assert "Subagent: Build issue 4" in names + teammate_span = next(o for o in fake_langfuse.observations if o.name == "Subagent: Build issue 4") + # Real teammate metas set agentType to the teammate name (config/launch + # result say "general-purpose"; the meta does not). + assert teammate_span.kwargs["metadata"]["agent_type"] == "builder-issue-4" + assert teammate_span.output == {"role": "assistant", "content": "Patch ready."} + # The trace contains the teammate tokens. + teammate_generation = next(o for o in fake_langfuse.observations if o.name == "Subagent LLM Call") + assert teammate_generation.kwargs["usage_details"] == {"input": 12, "output": 34} + + def test_only_the_turn_root_span_is_marked_as_root( hook_module, fixture_transcript_path, diff --git a/tests/integration/test_emit_turn_observations.py b/tests/integration/test_emit_turn_observations.py index a033872..219d1ca 100644 --- a/tests/integration/test_emit_turn_observations.py +++ b/tests/integration/test_emit_turn_observations.py @@ -22,10 +22,10 @@ def test_emit_turn_observations_creates_generation_tool_and_subagent_observation ) names = [observation.name for observation in fake_langfuse.observations] - assert "LLM Call 1" in names + assert "LLM Call" in names assert "Tool: Agent" in names assert "Subagent: Summarize docs" in names - assert "Subagent LLM Call 1" in names + assert "Subagent LLM Call" in names assert "Tool: ToolSearch" in names assert latest_end_timestamp.isoformat() == "2026-01-01T00:02:06+00:00" @@ -34,7 +34,7 @@ def test_emit_turn_observations_creates_generation_tool_and_subagent_observation assert "Async agent launched successfully." in agent_tool.output -def test_nested_subagents_document_current_non_recursive_emission_behavior( +def test_nested_subagents_emit_inner_agent_under_outer_agent( hook_module, fixture_transcript_path, read_fixture_jsonl, @@ -54,7 +54,83 @@ def test_nested_subagents_document_current_non_recursive_emission_behavior( subagent_transcripts_by_tool_use_id=subagents, ) - names = [observation.name for observation in fake_langfuse.observations] - assert "Subagent: Outer agent" in names - assert "Tool: Agent" in names - assert "Subagent: Inner agent" not in names + by_name = {observation.name: observation for observation in fake_langfuse.observations} + outer = by_name["Subagent: Outer agent"] + inner = by_name["Subagent: Inner agent"] + + ancestors = [] + span = inner._otel_span.parent + while span is not None: + ancestors.append(span) + span = span.parent + assert outer._otel_span in ancestors + assert any( + observation._otel_span.parent is inner._otel_span and observation.as_type == "generation" + for observation in fake_langfuse.observations + ) + + +# The fixture's human wait: the AskUserQuestion tool_use is written at +# 00:01:08 and its result only at 03:01:08, when the human answered. +WAIT_TURN_START = "2026-01-01T00:01:00.000Z" +WAIT_QUESTION_ASKED = "2026-01-01T00:01:08.000Z" +WAIT_ANSWER_ARRIVED = "2026-01-01T03:01:08.000Z" +WAIT_TURN_END = "2026-01-01T03:01:14.000Z" + + +def test_a_human_wait_on_a_tool_stays_off_the_generation_that_called_it( + hook_module, + fixture_transcript_path, + read_fixture_jsonl, + fake_langfuse, +): + """A generation must not absorb the wait of a tool it launched. + + The tool result timestamp is when the tool finished, which for + AskUserQuestion is when the human answered. Ending the generation there + reported three hours of model latency for 1742 output tokens. + """ + def to_ns(iso_timestamp: str) -> int: + return hook_module.to_otel_nanoseconds(hook_module.parse_timestamp(iso_timestamp)) + + transcript = fixture_transcript_path("human_wait_question") + rows = read_fixture_jsonl(transcript) + turns = hook_module.build_turns(rows, {}) + + latest_end_timestamp = hook_module.emit_turn_observations( + fake_langfuse, + fake_langfuse._otel_tracer.start_span(name="parent", start_time=None), + turns[0], + hook_module.parse_timestamp(turns[0].user_msg), + ) + + generations = [o for o in fake_langfuse.observations if o.as_type == "generation"] + asking_generation, answering_generation = generations + question_tool = next( + o for o in fake_langfuse.observations if o.name == "Tool: AskUserQuestion" + ) + + # The generation that asked ends at its own assistant row, so its + # duration stays the eight seconds the model took. + assert asking_generation.kwargs["usage_details"]["output"] == 1742 + assert asking_generation._otel_span.start_time == to_ns(WAIT_TURN_START) + assert asking_generation.end_time == to_ns(WAIT_QUESTION_ASKED) + + # The three-hour wait belongs to the tool alone. + assert question_tool._otel_span.start_time == to_ns(WAIT_QUESTION_ASKED) + assert question_tool.end_time == to_ns(WAIT_ANSWER_ARRIVED) + + # No generation overlaps the wait interval. + assert all( + generation.end_time <= to_ns(WAIT_QUESTION_ASKED) + or generation._otel_span.start_time >= to_ns(WAIT_ANSWER_ARRIVED) + for generation in generations + ) + + # The answer is the input of the next generation, which starts when the + # answer arrived. + assert answering_generation._otel_span.start_time == to_ns(WAIT_ANSWER_ARRIVED) + assert answering_generation.end_time == to_ns(WAIT_TURN_END) + + # The turn span still covers the whole turn, wait included. + assert latest_end_timestamp == hook_module.parse_timestamp(WAIT_TURN_END) diff --git a/tests/integration/test_forked_session_history.py b/tests/integration/test_forked_session_history.py new file mode 100644 index 0000000..9a48a02 --- /dev/null +++ b/tests/integration/test_forked_session_history.py @@ -0,0 +1,251 @@ +from __future__ import annotations + +import hashlib +import json +import shutil +from pathlib import Path +from typing import Any + +ORIGINAL_SESSION = "aaaaaaaa-1111-1111-1111-111111111111" +FORK_SESSION = "bbbbbbbb-2222-2222-2222-222222222222" + + +def turn_rows(session_id: str, index: int) -> list[dict[str, Any]]: + return [ + { + "type": "user", + "timestamp": f"2026-08-11T14:4{index}:00.000Z", + "sessionId": session_id, + "uuid": f"user-row-{index}", + "message": {"role": "user", "content": f"Question {index}?"}, + }, + { + "type": "assistant", + "timestamp": f"2026-08-11T14:4{index}:05.000Z", + "sessionId": session_id, + "uuid": f"assistant-row-{index}", + "message": { + "id": f"msg-{index}", + "role": "assistant", + "model": "claude-test", + "content": [{"type": "text", "text": f"Answer {index}."}], + "usage": {"input_tokens": 2, "output_tokens": 540}, + }, + }, + ] + + +def write_rows(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def append_rows(path: Path, rows: list[dict[str, Any]]) -> None: + with path.open("a", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + + +def write_original_transcript(tmp_path: Path, turns: int = 3) -> Path: + """A transcript named after its session, holding `turns` complete turns.""" + transcript = tmp_path / f"{ORIGINAL_SESSION}.jsonl" + rows: list[dict[str, Any]] = [] + for index in range(1, turns + 1): + rows.extend(turn_rows(ORIGINAL_SESSION, index)) + write_rows(transcript, rows) + return transcript + + +def fork_transcript(original: Path, tmp_path: Path) -> Path: + """Copy the history verbatim into a file named after the new session id. + + Mirrors what --fork-session writes: the copied rows keep the original + session id, only the file name carries the new one. + """ + fork = tmp_path / f"{FORK_SESSION}.jsonl" + shutil.copyfile(original, fork) + return fork + + +def config_for(hook_module: Any, trace_seed: str | None = None) -> Any: + return hook_module.LangfuseConfig( + "public", "secret", "https://example.test", "user-1", trace_seed=trace_seed + ) + + +def roots(fake_langfuse: Any) -> list[Any]: + return [o for o in fake_langfuse.observations if o.name == "Conversational Turn"] + + +def generations(fake_langfuse: Any) -> list[Any]: + return [o for o in fake_langfuse.observations if o.as_type == "generation"] + + +def turn_numbers(fake_langfuse: Any) -> list[int]: + return [(o.kwargs.get("metadata") or {}).get("turn_number") for o in roots(fake_langfuse)] + + +def test_forked_session_does_not_re_export_the_copied_history( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + original = write_original_transcript(tmp_path) + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, ORIGINAL_SESSION, original + ) + after_original = len(fake_langfuse.observations) + assert len(generations(fake_langfuse)) == 3 + + # The fork copies the history and the user asks one more question in it. + fork = fork_transcript(original, tmp_path) + append_rows(fork, turn_rows(FORK_SESSION, 4)) + append_rows(fork, turn_rows(FORK_SESSION, 5)) + + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, FORK_SESSION, fork) + + # Only the two turns the fork itself produced are exported; the three + # copied ones were already exported by the session they came from. + fork_observations = fake_langfuse.observations[after_original:] + fork_roots = [o for o in fork_observations if o.name == "Conversational Turn"] + assert len(fork_roots) == 2 + assert len(generations(fake_langfuse)) == 5 + + +def test_forked_turns_continue_the_turn_numbering_of_the_copied_history( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + original = write_original_transcript(tmp_path) + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, ORIGINAL_SESSION, original + ) + assert turn_numbers(fake_langfuse) == [1, 2, 3] + + fork = fork_transcript(original, tmp_path) + append_rows(fork, turn_rows(FORK_SESSION, 4)) + append_rows(fork, turn_rows(FORK_SESSION, 5)) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, FORK_SESSION, fork) + + # Restarting at 1 would collide with the original turns under + # CC_LANGFUSE_TRACE_SEED, whose trace ids derive from the turn number. + assert turn_numbers(fake_langfuse) == [1, 2, 3, 4, 5] + + +def test_seeded_fork_turns_get_trace_ids_of_their_own( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + seed = "deployment-seed" + original = write_original_transcript(tmp_path) + config = config_for(hook_module, trace_seed=seed) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, ORIGINAL_SESSION, original + ) + original_trace_ids = { + o._otel_span.context["current_span"].get_span_context().trace_id + for o in roots(fake_langfuse) + if o._otel_span.context + } + + fork = fork_transcript(original, tmp_path) + append_rows(fork, turn_rows(FORK_SESSION, 4)) + append_rows(fork, turn_rows(FORK_SESSION, 5)) + before = len(fake_langfuse.observations) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, FORK_SESSION, fork) + + fork_trace_ids = { + o._otel_span.context["current_span"].get_span_context().trace_id + for o in fake_langfuse.observations[before:] + if o.name == "Conversational Turn" and o._otel_span.context + } + assert fork_trace_ids + assert not (fork_trace_ids & original_trace_ids) + expected = int(hashlib.sha256(f"{seed}:4".encode("utf-8")).hexdigest()[:32], 16) + assert expected in fork_trace_ids + + +def test_plain_session_keeps_every_row( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # A session whose rows carry its own id must be untouched by the check. + original = write_original_transcript(tmp_path) + config = config_for(hook_module) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, ORIGINAL_SESSION, original + ) + + assert len(generations(fake_langfuse)) == 3 + assert turn_numbers(fake_langfuse) == [1, 2, 3] + + +def test_rows_without_a_session_id_are_kept( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{ORIGINAL_SESSION}.jsonl" + rows = turn_rows(ORIGINAL_SESSION, 1) + for row in rows: + del row["sessionId"] + write_rows(transcript, rows) + write_rows(transcript, rows + turn_rows(ORIGINAL_SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), ORIGINAL_SESSION, transcript + ) + + assert len(generations(fake_langfuse)) == 2 + + +def test_transcript_not_named_after_the_session_is_left_alone( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # If the payload id and the file name ever stop agreeing, exporting + # duplicates is better than silently exporting nothing at all. + transcript = tmp_path / "some-other-name.jsonl" + write_rows(transcript, turn_rows(ORIGINAL_SESSION, 1) + turn_rows(ORIGINAL_SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), "a-different-session-id", transcript + ) + + assert len(generations(fake_langfuse)) == 2 + + +def test_fork_without_any_new_activity_exports_nothing( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + original = write_original_transcript(tmp_path) + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, ORIGINAL_SESSION, original + ) + before = len(fake_langfuse.observations) + + # A fork that was opened but never continued holds copied history only. + fork = fork_transcript(original, tmp_path) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, FORK_SESSION, fork) + + assert len(fake_langfuse.observations) == before + + +def test_fork_turns_see_the_copied_history_in_their_generation_input( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # The copied rows are never exported a second time, but the model did + # read them, so they must still appear in the input of the fork's turns. + original = write_original_transcript(tmp_path) + fork = fork_transcript(original, tmp_path) + append_rows(fork, turn_rows(FORK_SESSION, 4)) + append_rows(fork, turn_rows(FORK_SESSION, 5)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), FORK_SESSION, fork + ) + + inputs = [generation.kwargs["input"] for generation in generations(fake_langfuse)] + assert len(inputs) == 2 + # The fork's first turn opens with the copied question, not its own. + assert inputs[0][0] == {"role": "user", "content": "Question 1?"} + assert inputs[0][-1] == {"role": "user", "content": "Question 4?"} + assert len(inputs[0]) == 7 + # The next turn adds its predecessor, so the history keeps accumulating. + assert inputs[1][-1] == {"role": "user", "content": "Question 5?"} + assert len(inputs[1]) == 9 diff --git a/tests/integration/test_generation_history.py b/tests/integration/test_generation_history.py new file mode 100644 index 0000000..36b738f --- /dev/null +++ b/tests/integration/test_generation_history.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any + +SESSION = "cccccccc-3333-3333-3333-333333333333" + + +def user_row(session_id: str, index: int, text: str) -> dict[str, Any]: + return { + "type": "user", + "timestamp": f"2026-08-11T14:{index:02d}:00.000Z", + "sessionId": session_id, + "uuid": f"user-row-{index}", + "message": {"role": "user", "content": text}, + } + + +def assistant_row( + session_id: str, index: int, content: Any, *, part: int = 0 +) -> dict[str, Any]: + return { + "type": "assistant", + "timestamp": f"2026-08-11T14:{index:02d}:{10 + part}.000Z", + "sessionId": session_id, + "uuid": f"assistant-row-{index}-{part}", + "message": { + "id": f"msg-{index}-{part}", + "role": "assistant", + "model": "claude-test", + "content": content, + "usage": {"input_tokens": 2, "output_tokens": 540}, + }, + } + + +def user_row_with_image(session_id: str, index: int, text: str) -> dict[str, Any]: + """A user row with a pasted image, like Claude Code writes it.""" + return { + "type": "user", + "timestamp": f"2026-08-11T14:{index:02d}:00.000Z", + "sessionId": session_id, + "uuid": f"user-row-{index}", + "message": {"role": "user", "content": [ + {"type": "text", "text": text}, + {"type": "image", "source": { + "type": "base64", "media_type": "image/png", "data": "aGVsbG8="}}, + ]}, + } + + +def tool_result_row(session_id: str, index: int, tool_use_id: str, text: str) -> dict[str, Any]: + return { + "type": "user", + "timestamp": f"2026-08-11T14:{index:02d}:20.000Z", + "sessionId": session_id, + "uuid": f"tool-result-{tool_use_id}", + "message": { + "role": "user", + "content": [ + {"type": "tool_result", "tool_use_id": tool_use_id, "content": text} + ], + }, + } + + +def simple_turn_rows(session_id: str, index: int) -> list[dict[str, Any]]: + return [ + user_row(session_id, index, f"Question {index}?"), + assistant_row(session_id, index, [{"type": "text", "text": f"Answer {index}."}]), + ] + + +def tool_turn_rows(session_id: str, index: int) -> list[dict[str, Any]]: + """A turn with two generations: tool call first, then the answer.""" + tool_use_id = f"toolu-{index}" + return [ + user_row(session_id, index, f"Question {index}?"), + assistant_row( + session_id, index, + [{"type": "tool_use", "id": tool_use_id, "name": "Read", + "input": {"file_path": "README.md"}}], + part=0, + ), + tool_result_row(session_id, index, tool_use_id, "file content"), + assistant_row( + session_id, index, + [{"type": "text", "text": f"Answer {index}."}], + part=1, + ), + ] + + +def write_rows(path: Path, rows: list[dict[str, Any]]) -> None: + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + + +def append_rows(path: Path, rows: list[dict[str, Any]]) -> None: + with path.open("a", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + + +def config_for(hook_module: Any) -> Any: + return hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + +def generations(fake_langfuse: Any) -> list[Any]: + return [o for o in fake_langfuse.observations if o.as_type == "generation"] + + +def generation_inputs(fake_langfuse: Any) -> list[Any]: + return [g.kwargs.get("input") for g in generations(fake_langfuse)] + + +def test_generation_input_contains_prior_turns( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + rows: list[dict[str, Any]] = [] + for index in (1, 2, 3): + rows.extend(simple_turn_rows(SESSION, index)) + write_rows(transcript, rows) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + inputs = generation_inputs(fake_langfuse) + assert len(inputs) == 3 + # Turn 1 has no history: only its own user message. + assert inputs[0] == [{"role": "user", "content": "Question 1?"}] + # Turn 3 sees turns 1 and 2 in order, then its own user message. + third = inputs[2] + assert [m.get("role") for m in third] == ["user", "assistant", "user", "assistant", "user"] + assert third[0]["content"] == "Question 1?" + assert third[1]["content"] == "Answer 1." + assert third[-1] == {"role": "user", "content": "Question 3?"} + + +def test_generation_input_within_turn_contains_tool_steps( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, tool_turn_rows(SESSION, 1) + simple_turn_rows(SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + inputs = generation_inputs(fake_langfuse) + assert len(inputs) == 3 + # Second generation of turn 1: user, tool-call assistant, tool results. + second = inputs[1] + assert [m.get("role") for m in second] == ["user", "assistant", "tool"] + assert second[1]["tool_calls"] == [ + {"id": "toolu-1", "type": "function", "function": {"name": "Read"}} + ] + assert second[2]["tool_results"][0]["tool_use_id"] == "toolu-1" + # Turn 2 sees the full tool exchange of turn 1. + third = inputs[2] + assert [m.get("role") for m in third] == ["user", "assistant", "tool", "assistant", "user"] + + +def test_history_survives_separate_firings( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + rows = simple_turn_rows(SESSION, 1) + simple_turn_rows(SESSION, 2) + write_rows(transcript, rows) + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, SESSION, transcript) + + # The next firing reads only new bytes for emission, but the history + # comes from the whole file. + append_rows(transcript, simple_turn_rows(SESSION, 3)) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, SESSION, transcript) + + third = generation_inputs(fake_langfuse)[-1] + assert third[0] == {"role": "user", "content": "Question 1?"} + assert third[-1] == {"role": "user", "content": "Question 3?"} + assert len(third) == 5 + + +def test_root_span_input_stays_this_turns_question( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # The trace list must stay scannable, so the root shows the new question + # only. The conversation history belongs on the LLM call inputs. + transcript = tmp_path / f"{SESSION}.jsonl" + rows: list[dict[str, Any]] = [] + for index in (1, 2, 3): + rows.extend(simple_turn_rows(SESSION, index)) + write_rows(transcript, rows) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + roots = [o for o in fake_langfuse.observations if o.name == "Conversational Turn"] + assert [r.kwargs["input"] for r in roots] == [ + {"role": "user", "content": "Question 1?"}, + {"role": "user", "content": "Question 2?"}, + {"role": "user", "content": "Question 3?"}, + ] + for root in roots: + assert "history" not in root.kwargs["metadata"] + # The third turn's generation does carry the history. + assert len(generation_inputs(fake_langfuse)[2]) == 5 + + +def test_history_keeps_images_at_their_turn( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + rows: list[dict[str, Any]] = [ + user_row_with_image(SESSION, 1, "Look at this."), + assistant_row(SESSION, 1, [{"type": "text", "text": "Answer 1."}]), + ] + rows.extend(simple_turn_rows(SESSION, 2)) + write_rows(transcript, rows) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + # Turn 2's input repeats turn 1 - including its image, at turn 1's position. + second = generation_inputs(fake_langfuse)[1] + first_message_content = second[0]["content"] + assert isinstance(first_message_content, list) + # The text keeps its image marker, the media object follows it. + assert first_message_content[0].startswith("Look at this.") + assert "[image image/png" in first_message_content[0] + assert type(first_message_content[1]).__name__ == "LangfuseMedia" + # The root span of turn 1 shows the image too, but without the history. + # Its media object is a separate instance with the same content; Langfuse + # deduplicates uploads by content hash. + roots = [o for o in fake_langfuse.observations if o.name == "Conversational Turn"] + root_content = roots[0].kwargs["input"]["content"] + assert root_content[0] == first_message_content[0] + assert type(root_content[1]).__name__ == "LangfuseMedia" + assert root_content[1].base64_data_uri == first_message_content[1].base64_data_uri + assert roots[1].kwargs["input"] == {"role": "user", "content": "Question 2?"} + + +def test_history_has_no_length_limit( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # Long tool outputs used to be cut by a byte budget; nothing is dropped now. + transcript = tmp_path / f"{SESSION}.jsonl" + rows: list[dict[str, Any]] = [] + for index in (1, 2, 3, 4): + tool_use_id = f"toolu-{index}" + rows.extend([ + user_row(SESSION, index, f"Question {index}?"), + assistant_row(SESSION, index, [{"type": "tool_use", "id": tool_use_id, + "name": "Read", "input": {}}], part=0), + tool_result_row(SESSION, index, tool_use_id, "x" * 30000), + assistant_row(SESSION, index, [{"type": "text", "text": f"Answer {index}."}], part=1), + ]) + write_rows(transcript, rows) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + last_input = generation_inputs(fake_langfuse)[-1] + # 4 turns x 4 messages, minus the last turn's unfinished steps. + assert len(last_input) >= 13 + assert last_input[0] == {"role": "user", "content": "Question 1?"} + # Each tool result obeys CC_LANGFUSE_MAX_CHARS on its own, but the + # history keeps every one of them, so it grows past that budget. + tool_output_chars = sum( + len(result["output"]) + for message in last_input + if message.get("role") == "tool" + for result in message["tool_results"] + ) + assert tool_output_chars > hook_module.MAX_CHARS + # The reported message count is the real one, so nothing was dropped. + for generation in generations(fake_langfuse): + assert generation.kwargs["metadata"]["history"]["messages"] == len( + generation.kwargs["input"] + ) + + +def test_delta_inputs_when_the_history_cannot_be_rebuilt( + hook_module, fake_langfuse, isolated_hook_state, tmp_path, monkeypatch +): + # A transcript the history pass cannot read leaves no messages to attach. + # Emission then continues with the delta inputs. + monkeypatch.setattr(hook_module, "build_session_history", lambda *a, **kw: None) + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, simple_turn_rows(SESSION, 1) + simple_turn_rows(SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + inputs = generation_inputs(fake_langfuse) + assert inputs[0] == {"role": "user", "content": "Question 1?"} + assert inputs[1] == {"role": "user", "content": "Question 2?"} + for generation in generations(fake_langfuse): + assert "history" not in generation.kwargs["metadata"] + + +def test_subagent_generations_get_history_of_their_own_transcript( + hook_module, fake_langfuse, tmp_path +): + subagent_path = tmp_path / "agent.jsonl" + write_rows( + subagent_path, + simple_turn_rows("agent-session", 1) + simple_turn_rows("agent-session", 2), + ) + + hook_module.emit_subagent_observations( + fake_langfuse, + None, + {"path": subagent_path, "agent_type": "test", "description": "demo"}, + None, + ) + + inputs = generation_inputs(fake_langfuse) + assert len(inputs) == 2 + assert inputs[0] == [{"role": "user", "content": "Question 1?"}] + assert inputs[1][0] == {"role": "user", "content": "Question 1?"} + assert inputs[1][-1] == {"role": "user", "content": "Question 2?"} + + +def test_the_turn_with_the_image_sees_it_in_its_own_input( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # The turn that received the image must show it in its own generation + # input, in the same shape later turns see it in the history. + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, [ + user_row_with_image(SESSION, 1, "Look at this."), + assistant_row(SESSION, 1, [{"type": "text", "text": "Answer 1."}]), + ]) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + own_input = generation_inputs(fake_langfuse)[0] + content = own_input[0]["content"] + assert isinstance(content, list) + assert content[0].startswith("Look at this.") + assert type(content[1]).__name__ == "LangfuseMedia" + + +def test_generation_emitted_in_a_later_firing_carries_the_full_history( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + # A turn can span several Stop firings. The generation that ships in the + # second firing must see the steps that shipped in the first one. + transcript = tmp_path / f"{SESSION}.jsonl" + tool_use_id = "toolu-1" + write_rows(transcript, [ + user_row(SESSION, 1, "Question 1?"), + assistant_row(SESSION, 1, [{"type": "tool_use", "id": tool_use_id, "name": "Read", + "input": {"file_path": "README.md"}}], part=0), + tool_result_row(SESSION, 1, tool_use_id, "file content"), + ]) + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, SESSION, transcript) + first_firing = [g.kwargs["input"] for g in generations(fake_langfuse)] + assert first_firing == [[{"role": "user", "content": "Question 1?"}]] + + append_rows(transcript, [ + assistant_row(SESSION, 1, [{"type": "text", "text": "Answer 1."}], part=1), + ] + simple_turn_rows(SESSION, 2)) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, SESSION, transcript) + + inputs = generation_inputs(fake_langfuse) + # The first generation is not emitted again; the second one sees the + # tool exchange; turn 2 sees the whole of turn 1. + assert len(inputs) == 3 + assert [m["role"] for m in inputs[1]] == ["user", "assistant", "tool"] + assert [m["role"] for m in inputs[2]] == ["user", "assistant", "tool", "assistant", "user"] + + +def test_deferred_agent_turn_flushed_at_session_end_gets_history( + hook_module, fake_langfuse, isolated_hook_state, fixture_transcript_path +): + transcript = fixture_transcript_path("async_agent_deferred") + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-deferred", transcript) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-deferred", transcript, flush_deferred_agent_turns=True + ) + + inputs = generation_inputs(fake_langfuse) + assert inputs + for generation_input in inputs: + assert isinstance(generation_input, list) + assert generation_input[0]["role"] == "user" + + +def test_workflow_agent_generations_carry_history( + hook_module, fake_langfuse, isolated_hook_state, fixture_transcript_path +): + transcript = fixture_transcript_path("workflow_subagents") + config = config_for(hook_module) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow", transcript) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-workflow", transcript, flush_deferred_agent_turns=True + ) + + inputs = generation_inputs(fake_langfuse) + assert inputs + for generation_input in inputs: + assert isinstance(generation_input, list) + for generation in generations(fake_langfuse): + assert "history" in generation.kwargs["metadata"] + + +# ---- tool_calls shape: one producer, every path ---- + +def test_history_tool_calls_come_from_the_one_shape_builder( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + """Pins the history path to build_generation_output.""" + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, tool_turn_rows(SESSION, 1) + simple_turn_rows(SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + expected = hook_module.build_generation_output("", [{"id": "toolu-1", "name": "Read"}]) + live_generation = generation_inputs(fake_langfuse)[1][1] + later_turn_history = generation_inputs(fake_langfuse)[2][1] + + assert live_generation["tool_calls"] == expected["tool_calls"] + assert later_turn_history["tool_calls"] == expected["tool_calls"] + + +def test_every_tool_call_the_hook_emits_has_the_nested_keys( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, tool_turn_rows(SESSION, 1) + tool_turn_rows(SESSION, 2)) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + seen = 0 + for messages in generation_inputs(fake_langfuse): + for message in messages or []: + for tool_call in (message or {}).get("tool_calls", []): + assert set(tool_call) == {"id", "type", "function"} + assert tool_call["type"] == "function" + assert set(tool_call["function"]) == {"name"} + seen += 1 + assert seen > 0, "no tool_calls reached the history, the assertion proved nothing" + + +def test_a_tool_use_without_a_name_keeps_the_nested_shape_in_the_history( + hook_module, fake_langfuse, isolated_hook_state, tmp_path +): + transcript = tmp_path / f"{SESSION}.jsonl" + write_rows(transcript, [ + user_row(SESSION, 1, "Question 1?"), + assistant_row(SESSION, 1, [{"type": "tool_use", "id": "toolu-x"}], part=0), + tool_result_row(SESSION, 1, "toolu-x", "result"), + assistant_row(SESSION, 1, [{"type": "text", "text": "Answer 1."}], part=1), + *simple_turn_rows(SESSION, 2), + ]) + + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config_for(hook_module), SESSION, transcript + ) + + assert generation_inputs(fake_langfuse)[1][1]["tool_calls"] == [ + {"id": "toolu-x", "type": "function", "function": {"name": None}} + ] diff --git a/tests/integration/test_observation_levels.py b/tests/integration/test_observation_levels.py new file mode 100644 index 0000000..b96b9bf --- /dev/null +++ b/tests/integration/test_observation_levels.py @@ -0,0 +1,283 @@ +from __future__ import annotations + +import json + +import pytest + + +@pytest.fixture +def level_turns(hook_module, fixture_transcript_path, read_fixture_jsonl): + transcript = fixture_transcript_path("observation_levels") + rows = read_fixture_jsonl(transcript) + return hook_module.build_turns(rows, {}) + + +def emit(hook_module, fake_langfuse, turn): + parent_span = fake_langfuse._otel_tracer.start_span(name="parent", start_time=None) + hook_module.emit_turn_observations( + fake_langfuse, + parent_span, + turn, + hook_module.parse_timestamp(turn.user_msg), + ) + return fake_langfuse.observations + + +def tool_by_id(observations, tool_id): + return next( + observation + for observation in observations + if observation.as_type == "tool" and observation.kwargs["metadata"]["tool_id"] == tool_id + ) + + +def test_a_failed_tool_result_marks_its_tool_observation_as_error( + hook_module, fake_langfuse, level_turns +): + observations = emit(hook_module, fake_langfuse, level_turns[0]) + + failed = tool_by_id(observations, "toolu_fail") + assert failed.kwargs["level"] == "ERROR" + assert failed.kwargs["status_message"] == "cat: /nope: No such file or directory" + + +@pytest.mark.parametrize("tool_id", ["toolu_false", "toolu_absent"]) +def test_a_tool_result_without_an_error_flag_keeps_the_default_level( + hook_module, fake_langfuse, level_turns, tool_id +): + observations = emit(hook_module, fake_langfuse, level_turns[0]) + + succeeded = tool_by_id(observations, tool_id) + assert "level" not in succeeded.kwargs + assert "status_message" not in succeeded.kwargs + + +def test_an_interrupted_turn_marks_its_root_span_as_warning( + hook_module, fake_langfuse, level_turns, tmp_path +): + interrupted = level_turns[1] + + root = hook_module.open_turn_root_span( + fake_langfuse, "session-levels", 2, interrupted, tmp_path / "t.jsonl" + ) + + assert root.kwargs["level"] == "WARNING" + assert root.kwargs["status_message"] == "Turn interrupted by user" + + +def test_the_interrupt_marker_stays_in_the_turn_it_ended(level_turns): + # A turn of its own would carry no assistant row and be dropped, losing + # both the marker and the level it implies. + assert len(level_turns) == 4 + interrupted = level_turns[1] + assert interrupted.user_msg["uuid"] == "u2" + assert any(row["uuid"] == "int1" for row in interrupted.rows) + + +def test_an_interrupt_bounds_the_turn_it_ended(hook_module, level_turns): + end_timestamp = hook_module.get_turn_end_timestamp(level_turns[1]) + + assert end_timestamp.isoformat() == "2026-01-01T00:02:02+00:00" + + +def test_an_api_error_that_ends_the_turn_marks_the_turn_as_error( + hook_module, fake_langfuse, level_turns, tmp_path +): + root = hook_module.open_turn_root_span( + fake_langfuse, "session-levels", 3, level_turns[2], tmp_path / "t.jsonl" + ) + + assert root.kwargs["level"] == "ERROR" + assert root.kwargs["status_message"] == "API Error: 500 Internal Server Error" + + +def test_a_failed_api_call_marks_its_own_generation_as_error( + hook_module, fake_langfuse, level_turns +): + observations = emit(hook_module, fake_langfuse, level_turns[2]) + + generation = next( + observation for observation in observations if observation.as_type == "generation" + ) + assert generation.kwargs["level"] == "ERROR" + assert generation.kwargs["status_message"] == "API Error: 500 Internal Server Error" + + +def test_an_api_error_the_turn_recovered_from_leaves_the_turn_at_the_default_level( + hook_module, fake_langfuse, level_turns, tmp_path +): + recovered = level_turns[3] + + root = hook_module.open_turn_root_span( + fake_langfuse, "session-levels", 4, recovered, tmp_path / "t.jsonl" + ) + generations = [ + observation + for observation in emit(hook_module, fake_langfuse, recovered) + if observation.as_type == "generation" + ] + + assert "level" not in root.kwargs + assert [generation.kwargs.get("level") for generation in generations] == ["ERROR", None] + + +def test_an_error_that_ends_the_turn_outranks_an_interrupt(hook_module, level_turns): + interrupted = level_turns[1] + interrupted.assistant_msgs.append( + {"isApiErrorMessage": True, "message": {"role": "assistant", "content": "API Error: boom"}} + ) + + assert hook_module.get_turn_status(interrupted) == ("ERROR", "API Error: boom") + + +def test_a_status_message_keeps_the_whole_error_text(hook_module): + message = hook_module.build_status_message("x" * 5000, "fallback") + + assert message == "x" * 5000 + + +def test_a_status_message_falls_back_when_there_is_no_text(hook_module): + assert hook_module.build_status_message("", "fallback") == "fallback" + assert hook_module.build_status_message(None, "fallback") == "fallback" + + +def subagent_prompt(uuid, text, timestamp): + return { + "type": "user", + "agentId": "a0", + "sessionId": "session-sub", + "uuid": uuid, + "timestamp": timestamp, + "message": {"role": "user", "content": text}, + } + + +def subagent_answer(uuid, text, timestamp, api_error=False): + row = { + "type": "assistant", + "agentId": "a0", + "sessionId": "session-sub", + "uuid": uuid, + "requestId": f"req-{uuid}", + "timestamp": timestamp, + "message": { + "id": f"msg-{uuid}", + "role": "assistant", + "model": "" if api_error else "claude-test", + "content": [{"type": "text", "text": text}], + }, + } + if api_error: + row["isApiErrorMessage"] = True + return row + + +def subagent_interrupt(uuid, timestamp): + return { + "type": "user", + "agentId": "a0", + "sessionId": "session-sub", + "uuid": uuid, + "timestamp": timestamp, + "message": { + "role": "user", + "content": [{"type": "text", "text": "[Request interrupted by user]"}], + }, + } + + +def emit_subagent(hook_module, fake_langfuse, tmp_path, rows): + path = tmp_path / "agent-sub.jsonl" + path.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + parent_span = fake_langfuse._otel_tracer.start_span(name="parent", start_time=None) + hook_module.emit_subagent_observations( + fake_langfuse, + parent_span, + {"path": path, "description": "Check things", "agent_type": "general-purpose"}, + None, + ) + return next( + observation + for observation in fake_langfuse.observations + if observation.name == "Subagent: Check things" + ) + + +def test_an_interrupted_subagent_marks_its_own_span_as_warning( + hook_module, fake_langfuse, tmp_path +): + # The subagent answered before it was stopped, so the turn exists and the + # interrupt has somewhere to land. + span = emit_subagent( + hook_module, + fake_langfuse, + tmp_path, + [ + subagent_prompt("s1", "Check the repo.", "2026-01-01T00:00:00.000Z"), + subagent_answer("s2", "Looking now.", "2026-01-01T00:00:01.000Z"), + subagent_interrupt("s3", "2026-01-01T00:00:02.000Z"), + ], + ) + + assert span.kwargs["level"] == "WARNING" + assert span.kwargs["status_message"] == "Turn interrupted by user" + + +def test_a_failed_subagent_marks_its_own_span_as_error(hook_module, fake_langfuse, tmp_path): + span = emit_subagent( + hook_module, + fake_langfuse, + tmp_path, + [ + subagent_prompt("s1", "Check the repo.", "2026-01-01T00:00:00.000Z"), + subagent_answer("s2", "API Error: 500", "2026-01-01T00:00:01.000Z", api_error=True), + ], + ) + + assert span.kwargs["level"] == "ERROR" + assert span.kwargs["status_message"] == "API Error: 500" + + +def test_a_subagent_span_reports_its_worst_turn(hook_module, fake_langfuse, tmp_path): + span = emit_subagent( + hook_module, + fake_langfuse, + tmp_path, + [ + subagent_prompt("s1", "First job.", "2026-01-01T00:00:00.000Z"), + subagent_answer("s2", "Working.", "2026-01-01T00:00:01.000Z"), + subagent_interrupt("s3", "2026-01-01T00:00:02.000Z"), + subagent_prompt("s4", "Second job.", "2026-01-01T00:00:03.000Z"), + subagent_answer("s5", "API Error: 529", "2026-01-01T00:00:04.000Z", api_error=True), + ], + ) + + assert span.kwargs["level"] == "ERROR" + + +def test_a_healthy_subagent_span_keeps_the_default_level(hook_module, fake_langfuse, tmp_path): + span = emit_subagent( + hook_module, + fake_langfuse, + tmp_path, + [ + subagent_prompt("s1", "Check the repo.", "2026-01-01T00:00:00.000Z"), + subagent_answer("s2", "All good.", "2026-01-01T00:00:01.000Z"), + ], + ) + + assert "level" not in span.kwargs + + +def test_a_denied_tool_does_not_interrupt_the_whole_turn(hook_module): + denial = { + "type": "user", + "message": { + "role": "user", + "content": [ + {"type": "text", "text": "[Request interrupted by user for tool use]"} + ], + }, + } + + assert hook_module.is_interrupted_turn_row(denial) is False diff --git a/tests/integration/test_parent_trace_attachment.py b/tests/integration/test_parent_trace_attachment.py index 4b5f385..ee792fb 100644 --- a/tests/integration/test_parent_trace_attachment.py +++ b/tests/integration/test_parent_trace_attachment.py @@ -146,7 +146,8 @@ def recording_propagate(**kwargs: Any): fake_langfuse, standalone, "session-standalone", write_two_turn_transcript(standalone_dir) ) assert calls - assert all("trace_name" in call for call in calls) + assert all(call.get("trace_name") == "Claude Code Turn" for call in calls) + assert all(call.get("session_id") == "session-standalone" for call in calls) def test_attached_open_turn_resumes_without_reopening_its_root( diff --git a/tests/integration/test_state_lock_contention.py b/tests/integration/test_state_lock_contention.py new file mode 100644 index 0000000..899105b --- /dev/null +++ b/tests/integration/test_state_lock_contention.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +import io +import json +import os +from datetime import datetime, timedelta, timezone +from pathlib import Path +from typing import Any + +import pytest + + +def _now_iso() -> str: + return datetime.now(timezone.utc).isoformat() + + +def _run_emit(hook_module: Any, fake_langfuse: Any, session_id: str, transcript: Path) -> int: + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + return hook_module.emit_new_turns_from_transcript( + fake_langfuse, + config, + session_id, + transcript, + flush_deferred_agent_turns=True, + ) + + +def _during_emit(hook_module: Any, monkeypatch: pytest.MonkeyPatch, callback: Any) -> None: + """Run callback in the middle of the emit, where the old code held the lock.""" + original = hook_module.emit_and_close_ready_turns + + def wrapped(*args: Any, **kwargs: Any) -> int: + callback() + return original(*args, **kwargs) + + monkeypatch.setattr(hook_module, "emit_and_close_ready_turns", wrapped) + + +def test_emit_leaves_the_global_lock_free( + hook_module: Any, + fixture_transcript_path: Any, + fake_langfuse: Any, + isolated_hook_state: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The emit must not hold the global lock, or a second session is starved. + + The emit takes seconds on a large transcript. While the global lock covered + it, every other session waited for the whole emit, gave up and dropped its + turn without a trace in the log. + """ + acquired: list[bool] = [] + + def try_global_lock() -> None: + try: + with hook_module.FileLock(hook_module.LOCK_FILE, timeout_s=0.05): + acquired.append(True) + except TimeoutError: + acquired.append(False) + + _during_emit(hook_module, monkeypatch, try_global_lock) + + emitted = _run_emit(hook_module, fake_langfuse, "session-simple", fixture_transcript_path("simple_turn")) + + assert emitted == 1 + assert acquired == [True], "the global lock was held across the emit" + + +def test_the_session_lock_is_held_across_the_emit( + hook_module: Any, + fixture_transcript_path: Any, + fake_langfuse: Any, + isolated_hook_state: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """One session still may not emit twice at the same time. + + Narrowing the global lock must not drop that guarantee: two firings of the + same session would read one offset and emit the same turn twice. + """ + acquired: list[bool] = [] + key = hook_module.get_session_state_key("session-simple", str(fixture_transcript_path("simple_turn"))) + + def try_session_lock() -> None: + try: + with hook_module.FileLock(hook_module.get_session_lock_path(key), timeout_s=0.05): + acquired.append(True) + except TimeoutError: + acquired.append(False) + + _during_emit(hook_module, monkeypatch, try_session_lock) + + _run_emit(hook_module, fake_langfuse, "session-simple", fixture_transcript_path("simple_turn")) + + assert acquired == [False], "a second firing of the same session could emit in parallel" + + +def test_entry_of_another_session_survives_the_emit( + hook_module: Any, + fixture_transcript_path: Any, + fake_langfuse: Any, + isolated_hook_state: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The write must re-read the file, or it deletes a concurrent entry. + + The state file holds every session. A session that saves from the copy it + loaded before its emit writes back a file without the entries that other + sessions saved meanwhile. + """ + def other_session_saves() -> None: + with hook_module.FileLock(hook_module.LOCK_FILE): + state = hook_module.load_hook_state() + state["other-session-key"] = {"offset": 42, "updated": "2026-09-10T00:00:00+00:00"} + hook_module.save_hook_state(state) + + _during_emit(hook_module, monkeypatch, other_session_saves) + + _run_emit(hook_module, fake_langfuse, "session-simple", fixture_transcript_path("simple_turn")) + + state = json.loads((isolated_hook_state / "langfuse_state.json").read_text(encoding="utf-8")) + assert "other-session-key" in state, "the emit deleted the entry of a concurrent session" + assert len(state) == 2, "the session under test did not save its own entry" + + +def _age(path: Path, days: int) -> None: + stamp = (datetime.now(timezone.utc) - timedelta(days=days)).timestamp() + os.utime(path, (stamp, stamp)) + + +def test_a_stale_session_lock_file_is_swept(hook_module: Any, isolated_hook_state: Path) -> None: + """A run that fails before it saves leaves a lock file with no entry to prune it.""" + orphan = hook_module.get_session_lock_path("a" * 64) + orphan.parent.mkdir(parents=True, exist_ok=True) + orphan.touch() + _age(orphan, 365) + + hook_module.save_hook_state({}) + + assert not orphan.exists(), "orphaned lock files accumulate in the state directory forever" + + +def test_a_fresh_session_lock_file_is_never_swept( + hook_module: Any, isolated_hook_state: Path +) -> None: + """The sweep must not delete a lock a running session still holds. + + Deleting it lets the next run of that session create its own file and take a + second, independent lock, so the session would emit twice in parallel. + """ + running = hook_module.get_session_lock_path("b" * 64) + running.parent.mkdir(parents=True, exist_ok=True) + running.touch() + + hook_module.save_hook_state({}) + + assert running.exists(), "the sweep deleted a lock file that a live run may hold" + + +def test_the_sweep_keeps_the_lock_of_a_live_entry( + hook_module: Any, isolated_hook_state: Path +) -> None: + """An old lock file still belonging to a live state entry stays.""" + key = "c" * 64 + lock_path = hook_module.get_session_lock_path(key) + lock_path.parent.mkdir(parents=True, exist_ok=True) + lock_path.touch() + _age(lock_path, 365) + + hook_module.save_hook_state({key: {"offset": 1, "updated": _now_iso()}}) + + assert lock_path.exists(), "the sweep removed the lock file of a live session" + + +def test_the_sweep_leaves_the_global_lock_alone( + hook_module: Any, isolated_hook_state: Path +) -> None: + """The global lock file must never be swept.""" + global_lock = Path(hook_module.LOCK_FILE) + global_lock.parent.mkdir(parents=True, exist_ok=True) + global_lock.touch() + _age(global_lock, 365) + + hook_module.save_hook_state({}) + + assert global_lock.exists(), "the sweep deleted the global lock file" + + +def test_main_logs_a_lock_timeout_without_debug( + hook_module: Any, isolated_hook_state: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A skipped export must say so in the log under default settings. + + The symptom reaches the user as missing data in Langfuse, while the cause + is local. At debug level the skip left no record at all. + """ + monkeypatch.setattr(hook_module, "DEBUG", False) + monkeypatch.setattr(hook_module, "get_langfuse_config", lambda: hook_module.LangfuseConfig( + "public", "secret", "https://example.test", None + )) + monkeypatch.setattr(hook_module, "create_langfuse_client", lambda _config: object()) + monkeypatch.setattr(hook_module, "flush_and_shutdown_langfuse_client", lambda _client: None) + + def refuse(*_args: Any, **_kwargs: Any) -> int: + raise TimeoutError("could not acquire lock within 2.0s") + + monkeypatch.setattr(hook_module, "emit_new_turns_from_transcript", refuse) + + transcript = isolated_hook_state / "transcript.jsonl" + transcript.parent.mkdir(parents=True, exist_ok=True) + transcript.write_text("", encoding="utf-8") + payload = json.dumps( + {"session_id": "s", "transcript_path": str(transcript), "hook_event_name": "Stop"} + ) + monkeypatch.setattr("sys.stdin", io.StringIO(payload)) + + assert hook_module.main() == 0 + + log_file = Path(hook_module.LOG_FILE) + log = log_file.read_text(encoding="utf-8") if log_file.exists() else "" + assert "[INFO]" in log, "the skip left no log line: the cause stays invisible" + assert "lock timeout" in log diff --git a/tests/integration/test_workflow_subagent_emission.py b/tests/integration/test_workflow_subagent_emission.py new file mode 100644 index 0000000..9a78e7e --- /dev/null +++ b/tests/integration/test_workflow_subagent_emission.py @@ -0,0 +1,592 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any + + +def copy_workflow_fixture(fixture_transcript_path: Any, tmp_path: Path) -> tuple[Path, list[dict[str, Any]]]: + """Copy the workflow fixture's transcript directory (agent transcripts, + metas, journal) to tmp_path and return the parent rows for incremental + appends — the fixture file itself must stay untouched so the offset-based + incremental reader can be driven per firing.""" + source_transcript = fixture_transcript_path("workflow_subagents") + shutil.copytree(source_transcript.with_suffix(""), tmp_path / "transcript") + rows = [ + json.loads(line) + for line in source_transcript.read_text(encoding="utf-8").splitlines() + if line.strip() + ] + return tmp_path / "transcript.jsonl", rows + + +def append_rows(transcript: Path, rows: list[dict[str, Any]]) -> None: + with transcript.open("a", encoding="utf-8") as f: + for row in rows: + f.write(json.dumps(row) + "\n") + + +def ts_ns(hook_module: Any, iso_timestamp: str) -> int: + return hook_module.to_otel_nanoseconds(hook_module.parse_timestamp(iso_timestamp)) + + +def record_scanned_run_dirs(hook_module: Any, monkeypatch: Any) -> list[str]: + """Record the run directories that an emission reads from disk. + + The returned list stays live. Clear it between firings to limit an assertion + to one firing.""" + scanned: list[str] = [] + real_scan = hook_module.get_workflow_agents_in_run_dir + + def recording_scan(run_dir: Path) -> Any: + scanned.append(run_dir.name) + return real_scan(run_dir) + + monkeypatch.setattr(hook_module, "get_workflow_agents_in_run_dir", recording_scan) + return scanned + + +def plain_turn_rows(index: int, minute: int) -> list[dict[str, Any]]: + """An ordinary turn of two rows. The session continues after a workflow.""" + return [ + {"type": "user", "timestamp": f"2026-07-23T10:{minute:02d}:00.000Z", + "sessionId": "session-workflow", "uuid": f"user-plain-{index}", "cwd": "/repo", + "gitBranch": "main", "origin": {"kind": "human"}, "parentUuid": None, + "permissionMode": "default", "promptId": f"prompt-plain-{index}", + "promptSource": "sdk", "entrypoint": "cli", "userType": "external", + "version": "2.1.215", "isSidechain": False, + "message": {"role": "user", "content": f"Plain question {index}?"}}, + {"type": "assistant", "timestamp": f"2026-07-23T10:{minute:02d}:20.000Z", + "sessionId": "session-workflow", "uuid": f"assistant-plain-{index}", + "parentUuid": f"user-plain-{index}", "requestId": f"req-plain-{index}", + "entrypoint": "cli", "userType": "external", "version": "2.1.215", + "isSidechain": False, + "message": {"id": f"msg-plain-{index}", "type": "message", "role": "assistant", + "model": "claude-test", + "content": [{"type": "text", "text": f"Plain answer {index}."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 5, "output_tokens": 5}}}, + ] + + +# The last workflow-agent activity on disk (r2's trailing StructuredOutput +# tool_result row): the only timestamp that can reach the tool/root span ends +# via workflow_end_timestamp. +LAST_AGENT_ACTIVITY = "2026-07-23T10:01:55.000Z" + +# Output-token arithmetic of the fixture's agent generations: r1 is two whole +# messages (210 + 340); r2's first message is SPLIT across two rows sharing +# message.id and the merge keeps the LAST row's usage (tool_use row: 316 — the +# text row's 2 output tokens are dropped), then its retry message adds 273. +# 210 + 340 + 316 + 273 = 1139. +TOTAL_AGENT_OUTPUT_TOKENS = 210 + 340 + 316 + 273 + + +def test_workflow_turn_defers_and_emits_agents_nested_under_the_tool_span( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + def roots(): + return [o for o in fake_langfuse.observations if o.name == "Conversational Turn"] + + # Firing 1: launch ack + Claude's end_turn hand-back (the real transcript + # shape) — the workflow has not notified, so the turn provably continues + # and nothing may be exported yet (the root is immutable). + append_rows(transcript, rows[:4]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow", transcript) + assert fake_langfuse.observations == [] + + # Firing 2: notification delivered + Claude's summary — everything ships. + append_rows(transcript, rows[4:]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow", transcript) + + names = [o.name for o in fake_langfuse.observations] + assert len(roots()) == 1 + assert "Tool: Workflow" in names + assert "Workflow agent: verify-claims/r1" in names + assert "Workflow agent: verify-claims/r2" in names + + tool_span = next(o for o in fake_langfuse.observations if o.name == "Tool: Workflow") + assert tool_span._otel_span.parent is roots()[0]._otel_span + assert tool_span.kwargs["metadata"]["workflow_run_id"] == "wf_test001" + assert tool_span.kwargs["metadata"]["workflow_name"] == "verify-claims" + assert tool_span.kwargs["metadata"]["workflow_agent_count"] == 2 + + # Both agent spans nest under the launching "Tool: Workflow" span and + # carry the journal's per-agent result in their metadata. + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + assert len(agent_spans) == 2 + assert all(o._otel_span.parent is tool_span._otel_span for o in agent_spans) + r1_span = next(o for o in agent_spans if o.name.endswith("/r1")) + assert r1_span.kwargs["metadata"]["workflow_agent_id"] == "r1" + assert json.loads(r1_span.kwargs["metadata"]["workflow_agent_result"]) == { + "verdict": "claim A holds", + "confidence": "high", + } + + # The tool span's exported interval must contain its agent children — + # spans are immutable once exported, so the end must already cover the + # agents' last activity, not just the seconds-long launch ack. + assert tool_span.end_time == ts_ns(hook_module, LAST_AGENT_ACTIVITY) + assert all(o._otel_span.start_time >= tool_span._otel_span.start_time for o in agent_spans) + assert all(o.end_time <= tool_span.end_time for o in agent_spans) + + # Generations nest under their agent spans and share the plain "LLM Call" + # name (context lives on the parent); token math: TOTAL_AGENT_OUTPUT_TOKENS. + agent_otel_spans = {o._otel_span for o in agent_spans} + agent_generations = [ + o for o in fake_langfuse.observations + if o.as_type == "generation" and o._otel_span.parent in agent_otel_spans + ] + assert len(agent_generations) == 4 + assert all(o.name == "LLM Call" for o in agent_generations) + assert sum(g.kwargs["usage_details"]["output"] for g in agent_generations) == TOTAL_AGENT_OUTPUT_TOKENS + + observation_count_after_firing_2 = len(fake_langfuse.observations) + + # SessionEnd: the emission cursor neither duplicates nor starves anything. + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-workflow", transcript, flush_deferred_agent_turns=True + ) + assert len(fake_langfuse.observations) == observation_count_after_firing_2 + assert len(roots()) == 1 + assert roots()[0].output == { + "role": "assistant", + "content": "Both workflow agents completed; claims A and B hold.", + } + + +def test_session_end_flush_emits_never_completed_workflow_turn_with_agents( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + # Only the launch + end_turn hand-back reach the transcript; the + # notification never arrives (killed session). Stop holds the turn ... + append_rows(transcript, rows[:4]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-dead", transcript) + assert fake_langfuse.observations == [] + + # ... and SessionEnd must still flush it, including whatever workflow + # agent transcripts exist on disk at that point. + emitted = hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-workflow-dead", transcript, flush_deferred_agent_turns=True + ) + + assert emitted == 1 + names = [o.name for o in fake_langfuse.observations] + assert names.count("Conversational Turn") == 1 + assert "Tool: Workflow" in names + assert "Workflow agent: verify-claims/r1" in names + assert "Workflow agent: verify-claims/r2" in names + tool_span = next(o for o in fake_langfuse.observations if o.name == "Tool: Workflow") + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + assert all(o._otel_span.parent is tool_span._otel_span for o in agent_spans) + + # The agents' end timestamps must propagate through workflow_end_timestamp + # to both the tool span and the root: every parent-turn row predates the + # agents here (last assistant 10:00:15, launch ack 10:00:08), so only the + # r2 generation end can produce these values — dropping + # workflow_end_timestamp from either fold would truncate the trace to the + # launch ack and no other fixture timestamp could mask it. + root_span = next(o for o in fake_langfuse.observations if o.name == "Conversational Turn") + assert tool_span.end_time == ts_ns(hook_module, LAST_AGENT_ACTIVITY) + assert root_span.end_time == ts_ns(hook_module, LAST_AGENT_ACTIVITY) + + +def test_structured_output_ending_agent_falls_back_to_journal_result_output( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + """r2 ends the real way: a message.id-split final message (text row + + StructuredOutput tool_use row), a schema-error retry with a new + message.id, and a trailing tool_result row with toolEndsTurn:true.""" + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-so", transcript) + + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + r1_span = next(o for o in agent_spans if o.name.endswith("/r1")) + r2_span = next(o for o in agent_spans if o.name.endswith("/r2")) + + # r2's StructuredOutput tool calls render as tool spans nested under its + # agent span (both the schema-error attempt and the accepted retry). + structured_output_spans = [ + o for o in fake_langfuse.observations if o.name == "Tool: StructuredOutput" + ] + assert len(structured_output_spans) == 2 + assert all(o._otel_span.parent is r2_span._otel_span for o in structured_output_spans) + + # D1: r2's final message is tool_use-only, so its text extracts empty; the + # span output falls back to the agent's journal result as a JSON string. + expected_r2_output = json.dumps( + {"verdict": "claim B holds", "confidence": "high"}, ensure_ascii=False + ) + assert r2_span.output == {"role": "assistant", "content": expected_r2_output} + assert json.loads(r2_span.output["content"]) == {"verdict": "claim B holds", "confidence": "high"} + + # A text-ending agent (r1) keeps its extracted text output — the fallback + # only fills in when extraction comes back empty. + assert r1_span.output == {"role": "assistant", "content": "Claim A holds."} + + # Generation split/merge arithmetic stays documented: r1 = 2 whole + # messages, r2 = 2 merged messages (split rows share message.id). + agent_otel_spans = {o._otel_span for o in agent_spans} + agent_generations = [ + o for o in fake_langfuse.observations + if o.as_type == "generation" and o._otel_span.parent in agent_otel_spans + ] + assert len(agent_generations) == 4 + assert sum(g.kwargs["usage_details"]["output"] for g in agent_generations) == TOTAL_AGENT_OUTPUT_TOKENS + # The r2 tool_use rows carry the real usage shape incl. usage.speed, which + # surfaces as generation metadata. + assert any(g.kwargs["metadata"].get("speed") == "standard" for g in agent_generations) + + +def test_second_workflow_run_on_disk_stays_out_of_the_launching_turn( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + # A second, unrelated run directory exists on disk (e.g. an earlier or + # later /workflows invocation in the same session) with one finished agent + # whose activity postdates wf_test001's. + other_run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_test002" + other_run_dir.mkdir(parents=True) + (other_run_dir / "agent-x9.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + append_rows(other_run_dir / "agent-x9.jsonl", [ + {"type": "user", "timestamp": "2026-07-23T10:03:00.000Z", "sessionId": "session-workflow", + "agentId": "x9", "uuid": "wf-x9-user-1", "parentUuid": None, "isSidechain": True, + "message": {"role": "user", "content": "Verify claim X."}}, + {"type": "assistant", "timestamp": "2026-07-23T10:03:30.000Z", "sessionId": "session-workflow", + "agentId": "x9", "uuid": "wf-x9-assistant-1", "parentUuid": "wf-x9-user-1", "isSidechain": True, + "message": {"id": "msg-wf-x9-1", "type": "message", "role": "assistant", "model": "claude-test", + "content": [{"type": "text", "text": "Claim X holds."}], "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 50}}}, + ]) + append_rows(other_run_dir / "journal.jsonl", [ + {"type": "started", "key": "v2:" + "d4" * 32, "agentId": "x9"}, + {"type": "result", "key": "v2:" + "d4" * 32, "agentId": "x9", "result": {"verdict": "claim X holds"}}, + ]) + + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-multi", transcript) + + # Only the launching tool_result's runId (wf_test001) attaches: its two + # agents nest under the tool span, and nothing from wf_test002 appears in + # this turn — not as a span, not in any metadata, not in the agent count. + names = [o.name for o in fake_langfuse.observations] + assert "Workflow agent: verify-claims/r1" in names + assert "Workflow agent: verify-claims/r2" in names + assert not any("x9" in name for name in names) + assert all( + o.kwargs.get("metadata", {}).get("workflow_run_id") != "wf_test002" + for o in fake_langfuse.observations + ) + tool_span = next(o for o in fake_langfuse.observations if o.name == "Tool: Workflow") + assert tool_span.kwargs["metadata"]["workflow_agent_count"] == 2 + # x9's later activity (10:03:30) must not leak into the tool span's end. + assert tool_span.end_time == ts_ns(hook_module, LAST_AGENT_ACTIVITY) + + +def test_interrupted_agent_emits_no_span_but_stays_in_the_agent_count( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + # Real interrupted-agent shape (harvested): user prompt, the two initial + # attachment rows, then a user "[Request interrupted by user]" row — the + # agent never produced a single assistant row. + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_test001" + (run_dir / "agent-r0.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + append_rows(run_dir / "agent-r0.jsonl", [ + {"parentUuid": None, "isSidechain": True, "promptId": "prompt-wf-r0-1", "agentId": "r0", + "type": "user", "message": {"role": "user", "content": "Verify claim C."}, + "uuid": "wf-r0-user-1", "timestamp": "2026-07-23T10:00:14.000Z", "userType": "external", + "entrypoint": "cli", "cwd": "/repo", "sessionId": "session-workflow", + "version": "2.1.215", "gitBranch": "main"}, + {"parentUuid": "wf-r0-user-1", "isSidechain": True, "agentId": "r0", + "attachment": {"type": "deferred_tools_delta", "addedNames": ["WebFetch"], + "addedLines": ["WebFetch"], "removedNames": [], "readdedNames": []}, + "type": "attachment", "uuid": "wf-r0-attachment-1", "timestamp": "2026-07-23T10:00:14.001Z", + "userType": "external", "entrypoint": "cli", "cwd": "/repo", + "sessionId": "session-workflow", "version": "2.1.215", "gitBranch": "main"}, + {"parentUuid": "wf-r0-attachment-1", "isSidechain": True, "agentId": "r0", + "attachment": {"type": "skill_listing", "content": "- review: Review a GitHub pull request", + "skillCount": 1, "isInitial": True, "names": ["review"]}, + "type": "attachment", "uuid": "wf-r0-attachment-2", "timestamp": "2026-07-23T10:00:14.002Z", + "userType": "external", "entrypoint": "cli", "cwd": "/repo", + "sessionId": "session-workflow", "version": "2.1.215", "gitBranch": "main"}, + {"parentUuid": "wf-r0-attachment-2", "isSidechain": True, "promptId": "prompt-wf-r0-1", + "agentId": "r0", "type": "user", + "message": {"role": "user", "content": [{"type": "text", "text": "[Request interrupted by user]"}]}, + "uuid": "wf-r0-user-2", "timestamp": "2026-07-23T10:00:20.000Z", "userType": "external", + "entrypoint": "cli", "cwd": "/repo", "sessionId": "session-workflow", + "version": "2.1.215", "gitBranch": "main"}, + ]) + + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-int", transcript) + + # Decided behavior (D2): an agent whose transcript yields zero turns emits + # NO span — with no assistant rows there is nothing to backdate a span + # interval or any generation from, so an empty box would be pure noise. + # The emission consumes its cursor key and moves on without crashing. + names = [o.name for o in fake_langfuse.observations] + assert names.count("Conversational Turn") == 1 + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + assert sorted(o.name for o in agent_spans) == [ + "Workflow agent: verify-claims/r1", + "Workflow agent: verify-claims/r2", + ] + + # ... while workflow_agent_count keeps counting every DISCOVERED agent + # (r0 included). The mismatch is deliberate: the count reports what the + # run spawned on disk, so a reader can see that an agent existed but left + # no observable turns — hiding r0 from the count would mask the + # interruption entirely. + tool_span = next(o for o in fake_langfuse.observations if o.name == "Tool: Workflow") + assert tool_span.kwargs["metadata"]["workflow_agent_count"] == 3 + + +def test_truncated_non_json_result_notification_still_resolves_the_turn( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + # Real shape: workflows returning large prose get their cut at + # ~8k chars with a trailing truncation marker — not JSON, not resumable. + # The harvested original was 8198 chars; rebuild that length here. + marker = "\n... (truncated 39258 chars, full result in /repo/tasks/wtest001.output)" + prose = "VERIFICATION REPORT: claim A cross-checked against three primary sources; claim B confirmed. " + opaque_result = (prose * 90)[: 8198 - len(marker)] + marker + assert len(opaque_result) == 8198 + + def swap_result(notification_text: str) -> str: + start = notification_text.index("") + len("") + end = notification_text.index("") + return notification_text[:start] + opaque_result + notification_text[end:] + + rows[4]["content"] = swap_result(rows[4]["content"]) + rows[6]["message"]["content"] = swap_result(rows[6]["message"]["content"]) + + # The opaque result is carried verbatim as the launch's final_content — + # nothing ever tries to parse it. + turns = hook_module.build_turns(rows) + assert turns[0].tool_results_by_id["toolu_workflow_launch"]["final_content"] == opaque_result + + # Firing 1 (launch + hand-back): held. Firing 2 (notification + summary): + # the turn still resolves — resolution keys off the notification's + # tool-use-id, never off the result payload. + append_rows(transcript, rows[:4]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-trunc", transcript) + assert fake_langfuse.observations == [] + + append_rows(transcript, rows[4:]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-trunc", transcript) + + names = [o.name for o in fake_langfuse.observations] + assert names.count("Conversational Turn") == 1 + assert "Tool: Workflow" in names + + # The agents are untouched by the opaque notification: their spans emit + # with the per-agent journal results (metadata and, for r2, the D1 output). + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + assert len(agent_spans) == 2 + r1_span = next(o for o in agent_spans if o.name.endswith("/r1")) + assert json.loads(r1_span.kwargs["metadata"]["workflow_agent_result"]) == { + "verdict": "claim A holds", + "confidence": "high", + } + r2_span = next(o for o in agent_spans if o.name.endswith("/r2")) + assert json.loads(r2_span.output["content"]) == {"verdict": "claim B holds", "confidence": "high"} + + +def test_workflow_launch_without_workflow_name_falls_back_to_run_id_labels( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, +): + # Real launches carry toolUseResult.workflowName, but the discovery is + # tolerant of its absence — span names then fall back to the run id and + # the metadata omits workflow_name rather than carrying a null. + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + for row in rows: + tool_use_result = row.get("toolUseResult") + if isinstance(tool_use_result, dict): + tool_use_result.pop("workflowName", None) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-noname", transcript) + + names = [o.name for o in fake_langfuse.observations] + assert names.count("Conversational Turn") == 1 + assert "Workflow agent: wf_test001/r1" in names + assert "Workflow agent: wf_test001/r2" in names + + tool_span = next(o for o in fake_langfuse.observations if o.name == "Tool: Workflow") + assert tool_span.kwargs["metadata"]["workflow_run_id"] == "wf_test001" + assert "workflow_name" not in tool_span.kwargs["metadata"] + agent_spans = [o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")] + assert all(o._otel_span.parent is tool_span._otel_span for o in agent_spans) + assert all("workflow_name" not in o.kwargs["metadata"] for o in agent_spans) + + +def test_only_the_run_directory_the_turn_references_is_read( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, + monkeypatch, +): + + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + other_run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_test002" + other_run_dir.mkdir(parents=True) + (other_run_dir / "agent-x9.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + (other_run_dir / "agent-x9.jsonl").write_text("", encoding="utf-8") + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + scanned = record_scanned_run_dirs(hook_module, monkeypatch) + + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-scope", transcript) + + assert scanned == ["wf_test001"] + + +def test_closed_workflow_turn_is_not_reread_by_later_firings( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, + monkeypatch, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + scanned = record_scanned_run_dirs(hook_module, monkeypatch) + + # Firing 1 resolves the workflow turn and emits its agents. This reads the + # run directory. + append_rows(transcript, rows) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-closed", transcript) + assert scanned == ["wf_test001"] + assert len([o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")]) == 2 + + # The workflow turn is still the open turn. The next turn completes it, and + # that close walks its rows once more and reads the run again. + scanned.clear() + append_rows(transcript, plain_turn_rows(1, 5)) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-closed", transcript) + assert scanned == ["wf_test001"] + + # A closed turn never enters emission again, so its run directory, whose + # journal holds every agent's return value, is not read again. + scanned.clear() + for index, minute in ((2, 6), (3, 7), (4, 8)): + append_rows(transcript, plain_turn_rows(index, minute)) + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-workflow-closed", transcript + ) + + assert scanned == [] + # No turn is a duplicate, and no turn is lost. The workflow turn and the + # four ordinary turns each reach Langfuse one time. + assert len([o for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ")]) == 2 + assert [o.name for o in fake_langfuse.observations].count("Conversational Turn") == 5 + + +def test_held_workflow_turn_defers_the_run_directory_read_too( + hook_module, + fake_langfuse, + isolated_hook_state, + fixture_transcript_path, + tmp_path, + monkeypatch, +): + transcript, rows = copy_workflow_fixture(fixture_transcript_path, tmp_path) + config = hook_module.LangfuseConfig("public", "secret", "https://example.test", "user-1") + scanned = record_scanned_run_dirs(hook_module, monkeypatch) + + append_rows(transcript, rows[:4]) + for _ in range(3): + hook_module.emit_new_turns_from_transcript( + fake_langfuse, config, "session-workflow-held", transcript + ) + assert fake_langfuse.observations == [] + assert scanned == [] + + # An agent that comes to disk after those firings is still found. The lookup + # keeps a result for one firing only. + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_test001" + (run_dir / "agent-r3.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + append_rows(run_dir / "agent-r3.jsonl", [ + {"type": "user", "timestamp": "2026-07-23T10:01:00.000Z", "sessionId": "session-workflow", + "agentId": "r3", "uuid": "wf-r3-user-1", "parentUuid": None, "isSidechain": True, + "message": {"role": "user", "content": "Verify claim C."}}, + {"type": "assistant", "timestamp": "2026-07-23T10:01:30.000Z", "sessionId": "session-workflow", + "agentId": "r3", "uuid": "wf-r3-assistant-1", "parentUuid": "wf-r3-user-1", + "isSidechain": True, + "message": {"id": "msg-wf-r3-1", "type": "message", "role": "assistant", + "model": "claude-test", + "content": [{"type": "text", "text": "Claim C holds."}], + "stop_reason": "end_turn", + "usage": {"input_tokens": 2, "output_tokens": 60}}}, + ]) + + append_rows(transcript, rows[4:]) + hook_module.emit_new_turns_from_transcript(fake_langfuse, config, "session-workflow-held", transcript) + + assert scanned == ["wf_test001"] + agent_spans = sorted( + o.name for o in fake_langfuse.observations if o.name.startswith("Workflow agent: ") + ) + assert agent_spans == [ + "Workflow agent: verify-claims/r1", + "Workflow agent: verify-claims/r2", + "Workflow agent: verify-claims/r3", + ] diff --git a/tests/unit/test_async_agent_deferral.py b/tests/unit/test_async_agent_deferral.py index b2b8910..53dd655 100644 --- a/tests/unit/test_async_agent_deferral.py +++ b/tests/unit/test_async_agent_deferral.py @@ -490,6 +490,82 @@ def test_launch_text_fallback_defers_rows_without_tool_use_result(hook_module): assert len(state.pending_agent_turns) == 1 +def workflow_launch_turn_rows( + tool_use_id: str = "toolu_wf", + run_id: str = "wf_run1", +) -> list[dict[str, Any]]: + return [ + make_user_row("user-1", "Run the workflow.", "2026-01-01T00:00:00.000Z"), + make_assistant_row( + "assistant-1", + "msg-1", + [{"type": "tool_use", "id": tool_use_id, "name": "Workflow", + "input": {"script": "await parallel([() => agent('x')])"}}], + "2026-01-01T00:00:01.000Z", + ), + make_agent_result_row( + "tool-result-1", + tool_use_id, + "Workflow launched in background. Task ID: t1", + "2026-01-01T00:00:02.000Z", + tool_use_result={ + "status": "async_launched", + "taskId": "t1", + "taskType": "local_workflow", + "workflowName": "wf-name", + "runId": run_id, + }, + ), + make_assistant_row( + "assistant-2", + "msg-2", + [{"type": "text", "text": "The workflow is running in the background."}], + "2026-01-01T00:00:03.000Z", + ), + ] + + +def test_running_workflow_defers_the_turn_until_notification(hook_module): + turns = hook_module.build_turns(workflow_launch_turn_rows()) + state = hook_module.SessionState() + + turns_to_emit = hook_module.get_turns_to_emit(turns, state) + + assert turns_to_emit == [] + assert len(state.pending_agent_turns) == 1 + assert state.pending_agent_turns[0]["pending_tool_use_ids"] == ["toolu_wf"] + + +def test_workflow_turn_resolves_via_embedded_tool_use_id_notification(hook_module): + rows = workflow_launch_turn_rows() + [ + make_notification_row("notif-1", "toolu_wf", "workflow done", "2026-01-01T00:02:00.000Z"), + ] + turns = hook_module.build_turns(rows) + state = hook_module.SessionState() + + turns_to_emit = hook_module.get_turns_to_emit(turns, state) + + assert len(turns_to_emit) == 1 + assert state.pending_agent_turns == [] + entry = turns_to_emit[0].tool_results_by_id["toolu_wf"] + assert entry["final_content"] == "workflow done" + # The run id linking emission to the workflow's agent transcripts comes + # from the structured toolUseResult, recorded at turn assembly. + assert entry["workflow_run_id"] == "wf_run1" + assert entry["workflow_name"] == "wf-name" + + +def test_workflow_link_requires_structured_tool_use_result(hook_module): + # Without toolUseResult (older Claude Code) the launch text is NEVER + # parsed for a run id; the entry simply carries no workflow link. + rows = workflow_launch_turn_rows() + rows[2].pop("toolUseResult") + turns = hook_module.build_turns(rows) + + entry = turns[0].tool_results_by_id["toolu_wf"] + assert "workflow_run_id" not in entry + + def make_task_id_notification_row(uuid: str, task_id: str, result: str, timestamp: str) -> dict[str, Any]: return { "type": "user", diff --git a/tests/unit/test_config_precedence.py b/tests/unit/test_config_precedence.py new file mode 100644 index 0000000..c37ac58 --- /dev/null +++ b/tests/unit/test_config_precedence.py @@ -0,0 +1,186 @@ +"""Pins the config precedence: plain env vars beat machine-wide wizard values, +and they beat them as a source tier — a wizard value under one spelling must +not shadow a repo env var under the other (LANGFUSE_X vs CC_LANGFUSE_X).""" + +from pathlib import Path +from typing import Any + +import pytest + + +CORE_NAMES = [ + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "LANGFUSE_BASE_URL", + "LANGFUSE_USER_ID", +] + + +@pytest.fixture(autouse=True) +def clean_config_env(monkeypatch: pytest.MonkeyPatch) -> None: + for name in CORE_NAMES: + for key in ( + name, + f"CC_{name}", + f"CLAUDE_PLUGIN_OPTION_{name}", + f"CLAUDE_PLUGIN_OPTION_CC_{name}", + ): + monkeypatch.delenv(key, raising=False) + + +def _read_log(hook_module: Any) -> str: + log = Path(hook_module.LOG_FILE) + return log.read_text(encoding="utf-8") if log.exists() else "" + + +# ----------------- _opt: plain env var wins over wizard ----------------- + +def test_plain_env_var_wins_over_wizard_option(hook_module: Any, monkeypatch): + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + + assert hook_module._opt("LANGFUSE_PUBLIC_KEY") == "pk-lf-repo" + + +def test_wizard_option_is_the_fallback_when_no_env_var_is_set(hook_module: Any, monkeypatch): + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + + assert hook_module._opt("LANGFUSE_PUBLIC_KEY") == "pk-lf-machine" + + +def test_empty_env_var_falls_through_to_wizard_option(hook_module: Any, monkeypatch): + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + + assert hook_module._opt("LANGFUSE_PUBLIC_KEY") == "pk-lf-machine" + + +def test_neither_set_yields_empty_string(hook_module: Any, monkeypatch): + assert hook_module._opt("LANGFUSE_PUBLIC_KEY") == "" + + +# ----------------- source tier beats spelling ----------------- + +def test_repo_cc_alias_beats_wizard_value(hook_module: Any, monkeypatch): + # The wizard materializes the plain names on every wizard install; a repo + # routing via the namespaced spelling must still win. + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_SECRET_KEY", "sk-machine") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_USER_ID", "user-machine") + monkeypatch.setenv("CC_LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("CC_LANGFUSE_SECRET_KEY", "sk-repo") + monkeypatch.setenv("LANGFUSE_USER_ID", "user-repo") + + config = hook_module.get_langfuse_config() + + assert config is not None + assert config.public_key == "pk-lf-repo" + assert config.secret_key == "sk-repo" + assert config.user_id == "user-repo" + + +def test_empty_env_vars_fall_through_to_wizard_for_core_opt(hook_module: Any, monkeypatch): + # A cleared placeholder ("") must count as unset on both env rungs, or an + # empty repo value would silently disable a valid wizard config. + name = "LANGFUSE_PUBLIC_KEY" + monkeypatch.setenv(name, "") + monkeypatch.setenv(f"CC_{name}", "") + monkeypatch.setenv(f"CLAUDE_PLUGIN_OPTION_{name}", "wizard-plain") + + assert hook_module._core_opt(name) == ("wizard-plain", "wizard") + + +def test_core_opt_returns_empty_pair_when_nothing_is_set(hook_module: Any): + assert hook_module._core_opt("LANGFUSE_PUBLIC_KEY") == ("", "") + + +def test_core_opt_resolves_the_full_ladder(hook_module: Any, monkeypatch): + name = "LANGFUSE_PUBLIC_KEY" + monkeypatch.setenv(f"CLAUDE_PLUGIN_OPTION_CC_{name}", "wizard-cc") + assert hook_module._core_opt(name) == ("wizard-cc", "wizard") + + monkeypatch.setenv(f"CLAUDE_PLUGIN_OPTION_{name}", "wizard-plain") + assert hook_module._core_opt(name) == ("wizard-plain", "wizard") + + monkeypatch.setenv(f"CC_{name}", "env-cc") + assert hook_module._core_opt(name) == ("env-cc", "env") + + monkeypatch.setenv(name, "env-plain") + assert hook_module._core_opt(name) == ("env-plain", "env") + + +# ----------------- mixed-source configs warn ----------------- + +def test_partial_repo_env_block_warns_mixed_source(hook_module: Any, monkeypatch): + # Keys from the repo, base URL left to the wizard's machine-wide value: + # the classic partial env block that 401s against the wrong host. + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-repo") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_BASE_URL", "https://cloud.langfuse.com") + + config = hook_module.get_langfuse_config() + + assert config is not None + assert config.host == "https://cloud.langfuse.com" + log = _read_log(hook_module) + assert "mixed-source" in log + assert "base_url from wizard" in log + + +def test_torn_key_pair_warns_mixed_source(hook_module: Any, monkeypatch): + # The most dangerous mix: the two keys themselves from different sources. + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_SECRET_KEY", "sk-machine") + + config = hook_module.get_langfuse_config() + + assert config is not None + log = _read_log(hook_module) + assert "mixed-source" in log + assert "secret_key from wizard" in log + assert "base_url from default" in log + + +def test_wizard_keys_with_env_host_warn_mixed_source(hook_module: Any, monkeypatch): + # The reverse direction: keys from the wizard, host from the environment. + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_SECRET_KEY", "sk-machine") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.com") + + config = hook_module.get_langfuse_config() + + assert config is not None + assert config.host == "https://langfuse.example.com" + assert "mixed-source" in _read_log(hook_module) + + +def test_wizard_only_config_does_not_warn(hook_module: Any, monkeypatch): + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_PUBLIC_KEY", "pk-lf-machine") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_SECRET_KEY", "sk-machine") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_LANGFUSE_BASE_URL", "https://cloud.langfuse.com") + + assert hook_module.get_langfuse_config() is not None + assert "mixed-source" not in _read_log(hook_module) + + +def test_single_source_config_does_not_warn(hook_module: Any, monkeypatch): + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-repo") + monkeypatch.setenv("LANGFUSE_BASE_URL", "https://langfuse.example.com") + + assert hook_module.get_langfuse_config() is not None + assert "mixed-source" not in _read_log(hook_module) + + +def test_env_keys_with_code_default_host_do_not_warn(hook_module: Any, monkeypatch): + # No wizard value anywhere: falling back to the built-in default is not + # a mixed config, it is the documented single-source setup. + monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-repo") + monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-repo") + + config = hook_module.get_langfuse_config() + + assert config is not None + assert config.host == "https://cloud.langfuse.com" + assert config.user_id is None + assert "mixed-source" not in _read_log(hook_module) diff --git a/tests/unit/test_config_visibility.py b/tests/unit/test_config_visibility.py index 782cb48..4624f0a 100644 --- a/tests/unit/test_config_visibility.py +++ b/tests/unit/test_config_visibility.py @@ -1,5 +1,6 @@ from __future__ import annotations +import io from pathlib import Path from typing import Any @@ -128,3 +129,26 @@ def __init__(self, **_: Any) -> None: log = _read_log(hook_module) assert "Langfuse client creation failed" in log assert "RuntimeError" in log + + +# ----------------- the reason reaches the log through main() ----------------- + +def test_main_logs_incomplete_config_without_debug( + hook_module: Any, clean_key_env: None, monkeypatch: pytest.MonkeyPatch +) -> None: + """Covers the call path, not only the function: main() must log the reason. + + The tests above call log_missing_langfuse_config() directly, so they stay + green even when main() stops calling it and users get silence. + """ + monkeypatch.setattr(hook_module, "DEBUG", False) + monkeypatch.setattr("sys.stdin", io.StringIO("{}")) + + assert hook_module.main() == 0 + + # Read tolerantly: without the call, no line is written and the log file is + # absent, which must fail as a readable assertion, not as a file error. + log_file = Path(hook_module.LOG_FILE) + log = log_file.read_text(encoding="utf-8") if log_file.exists() else "" + assert "[INFO]" in log, "main() wrote no log line: the reason stays invisible" + assert "Langfuse config incomplete" in log diff --git a/tests/unit/test_image_capture.py b/tests/unit/test_image_capture.py new file mode 100644 index 0000000..7ca5702 --- /dev/null +++ b/tests/unit/test_image_capture.py @@ -0,0 +1,263 @@ +from __future__ import annotations + +import base64 +import json +import os + + +PNG_B64 = base64.b64encode(b"\x89PNG" + b"x" * 3000).decode() + + +def image_block(data: str = PNG_B64, media_type: str = "image/png") -> dict: + return {"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}} + + +# ---- text extraction (direct message content) ---- + +def test_extract_text_replaces_image_blocks_with_markers(hook_module): + text = hook_module.extract_text_from_content( + [image_block(), {"type": "text", "text": "[Image #1] what is this?"}] + ) + + assert PNG_B64 not in text + assert "[image image/png ~2KB]" in text + assert "[Image #1] what is this?" in text + + +def test_extract_text_of_image_only_content_is_not_empty(hook_module): + text = hook_module.extract_text_from_content([image_block()]) + + assert text == "[image image/png ~2KB]" + + +def test_image_marker_survives_malformed_source(hook_module): + assert hook_module.extract_text_from_content([{"type": "image"}]) == "[image unknown type]" + assert ( + hook_module.extract_text_from_content([{"type": "image", "source": {"type": "base64"}}]) + == "[image unknown type]" + ) + + +# ---- tool_result serialization ---- + +def test_tool_result_images_become_markers_and_trailing_text_survives(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", False) + big_b64 = "A" * (hook_module.MAX_CHARS * 2) + result = hook_module.get_tool_result_for_observation( + { + "content": [ + {"type": "text", "text": "Screenshot captured"}, + image_block(data=big_b64), + {"type": "text", "text": "tab context after the image"}, + ], + "timestamp": "2026-01-01T00:00:01.000Z", + } + ) + + assert isinstance(result.output, str) + assert big_b64[:100] not in result.output + assert "Screenshot captured" in result.output + assert "tab context after the image" in result.output + assert result.output_meta == {"truncated": False, "orig_len": len(result.output)} + + +def test_tool_result_string_content_stays_plain(hook_module): + result = hook_module.get_tool_result_for_observation( + {"content": "plain text result", "timestamp": "2026-01-01T00:00:01.000Z"} + ) + + assert result.output == "plain text result" + + +def test_tool_result_capture_appends_media_after_text(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", True) + result = hook_module.get_tool_result_for_observation( + { + "content": [ + {"type": "text", "text": "Screenshot captured"}, + image_block(), + ], + "timestamp": "2026-01-01T00:00:01.000Z", + } + ) + + text, media = result.output[0], result.output[1:] + assert "Screenshot captured" in text + assert PNG_B64 not in text + assert [(m.content_type, m.content_bytes) for m in media] == [("image/png", base64.b64decode(PNG_B64))] + + +def test_tool_result_with_multiple_images_keeps_marker_and_media_order(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", True) + b64_a = base64.b64encode(b"\x89PNG-first" * 300).decode() + b64_b = base64.b64encode(b"RIFF-second" * 500).decode() + result = hook_module.get_tool_result_for_observation( + { + "content": [ + {"type": "text", "text": "first shot"}, + image_block(data=b64_a), + {"type": "text", "text": "second shot"}, + image_block(data=b64_b, media_type="image/webp"), + ], + "timestamp": "2026-01-01T00:00:01.000Z", + } + ) + + text, media = result.output[0], result.output[1:] + assert ( + text.index("first shot") + < text.index("[image image/png") + < text.index("second shot") + < text.index("[image image/webp") + ) + assert [(m.content_type, m.content_bytes) for m in media] == [ + ("image/png", base64.b64decode(b64_a)), + ("image/webp", base64.b64decode(b64_b)), + ] + + +def test_tool_result_final_content_images_become_markers(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", False) + result = hook_module.get_tool_result_for_observation( + { + "content": "Async agent launched successfully.", + "timestamp": "2026-01-01T00:00:01.000Z", + "final_content": [image_block(), {"type": "text", "text": "final answer"}], + "final_timestamp": "2026-01-01T00:00:05.000Z", + } + ) + + assert PNG_B64 not in json.dumps(result.final_output) + assert "final answer" in result.final_output + + +# ---- media creation guardrails ---- + +def test_media_creation_is_on_by_default(hook_module): + assert hook_module.CAPTURE_IMAGES is True + media = hook_module.media_from_image_block(image_block()) + assert media is not None and media.content_type == "image/png" + + +def test_media_creation_rejects_non_base64_and_survives_sdk_errors(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", True) + + url_block = {"type": "image", "source": {"type": "url", "url": "https://example.com/x.png"}} + assert hook_module.media_from_image_block(url_block) is None + + # The SDK does not raise on these, so the hook must reject them itself. + assert hook_module.media_from_image_block(image_block(data="!!!not-base64!!!")) is None + assert hook_module.media_from_image_block(image_block(media_type="")) is None + + # The SDK accepts whitespace-wrapped base64, so the hook must too. + wrapped = "\n".join(PNG_B64[i:i + 76] for i in range(0, len(PNG_B64), 76)) + media = hook_module.media_from_image_block(image_block(data=wrapped)) + assert media is not None and media.content_bytes == base64.b64decode(PNG_B64) + + class ExplodingMedia: + def __init__(self, **kwargs): + raise RuntimeError("boom") + + monkeypatch.setattr(hook_module, "LangfuseMedia", ExplodingMedia) + assert hook_module.media_from_image_block(image_block()) is None + + monkeypatch.setattr(hook_module, "LangfuseMedia", None) + assert hook_module.media_from_image_block(image_block()) is None + + +# ---- SDK media auto-upload hardening ---- + +def _config(hook_module): + return hook_module.LangfuseConfig(public_key="pk", secret_key="sk", host="http://x", user_id=None) + + +def test_capture_off_disables_sdk_media_auto_upload(hook_module, monkeypatch): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", False) + monkeypatch.delenv("LANGFUSE_MEDIA_UPLOAD_ENABLED", raising=False) + + try: + hook_module.create_langfuse_client(_config(hook_module)) + assert os.environ["LANGFUSE_MEDIA_UPLOAD_ENABLED"] == "false" + finally: + os.environ.pop("LANGFUSE_MEDIA_UPLOAD_ENABLED", None) + + +def test_capture_on_leaves_sdk_media_upload_untouched(hook_module, monkeypatch): + monkeypatch.delenv("LANGFUSE_MEDIA_UPLOAD_ENABLED", raising=False) + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", True) + + hook_module.create_langfuse_client(_config(hook_module)) + + assert "LANGFUSE_MEDIA_UPLOAD_ENABLED" not in os.environ + + +def test_user_set_media_upload_setting_wins_over_hardening(hook_module, monkeypatch): + monkeypatch.setenv("LANGFUSE_MEDIA_UPLOAD_ENABLED", "true") + + hook_module.create_langfuse_client(_config(hook_module)) + + assert os.environ["LANGFUSE_MEDIA_UPLOAD_ENABLED"] == "true" + + +# ---- turn root input ---- + +def test_turn_root_input_attaches_media_when_capture_is_on(hook_module, fake_langfuse, monkeypatch, tmp_path): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", True) + turn = hook_module.build_turns( + [ + { + "type": "user", + "timestamp": "2026-01-01T00:01:00.000Z", + "uuid": "user-1", + "message": { + "role": "user", + "content": [image_block(), {"type": "text", "text": "what is this?"}], + }, + }, + { + "type": "assistant", + "timestamp": "2026-01-01T00:01:01.000Z", + "uuid": "assistant-1", + "message": {"role": "assistant", "content": [{"type": "text", "text": "a png"}]}, + }, + ] + )[0] + + hook_module.open_turn_root_span(fake_langfuse, "session-img", 1, turn, tmp_path / "t.jsonl") + + root = fake_langfuse.observations[-1] + text, media = root.kwargs["input"]["content"][0], root.kwargs["input"]["content"][1:] + assert "what is this?" in text + assert PNG_B64 not in text + assert [(m.content_type, m.content_bytes) for m in media] == [("image/png", base64.b64decode(PNG_B64))] + + +def test_turn_root_input_stays_text_when_capture_is_off(hook_module, fake_langfuse, monkeypatch, tmp_path): + monkeypatch.setattr(hook_module, "CAPTURE_IMAGES", False) + turn = hook_module.build_turns( + [ + { + "type": "user", + "timestamp": "2026-01-01T00:01:00.000Z", + "uuid": "user-1", + "message": { + "role": "user", + "content": [image_block(), {"type": "text", "text": "what is this?"}], + }, + }, + { + "type": "assistant", + "timestamp": "2026-01-01T00:01:01.000Z", + "uuid": "assistant-1", + "message": {"role": "assistant", "content": [{"type": "text", "text": "a png"}]}, + }, + ] + )[0] + + hook_module.open_turn_root_span(fake_langfuse, "session-img", 1, turn, tmp_path / "t.jsonl") + + root = fake_langfuse.observations[-1] + content = root.kwargs["input"]["content"] + assert isinstance(content, str) + assert "[image image/png ~2KB]" in content + assert "what is this?" in content diff --git a/tests/unit/test_langfuse_payload_builders.py b/tests/unit/test_langfuse_payload_builders.py index 75a7e1d..f1e71e2 100644 --- a/tests/unit/test_langfuse_payload_builders.py +++ b/tests/unit/test_langfuse_payload_builders.py @@ -1,5 +1,7 @@ from __future__ import annotations +import json + def test_generation_input_tracks_user_tool_and_async_tool_contexts(hook_module): assert hook_module.build_generation_input(0, "hello", [], []) == { @@ -48,10 +50,39 @@ def test_generation_output_lists_tool_calls_without_full_tool_input(hook_module) assert output == { "role": "assistant", "content": "I will read it.", - "tool_calls": [{"id": "toolu_read", "name": "Read"}], + "tool_calls": [ + {"id": "toolu_read", "type": "function", "function": {"name": "Read"}} + ], + } + assert "README.md" not in json.dumps(output) + + +def test_generation_output_keeps_the_order_of_parallel_tool_calls(hook_module): + output = hook_module.build_generation_output( + "", + [ + {"id": "toolu_a", "name": "Bash", "input": {"command": "ls"}}, + {"id": "toolu_b", "name": "Read", "input": {"file_path": "a.py"}}, + ], + ) + + assert output == { + "role": "assistant", + "tool_calls": [ + {"id": "toolu_a", "type": "function", "function": {"name": "Bash"}}, + {"id": "toolu_b", "type": "function", "function": {"name": "Read"}}, + ], } +def test_generation_output_of_malformed_tool_use_keeps_the_nested_shape(hook_module): + output = hook_module.build_generation_output("", [{}]) + + assert output["tool_calls"] == [ + {"id": None, "type": "function", "function": {"name": None}} + ] + + def test_tool_result_payload_preserves_initial_and_final_async_outputs(hook_module): result = hook_module.get_tool_result_for_observation( { @@ -111,6 +142,7 @@ def test_trace_metadata_includes_session_turn_project_and_branch( assert metadata["session_id"] == "12345678-abcd-4000-8000-123456789abc" assert metadata["turn_number"] == 7 assert metadata["transcript_path"] == "transcript.jsonl" + assert metadata["project"] == "repo" assert metadata["cwd"] == "/repo" assert metadata["git_branch"] == "feature/test" diff --git a/tests/unit/test_operator_tags.py b/tests/unit/test_operator_tags.py new file mode 100644 index 0000000..3366d2e --- /dev/null +++ b/tests/unit/test_operator_tags.py @@ -0,0 +1,147 @@ +"""Pins the caller-supplied trace tag contract: both input shapes are accepted, +each tag is bounded, and bad input is ignored with a diagnostic.""" + +from typing import Any + + +def tags(hook_module: Any, raw: str) -> list: + return hook_module.parse_operator_tags(raw)[0] + + +def warning(hook_module: Any, raw: str) -> str: + return hook_module.parse_operator_tags(raw)[1] + + +def a_turn(hook_module: Any, read_fixture_jsonl: Any, fixture_transcript_path: Any) -> Any: + return hook_module.build_turns(read_fixture_jsonl(fixture_transcript_path("simple_turn")))[0] + + +# ----------------- both input shapes ----------------- + +def test_a_comma_separated_list_becomes_tags(hook_module): + assert tags(hook_module, "env:prod,team:platform") == ["env:prod", "team:platform"] + + +def test_a_json_array_becomes_tags(hook_module): + assert tags(hook_module, '["env:prod", "lane:review"]') == ["env:prod", "lane:review"] + + +def test_json_array_entries_are_coerced_to_strings(hook_module): + assert tags(hook_module, "[1, true]") == ["1", "True"] + + +def test_tags_keep_their_input_order(hook_module): + assert tags(hook_module, "c,a,b") == ["c", "a", "b"] + + +def test_blank_and_duplicate_tags_are_dropped(hook_module): + assert tags(hook_module, "a, ,a, b ") == ["a", "b"] + + +def test_a_tag_may_hold_spaces_and_colons(hook_module): + assert tags(hook_module, "lane:review, two words") == ["lane:review", "two words"] + + +def test_no_tags_and_no_warning_for_empty_input(hook_module): + assert hook_module.parse_operator_tags("") == ([], "") + assert hook_module.parse_operator_tags(" ") == ([], "") + + +# ----------------- bad input is ignored, not fatal ----------------- + +def test_a_broken_json_array_is_ignored_with_a_diagnostic(hook_module): + assert tags(hook_module, '["unclosed"') == [] + assert "not valid JSON" in warning(hook_module, '["unclosed"') + + +def test_a_json_object_is_ignored_with_a_diagnostic(hook_module): + # Without the guard the object text becomes one tag, because it holds no + # comma. The likely cause is a value meant for a different variable. + raw = '{"lane": "review"}' + + assert tags(hook_module, raw) == [] + assert "not an array" in warning(hook_module, raw) + + +def test_an_over_long_tag_is_dropped_with_a_diagnostic(hook_module): + raw = "keep," + "x" * (hook_module.MAX_OPERATOR_TAG_CHARS + 1) + + assert tags(hook_module, raw) == ["keep"] + assert str(hook_module.MAX_OPERATOR_TAG_CHARS) in warning(hook_module, raw) + + +def test_tags_past_the_limit_are_dropped_with_a_diagnostic(hook_module): + raw = ",".join(f"t{i}" for i in range(hook_module.MAX_OPERATOR_TAGS + 5)) + + assert len(tags(hook_module, raw)) == hook_module.MAX_OPERATOR_TAGS + assert "past the first" in warning(hook_module, raw) + + +def test_a_non_scalar_json_entry_is_dropped_with_a_diagnostic(hook_module): + raw = '["keep", {"nested": 1}]' + + assert tags(hook_module, raw) == ["keep"] + assert "non-scalar" in warning(hook_module, raw) + + +def test_one_diagnostic_names_a_reason_once(hook_module): + raw = ",".join(["x" * 300] * 3) + + assert warning(hook_module, raw).count("characters") == 1 + + +# ----------------- reaching the trace ----------------- + +def test_caller_tags_join_the_hook_tags( + hook_module, monkeypatch, read_fixture_jsonl, fixture_transcript_path +): + monkeypatch.setattr(hook_module, "OPERATOR_TAGS", ["env:prod", "lane:review"]) + turn = a_turn(hook_module, read_fixture_jsonl, fixture_transcript_path) + + assert hook_module.get_trace_tags(turn) == ["claude-code", "env:prod", "lane:review"] + + +def test_caller_tags_survive_with_skill_tags_turned_off( + hook_module, monkeypatch, read_fixture_jsonl, fixture_transcript_path +): + # The skill gate must not take the operator's own labels with it. + monkeypatch.setattr(hook_module, "SKILL_TAGS", False) + monkeypatch.setattr(hook_module, "OPERATOR_TAGS", ["lane:executor"]) + turn = a_turn(hook_module, read_fixture_jsonl, fixture_transcript_path) + + assert hook_module.get_trace_tags(turn) == ["claude-code", "lane:executor"] + + +def test_a_caller_tag_never_duplicates_a_hook_tag( + hook_module, monkeypatch, read_fixture_jsonl, fixture_transcript_path +): + monkeypatch.setattr(hook_module, "OPERATOR_TAGS", ["claude-code", "lane:background"]) + turn = a_turn(hook_module, read_fixture_jsonl, fixture_transcript_path) + + assert hook_module.get_trace_tags(turn) == ["claude-code", "lane:background"] + + +def test_no_caller_tags_leaves_the_hook_tags_untouched( + hook_module, monkeypatch, read_fixture_jsonl, fixture_transcript_path +): + monkeypatch.setattr(hook_module, "OPERATOR_TAGS", []) + turn = a_turn(hook_module, read_fixture_jsonl, fixture_transcript_path) + + assert hook_module.get_trace_tags(turn) == ["claude-code"] + + +# ----------------- config precedence ----------------- + +def test_the_env_var_wins_over_the_wizard_option(hook_module, monkeypatch): + monkeypatch.setenv("CC_LANGFUSE_TRACE_TAGS", "env:repo") + monkeypatch.setenv("CLAUDE_PLUGIN_OPTION_CC_LANGFUSE_TRACE_TAGS", "env:machine") + + assert hook_module._opt(hook_module.OPERATOR_TAGS_VAR) == "env:repo" + + +def test_a_bare_langfuse_trace_tags_variable_is_not_read(hook_module, monkeypatch): + monkeypatch.delenv("CC_LANGFUSE_TRACE_TAGS", raising=False) + monkeypatch.delenv("CLAUDE_PLUGIN_OPTION_CC_LANGFUSE_TRACE_TAGS", raising=False) + monkeypatch.setenv("LANGFUSE_TRACE_TAGS", "should-be-ignored") + + assert hook_module._opt(hook_module.OPERATOR_TAGS_VAR) == "" diff --git a/tests/unit/test_state_dir.py b/tests/unit/test_state_dir.py new file mode 100644 index 0000000..4499a97 --- /dev/null +++ b/tests/unit/test_state_dir.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import importlib.util +import io +import os +import sys +from pathlib import Path +from typing import Any + +import pytest + + +OPTION = "CC_LANGFUSE_STATE_DIR" + + +@pytest.fixture(autouse=True) +def clean_state_dir_env(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv(OPTION, raising=False) + monkeypatch.delenv(f"CLAUDE_PLUGIN_OPTION_{OPTION}", raising=False) + + +def _assert_actionable_fallback(warning: str, rejected_value: str) -> None: + # A fallback warning is only useful if it names the option, the rejected + # value and where state actually went instead. + assert OPTION in warning + assert rejected_value in warning + assert str(Path.home() / ".claude" / "state") in warning + + +# ----------------- accepted values ----------------- + +def test_default_without_override(hook_module: Any) -> None: + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert warning == "" + + +def test_empty_value_counts_as_unset(hook_module: Any, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv(OPTION, "") + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert warning == "" + + +def test_plain_env_var_override( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + target = tmp_path / "profile-a" / "state" + monkeypatch.setenv(OPTION, str(target)) + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == target + assert warning == "" + assert target.is_dir() + + +def test_plugin_option_channel( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + target = tmp_path / "wizard-state" + monkeypatch.setenv(f"CLAUDE_PLUGIN_OPTION_{OPTION}", str(target)) + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == target + assert warning == "" + assert target.is_dir() + + +def test_tilde_expands_against_home( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) + monkeypatch.setenv(OPTION, "~/agent/state") + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == tmp_path / "agent" / "state" + assert warning == "" + + +# ----------------- rejected values fall back loudly ----------------- + +@pytest.mark.parametrize("value", ["relative/state", "./state", " "]) +def test_non_absolute_values_fall_back( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, value: str +) -> None: + monkeypatch.setenv(OPTION, value) + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert "not an absolute path" in warning + _assert_actionable_fallback(warning, value) + # The rejected value must not leave a directory behind in the hook's cwd. + assert not (Path.cwd() / "relative").exists() + + +def test_unknown_user_tilde_falls_back( + hook_module: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + # Path.expanduser raises RuntimeError for an unresolvable ~user prefix; + # that must degrade to the default, not crash the module import. + monkeypatch.setenv(OPTION, "~no-such-user-xyz/state") + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert "unusable" in warning + _assert_actionable_fallback(warning, "~no-such-user-xyz/state") + + +@pytest.mark.skipif(sys.platform == "win32", reason="permission bits are POSIX semantics") +def test_existing_unwritable_dir_falls_back( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # mkdir(exist_ok=True) succeeds on an existing read-only dir, so the probe + # alone would accept a directory in which log, lock and state all fail. + if hasattr(os, "geteuid") and os.geteuid() == 0: + pytest.skip("root bypasses permission checks") + ro = tmp_path / "ro-state" + ro.mkdir() + ro.chmod(0o555) + try: + monkeypatch.setenv(OPTION, str(ro)) + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert "unusable" in warning + assert "not writable" in warning + _assert_actionable_fallback(warning, str(ro)) + finally: + ro.chmod(0o755) + + +def test_uncreatable_path_falls_back( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + blocker = tmp_path / "blocker" + blocker.write_text("a file, not a directory", encoding="utf-8") + target = blocker / "state" + monkeypatch.setenv(OPTION, str(target)) + + state_dir, warning = hook_module._resolve_state_dir() + + assert state_dir == Path.home() / ".claude" / "state" + assert "unusable" in warning + _assert_actionable_fallback(warning, str(target)) + + +# ----------------- import-time wiring ----------------- + +def _fresh_hook_module(name: str) -> Any: + """Import a private module instance so the import-time path resolution + (STATE_DIR, _STATE_DIR_WARNING = _resolve_state_dir()) runs under the + current test environment. The shared hook_module fixture imports once per + session, so it can never observe env vars set inside a test.""" + module_path = Path(__file__).resolve().parents[2] / "hooks" / "langfuse_hook.py" + spec = importlib.util.spec_from_file_location(name, module_path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + try: + spec.loader.exec_module(module) + finally: + sys.modules.pop(name, None) + return module + + +def test_import_wires_globals_to_override( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + # hook_module guarantees the langfuse stubs are installed for the fresh import. + target = tmp_path / "profile" / "state" + monkeypatch.setenv(OPTION, str(target)) + + fresh = _fresh_hook_module("langfuse_hook_wiring_override") + + assert fresh.STATE_DIR == target + assert fresh._STATE_DIR_WARNING == "" + assert fresh.LOG_FILE == target / "langfuse_hook.log" + assert fresh.STATE_FILE == target / "langfuse_state.json" + assert fresh.LOCK_FILE == target / "langfuse_state.lock" + + +def test_import_wires_fallback_and_warning( + hook_module: Any, monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + monkeypatch.setenv("HOME", str(tmp_path)) # keep the fallback away from the real home + monkeypatch.setenv(OPTION, "relative/state") + + fresh = _fresh_hook_module("langfuse_hook_wiring_fallback") + + assert fresh.STATE_DIR == tmp_path / ".claude" / "state" + assert "not an absolute path" in fresh._STATE_DIR_WARNING + + +# ----------------- the warning reaches the log ----------------- + +def test_main_logs_warning_even_without_langfuse_keys( + hook_module: Any, monkeypatch: pytest.MonkeyPatch +) -> None: + monkeypatch.setattr(hook_module, "_STATE_DIR_WARNING", "CC_LANGFUSE_STATE_DIR test-warning") + for name in ( + "LANGFUSE_PUBLIC_KEY", + "LANGFUSE_SECRET_KEY", + "CC_LANGFUSE_PUBLIC_KEY", + "CC_LANGFUSE_SECRET_KEY", + ): + monkeypatch.delenv(name, raising=False) + monkeypatch.delenv(f"CLAUDE_PLUGIN_OPTION_{name}", raising=False) + monkeypatch.setattr("sys.stdin", io.StringIO("{}")) + + assert hook_module.main() == 0 + + log = Path(hook_module.LOG_FILE).read_text(encoding="utf-8") + assert "CC_LANGFUSE_STATE_DIR test-warning" in log + assert "[INFO]" in log diff --git a/tests/unit/test_subagent_discovery.py b/tests/unit/test_subagent_discovery.py index 78f9e3a..6a9355f 100644 --- a/tests/unit/test_subagent_discovery.py +++ b/tests/unit/test_subagent_discovery.py @@ -1,5 +1,6 @@ from __future__ import annotations +import errno import json @@ -33,6 +34,422 @@ def test_discovers_nested_subagent_metadata_seen_in_real_claude_code_transcripts assert subagents["toolu_inner_agent"]["agent_type"] == "fork" +def test_discovers_workflow_agent_transcripts_by_run_id(hook_module, fixture_transcript_path): + transcript = fixture_transcript_path("workflow_subagents") + + lookup = hook_module.WorkflowAgentTranscriptsByRunId(transcript) + + agents = lookup.get("wf_test001") + # The journal also holds a started-row for "r3" with no result and no + # meta/jsonl on disk (real partial-journal case: agent still running or + # torn down early) — discovery is meta-driven, so r3 never surfaces. + assert [agent["agent_id"] for agent in agents] == ["r1", "r2"] + assert [agent["path"].name for agent in agents] == ["agent-r1.jsonl", "agent-r2.jsonl"] + assert all(agent["agent_type"] == "workflow-subagent" for agent in agents) + # Per-agent return values come from the run's journal.jsonl. + assert agents[0]["result"] == {"verdict": "claim A holds", "confidence": "high"} + assert agents[1]["result"] == {"verdict": "claim B holds", "confidence": "high"} + + +def test_workflow_lookup_keeps_concurrent_runs_apart(hook_module, tmp_path): + # One firing can emit turns that use more than one run, so each run keeps + # its own entry. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + workflows_dir = tmp_path / "transcript" / "subagents" / "workflows" + for run_id, agent_id in (("wf_first", "f1"), ("wf_second", "s1")): + run_dir = workflows_dir / run_id + run_dir.mkdir(parents=True) + (run_dir / f"agent-{agent_id}.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + (run_dir / f"agent-{agent_id}.jsonl").write_text("", encoding="utf-8") + (run_dir / "journal.jsonl").write_text( + json.dumps({"type": "result", "key": "v2:abc", "agentId": agent_id, + "result": {"run": run_id}}) + "\n", + encoding="utf-8", + ) + lookup = hook_module.WorkflowAgentTranscriptsByRunId(transcript) + + first = lookup.get("wf_first") + second = lookup.get("wf_second") + + assert [agent["agent_id"] for agent in first] == ["f1"] + assert [agent["agent_id"] for agent in second] == ["s1"] + assert first[0]["result"] == {"run": "wf_first"} + assert second[0]["result"] == {"run": "wf_second"} + # A second lookup of each run gives the agents of that run. + assert lookup.get("wf_first") is first + assert lookup.get("wf_second") is second + + +def test_workflow_lookup_finds_a_run_directory_without_the_wf_prefix(hook_module, tmp_path): + # The run id names the directory, so a name without the earlier "wf_" glob + # pattern resolves too. This is wider on purpose. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "run_abc" + run_dir.mkdir(parents=True) + (run_dir / "agent-n1.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), encoding="utf-8" + ) + (run_dir / "agent-n1.jsonl").write_text("", encoding="utf-8") + + agents = hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("run_abc") + + assert [agent["agent_id"] for agent in agents] == ["n1"] + + +def test_workflow_lookup_memoizes_run_ids_that_have_no_agents( + hook_module, + tmp_path, + monkeypatch, +): + # A run directory can be absent. The lookup must keep that empty result too, + # or each observation that uses the run makes a new read. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + scanned: list[str] = [] + monkeypatch.setattr( + hook_module, + "get_workflow_agents_in_run_dir", + lambda run_dir: scanned.append(run_dir.name) or [], + ) + lookup = hook_module.WorkflowAgentTranscriptsByRunId(transcript) + + assert lookup.get("wf_gone") is None + assert lookup.get("wf_gone") is None + assert scanned == ["wf_gone"] + + +def test_workflow_lookup_rejects_run_ids_that_leave_the_workflows_directory( + hook_module, + fixture_transcript_path, + monkeypatch, +): + # The lookup makes a path from transcript data, so only a plain directory + # name must resolve, and a rejected run id must cause no read. + transcript = fixture_transcript_path("workflow_subagents") + scanned: list[str] = [] + monkeypatch.setattr( + hook_module, + "get_workflow_agents_in_run_dir", + lambda run_dir: scanned.append(str(run_dir)) or [], + ) + lookup = hook_module.WorkflowAgentTranscriptsByRunId(transcript) + + for hostile_run_id in ("..", "../wf_test001", "wf_test001/..", "nested/wf_test001", + "/etc", ".", "wf_test001/"): + assert lookup.get(hostile_run_id) is None, hostile_run_id + # An absent or incorrect run id also gives None, and makes no file read. + assert lookup.get(None) is None + assert lookup.get("") is None + assert lookup.get(123) is None + assert scanned == [] + + +def test_workflow_agents_stay_invisible_to_classic_subagent_discovery( + hook_module, + fixture_transcript_path, +): + # Documented split: the classic function keys strictly by toolUseId and + # stays non-recursive. Only the run-id lookup finds workflow agents. + transcript = fixture_transcript_path("workflow_subagents") + + assert hook_module.get_subagent_transcripts_by_tool_use_id(transcript) == {} + + +def test_discovers_named_teammate_meta_without_tool_use_id_via_launch_input_name(hook_module, tmp_path): + # A teammate meta has no toolUseId. Only input.name can attribute it. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + json.dumps({ + "type": "assistant", + "timestamp": "2026-01-01T00:00:01.000Z", + "message": { + "role": "assistant", + "content": [ + { + "type": "tool_use", + "id": "toolu_builder", + "name": "Agent", + "input": { + "name": "builder-issue-4", + "taskKind": "in_process_teammate", + "prompt": "Build the fix", + }, + }, + ], + }, + }) + "\n", + encoding="utf-8", + ) + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + # Real teammate file stems are "agent-a-". The stem never matches a launch. + (subagent_dir / "agent-abuilder-issue-4-03b96faf.meta.json").write_text( + json.dumps({ + "agentType": "builder-issue-4", + "description": "Build issue 4", + "name": "builder-issue-4", + "spawnDepth": 0, + "model": "sonnet", + "taskKind": "in_process_teammate", + "teamName": "session-test", + }), + encoding="utf-8", + ) + (subagent_dir / "agent-abuilder-issue-4-03b96faf.jsonl").write_text("", encoding="utf-8") + + subagents = hook_module.get_subagent_transcripts_by_tool_use_id(transcript) + + assert set(subagents) == {"toolu_builder"} + assert subagents["toolu_builder"]["agent_id"] == "abuilder-issue-4-03b96faf" + assert subagents["toolu_builder"]["agent_type"] == "builder-issue-4" + assert subagents["toolu_builder"]["description"] == "Build issue 4" + + +def test_named_teammate_fallback_ignores_ambiguous_names(hook_module, tmp_path): + transcript = tmp_path / "transcript.jsonl" + rows = [ + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_one", "name": "Agent", "input": {"name": "builder"}} + ], + }, + }, + { + "type": "assistant", + "message": { + "role": "assistant", + "content": [ + {"type": "tool_use", "id": "toolu_two", "name": "Agent", "input": {"name": "builder"}} + ], + }, + }, + ] + transcript.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + (subagent_dir / "agent-builder.meta.json").write_text( + json.dumps({"agentType": "builder", "name": "builder", "taskKind": "in_process_teammate"}), + encoding="utf-8", + ) + (subagent_dir / "agent-builder.jsonl").write_text("", encoding="utf-8") + + assert hook_module.get_subagent_transcripts_by_tool_use_id(transcript) == {} + + +def test_reused_teammate_name_drops_both_transcripts(hook_module, tmp_path): + # Real teammate launch results carry no per-spawn key (snake_case agent_id is + # "@"). A reused name is ambiguous, so both metas are skipped. + rows = [ + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_first", "name": "Agent", "input": {"name": "builder"}}]}}, + {"type": "user", "toolUseResult": {"status": "teammate_spawned", "agent_id": "builder@session-x", + "teammate_id": "builder@session-x", "name": "builder"}, + "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_first", + "content": "Spawned successfully. agent_id: builder@session-x"}]}}, + {"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_second", "name": "Agent", "input": {"name": "builder"}}]}}, + {"type": "user", "toolUseResult": {"status": "teammate_spawned", "agent_id": "builder@session-x", + "teammate_id": "builder@session-x", "name": "builder"}, + "message": {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_second", + "content": "Spawned successfully. agent_id: builder@session-x"}]}}, + ] + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("\n".join(json.dumps(row) for row in rows) + "\n", encoding="utf-8") + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + for stem in ("abuilder-11111111", "abuilder-22222222"): + (subagent_dir / f"agent-{stem}.meta.json").write_text( + json.dumps({"agentType": "builder", "name": "builder", "taskKind": "in_process_teammate"}), + encoding="utf-8", + ) + (subagent_dir / f"agent-{stem}.jsonl").write_text("", encoding="utf-8") + + assert hook_module.get_subagent_transcripts_by_tool_use_id(transcript) == {} + + +def test_two_metas_claiming_one_launch_drop_both(hook_module, tmp_path): + # A stale meta and a live one can share a name while only one launch row + # exists. Attribution is unknowable, so neither meta may win. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text( + json.dumps({"type": "assistant", "message": {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_only", "name": "Agent", "input": {"name": "reviewer"}}]}}) + "\n", + encoding="utf-8", + ) + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + for stem in ("areviewer-11111111", "areviewer-22222222"): + (subagent_dir / f"agent-{stem}.meta.json").write_text( + json.dumps({"agentType": "reviewer", "name": "reviewer", "taskKind": "in_process_teammate"}), + encoding="utf-8", + ) + (subagent_dir / f"agent-{stem}.jsonl").write_text("", encoding="utf-8") + + assert hook_module.get_subagent_transcripts_by_tool_use_id(transcript) == {} + + +def test_non_dict_meta_is_skipped_without_crashing(hook_module, tmp_path): + # A meta holding a JSON array raised AttributeError before the dict guard. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + (subagent_dir / "agent-broken.meta.json").write_text('["not", "a", "dict"]', encoding="utf-8") + (subagent_dir / "agent-broken.jsonl").write_text("", encoding="utf-8") + (subagent_dir / "agent-ok.meta.json").write_text( + json.dumps({"agentType": "general-purpose", "toolUseId": "toolu_ok", "spawnDepth": 1}), + encoding="utf-8", + ) + (subagent_dir / "agent-ok.jsonl").write_text("", encoding="utf-8") + + assert set(hook_module.get_subagent_transcripts_by_tool_use_id(transcript)) == {"toolu_ok"} + + +def test_unattributable_teammate_meta_does_not_break_classic_discovery(hook_module, tmp_path): + # A meta with no match must not stop classic discovery in the same directory. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + subagent_dir = tmp_path / "transcript" / "subagents" + subagent_dir.mkdir(parents=True) + (subagent_dir / "agent-classic.meta.json").write_text( + json.dumps({"agentType": "general-purpose", "toolUseId": "toolu_classic", "spawnDepth": 1}), + encoding="utf-8", + ) + (subagent_dir / "agent-classic.jsonl").write_text("", encoding="utf-8") + (subagent_dir / "agent-orphan.meta.json").write_text( + json.dumps({"agentType": "orphan", "name": "orphan", "taskKind": "in_process_teammate"}), + encoding="utf-8", + ) + (subagent_dir / "agent-orphan.jsonl").write_text("", encoding="utf-8") + + subagents = hook_module.get_subagent_transcripts_by_tool_use_id(transcript) + + assert set(subagents) == {"toolu_classic"} + + +def test_workflow_discovery_ignores_bad_or_foreign_metadata(hook_module, tmp_path): + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_x1" + run_dir.mkdir(parents=True) + (run_dir / "agent-bad.meta.json").write_text("{not-json", encoding="utf-8") + (run_dir / "agent-classic.meta.json").write_text( + json.dumps({"agentType": "general-purpose", "toolUseId": "toolu_x"}), + encoding="utf-8", + ) + (run_dir / "agent-classic.jsonl").write_text("", encoding="utf-8") + (run_dir / "agent-no-jsonl.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), + encoding="utf-8", + ) + + assert hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_x1") is None + + +def test_workflow_discovery_drops_run_dirs_with_journal_but_no_agents(hook_module, tmp_path): + # A run dir can exist with only its journal (e.g. the workflow crashed + # before spawning agents, or agents were cleaned up): no agent metas means + # nothing to emit, so the run must not surface at all. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_journal_only" + run_dir.mkdir(parents=True) + (run_dir / "journal.jsonl").write_text( + json.dumps({"type": "started", "key": "v2:abc", "agentId": "r1"}) + "\n", + encoding="utf-8", + ) + + assert hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_journal_only") is None + + +def test_workflow_discovery_ignores_orphan_agent_jsonl_without_meta(hook_module, tmp_path): + # Discovery is meta-driven: an agent transcript without its meta.json + # (half-written run, partial cleanup) stays invisible instead of crashing + # or emitting an unidentifiable agent. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_orphan" + run_dir.mkdir(parents=True) + (run_dir / "agent-orphan.jsonl").write_text( + json.dumps({"type": "user", "message": {"role": "user", "content": "hi"}}) + "\n", + encoding="utf-8", + ) + + assert hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_orphan") is None + + +def test_workflow_discovery_accepts_metas_with_extra_real_world_keys(hook_module, tmp_path): + # Harvested ground truth: real metas carry only agentType/spawnDepth, plus + # worktreePath when the run launched from a worktree (key order + # agentType -> worktreePath -> spawnDepth, matching the real meta). + # "model" is NOT harvested from any real meta — it is a hypothetical extra + # key probing that discovery keys on agentType only and tolerates unknown + # additions. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_x3" + run_dir.mkdir(parents=True) + (run_dir / "agent-a1.meta.json").write_text( + json.dumps({ + "agentType": "workflow-subagent", + "worktreePath": "/repo/.claude/worktrees/wf_x3-11", + "spawnDepth": 1, + "model": "claude-test", + }), + encoding="utf-8", + ) + (run_dir / "agent-a1.jsonl").write_text("", encoding="utf-8") + + agents = hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_x3") + + assert [agent["agent_id"] for agent in agents] == ["a1"] + assert agents[0]["agent_type"] == "workflow-subagent" + + +def test_workflow_discovery_without_journal_leaves_results_empty(hook_module, tmp_path): + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + run_dir = tmp_path / "transcript" / "subagents" / "workflows" / "wf_x2" + run_dir.mkdir(parents=True) + (run_dir / "agent-a1.meta.json").write_text( + json.dumps({"agentType": "workflow-subagent", "spawnDepth": 1}), + encoding="utf-8", + ) + (run_dir / "agent-a1.jsonl").write_text("", encoding="utf-8") + + agents = hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_x2") + + assert [agent["agent_id"] for agent in agents] == ["a1"] + assert agents[0]["result"] is None + + +def test_workflow_lookup_survives_a_run_directory_the_filesystem_rejects( + hook_module, + tmp_path, + monkeypatch, +): + # The hook runs in the Stop path of Claude Code. Thus a run path that the + # operating system refuses must not stop the turn. + transcript = tmp_path / "transcript.jsonl" + transcript.write_text("", encoding="utf-8") + + def raise_name_too_long(_run_dir): + raise OSError(errno.ENAMETOOLONG, "File name too long") + + monkeypatch.setattr(hook_module, "get_workflow_agents_in_run_dir", raise_name_too_long) + + assert hook_module.WorkflowAgentTranscriptsByRunId(transcript).get("wf_x") is None + + def test_ignores_bad_or_incomplete_subagent_metadata(hook_module, tmp_path): transcript = tmp_path / "transcript.jsonl" transcript.write_text("", encoding="utf-8") diff --git a/tests/unit/test_subagent_observations.py b/tests/unit/test_subagent_observations.py index 596043c..a4f2d84 100644 --- a/tests/unit/test_subagent_observations.py +++ b/tests/unit/test_subagent_observations.py @@ -30,7 +30,7 @@ def test_subagent_observations_skip_malformed_jsonl_lines( assert end_timestamp.isoformat() == "2026-01-01T00:02:04.500000+00:00" assert [observation.name for observation in fake_langfuse.observations] == [ "Subagent: Summarize docs", - "Subagent LLM Call 1", + "Subagent LLM Call", "Tool: ToolSearch", - "Subagent LLM Call 2", + "Subagent LLM Call", ] diff --git a/tests/unit/test_thinking_capture.py b/tests/unit/test_thinking_capture.py new file mode 100644 index 0000000..c725ae4 --- /dev/null +++ b/tests/unit/test_thinking_capture.py @@ -0,0 +1,152 @@ +from __future__ import annotations + + +def thinking_block(text: str, signature: str = "sig-abc") -> dict: + return {"type": "thinking", "thinking": text, "signature": signature} + + +def assistant_row(content: list) -> dict: + return { + "type": "assistant", + "message": {"role": "assistant", "id": "msg_1", "model": "claude-opus-5", "content": content}, + "timestamp": "2026-01-01T00:00:01.000Z", + } + + +def generation_kwargs(hook_module, content: list) -> dict: + kwargs, _ = hook_module.build_generation_kwargs(0, assistant_row(content), "do it", [], []) + return kwargs + + +# ---- thinking parts ---- + +def test_thinking_blocks_become_chatml_thinking_parts(hook_module): + kwargs = generation_kwargs( + hook_module, + [ + thinking_block("The user wants the config path, so I read the settings file."), + {"type": "text", "text": "Reading the settings file."}, + {"type": "tool_use", "id": "toolu_read", "name": "Read", "input": {"file_path": "settings.json"}}, + ], + ) + + assert kwargs["output"] == { + "role": "assistant", + "content": "Reading the settings file.", + "thinking": [ + { + "type": "thinking", + "content": "The user wants the config path, so I read the settings file.", + } + ], + "tool_calls": [{"id": "toolu_read", "type": "function", "function": {"name": "Read"}}], + } + assert kwargs["metadata"]["thinking"] == [{"truncated": False, "orig_len": 60}] + + +def test_thinking_text_stays_out_of_assistant_content(hook_module): + content = [thinking_block("secret reasoning"), {"type": "text", "text": "the answer"}] + + assert hook_module.extract_text_from_content(content) == "the answer" + assert generation_kwargs(hook_module, content)["output"]["content"] == "the answer" + + +def test_interleaved_thinking_blocks_keep_transcript_order(hook_module): + # Claude Code splits one assistant message across rows, so a merged + # message can hold more than one thinking block. + parts, metas = hook_module.build_thinking_parts( + [ + thinking_block("first thought"), + {"type": "tool_use", "id": "toolu_a", "name": "Bash", "input": {}}, + thinking_block("second thought"), + ] + ) + + assert [part["content"] for part in parts] == ["first thought", "second thought"] + assert len(metas) == 2 + + +def test_thinking_only_message_keeps_the_reasoning(hook_module): + output = generation_kwargs(hook_module, [thinking_block("only reasoning")])["output"] + + assert output == {"role": "assistant", "thinking": [{"type": "thinking", "content": "only reasoning"}]} + + +# ---- blocks that hold no reasoning ---- + +def test_blocks_without_thinking_text_are_skipped(hook_module): + # Claude Code writes the block with an empty text when the + # showThinkingSummaries setting is off. + kwargs = generation_kwargs( + hook_module, + [ + thinking_block(""), + thinking_block(" \n "), + {"type": "thinking", "signature": "sig-only"}, + {"type": "thinking", "thinking": {"not": "a string"}}, + {"type": "text", "text": "the answer"}, + ], + ) + + assert kwargs["output"] == {"role": "assistant", "content": "the answer"} + assert "thinking" not in kwargs["metadata"] + + +def test_thinking_parts_of_string_content_are_empty(hook_module): + assert hook_module.build_thinking_parts("plain string content") == ([], []) + assert hook_module.build_thinking_parts(None) == ([], []) + + +# ---- truncation ---- + +def test_long_thinking_is_truncated_and_recorded(hook_module): + long_thinking = "t" * (hook_module.MAX_CHARS + 500) + kwargs = generation_kwargs(hook_module, [thinking_block(long_thinking)]) + + part = kwargs["output"]["thinking"][0] + assert len(part["content"]) == hook_module.MAX_CHARS + meta = kwargs["metadata"]["thinking"][0] + assert meta["truncated"] is True + assert meta["orig_len"] == len(long_thinking) + assert meta["kept_len"] == hook_module.MAX_CHARS + + +# ---- signature ---- + +def test_thinking_signature_is_not_traced(hook_module): + parts, _ = hook_module.build_thinking_parts([thinking_block("reasoning", signature="long-attestation-blob")]) + + assert parts == [{"type": "thinking", "content": "reasoning"}] + + +def test_thinking_sits_beside_parallel_tool_calls(hook_module): + kwargs = generation_kwargs( + hook_module, + [ + thinking_block("Two files to read, so two calls."), + {"type": "tool_use", "id": "toolu_a", "name": "Read", "input": {"file_path": "a.py"}}, + {"type": "tool_use", "id": "toolu_b", "name": "Read", "input": {"file_path": "b.py"}}, + ], + ) + + assert kwargs["output"]["thinking"] == [ + {"type": "thinking", "content": "Two files to read, so two calls."} + ] + assert kwargs["output"]["tool_calls"] == [ + {"id": "toolu_a", "type": "function", "function": {"name": "Read"}}, + {"id": "toolu_b", "type": "function", "function": {"name": "Read"}}, + ] + # A message of only thinking and tool_use extracts no text. + assert "content" not in kwargs["output"] + + +def test_thinking_and_tool_calls_use_the_shared_shape_builder(hook_module): + content = [ + thinking_block("Reading it."), + {"type": "tool_use", "id": "toolu_read", "name": "Read", "input": {"file_path": "a.py"}}, + ] + + kwargs = generation_kwargs(hook_module, content) + expected = hook_module.build_generation_output("", [{"id": "toolu_read", "name": "Read"}]) + + assert kwargs["output"]["tool_calls"] == expected["tool_calls"]