diff --git a/.github/workflows/demo-assert.yml b/.github/workflows/demo-assert.yml new file mode 100644 index 0000000..4ce661c --- /dev/null +++ b/.github/workflows/demo-assert.yml @@ -0,0 +1,49 @@ +# Planted-bug assert gate (TDD §18 / §19). Runs after the demo dual_run job via +# workflow_call from demo-leanci.yml, or standalone against a metrics file. +# Kept as a separate workflow file so the assert contract is discoverable. +name: LeanCI Demo Assert + +on: + workflow_call: + inputs: + metrics_artifact: + description: Artifact name containing leanci-metrics.json + required: false + type: string + default: leanci-demo-metrics + workflow_dispatch: + inputs: + metrics_path: + description: Path to a local metrics JSON (dispatch with checkout only) + required: false + default: leanci-metrics.json + +permissions: + contents: read + +jobs: + assert-planted-bug: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Download LeanCI metrics + if: ${{ github.event_name != 'workflow_dispatch' }} + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.metrics_artifact }} + path: artifacts + + - name: Assert planted bug detected + run: | + if [ -f artifacts/leanci-metrics.json ]; then + python scripts/assert_planted_bug.py --metrics artifacts/leanci-metrics.json + else + python scripts/assert_planted_bug.py --metrics "${{ inputs.metrics_path || 'leanci-metrics.json' }}" + fi diff --git a/.github/workflows/demo-leanci.yml b/.github/workflows/demo-leanci.yml new file mode 100644 index 0000000..4cac4bf --- /dev/null +++ b/.github/workflows/demo-leanci.yml @@ -0,0 +1,77 @@ +# Monorepo demo gate: dual_run LeanCI + planted-bug assert (TDD §2.16 / §19 / M4). +# Does not replace the consumer template in leanci.yml. +name: LeanCI Demo + +on: + pull_request: + types: [opened, synchronize, reopened] + paths: + - "demo/**" + - "scripts/assert_planted_bug.py" + - ".github/workflows/demo-leanci.yml" + - ".github/workflows/demo-assert.yml" + - "action/**" + - "src/leanci/**" + workflow_dispatch: {} + +permissions: + contents: read + pull-requests: write + actions: read + +jobs: + leanci-dual: + runs-on: ubuntu-latest + steps: + - name: Checkout pull request head + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Review pull request (dual_run) + uses: ./action + with: + mode: dual_run + openai_url: https://api.groq.com/openai + model: llama-3.1-8b-instant + max_files: "8" + max_bytes: "40000" + max_tool_turns: "8" + max_findings: "5" + severity_floor: medium + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PARITOK_API_KEY: ${{ secrets.PARITOK_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LEANCI_PLANTED_BUG_MARKER: validate_charge + LEANCI_PATH_PREFIX: demo/ + + - name: Upload metrics for assert job + if: always() + uses: actions/upload-artifact@v4 + with: + name: leanci-demo-metrics + path: ${{ github.workspace }}/leanci-metrics.json + if-no-files-found: error + + assert-planted-bug: + needs: leanci-dual + if: always() && needs.leanci-dual.result != 'cancelled' + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Download LeanCI metrics + uses: actions/download-artifact@v4 + with: + name: leanci-demo-metrics + path: artifacts + + - name: Assert planted bug detected + run: python scripts/assert_planted_bug.py --metrics artifacts/leanci-metrics.json diff --git a/.github/workflows/leanci.yml b/.github/workflows/leanci.yml index 404a22d..fee463e 100644 --- a/.github/workflows/leanci.yml +++ b/.github/workflows/leanci.yml @@ -24,7 +24,7 @@ jobs: # Groq OpenAI-compatible upstream (Paritok --openai-url). # Local UA forwarder unwraps Cloudflare; keep context under free TPM. openai_url: https://api.groq.com/openai - model: llama-3.3-70b-versatile + model: llama-3.1-8b-instant max_files: '2' max_bytes: '20000' max_tool_turns: '8' diff --git a/.gitignore b/.gitignore index 03d8d87..39f7cd2 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,14 @@ dist/ .DS_Store leanci-metrics.json paritok.yaml +.env +.env.* +!.env.example +*.log +.coverage +htmlcov/ +.mypy_cache/ +.ruff_cache/ +.idea/ +.vscode/ +uv.lock diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..5b4261c --- /dev/null +++ b/LICENSE @@ -0,0 +1,204 @@ +Copyright 2026 Priyanshu Jha + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index 24587ad..c0bca46 100644 --- a/README.md +++ b/README.md @@ -1,35 +1,329 @@ # LeanCI -Dependency-aware AI pull request review with measured Paritok compression. +**Dependency-aware AI pull request review with measured [Paritok](https://github.com/Paritok) compression.** -## First end-to-end run (M3.6) +LeanCI is a GitHub Action that reviews PRs beyond the diff: it expands Python imports, call sites, and related tests, runs an OpenAI-compatible agent through a local Paritok proxy, then posts findings plus an honest **cost receipt** (tokens / estimated USD / reduction %). -### Required repository secrets +[![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) +[![Paritok](https://img.shields.io/badge/compression-Paritok-0B5FFF)](https://github.com/Paritok) -Configure these under **Settings → Secrets and variables → Actions**: +--- + +## Project overview + +LeanCI makes deep, cross-file PR review affordable by putting **Paritok token compression** on the agent tool path. Judges and adopters can verify savings with a dual-run receipt, not marketing claims. + +| | | +| --- | --- | +| **Type** | Composite GitHub Action + Python package (`leanci`) | +| **Language (MVP)** | Python 3.11+ | +| **License** | Apache-2.0 | +| **Demo** | `demo/` planted cross-file bug + `mode=dual_run` workflow | + +--- + +## Problem statement + +AI PR reviewers are useful but expensive: tool-heavy agent loops send large contexts (files, search hits, history) to the LLM on every turn. Teams either: + +1. **Review only the diff** and miss cross-file contract breaks, or +2. **Expand context aggressively** and burn tokens until the review no longer pencils out. + +Cross-file bugs (e.g. a helper’s return type changes while callers still assume the old shape) are exactly what shallow diff review misses—and what deep review costs the most to find. + +--- + +## Solution + +LeanCI combines three ideas: + +1. **Bounded dependency expansion** — seed from the PR diff, then add imports, reverse call sites, and related tests under hard caps. +2. **Paritok on the compressed path** — the agent talks to a local Paritok OpenAI-compatible proxy; Paritok compresses eligible tool/content traffic to the upstream model. +3. **Measured economics** — optional `dual_run` executes an uncompressed baseline on the same expansion set and publishes a side-by-side **cost receipt** with honesty labels when compression is weak. + +The demo proves the loop: a planted `validate_charge` tuple/bool contract break is caught citing both `payments/charge.py` and `api/checkout.py`, with metrics + assert gate. + +--- + +## Architecture + +```text +GitHub Action (action/) + └─ ParitokGateway (local proxy) ──► Paritok GPU + upstream LLM + └─ python -m leanci + ├─ DiffCollector + ├─ DependencyExpander → ContextManifest + ├─ AgentRuntime ↔ ToolHost (read/search) + │ └─ LLMClient(compressed via Paritok) + ├─ DualRunController (optional) + │ └─ LLMClient(uncompressed provider URL) + ├─ FindingNormalizer + ├─ ReceiptBuilder + PricingTable + ├─ GitHubPublisher (idempotent PR comment) + └─ MetricsExporter (leanci-metrics.json artifact) +``` + +**Data that leaves the runner:** PR source needed for the review (to Paritok GPU + upstream LLM), GitHub API comment traffic, and Actions logs/artifacts. There is no LeanCI-hosted database. + +--- + +## Workflow + +### Consumer (default): Paritok-only + +1. PR opened / synchronized → `.github/workflows/leanci.yml` +2. Checkout with `fetch-depth: 0` +3. Start Paritok proxy (`OPENAI_BASE_URL=http://127.0.0.1:8080/v1`) +4. Diff → expand → agent (compressed) → normalize → receipt → PR comment +5. Upload `leanci-metrics.json` + +### Demo: dual-run + planted-bug assert + +1. Changes under `demo/` (and related paths) → `.github/workflows/demo-leanci.yml` +2. Same pipeline with `mode: dual_run` and `LEANCI_PATH_PREFIX=demo/` +3. Assert job runs `scripts/assert_planted_bug.py` on the metrics artifact + +--- + +## Screenshots / GIF + +> Placeholders for the Devpost / README media. Drop real assets under `docs/media/` and replace links. + +| Asset | Path | Caption | +| --- | --- | --- | +| Architecture diagram | `docs/media/architecture.png` *(add)* | LeanCI + Paritok data flow | +| PR comment screenshot | `docs/media/pr-comment.png` *(add)* | Findings + cost receipt | +| Dual-run receipt | `docs/media/receipt.gif` *(add)* | Side-by-side token/cost columns | +| Actions run | `docs/media/actions-run.png` *(add)* | Green demo dual_run + assert | + +```markdown + +``` + +Live reference (example dual-run demo): see PR discussion and Actions on this repository’s demo PR / [LeanCI Demo workflow](.github/workflows/demo-leanci.yml). + +--- + +## Installation + +### Local package (dev / tests) + +```bash +git clone https://github.com/CodewithJha/leanci.git +cd leanci +python -m pip install -e ".[dev]" +python -m pytest +``` + +Optional: `uv sync` / `uv run pytest` if you use uv. + +### As a GitHub Action (this repo) + +The composite action lives at [`action/`](action/) and is invoked with `uses: ./action` from workflows in this repository. + +### As a reusable Action (consumer repos) + +After tagging a release (e.g. `v0.1.0-mvp`): + +```yaml +uses: CodewithJha/leanci/action@v0.1.0-mvp +``` + +--- + +## GitHub Action setup + +### 1. Add a workflow + +Minimal consumer template (also see [`.github/workflows/leanci.yml`](.github/workflows/leanci.yml)): + +```yaml +name: LeanCI + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: {} + +permissions: + contents: read + pull-requests: write + +jobs: + leanci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: CodewithJha/leanci/action@v0.1.0-mvp # or ./action in this repo + with: + # mode: paritok # default + # mode: dual_run # demo / measurement only + openai_url: https://api.groq.com/openai # example upstream + model: llama-3.1-8b-instant + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PARITOK_API_KEY: ${{ secrets.PARITOK_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} +``` + +### 2. Action inputs (selected) + +| Input | Default | Notes | +| --- | --- | --- | +| `mode` | `paritok` | `paritok` \| `dual_run` | +| `model` | `gpt-4.1-mini` | Upstream model id | +| `openai_url` | _(empty)_ | Passed to Paritok as `--openai-url` | +| `max_files` / `max_bytes` | `20` / `400000` | Expansion caps | +| `max_tool_turns` / `max_findings` | `24` / `8` | Agent / publish caps | +| `severity_floor` | `medium` | Drop findings below floor | +| `fail_on_error` | `true` | Fail job on hard review errors | + +### 3. Confirm success + +- Workflow green +- PR comment containing `` +- Artifact `leanci-metrics` → `leanci-metrics.json` + +Target setup time: **≤ 10 minutes** once secrets exist. + +--- + +## Required secrets + +Configure under **Settings → Secrets and variables → Actions**: | Secret | Required | Purpose | | --- | --- | --- | -| `PARITOK_API_KEY` | Yes | Auth for the Paritok GPU proxy (`use_gpu_server: true`) | -| `OPENAI_API_KEY` | Yes | Upstream model provider key (forwarded through Paritok) | -| `GITHUB_TOKEN` | Automatic | Provided by Actions; workflow passes it for PR comments | +| `PARITOK_API_KEY` | Yes | Paritok GPU proxy (`use_gpu_server: true`) | +| `OPENAI_API_KEY` | Yes | Upstream provider key (OpenAI-compatible) | +| `GITHUB_TOKEN` | Automatic | Actions-provided; used for PR comments | + +Do **not** commit keys. Runtime config is written under `$RUNNER_TEMP` (see [`paritok.yaml.example`](paritok.yaml.example)); `PARITOK_API_KEY` is injected via the process environment. + +--- + +## Supported providers + +LeanCI’s LLM client is **OpenAI-compatible** (`/v1/chat/completions` + tools). Anything Paritok can forward works in principle. + +| Provider | Notes | +| --- | --- | +| **Groq** | Used in this repo’s CI demos (`openai_url: https://api.groq.com/openai`). Cloudflare UA quirks handled via a small local forwarder when needed. | +| **OpenAI** | Default uncompressed dual-run base when `LEANCI_OPENAI_URL` is unset. | +| **OpenRouter / others** | Set `openai_url` + matching `OPENAI_API_KEY`; watch rate limits and tool-calling quality. | + +Dual-run uncompressed traffic uses `LEANCI_OPENAI_URL` (provider base, **not** the Paritok proxy). + +--- + +## Demo instructions + +### Planted bug + +See [`demo/README.md`](demo/README.md): + +- `demo/src/payments/charge.py` — `validate_charge` returns `(ok, reason)` +- `demo/src/api/checkout.py` — still treats the return as a boolean (tuple is always truthy) +- `demo/tests/test_checkout_happy.py` — happy path only; CI stays green + +### Run the demo gate + +1. Ensure `PARITOK_API_KEY` and `OPENAI_API_KEY` are set. +2. Open a PR that touches `demo/` (or re-run **LeanCI Demo** via `workflow_dispatch`). +3. Wait for `leanci-dual` then `assert-planted-bug`. +4. Inspect the PR comment receipt and download the metrics artifact. +5. Locally (optional): + +```bash +python scripts/assert_planted_bug.py --metrics path/to/leanci-metrics.json +``` + +### Demo video script + +Recording beats: [`scripts/demo_video_script.md`](scripts/demo_video_script.md) (≤ 3 minutes). + +--- + +## Repository structure + +```text +leanci/ +├── LICENSE +├── README.md +├── pyproject.toml +├── paritok.yaml.example +├── action/ # Composite GitHub Action +├── src/leanci/ # Package: orchestrator, expand, agent, dual, receipt… +├── demo/ # Planted-bug toy service + dual_run template +├── scripts/ +│ ├── assert_planted_bug.py +│ └── demo_video_script.md +├── docs/ +│ ├── PRD.md +│ ├── devpost_submission.md +│ ├── RELEASE_CHECKLIST.md +│ ├── media/ # Screenshot/GIF placeholders +│ └── superpowers/specs/ # TDD +├── tests/ +└── .github/workflows/ + ├── leanci.yml # Consumer template (paritok) + ├── demo-leanci.yml # dual_run + assert + └── demo-assert.yml +``` + +--- + +## Limitations + +- **Python-first expansion** — AST imports + ripgrep heuristics; not full program analysis. +- **Heuristic, capped context** — may miss distant callers when caps hit. +- **Line numbers** may be approximate; findings prefer `file` + `symbol` when unsure. +- **Dual-run doubles LLM spend** — opt-in; intended for demos/measurement. +- **Provider tool-calling quality varies** — some models emit malformed tool XML (retried / nudged). +- **Compression % is workload-dependent** — tiny demos may show ~0% reduction; receipts label near-ineffective compression honestly (do not market a fixed %). +- **Private repos** — code leaves the runner to Paritok + the LLM provider; hackathon demo is public. + +### Threat model (brief) + +| Trust boundary | What crosses it | +| --- | --- | +| GitHub runner → Paritok GPU | Review context / tool payloads (compressed path) | +| Runner / forwarder → upstream LLM | Chat + tools (compressed via Paritok; uncompressed on dual_run) | +| Runner → GitHub API | PR comment bodies (no secrets) | + +Secrets stay in Actions secrets / env; never in committed YAML. + +--- + +## Future work + +Aligned with PRD stretch goals: + +- TypeScript expansion (S1) +- Inline review comments (S2) +- Hard token/budget ceilings (S3) +- `.leanci.yml` policy packs (S4) +- Org cost rollups (S5) +- Cross-PR compression cache (S6) +- Check-run gating on severity (S8) -Do not put secrets in `paritok.yaml` or commit them. The Action writes a key-free `paritok.yaml` under `$RUNNER_TEMP` and injects `PARITOK_API_KEY` via the process environment. See `paritok.yaml.example`. +--- -### Workflow +## Acknowledgements -- File: `.github/workflows/leanci.yml` -- Triggers: `pull_request` (`opened`, `synchronize`, `reopened`) and `workflow_dispatch` -- Permissions: `contents: read`, `pull-requests: write` -- Runs composite action `./action` (starts Paritok proxy, then `python -m leanci`) +- **[Paritok](https://github.com/Paritok)** — token compression middleware that makes tool-heavy review economically viable; LeanCI’s compressed path is designed around Paritok’s OpenAI-compatible proxy and `/stats` attribution. +- GitHub Actions / checkout / artifact ecosystem. +- Upstream model providers used in demos (e.g. Groq OpenAI-compatible API). +- Spec and product direction in [`docs/PRD.md`](docs/PRD.md) and the MVP TDD under `docs/superpowers/specs/`. -### Minimal PR checklist +--- -1. Commit and push LeanCI pipeline code to the default branch (or the branch the workflow runs from). -2. Set `PARITOK_API_KEY` and `OPENAI_API_KEY` repository secrets. -3. Open a pull request (or push to an existing PR) that changes Python files. -4. Confirm the **LeanCI** workflow run succeeds. -5. Confirm the PR has a LeanCI review comment (``) with findings and a cost receipt. -6. Confirm the `leanci-metrics` artifact (`leanci-metrics.json`) was uploaded. +## License -Default mode is `paritok` (single compressed run). Dual-run is opt-in via Action input `mode: dual_run`. +Copyright 2026 Priyanshu Jha. Licensed under the [Apache License 2.0](LICENSE). diff --git a/demo/.github/workflows/leanci.yml b/demo/.github/workflows/leanci.yml new file mode 100644 index 0000000..f2e0284 --- /dev/null +++ b/demo/.github/workflows/leanci.yml @@ -0,0 +1,39 @@ +# Template for running LeanCI against this demo tree (dual_run). +# In the LeanCI monorepo, the live gate is /.github/workflows/demo-leanci.yml. +name: LeanCI (demo dual_run) + +on: + pull_request: + types: [opened, synchronize, reopened] + workflow_dispatch: {} + +permissions: + contents: read + pull-requests: write + +jobs: + leanci: + runs-on: ubuntu-latest + steps: + - name: Checkout pull request head + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Review pull request (dual_run) + uses: ./action + with: + mode: dual_run + openai_url: https://api.groq.com/openai + model: llama-3.1-8b-instant + max_files: '8' + max_bytes: '40000' + max_tool_turns: '8' + max_findings: '5' + severity_floor: medium + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PARITOK_API_KEY: ${{ secrets.PARITOK_API_KEY }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + LEANCI_PLANTED_BUG_MARKER: validate_charge + LEANCI_PATH_PREFIX: demo/ diff --git a/demo/.gitignore b/demo/.gitignore new file mode 100644 index 0000000..5090e02 --- /dev/null +++ b/demo/.gitignore @@ -0,0 +1,3 @@ +uv.lock +.venv/ +__pycache__/ diff --git a/demo/README.md b/demo/README.md new file mode 100644 index 0000000..300d3bf --- /dev/null +++ b/demo/README.md @@ -0,0 +1,14 @@ +# LeanCI demo service + +Tiny checkout/payments toy service used to prove LeanCI’s dependency-aware review. + +## Planted cross-file bug + +`payments.charge.validate_charge` was changed to return `(ok: bool, reason: str)`, +but `api.checkout.place_order` still treats the return value as a plain boolean. + +In Python a non-empty tuple is always truthy, so failed validations still proceed +to charge. The happy-path unit test does not cover a failing validation, so CI stays green. + +LeanCI must expand from the changed payments module into the checkout caller to +report the contract break (AC-6/7). diff --git a/demo/pyproject.toml b/demo/pyproject.toml new file mode 100644 index 0000000..4569bf7 --- /dev/null +++ b/demo/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "leanci-demo-checkout" +version = "0.1.0" +description = "Planted cross-file bug demo for LeanCI" +requires-python = ">=3.11" +dependencies = [] + +[project.optional-dependencies] +dev = ["pytest>=8.0"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/demo/src/api/__init__.py b/demo/src/api/__init__.py new file mode 100644 index 0000000..20ca0ae --- /dev/null +++ b/demo/src/api/__init__.py @@ -0,0 +1 @@ +"""HTTP-ish API layer for the LeanCI demo checkout service.""" diff --git a/demo/src/api/checkout.py b/demo/src/api/checkout.py new file mode 100644 index 0000000..0b9a18f --- /dev/null +++ b/demo/src/api/checkout.py @@ -0,0 +1,29 @@ +"""Checkout flow that still assumes ``validate_charge`` returns a bool. + +Because a non-empty tuple is always truthy, rejected charges still call +``capture_charge`` — the planted cross-file contract break. +""" + +from __future__ import annotations + +from payments.charge import capture_charge, validate_charge + + +class Cart: + def __init__(self, amount_cents: int, currency: str = "USD") -> None: + self.amount_cents = amount_cents + self.currency = currency + + +class PaymentError(Exception): + """Raised when checkout should refuse payment.""" + + +def place_order(cart: Cart) -> dict[str, object]: + """Place an order after validating the charge. + + Bug: treats ``validate_charge``'s ``(ok, reason)`` tuple as a boolean. + """ + if validate_charge(cart.amount_cents, cart.currency): + return capture_charge(cart.amount_cents, cart.currency) + raise PaymentError("charge validation failed") diff --git a/demo/src/payments/__init__.py b/demo/src/payments/__init__.py new file mode 100644 index 0000000..1e47ce1 --- /dev/null +++ b/demo/src/payments/__init__.py @@ -0,0 +1 @@ +"""Payment helpers for the LeanCI demo checkout service.""" diff --git a/demo/src/payments/charge.py b/demo/src/payments/charge.py new file mode 100644 index 0000000..c06eac0 --- /dev/null +++ b/demo/src/payments/charge.py @@ -0,0 +1,25 @@ +"""Charge validation and capture (demo). + +Planted bug: ``validate_charge`` used to return ``bool``. It now returns +``(ok, reason)`` but callers in ``api.checkout`` were not updated. +""" + +from __future__ import annotations + + +def validate_charge(amount_cents: int, currency: str) -> tuple[bool, str]: + """Return ``(ok, reason)`` for whether a charge may proceed.""" + if amount_cents <= 0: + return False, "amount must be positive" + if currency.upper() != "USD": + return False, "only USD is supported" + return True, "ok" + + +def capture_charge(amount_cents: int, currency: str) -> dict[str, object]: + """Pretend to capture funds after validation.""" + return { + "status": "captured", + "amount_cents": amount_cents, + "currency": currency.upper(), + } diff --git a/demo/tests/test_checkout_happy.py b/demo/tests/test_checkout_happy.py new file mode 100644 index 0000000..c4a694c --- /dev/null +++ b/demo/tests/test_checkout_happy.py @@ -0,0 +1,18 @@ +"""Intentional weak coverage: happy path only; does not catch the tuple/bool bug.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "src")) + +from api.checkout import Cart, place_order # noqa: E402 + + +def test_place_order_happy_path_usd() -> None: + result = place_order(Cart(amount_cents=2500, currency="USD")) + + assert result["status"] == "captured" + assert result["amount_cents"] == 2500 diff --git a/docs/RELEASE_CHECKLIST.md b/docs/RELEASE_CHECKLIST.md new file mode 100644 index 0000000..fb1690f --- /dev/null +++ b/docs/RELEASE_CHECKLIST.md @@ -0,0 +1,42 @@ +# Release / submission checklist (M5) + +Use before Devpost final submit and before tagging `v0.1.0-mvp`. + +## Legal & repo hygiene + +- [ ] `LICENSE` is Apache-2.0 with correct copyright year/name +- [ ] README documents secrets, threat model, Paritok credit +- [ ] No secrets, API keys, or `.env` files committed +- [ ] `paritok.yaml.example` remains key-free +- [ ] Public GitHub repository + +## Product / demo proof + +- [ ] Consumer workflow `.github/workflows/leanci.yml` runs on a sample PR +- [ ] Demo workflow dual_run + assert green on `demo/` PR +- [ ] PR comment contains `` + cost receipt +- [ ] `leanci-metrics` / `leanci-demo-metrics` artifact present +- [ ] `scripts/assert_planted_bug.py` PASS on demo metrics +- [ ] Dual-run soft-fail behavior understood if uncompressed leg fails + +## Documentation assets + +- [ ] README complete (overview → acknowledgements) +- [ ] `scripts/demo_video_script.md` reviewed (≤3 min) +- [ ] `docs/devpost_submission.md` copied into Devpost; placeholders filled +- [ ] Screenshots/GIFs added under `docs/media/` (replace placeholders) +- [ ] Demo video recorded and uploaded (≤3 min) +- [ ] Paritok email entered on Devpost +- [ ] “How Paritok was used” paragraph pasted from Devpost draft + +## Optional polish + +- [ ] Pin Action / Paritok versions for reproducibility +- [ ] Tag `v0.1.0-mvp` after M4+M5 merge to `main` +- [ ] Final timed demo run ≤8 minutes wall clock +- [ ] Paritok GitHub issue with repro (stretch S7) + +## Do not do at freeze + +- [ ] No engine feature work after freeze without a release note +- [ ] Do not claim fixed token-savings % unless the receipt on screen shows it diff --git a/docs/devpost_submission.md b/docs/devpost_submission.md new file mode 100644 index 0000000..8b67648 --- /dev/null +++ b/docs/devpost_submission.md @@ -0,0 +1,132 @@ +# Devpost submission draft — LeanCI + +Fill these fields on Devpost. Replace bracketed placeholders before submit. + +--- + +## Title + +**LeanCI: Dependency-aware PR review with measured Paritok compression** + +--- + +## One-line pitch (tagline) + +Deep, cross-file AI PR review that stays affordable—because Paritok compression is in the agent loop, and every run ships an honest cost receipt. + +--- + +## Elevator pitch (short) + +LeanCI is a GitHub Action that expands beyond the PR diff (imports, call sites, tests), runs a tool-using review agent through a local Paritok proxy, and posts findings plus a dual-run cost receipt. A planted cross-file bug demo proves both quality and measurable economics. + +--- + +## Problem + +CI-friendly AI review usually means **diff-only** context. Real regressions often live in **callers and contracts** outside the changed lines. Expanding context and tool traffic makes reviews better—and burns tokens until teams turn the feature off. Existing “AI reviewer” tools rarely prove **token economics** with a reproducible baseline. + +--- + +## Solution + +LeanCI packages: + +1. **Bounded dependency expansion** for Python PRs +2. **Paritok-backed compressed agent path** (OpenAI-compatible local proxy) +3. **Optional dual-run** uncompressed baseline on the same expansion set +4. **PR comment + metrics artifact** with findings, receipt, and planted-bug parity +5. **Demo harness + assert script** so judges can verify the planted contract break + +--- + +## Features + +- Composite GitHub Action (`action/`) — setup Python, Paritok, ripgrep, run review +- Diff → expand → agent tools (`read_file`, `search_repo`, …) → JSON findings +- Cost receipt: original vs compressed input tokens, estimated USD, reduction %, honesty labels +- Dual-run mode for demos (`mode: dual_run`) +- Idempotent PR comment marker `` +- Metrics JSON artifact (`leanci-metrics.json`) +- `demo/` planted tuple/bool charge validation bug + weak test +- `scripts/assert_planted_bug.py` PASS/FAIL gate + +--- + +## Technical architecture + +**Stack:** Python 3.11+, GitHub Actions composite action, Paritok proxy (`paritok[proxy]`), OpenAI-compatible upstream (e.g. Groq), stdlib + ripgrep expansion. + +**Pipeline:** Action starts Paritok → `python -m leanci` loads config from env → collect diff → expand/rank/cap → build manifest → agent loop via `LLMClient` (compressed `OPENAI_BASE_URL`) → optional dual uncompressed client via provider `LEANCI_OPENAI_URL` → normalize findings → Paritok `/stats` → receipt → GitHub comment → metrics file → artifact upload. + +**How Paritok is used:** LeanCI does not reimplement compression. The compressed review binding points at the local Paritok OpenAI-compatible endpoint; tool-heavy chat traffic is eligible for Paritok compression; receipts read Paritok stats for measured original vs compressed input tokens. + +--- + +## Challenges we ran into + +- Provider rate limits (TPM/TPD) and Cloudflare/User-Agent quirks on some OpenAI-compatible hosts +- Models occasionally emitting malformed tool-call markup (`tool_use_failed`) mid-loop +- Tiny demo workloads can show **near-zero** compression—forcing honest labeling instead of vanity % +- Keeping dual-run opt-in so consumer CI does not silently 2× spend + +--- + +## Accomplishments that we're proud of + +- End-to-end Action on real PRs: comment + receipt + metrics artifact +- Planted cross-file bug caught with dual-run parity on compressed and uncompressed paths +- Assert script green in CI +- Honesty-first receipts when compression is ineffective +- Clear separation: consumer `paritok` workflow vs demo `dual_run` gate + +--- + +## What we learned + +- Economics and quality must be demonstrated together; a receipt without a finding (or vice versa) is a weak hackathon story +- Expansion caps and path scoping matter as much as the LLM when free-tier limits are tight +- Judges need reproducibility: public repo, Apache-2.0, secrets documented, assert script + +--- + +## What's next (future improvements) + +- Multi-language expansion (TypeScript) +- Inline GitHub review comments +- Hard budget ceilings with partial results +- Policy file (`.leanci.yml`) +- Org-level cost rollups +- Stronger compression demos on tool-heavy fixtures + +--- + +## Built with + +- Python +- GitHub Actions +- Paritok +- OpenAI-compatible APIs (e.g. Groq) +- ripgrep + +--- + +## Submission metadata (fill in) + +| Field | Value | +| --- | --- | +| **Project URL / repo** | https://github.com/CodewithJha/leanci | +| **Demo video** | _(upload ≤3 min; script in `scripts/demo_video_script.md`)_ | +| **Paritok account email** | `[YOUR_PARITOK_EMAIL]` | +| **How Paritok was used** | Local OpenAI-compatible proxy in the Action; compressed agent path; `/stats` for receipt token columns; dual-run compares against uncompressed provider traffic | +| **License** | Apache-2.0 | +| **Team** | Priyanshu Jha | + +--- + +## Suggested Devpost “Try it out” steps + +1. Open the public repo README secrets + workflow sections. +2. Fork or use workflow_dispatch on **LeanCI Demo**. +3. Open/inspect a demo PR comment and download `leanci-metrics`. +4. Run `python scripts/assert_planted_bug.py --metrics leanci-metrics.json`. diff --git a/docs/media/README.md b/docs/media/README.md new file mode 100644 index 0000000..448c645 --- /dev/null +++ b/docs/media/README.md @@ -0,0 +1,12 @@ +# Media placeholders + +Add Devpost / README visuals here before final submit: + +| File | Suggested content | +| --- | --- | +| `architecture.png` | Diff → expand → Paritok agent → receipt | +| `pr-comment.png` | LeanCI PR comment with findings + cost table | +| `receipt.gif` | Short scroll of dual-run receipt columns | +| `actions-run.png` | Green LeanCI Demo jobs (dual + assert) | + +Keep secrets out of frames. Prefer 1280×720 or larger. diff --git a/scripts/assert_planted_bug.py b/scripts/assert_planted_bug.py new file mode 100644 index 0000000..e0290fb --- /dev/null +++ b/scripts/assert_planted_bug.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 +"""Assert LeanCI detected the demo planted cross-file bug (TDD §2.16 / §19). + +Exit codes: + 0 — PASS + 1 — FAIL +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +# Paths the planted contract-break must cite (relative to repo root). +REQUIRED_PATH_FRAGMENTS = ( + "payments/charge.py", + "api/checkout.py", +) + +MARKER_HINTS = ( + "validate_charge", + "tuple", + "bool", + "truthy", + "contract", + "return", +) + + +def main(argv: Sequence[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--metrics", + required=True, + type=Path, + help="Path to leanci-metrics.json artifact", + ) + parser.add_argument( + "--require-uncompressed", + action="store_true", + help="Fail if dual_run uncompressed parity is not True", + ) + args = parser.parse_args(list(argv) if argv is not None else None) + + try: + payload = json.loads(args.metrics.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + return _fail(f"cannot read metrics JSON: {exc}") + + if not isinstance(payload, Mapping): + return _fail("metrics root must be a JSON object") + + return assert_planted_bug( + payload, + require_uncompressed=args.require_uncompressed, + ) + + +def assert_planted_bug( + metrics: Mapping[str, Any], + *, + require_uncompressed: bool = False, +) -> int: + """Validate metrics payload; print PASS/FAIL and return exit code.""" + status = str(metrics.get("status") or "") + mode = str(metrics.get("mode") or "") + findings = metrics.get("findings") or [] + if not isinstance(findings, list): + return _fail("metrics.findings must be a list") + + receipt = metrics.get("receipt") if isinstance(metrics.get("receipt"), Mapping) else {} + parity = metrics.get("parity") if isinstance(metrics.get("parity"), Mapping) else {} + errors = metrics.get("errors") if isinstance(metrics.get("errors"), list) else [] + + problems: list[str] = [] + + if status not in {"succeeded", "partial"}: + problems.append(f"run status is {status!r} (want succeeded|partial)") + + if not _findings_cover_planted_bug(findings): + problems.append( + "compressed findings must cite both demo/src/payments/charge.py and " + "demo/src/api/checkout.py (title/file/rationale), or match the " + "validate_charge contract-break heuristic" + ) + + compressed_parity = parity.get("planted_bug_found_compressed") + if compressed_parity is False: + problems.append("parity.planted_bug_found_compressed is false") + + uncompressed_parity = parity.get("planted_bug_found_uncompressed") + uncompressed_soft_failed = any( + isinstance(err, Mapping) + and "uncompressed" in str(err.get("message") or "").lower() + for err in errors + ) + + if mode == "dual_run": + if require_uncompressed: + if uncompressed_parity is not True and not uncompressed_soft_failed: + problems.append( + "parity.planted_bug_found_uncompressed is not true " + "(required by --require-uncompressed)" + ) + elif uncompressed_soft_failed: + problems.append( + "uncompressed baseline soft-failed but --require-uncompressed was set" + ) + # Default: compressed path is the hard gate (TDD §19); uncompressed soft-fail OK. + + if problems: + _print_summary(metrics, receipt, parity, errors, ok=False) + for item in problems: + print(f"FAIL: {item}", file=sys.stderr) + return 1 + + _print_summary(metrics, receipt, parity, errors, ok=True) + print("PASS: planted cross-file bug detected") + if mode == "dual_run" and uncompressed_soft_failed: + print("NOTE: uncompressed baseline soft-failed; compressed path asserted") + return 0 + + +def _findings_cover_planted_bug(findings: Sequence[Any]) -> bool: + if not findings: + return False + + blob_parts: list[str] = [] + files_hit = set() + for item in findings: + if not isinstance(item, Mapping): + continue + file_path = str(item.get("file") or "") + title = str(item.get("title") or "") + rationale = str(item.get("rationale") or "") + symbol = str(item.get("symbol") or "") + finding_id = str(item.get("id") or "") + text = " ".join([file_path, title, rationale, symbol, finding_id]).lower() + blob_parts.append(text) + for frag in REQUIRED_PATH_FRAGMENTS: + if frag in file_path.replace("\\", "/") or frag in text: + files_hit.add(frag) + + blob = "\n".join(blob_parts) + if files_hit == set(REQUIRED_PATH_FRAGMENTS): + return True + # Single finding may cite both modules in rationale while file= one path. + if all(frag in blob for frag in REQUIRED_PATH_FRAGMENTS): + return True + # Marker / heuristic fallback when paths are present via expansion notes. + if "validate_charge" in blob and any(h in blob for h in MARKER_HINTS): + if any(frag in blob for frag in REQUIRED_PATH_FRAGMENTS): + return True + return False + + +def _print_summary( + metrics: Mapping[str, Any], + receipt: Mapping[str, Any], + parity: Mapping[str, Any], + errors: Sequence[Any], + *, + ok: bool, +) -> None: + reduction = receipt.get("token_reduction_pct") + reduction_s = ( + f"{float(reduction) * 100:.1f}%" + if isinstance(reduction, (int, float)) + else "n/a" + ) + print("=== LeanCI planted-bug assert ===") + print(f"result: {'PASS' if ok else 'FAIL'}") + print(f"status: {metrics.get('status')}") + print(f"mode: {metrics.get('mode')}") + print(f"model: {metrics.get('model')}") + print(f"findings: {len(metrics.get('findings') or [])}") + print(f"token_reduction_pct: {reduction_s}") + print(f"est_cost_compressed_usd: {receipt.get('est_cost_compressed_usd')}") + print(f"est_cost_uncompressed_usd: {receipt.get('est_cost_uncompressed_usd')}") + print(f"parity: {dict(parity) if parity else None}") + if errors: + print(f"errors: {len(errors)}") + for err in errors[:5]: + print(f" - {err}") + + +def _fail(message: str) -> int: + print(f"FAIL: {message}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/demo_video_script.md b/scripts/demo_video_script.md new file mode 100644 index 0000000..cb6db8b --- /dev/null +++ b/scripts/demo_video_script.md @@ -0,0 +1,63 @@ +# LeanCI demo video script (≤ 3 minutes) + +**Target length:** 2:45–3:00 +**Tone:** economics-first, show don’t tell +**Visuals:** IDE/PR → Actions → PR comment receipt → optional Paritok dashboard + +--- + +## Beat sheet + +| Time | Section | On screen | Voiceover (approx.) | +| --- | ---: | --- | --- | +| 0:00–0:25 | **Problem** | Innocent-looking PR diff on `charge.py` | “AI PR review is either shallow—diff only—or deep and expensive. Cross-file contract bugs hide outside the diff, and that’s exactly where token costs explode.” | +| 0:25–0:55 | **Architecture** | Simple diagram: Diff → Expand → Agent via Paritok → Receipt | “LeanCI expands imports and call sites under caps, runs the agent through a local Paritok proxy for compression, then posts findings with a measured cost receipt—not a marketing percentage.” | +| 0:55–2:10 | **Live demo** | Open demo PR → Actions **LeanCI Demo** green → PR comment | “Here’s our planted bug: `validate_charge` now returns a tuple, but checkout still treats it as a bool. Python tuples are always truthy—failed charges still go through. Tests stay green. LeanCI dual-run expands to both files, catches the break on compressed and uncompressed paths, and updates one PR comment.” | +| 2:10–2:40 | **Results** | Zoom receipt + metrics / assert PASS | “Parity: planted bug found on both paths. Metrics artifact uploaded. Assert script PASS. On tiny demos compression may be modest—we label that honestly. On tool-heavy loops, Paritok is why deep review can pencil out.” | +| 2:40–3:00 | **Key innovations** | Title cards | “Three ideas: dependency-aware expansion, Paritok on the agent path, and dual-run receipts with honesty labels. LeanCI—deep review you can measure.” | + +--- + +## Spoken script (full) + +### 1. Problem (~25s) + +AI pull request reviewers force a bad tradeoff. If you only read the diff, you miss cross-file bugs—like a helper’s return type changing while callers keep the old assumption. If you expand context and tool-loop aggressively, token costs spike until the review no longer makes sense for CI. + +### 2. Architecture (~30s) + +LeanCI is a GitHub Action. It collects the PR diff, expands Python imports, reverse call sites, and related tests under hard caps, then runs a tool-using agent. On the compressed path, the agent talks to a **local Paritok proxy**, which compresses eligible traffic to your upstream OpenAI-compatible model. Findings are normalized, a cost receipt is built from Paritok stats and a dated price table, and one idempotent PR comment is published—plus a metrics JSON artifact. + +### 3. Live demo (~75s) + +Open the demo PR in `demo/`. The change looks small: `validate_charge` returns `(ok, reason)`. Checkout still does `if validate_charge(...):`—and because a non-empty tuple is always truthy, invalid amounts still capture. The happy-path test doesn’t catch it. + +Trigger **LeanCI Demo** with `mode=dual_run`. Watch the Action start Paritok, run compressed then uncompressed on the same expansion set, and post the review. The comment cites **both** `payments/charge.py` and `api/checkout.py`. The assert job downloads metrics and passes. + +### 4. Results (~30s) + +Show: findings severity, parity flags true/true when both legs succeed, artifact upload, wall time under the demo budget when possible. Call out the receipt columns. If reduction is low on a tiny fixture, say so—honesty is part of the product. + +### 5. Key innovations (~20s) + +1. **Dependency-aware expansion** for bugs the diff alone won’t show. +2. **Paritok in the loop** so deep agent review can be economical. +3. **Dual-run + assert** so savings and finding quality are measurable, not claimed. + +Close on logo / repo URL / Apache-2.0. + +--- + +## B-roll checklist + +- [ ] Diff view of `charge.py` + `checkout.py` +- [ ] Actions run (Paritok start → dual_run → assert) +- [ ] PR comment with `` receipt table +- [ ] `leanci-metrics.json` snippet (parity + tokens) +- [ ] Optional: Paritok dashboard attribution + +## Recording notes + +- Prefer 1080p; zoom browser to 125% for readability. +- Blur or omit secret values in the Actions env panel. +- Do not claim a fixed “70% savings” unless the on-screen receipt shows it. diff --git a/src/leanci/agent.py b/src/leanci/agent.py index 5c4f0ef..103b0d2 100644 --- a/src/leanci/agent.py +++ b/src/leanci/agent.py @@ -11,11 +11,19 @@ from pathlib import Path from typing import Any, Protocol +from leanci.errors import LLMError from leanci.llm import ChatResult, LLMClient, ToolCall -from leanci.models import ContextManifest, DiffBundle, RunConfig +from leanci.models import ContextManifest, DiffBundle, FileRole, RunConfig from leanci.tools import ToolHost, ToolResult _DIFF_CHAR_CAP = 8_000 +_SEED_PRELOAD_CHAR_CAP = 6_000 +_SEED_PRELOAD_FILE_CAP = 2_000 +_TOOL_USE_FAILED_NUDGE = ( + "Your previous tool call was rejected as malformed. Call tools using the " + "API tool_calls format only (valid JSON arguments). Do not emit custom XML. " + "Then continue the review." +) _REPAIR_INSTRUCTION = ( "Return only valid JSON matching the schema from the system prompt. " "No markdown, no commentary." @@ -130,9 +138,22 @@ def run_review( last_turn = turn_idx >= max_turns - 1 if last_turn: messages.append({"role": "user", "content": _FINALIZE_INSTRUCTION}) - resp = llm.chat(messages, tools=None) + try: + resp = llm.chat(messages, tools=None) + except LLMError as exc: + if _is_tool_use_failed(exc) and turn_idx < max_turns - 1: + messages.append({"role": "user", "content": _TOOL_USE_FAILED_NUDGE}) + continue + raise else: - resp = llm.chat(messages, tools=TOOL_SCHEMAS) + try: + resp = llm.chat(messages, tools=TOOL_SCHEMAS) + except LLMError as exc: + if _is_tool_use_failed(exc): + messages.append({"role": "user", "content": _TOOL_USE_FAILED_NUDGE}) + turns += 1 + continue + raise turns += 1 messages.append(_assistant_message(resp)) @@ -206,9 +227,37 @@ def render_user_prompt(cfg: RunConfig, manifest: ContextManifest, diff: DiffBund .replace("{{max_tool_turns}}", str(cfg.caps.max_tool_turns)) .replace("{{diff}}", patch or "(empty diff)") .replace("{{manifest_json}}", manifest_json) + + _preload_seed_contents(cfg, manifest) ) +def _preload_seed_contents(cfg: RunConfig, manifest: ContextManifest) -> str: + """Embed small seed/call-site files so Groq needs fewer fragile tool calls.""" + root = Path(cfg.repo_root) + parts: list[str] = [] + used = 0 + for entry in manifest.entries: + if entry.role not in {FileRole.SEED, FileRole.CALL_SITE, FileRole.TEST}: + continue + if entry.estimated_bytes > _SEED_PRELOAD_FILE_CAP: + continue + path = root / entry.path + try: + text = path.read_text(encoding="utf-8") + except OSError: + continue + if len(text) > _SEED_PRELOAD_FILE_CAP: + text = text[:_SEED_PRELOAD_FILE_CAP] + "\n… truncated …\n" + chunk = f"\n### `{entry.path}` ({entry.role})\n```python\n{text}\n```\n" + if used + len(chunk) > _SEED_PRELOAD_CHAR_CAP: + break + parts.append(chunk) + used += len(chunk) + if not parts: + return "" + return "\n## Preloaded priority files\n" + "".join(parts) + + def _read_prompt(name: str) -> str: # Prefer package resources; fall back to source tree for editable installs. try: @@ -249,6 +298,10 @@ def _dispatch_tool(tools: SupportsTools, call: ToolCall) -> ToolResult: return tools.dispatch(call.name, args) +def _is_tool_use_failed(exc: BaseException) -> bool: + return "tool_use_failed" in str(exc).lower() + + def _try_parse_findings_payload(content: str | None) -> dict[str, Any] | None: if content is None: return None diff --git a/src/leanci/dual.py b/src/leanci/dual.py index 573f1cc..7f53487 100644 --- a/src/leanci/dual.py +++ b/src/leanci/dual.py @@ -118,8 +118,15 @@ def _findings_match_marker( for item in findings: if not isinstance(item, Mapping): continue - title = str(item.get("title") or "").lower() - finding_id = str(item.get("id") or "").lower() - if needle in title or needle in finding_id: + blob = " ".join( + [ + str(item.get("title") or ""), + str(item.get("id") or ""), + str(item.get("file") or ""), + str(item.get("rationale") or ""), + str(item.get("symbol") or ""), + ] + ).lower() + if needle in blob: return True return False diff --git a/src/leanci/llm.py b/src/leanci/llm.py index 0e14aec..a0b4f06 100644 --- a/src/leanci/llm.py +++ b/src/leanci/llm.py @@ -102,7 +102,8 @@ def from_env( base = (env.get("OPENAI_BASE_URL") or DEFAULT_COMPRESSED_BASE_URL).rstrip("/") else: # Dual-run baseline must not ride the Paritok proxy (TDD §10.5). - base = DEFAULT_UNCOMPRESSED_BASE_URL + # Prefer LEANCI_OPENAI_URL so Groq/OpenRouter demos match the compressed upstream. + base = _uncompressed_base_url(env) timeout_raw = (env.get("LEANCI_LLM_TIMEOUT_S") or "").strip() timeout_s = float(timeout_raw) if timeout_raw else _DEFAULT_TIMEOUT_S return cls( @@ -140,6 +141,11 @@ def _post_chat(self, body: dict[str, Any]) -> dict[str, Any]: headers = { "Authorization": f"Bearer {self.api_key}", "Content-Type": "application/json", + # Cloudflare (e.g. Groq) rejects default urllib User-Agent. + "User-Agent": ( + "Mozilla/5.0 (compatible; LeanCI/0.1; " + "+https://github.com/CodewithJha/leanci)" + ), } attempts = 1 + max(len(_RETRY_BACKOFF_S), len(_RATE_LIMIT_BACKOFF_S)) last_error: Exception | None = None @@ -167,6 +173,16 @@ def _post_chat(self, body: dict[str, Any]) -> dict[str, Any]: raise LLMError(f"LLM request to {url} failed: {last_error}") +def _uncompressed_base_url(env: Mapping[str, str]) -> str: + """Provider OpenAI-compatible ``.../v1`` base; never the Paritok proxy.""" + raw = (env.get("LEANCI_OPENAI_URL") or "").strip().rstrip("/") + if not raw: + return DEFAULT_UNCOMPRESSED_BASE_URL + if raw.endswith("/v1"): + return raw + return f"{raw}/v1" + + def _is_retriable_http(exc: HTTPError) -> bool: """Transient upstream failures worth retrying (incl. Groq tool_use_failed).""" if exc.code in {429, 500, 502, 503, 504}: diff --git a/src/leanci/orchestrator.py b/src/leanci/orchestrator.py index 75501f0..32405dd 100644 --- a/src/leanci/orchestrator.py +++ b/src/leanci/orchestrator.py @@ -11,6 +11,7 @@ from __future__ import annotations import os +import re import time import uuid from collections.abc import Mapping, MutableMapping @@ -125,7 +126,14 @@ async def _run_stages( record: RunRecord, ) -> None: diff: DiffBundle = hooks["collect_diff"](cfg.repo_root, cfg.base_sha, cfg.head_sha) + # Optional demo/scope filter (e.g. LEANCI_PATH_PREFIX=demo/) — keeps dual_run + # focused on the planted-bug tree when the PR also touches LeanCI itself. + path_prefix = (env.get("LEANCI_PATH_PREFIX") or "").strip() + if path_prefix: + diff = _filter_diff_by_prefix(diff, path_prefix) expansion: ExpansionSet = hooks["expand"](diff, cfg) + if path_prefix: + expansion = _filter_expansion_by_prefix(expansion, path_prefix) manifest: ContextManifest = hooks["build_manifest"]( expansion, diff, repo_root=cfg.repo_root ) @@ -375,6 +383,53 @@ def _actions_url(env: Mapping[str, str]) -> str | None: return f"{server}/{repo}/actions/runs/{run_id}" +def _filter_expansion_by_prefix(expansion: ExpansionSet, prefix: str) -> ExpansionSet: + """Drop expanded paths outside the demo/scope prefix.""" + needle = prefix.strip() + + def keep(path: str) -> bool: + return path == needle.rstrip("/") or path.startswith(needle) + + ranked = [item for item in expansion.ranked_files if keep(item.path)] + return ExpansionSet( + seed_files=[p for p in expansion.seed_files if keep(p)], + import_deps=[p for p in expansion.import_deps if keep(p)], + call_sites=[p for p in expansion.call_sites if keep(p)], + tests=[p for p in expansion.tests if keep(p)], + ranked_files=ranked, + bytes_total=sum(item.bytes for item in ranked), + cap_hits=list(expansion.cap_hits), + ) + + +def _filter_diff_by_prefix(diff: DiffBundle, prefix: str) -> DiffBundle: + """Keep only changed paths under ``prefix`` (demo dual_run scoping).""" + needle = prefix.strip() + kept_files = [ + path + for path in diff.changed_files + if path == needle.rstrip("/") or path.startswith(needle) + ] + if not kept_files: + return DiffBundle() + kept_set = set(kept_files) + hunks = [hunk for hunk in diff.hunks if hunk.file in kept_set] + patch_parts: list[str] = [] + if diff.patch_text: + blocks = re.split(r"(?=^diff --git )", diff.patch_text, flags=re.MULTILINE) + for block in blocks: + if not block.strip(): + continue + match = re.match(r"^diff --git a/.*? b/(?P.*)$", block, re.MULTILINE) + if match and match.group("path") in kept_set: + patch_parts.append(block if block.endswith("\n") else f"{block}\n") + return DiffBundle( + changed_files=kept_files, + hunks=hunks, + patch_text="".join(patch_parts), + ) + + def _owner(repo: str) -> str: if "/" not in repo: raise ConfigError(f"GITHUB_REPOSITORY must be owner/name, got {repo!r}") diff --git a/src/leanci/prompts/review_user.md b/src/leanci/prompts/review_user.md index 4eb9e96..2d1a613 100644 --- a/src/leanci/prompts/review_user.md +++ b/src/leanci/prompts/review_user.md @@ -15,6 +15,7 @@ {{manifest_json}} ## Instructions -1. Use tools to read the changed file(s) and at least one related caller/import/test from the manifest. -2. Focus on high-signal issues only. -3. Return JSON only matching the system schema when done. +1. Prefer the preloaded priority files and the diff; only use tools if those are insufficient. +2. Read the changed file(s) and at least one related caller/import/test from the manifest. +3. Focus on high-signal issues only — especially API return-type / contract mismatches. +4. Return JSON only matching the system schema when done. diff --git a/tests/test_agent.py b/tests/test_agent.py index 2d5c866..32a36c1 100644 --- a/tests/test_agent.py +++ b/tests/test_agent.py @@ -310,6 +310,46 @@ def test_max_tool_turns_forces_json_finalization() -> None: assert "Stop calling tools" in llm.calls[1]["messages"][-1]["content"] +def test_tool_use_failed_nudges_and_continues() -> None: + from leanci.errors import LLMError + + class FlakyThenOk(ScriptedLLM): + def chat(self, messages, tools=None, **opts): # type: ignore[no-untyped-def] + if not getattr(self, "_failed_once", False): + self._failed_once = True + self.calls.append({"messages": list(messages), "tools": tools, "opts": opts}) + raise LLMError( + "LLM HTTP 400 from http://x/v1/chat/completions: " + 'Bad Request — {"error":{"code":"tool_use_failed"}}' + ) + return super().chat(messages, tools=tools, **opts) + + llm = FlakyThenOk( + [ + ChatResult( + content='{"findings": [{"severity": "high", "title": "ok", ' + '"file": "a.py", "rationale": "r", "category": "api_contract"}], ' + '"notes": null}', + finish_reason="stop", + ) + ] + ) + + result = run_review( + _config(), + _manifest("src/a.py"), + _diff(), + llm, # type: ignore[arg-type] + RecordingTools(), # type: ignore[arg-type] + ) + + assert result.stop_reason == "success" + assert len(result.findings) == 1 + nudge = llm.calls[1]["messages"][-1] + assert nudge["role"] == "user" + assert "malformed" in nudge["content"].lower() + + def test_prompts_are_loaded_from_package_templates() -> None: from leanci.agent import load_system_prompt, render_user_prompt diff --git a/tests/test_assert_planted_bug.py b/tests/test_assert_planted_bug.py new file mode 100644 index 0000000..9583165 --- /dev/null +++ b/tests/test_assert_planted_bug.py @@ -0,0 +1,114 @@ +"""Tests for scripts/assert_planted_bug.py (TDD §18 planted-bug assert).""" + +from __future__ import annotations + +import importlib.util +import json +from pathlib import Path + +import pytest + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "assert_planted_bug.py" + + +def _load_assert_module(): + spec = importlib.util.spec_from_file_location("assert_planted_bug", SCRIPT) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +@pytest.fixture(scope="module") +def assert_mod(): + return _load_assert_module() + + +def _metrics(**overrides: object) -> dict[str, object]: + base: dict[str, object] = { + "status": "succeeded", + "mode": "dual_run", + "model": "llama-3.3-70b-versatile", + "findings": [ + { + "id": "f1", + "severity": "high", + "title": "validate_charge return contract break", + "file": "demo/src/payments/charge.py", + "rationale": ( + "api/checkout.py still treats validate_charge as bool; " + "tuple is always truthy so failed charges proceed" + ), + "category": "api_contract", + } + ], + "receipt": { + "token_reduction_pct": 0.55, + "est_cost_compressed_usd": 0.001, + "est_cost_uncompressed_usd": 0.002, + }, + "parity": { + "planted_bug_found_compressed": True, + "planted_bug_found_uncompressed": True, + }, + "errors": [], + } + base.update(overrides) + return base + + +def test_assert_pass_on_dual_findings(assert_mod, tmp_path: Path) -> None: + path = tmp_path / "leanci-metrics.json" + path.write_text(json.dumps(_metrics()), encoding="utf-8") + + assert assert_mod.main(["--metrics", str(path)]) == 0 + + +def test_assert_fail_without_cross_file_evidence(assert_mod, tmp_path: Path) -> None: + path = tmp_path / "leanci-metrics.json" + path.write_text( + json.dumps( + _metrics( + findings=[ + { + "severity": "high", + "title": "unrelated", + "file": "demo/src/payments/charge.py", + "rationale": "style only", + "category": "correctness", + } + ], + parity={ + "planted_bug_found_compressed": False, + "planted_bug_found_uncompressed": False, + }, + ) + ), + encoding="utf-8", + ) + + assert assert_mod.main(["--metrics", str(path)]) == 1 + + +def test_assert_allows_uncompressed_soft_fail(assert_mod, tmp_path: Path) -> None: + path = tmp_path / "leanci-metrics.json" + path.write_text( + json.dumps( + _metrics( + parity={ + "planted_bug_found_compressed": True, + "planted_bug_found_uncompressed": None, + }, + errors=[ + { + "error_class": "LLMError", + "message": "uncompressed baseline failed: provider 429", + } + ], + ) + ), + encoding="utf-8", + ) + + assert assert_mod.main(["--metrics", str(path)]) == 0 diff --git a/tests/test_llm.py b/tests/test_llm.py index eb42107..9686ac2 100644 --- a/tests/test_llm.py +++ b/tests/test_llm.py @@ -104,6 +104,20 @@ def test_from_env_uncompressed_ignores_proxy_base_url() -> None: assert client.base_url == DEFAULT_UNCOMPRESSED_BASE_URL +def test_from_env_uncompressed_uses_leanci_openai_url() -> None: + client = LLMClient.from_env( + { + "OPENAI_API_KEY": "sk", + "OPENAI_BASE_URL": "http://127.0.0.1:8080/v1", + "LEANCI_OPENAI_URL": "https://api.groq.com/openai", + "LEANCI_MODEL": "llama-3.3-70b-versatile", + }, + binding=Binding.UNCOMPRESSED, + ) + + assert client.base_url == "https://api.groq.com/openai/v1" + + def test_chat_returns_typed_result_with_content_and_usage() -> None: client = LLMClient(api_key="sk", model="gpt-4.1-mini") payload = _assistant_payload(content="review ok")