diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..6a1c899 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,46 @@ +name: CI + +on: + push: + branches: [ main, fix/truthful-gate ] + pull_request: + branches: [ main ] + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - name: Check out repository + uses: actions/checkout@v4 + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'pip' + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install torch --index-url https://download.pytorch.org/whl/cpu + pip install -r requirements-dev.txt + + - name: Run test suite with daemon UNREACHABLE (fail-closed verification) + run: | + SYSTEM_ONE_URL=http://127.0.0.1:9 python3 tests/test_live.py + + - name: Start HardTruth Daemon in background + run: | + python3 daemon/app.py & + for i in {1..30}; do + if curl -s http://127.0.0.1:8000/health | grep -q "healthy"; then + echo "Daemon is healthy" + break + fi + sleep 1 + done + + - name: Run test suite with daemon UP (live NLI verification) + run: | + SYSTEM_ONE_URL=http://127.0.0.1:8000 python3 tests/test_live.py diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..a4ff21a --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2026 Vibherpunk + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/LICENSE.md b/LICENSE.md deleted file mode 100644 index 27fd1b6..0000000 --- a/LICENSE.md +++ /dev/null @@ -1,41 +0,0 @@ -# PolyForm Noncommercial License 1.0.0 - -**Source:** - ---- - -## 1. Grant of Rights for Individuals & Noncommercial Use - -Each contributor grants you a worldwide, royalty-free, non-sublicensable, non-exclusive license to use, reproduce, modify, display, perform, and distribute the Software, subject to the conditions and limitations below, **FOR NONCOMMERCIAL PURPOSES ONLY**. - -### Permitted Noncommercial Use (100% Free): -* **Individual Personal Use:** Individual developers, students, researchers, and hobbyists using the Software on personal workstations for personal, educational, or exploratory projects. -* **Academic & Non-Profit Research:** Use by accredited educational institutions and non-profit research organizations solely for academic research and instruction. -* **Open-Source Contribution:** Evaluating and contributing improvements back to the upstream HardTruth open-source project. - ---- - -## 2. Commercial Use Prohibition - -**Any Commercial Use is strictly prohibited under this License without a valid HardTruth Commercial Enterprise License.** - -### Commercial Use Includes (Without Limitation): -1. **Any use by an organization, company, corporation, partnership, LLC, sole proprietorship, agency, or commercial entity.** -2. Use by an individual or contractor acting on behalf of, or in support of, any business or commercial organization. -3. Use in connection with any revenue-generating product, service, subscription, API, client delivery, or consulting engagement. -4. Internal operational use within a company (including CI/CD pipelines, production server monitoring, automated agent workflows, or employee development environments). -5. Hosting, distributing, or offering the Software as a managed service, cloud platform, or SaaS feature. - ---- - -## 3. Commercial Licensing Inquiries - -To obtain a commercial license for business, enterprise, or production use, contact: -* Email: `licensing@tdhlabs.com` -* Enterprise Portal: - ---- - -## 4. No Warranty; Limitation of Liability - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. diff --git a/README.md b/README.md index d832329..4800cd1 100644 --- a/README.md +++ b/README.md @@ -1,227 +1,188 @@ -# HardTruth: Autonomous Anti-Hallucination & Truth-Enforcement Engine +# HardTruth: Autonomous Anti-Hallucination & Truth-Enforcement Gate -> **Immunity from AI Catastrophes.** A 10ms local neurosymbolic circuit breaker that physically halts AI coding agents when they lie, stub code, or claim unverified success. +> **Fail-Closed Verification for Coding Agents.** A local neurosymbolic circuit breaker that halts AI coding agents when they claim unverified test passes, emit empty stubs, or contradict physical execution facts. -[![Tests](https://img.shields.io/badge/tests-9%20passed-brightgreen.svg)]() -[![Latency](https://img.shields.io/badge/latency-10.8ms%20MPS-blue.svg)]() +[![Tests](https://img.shields.io/badge/tests-16%20passed-brightgreen.svg)]() [![Model](https://img.shields.io/badge/model-DeBERTa--v3--small-orange.svg)]() -[![License](https://img.shields.io/badge/license-PolyForm%20Noncommercial%201.0.0-blue.svg)](LICENSE.md) +[![License](https://img.shields.io/badge/license-MIT-green.svg)](./LICENSE) [![Status](https://img.shields.io/badge/repo-public-success.svg)](https://github.com/Vibherpunk/hardtruth) --- -## The Problem: The "Tests Passed" Epidemic +## The Problem: The "Tests Passed" Lie -Large Language Models (LLMs) predict the next token based on statistical probability, not physical execution. +Large Language Models (LLMs) generate tokens based on text statistics, not physical operating system state. -When an AI coding agent finishes writing code, the most probable continuation in human language is: -> *"All 10 unit tests passed, the webhook is wired, and everything is working."* +When an AI coding agent finishes writing code, the most probable continuation in natural language is often: +> *"All 10 unit tests passed, the implementation is complete, and everything is working."* -The model does not know if the code compiled. It does not know if the tests ran. It has no nervous system connecting its text generation to your operating system's kernel. Standard prompting (*"Be honest"*, *"Never lie"*) fails because **it attempts to solve a physical verification problem with linguistic persuasion**. +The model has no intrinsic connection to your operating system's kernel. Standard prompting (*"Be honest"*, *"Never lie"*) fails because **linguistic persuasion cannot verify physical execution state**. -Once an agent hallucinates a fake success log into its context window, **that hallucination becomes the accepted ground truth for all subsequent turns**. The agent enters a compounding death spiral, writing further code on top of non-existent foundations. +Once an agent hallucinates a fake success statement into its context window, that claim becomes accepted context for subsequent turns, leading the agent into an compounding cycle of false assumptions. --- -## The Solution: HardTruth Neurosymbolic Circuit Breaker +## The Solution: Two-Phase Neurosymbolic Gate -**HardTruth separates execution from verification.** It couples an unforgeable physical fact ledger with a local, non-autoregressive Natural Language Inference (NLI) cross-encoder running in 10.8 milliseconds. +HardTruth separates execution from claim verification using two distinct layers: + +1. **Deterministic Execution Layer:** Captures real tool calls and exit statuses into a local append-only JSONL ledger (`ledger.jsonl`), and checks modified Python files for empty stubs using an Abstract Syntax Tree (AST) analyzer. +2. **System One NLI Layer:** Runs a local `cross-encoder/nli-deberta-v3-small` model (via PyTorch MPS or CPU) to evaluate whether the agent's natural-language completion assertions are entailed by or contradict the physical execution facts. +3. **Fail-Closed Behavior:** If the daemon is unreachable, the gate **fails closed**β€”refusing unverified factual claims with `decision: "continue"` so that unproven assertions degrade to manual verification rather than passing silently. ```mermaid sequenceDiagram - participant Agent as AI Coding Agent (Antigravity/Claude/Goose/Local LLM) + participant Agent as Antigravity Agent participant Ledger as Execution Ledger (ledger.jsonl) participant Hook as HardTruth Stop Hook - participant Engine as DeBERTa-v3 NLI (port 8000) + participant Daemon as HardTruth Daemon (port 8000) + + Agent->>Ledger: Executes tool (PostToolUse Hook) + Note over Ledger: Stores tool, target, status, and error - Agent->>Ledger: Executes bash command / edit (PostToolUse Hook) - Note over Ledger: Records exit code, stdout, and git diffs - Agent->>Hook: Attempts turn completion ("All tests passed!") - Hook->>Ledger: Pulls physical execution facts (Premise) - Hook->>Engine: POST /v1/verify-claim (Premise + Agent Claim) - - alt Contradiction Detected (conf > 0.60) - Engine-->>Hook: Contradiction: 0.99, Entailment: 0.00 (10.8ms) - Hook->>Agent: 🚨 REJECT TURN: Traps agent in verification loop with failure proof - Note over Agent: Agent is forced to run real commands or admit failure - else Entailment Verified (conf > 0.90) - Engine-->>Hook: Entailment: 0.98, Contradiction: 0.01 (10.4ms) - Hook->>Agent: ALLOW TERMINATION: Delivers verified receipts to user + Hook->>Hook: Phase 1: Pure-Python deterministic AST & exit-code check + alt Deterministic Violation (No tests ran / tests failed / stubs) + Hook->>Agent: 🚨 REJECT TURN (decision: continue) + else Clean Claims + Hook->>Daemon: Phase 2: POST /v1/verify-claim (Premise + Claim) + alt Contradiction (conf >= 0.70) or Daemon Offline + Daemon-->>Hook: Contradiction >= 0.70 or Unreachable + Hook->>Agent: 🚨 REJECT TURN: Requires manual proof + else Entailment Verified + Daemon-->>Hook: Entailment verified (contradiction < 0.70) + Hook->>Agent: ALLOW TERMINATION (decision: allow) + end end ``` --- -## Why DeBERTa-v3? The Architecture & Physics of Non-Autoregressive Truth - -A common question is: *Why not just prompt a fast small model like Llama 3.2 1B or Qwen 0.5B to check if the agent is lying? Why use a specialized cross-encoder like DeBERTa-v3?* - -The answer lies in the fundamental architectural difference between **autoregressive generation** and **discriminative sequence-pair cross-encoding**: - -### 1. Autoregressive LLMs vs. Sequence-Pair Cross-Encoders -* **Autoregressive Causal Models (GPT, Llama, Claude):** Process tokens sequentially left-to-right. A token at position $i$ can only attend to previous tokens $< i$. When asked *"Does Premise support Hypothesis?"*, an autoregressive model must generate text tokens (`"Yes"`, `"No"`) based on conditional probability, making it vulnerable to self-confirmation bias, sycophancy, and hallucinating answers. -* **DeBERTa-v3 Cross-Encoder:** Feeds `[CLS] Premise [SEP] Hypothesis [SEP]` simultaneously into all 12 bidirectional transformer layers. **Every single word in the agent's claim directly cross-attends to every single word in the bash exit code and test output simultaneously.** It computes a dense interaction matrix between claim and physical reality. - -### 2. Disentangled Attention Mechanism -Unlike standard BERT or RoBERTa where a token's content and position embeddings are crudely added into a single vector ($\vec{v} = \vec{c} + \vec{p}$), DeBERTa represents each word with **two separate vectors**: -* A **content vector** $\vec{c}_i$ representing the semantic meaning of the token. -* A **relative position vector** $\vec{p}_{i,j}$ representing its relative distance to other tokens. +## Capabilities & Enforced Invariants -Self-attention is calculated as the sum of four disentangled attention matrices: -$$\text{Attention}(i, j) = \underbrace{c_i c_j^T}_{\text{Content-to-Content}} + \underbrace{c_i p_{i,j}^T}_{\text{Content-to-Position}} + \underbrace{p_{i,j} c_j^T}_{\text{Position-to-Content}} + \underbrace{p_{i,j} p_{i,j}^T}_{\text{Position-to-Position}}$$ +### 1. Stopping the "Fake Test Pass" Lie +If an agent claims tests passed (e.g. *"All 10 unit tests passed"*), HardTruth inspects the execution ledger. +* If **no commands** ran in the session, the turn is immediately halted (`decision: "continue"`). +* If commands ran but **failed** (non-zero exit code or error output), the turn is immediately halted with a contradiction warning. +* If matching test commands executed with exit code 0, the claim is verified. -This allows DeBERTa-v3 to possess extraordinary syntactic sensitivity to temporal and logical ordering: it understands the exact logical difference between *"Tests passed after failure"* vs. *"Tests failed after passing"*, which bi-encoders and naive embedding distance metrics completely blur. +### 2. AST Anti-Stubbing Linter +HardTruth's AST analyzer (`client/ast_checker.py`) checks every modified Python file. Any function reduced to a vacuous body is rejected before completion: +* `pass` +* `raise NotImplementedError` / `raise NotImplementedError()` / `raise NotImplementedError("...")` +* `...` (Ellipsis) +* `return True` or `return None` / bare `return` as the sole body statement -### 3. Enhanced Masked Decoder (EMD) -Standard relative-position models lose absolute position. DeBERTa re-injects absolute position embeddings immediately before the final classification head, ensuring that the boundary between `Premise` (the immutable OS facts) and `Hypothesis` (the agent's claim) remains razor-sharp. +**False Positive Suppression:** Legitimate abstract declarations are explicitly exempted: +* Classes inheriting from `typing.Protocol` +* Methods decorated with `@abstractmethod` +* Functions decorated with `@overload` -### 4. Mathematical Softmax (Impossible to Hallucinate Text) -DeBERTa-v3 does not generate text strings. Its classification head projects directly into a 3-dimensional logit space passed through a softmax function: -$$P(\text{Contradiction}), P(\text{Entailment}), P(\text{Neutral})$$ -Because it cannot output text, **it is mathematically incapable of hallucinating a conversational excuse**. It outputs pure probability. +### 3. Machine-Wide Git Pre-Commit Gate +`install.sh --global` configures a global git pre-commit hook (`~/.hardtruth/hooks/pre-commit`). Any commit across the entire machine containing empty stubs in staged `.py` files is rejected at the git level, protecting repositories from stubbed code regardless of which agent or tool wrote it. -### 5. Extreme Efficiency on Consumer Hardware -* **Parameters:** 141 Million (compact enough to live permanently in RAM). -* **RAM Footprint:** ~280 MB on Apple Silicon Metal (MPS); ~150 MB on CPU ONNX. -* **Latency:** **10.8 milliseconds** on Apple Silicon MPS / **18.5 ms** on Linux CPU. -* **Token Cost:** $0.00 (100% local, zero cloud network calls). +### 4. Circuit Breaker Escape Hatch +To prevent infinite loops when an agent cannot resolve an issue, HardTruth maintains a counter in `~/.hardtruth/halts/` (isolated with mode `0o700` and hashed conversation IDs). After 3 consecutive halts on the same conversation, the circuit breaker releases with a visible warning in the returned `reason`. --- -## Core Use Cases & Hero Applications - -HardTruth solves the reliability gap that prompt engineering, LLM-as-a-judge, and post-hoc observability fail to address: - -### 1. Upskilling Local Small Models (7B, 14B, 35B) on Long-Horizon Autonomous Runs -* **The Problem:** Smaller open models (Qwen 2.5 Coder, WangYang 35B, DeepSeek Coder) suffer from severe autoregressive drift during multi-turn tasks. By turn 6–10, models experience probability pull toward linguistic closure and hallucinate completion. Once a fake success log enters the context window, **that hallucination poisons all future reasoning steps**, triggering an inescapable death spiral. -* **The HardTruth Solution:** HardTruth acts as an external, non-autoregressive environmental oracle. The instant a small model asserts completion without physical receipts, HardTruth intercepts the exit, rejects termination, and feeds the exact physical failure trace back into the context window. This eliminates the "reasoning tax" and forces continuous exploration until working code is produced. -* **Synthetic DPO / RL Data Factory:** Every completed autonomous run under HardTruth produces an unforgeable, verified trajectory (`prompt -> tool calls -> bash exit code 0 -> clean diffs`) ready for high-fidelity model alignment without human labeling. - -### 2. Eliminating "The Tests Passed Lie" in Frontier Coding Agents -* **The Problem:** Even frontier models (Claude 3.5 Sonnet, GPT-4o, Gemini 1.5 Pro) in harnesses like Devin, Cursor, Claude Code, and Antigravity routinely claim: - > *"All 10 unit tests in the test suite passed with 100% success."* - ...when tests never ran, or when `pytest` exited with status code 1. -* **The HardTruth Solution:** HardTruth strips away the agent's ability to terminate with verbal reassurance. If the agent makes a completion or verification claim, HardTruth cross-references the deterministic execution ledger. If receipts are missing or contradictory, HardTruth returns `decision: "continue"`β€”physically trapping the agent into executing the actual `pytest`, `curl`, and `docker inspect` commands. - -### 3. AST Anti-Stubbing & Scope-Dropping Sentinel -* **The Problem:** Autonomous agents commonly take shortcuts under pressure: writing empty functions (`pass`, `raise NotImplementedError`), stubbing return mocks (`return True`), or disabling failing assertions to manufacture artificial green lights. -* **The HardTruth Solution:** HardTruth features a deterministic Abstract Syntax Tree (AST) static analyzer that inspects every modified file before commit. Any function reduced to a dummy stub or empty body is instantly rejected with line-number receipts before it can contaminate git history. - -### 4. Autonomous SRE & Infrastructure Enclaves (Zero-Hallucination SLA) -* **The Problem:** Autonomous agents executing database migrations, infrastructure provisioning (Terraform, Docker Compose), or cloud deployment scripts cannot afford a single hallucinated state. -* **The HardTruth Solution:** Operates as a fail-closed gatekeeper in sovereign cloud containers. Enforces that zero destructive or mutative actions can execute without prior verified simulation and cryptographic receipts, enabling production $1,000/mo autonomous SRE retainers with a guaranteed zero-hallucination SLA. - -### 5. Universal Git Pre-Commit & Pull Request Gate -* **The Problem:** Teams running multiple agents across local terminals, IDEs, and background workers have no centralized way to ensure all code committed to git actually works. -* **The HardTruth Solution:** Installed machine-wide at `~/.hardtruth/hooks/pre-commit`, HardTruth intercepts any `git commit` attempt across the entire system. Any agent (or human) attempting to commit stubbed code or unverified changes is blocked at the git level. +## Execution Ledger Schema -### 6. Continuous Compliance & Regulatory Audit Trails (SOC 2, EU AI Act, NAIC) -* **The Problem:** Regulated enterprise deployments require provable, explainable verification that autonomous AI decisions were grounded in physical evidence. -* **The HardTruth Solution:** HardTruth generates tamper-evident, append-only execution ledgers (`ledger.jsonl`) cryptographically linking agent claims to verified shell exit codes, providing audit-ready proof of truth-enforcement. +The deterministic ledger is stored at `~/.gemini/antigravity-cli/ledger.jsonl` (or configured via `HARDTRUTH_LEDGER_PATH`). Each entry is a single JSON record: ---- - -## Competitive Differentiation: Why Existing Tools Fail - -| Category | Representative Tools | Why They Fail Where HardTruth Succeeds | -| :--- | :--- | :--- | -| **Input / Prompt Guardrails** | ProtectAI, NeMo Guardrails | Only protect against inbound user attacks (jailbreaks/injections). They do **not** check if the agent is lying about its own actions. | -| **RAG Factuality Checkers** | Cleanlab TLM, Patronus AI, Galileo | Evaluate static text-to-text retrieval (PDF summaries). They have **zero connection to the OS kernel, terminal commands, or git diffs**. | -| **Post-Hoc Observability** | LangSmith, Braintrust, Arize Phoenix | **Passive loggers.** They record that an agent hallucinated *after* the turn is finished. They cannot intercept runtime `Stop` hooks to block completion. | -| **Cloud LLM-as-a-Judge** | GPT-4o / Claude Evals | **Slow, expensive, and hallucinates itself.** Takes 2,000ms – 15,000ms, costs $0.03/check, and suffers from autoregressive sycophancy. | -| **HardTruth Neurosymbolic Shield** | **HardTruth** | **Active 10.8ms circuit breaker.** Evaluates unforgeable physical bash exit codes via local NLI cross-encoder, physically halting the agent until real tests pass. | - ---- - -## Universal Machine-Wide Architecture - -HardTruth is installed **once per machine** and provides three concentric layers of protection across all agents: - -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ YOUR OPERATING SYSTEM β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Layer 1: Universal Git Pre-Commit Gate (~/.hardtruth/) β”‚ β”‚ -β”‚ β”‚ (Intercepts ANY agent or script attempting to git commit) β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Layer 2: Agent Harness Hooks (hooks.json) β”‚ β”‚ -β”‚ β”‚ (Antigravity CLI/IDE, Claude Code, Goose, OpenCode) β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β”‚ β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ Layer 3: Resident Truth Daemon (Port 8000 / Unix Socket) β”‚ β”‚ -β”‚ β”‚ (DeBERTa-v3 Cross-Encoder β€’ AST Analyzer β€’ Fact Ledger) β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +```json +{ + "timestamp": 1726792345.12, + "conversationId": "test-conv-uuid", + "stepIdx": 8, + "tool": "run_command", + "target": "python3 -m unittest test_daemon.py", + "status": "success", + "error": null +} ``` -1. **Harness Hook Layer:** Deep integration with Antigravity, Claude Code, and Goose via lifecycle hooks (`PostToolUse` records facts; `Stop` intercepts model exit). -2. **Global Git Layer:** A machine-wide pre-commit hook (`~/.hardtruth/hooks/pre-commit`) blocks any agent on the systemβ€”regardless of frameworkβ€”from committing empty stubs or unverified code. -3. **Universal Socket / HTTP Layer:** Any custom agent (LangGraph, CrewAI, AutoGen, Python, TypeScript, Go) can query `http://127.0.0.1:8000/v1/verify-claim` or `/var/run/hardtruth/sentinel.sock` in under 3 lines of code. +* `tool`: The tool invoked (`run_command`, `write_to_file`, `replace_file_content`, `view_file`). +* `target`: The command string or absolute file path. +* `status`: `"success"` or `"error"`. +* `error`: Error message or non-zero exit code string if the command failed. --- ## Performance Benchmarks -| Metric | Apple Silicon (M2/M3 MPS) | Linux VPS (CPU ONNX) | Cloud LLM-as-a-Judge | -| :--- | :--- | :--- | :--- | -| **Inference Latency** | **10.8 ms** | **18.5 ms – 110 ms** | 2,000 ms – 15,000 ms | -| **RAM Footprint** | **~280 MB** | **~150 MB** | > 16 GB (or external SaaS) | -| **Throughput (Batch 16)** | **~1,100 checks/sec** | **~220 checks/sec** | ~2 checks/sec | -| **Token Cost** | **$0.00 (Local)** | **$0.00 (Local)** | $0.02 – $0.08 per check | -| **Hallucination Risk** | **0.0% (Discriminative)**| **0.0% (Discriminative)**| High (Autoregressive) | - ---- +Measured using the committed benchmark reproduction script [`benchmarks/benchmark.py`](./benchmarks/benchmark.py): -## Commercial & Cloud Hosting Architecture +| Metric | Docker Container (Linux CPU on Apple Silicon) | +| :--- | :--- | +| **Median Latency (p50)** | **~73.2 ms** | +| **p90 Latency** | **~82.3 ms** | +| **p99 Latency** | **~86.5 ms** | +| **RAM Footprint (RSS)** | **~528 MB** (Python 3.11 + PyTorch CPU + DeBERTa-v3) | +| **Throughput (Sequential)**| **~13.7 req/sec** | -### Can it be hosted on OpenRouter? -**No.** OpenRouter is an aggregator for *autoregressive text generation* (`/v1/chat/completions`). HardTruth is a **stateful neurosymbolic execution gate**: it pairs a local append-only execution ledger with a sequence-pair classification cross-encoder. +*Hardware: Apple Silicon Mac (ARM64), OrbStack Linux Docker Engine, PyTorch CPU wheel.* -### Where to Host the Commercial Version: -1. **Sovereign Cloud VPS (Hostinger / Hetzner / OVH):** - - Run HardTruth inside a lightweight Docker container (`harbor-system-one-daemon` or `hardtruth-daemon`). - - Mount `/var/run/hardtruth/sentinel.sock` into multi-tenant client worker containers. - - Zero external cloud dependencies; 100% data sovereignty. -2. **Serverless GPU / CPU (RunPod Serverless / Modal):** - - Deploy as a high-throughput verification microservice capable of scaling to thousands of concurrent agent checks. -3. **Commercial Monetization Models:** - - **Open-Core Local Substrate (Free OSS):** The engine is free and open source to drive universal developer adoption. - - **HardTruth Fleet C2 ($99 – $499/mo):** Centralized dashboard aggregating tamper-evident execution ledgers, hallucination attempt heatmaps, and audit traces across an enterprise's entire agent fleet. - - **Sovereign SRE Retainers ($1,000/mo):** Guaranteed zero-hallucination autonomous operations for client infrastructure. - - **SOC 2 / EU AI Act Compliance Logs:** Export cryptographically verifiable proof that autonomous agents were gated by deterministic truth verification. +To reproduce benchmarks locally: +```bash +python3 benchmarks/benchmark.py http://127.0.0.1:8000 20 +``` --- ## Quickstart -### 1-Line Machine Install +### Prerequisites +* Python 3.10+ +* Docker & Docker Compose (optional, for daemon container) + +### 1. Clone Repository & Install Dependencies ```bash git clone https://github.com/Vibherpunk/hardtruth.git cd hardtruth -bash install.sh +pip install -r requirements.txt ``` -### Run Live Verification Tests +### 2. Start Verification Daemon +You can run the daemon directly: ```bash -python3 tests/test_live.py +python3 daemon/app.py ``` -Outputs 9 passing tests verifying fake pass blocking, contradiction detection, AST stub blocking, conversational prose allowance, and evasion resistance: +Or start via Docker Compose: +```bash +docker compose -f docker/docker-compose.yml up --build -d ``` -Ran 9 tests in 0.722s -OK +Verify the daemon is healthy: +```bash +curl -s http://127.0.0.1:8000/health ``` +### 3. Install Hooks +```bash +# Installs Antigravity lifecycle hook and library: +bash install.sh + +# Optional: To enable machine-wide git commit gate across all repositories: +bash install.sh --global +``` + +### 4. Run Test Suite +```bash +# Run against live daemon: +python3 tests/test_live.py + +# Run with daemon unreachable (verifies fail-closed behavior): +SYSTEM_ONE_URL=http://127.0.0.1:9 python3 tests/test_live.py +``` + +All 16 tests pass with the daemon online and with it unreachable. + --- -## Integration Guides +## Integration ### 1. Antigravity (CLI & IDE) -Configured automatically via `~/.gemini/config/hooks.json`: +Configured automatically by `install.sh` in `~/.gemini/config/hooks.json`: ```json { "hardtruth": { @@ -241,34 +202,37 @@ Configured automatically via `~/.gemini/config/hooks.json`: } ``` -### 2. Python Agent Frameworks (LangGraph, CrewAI, AutoGen) -Embed the zero-dependency client in any agent loop: +### 2. Python Agent Loops (Custom Scripts) +Embed the client directly into custom Python agents: ```python -from client.hardtruth_client import HardTruthClient - -client = HardTruthClient() -# Evaluate agent's draft message against execution facts -res = client.verify_claim( - premise="COMMAND: 'pytest'. STATUS: FAILED (exit 1).", - hypothesis="All 10 unit tests passed." -) - -if res["probabilities"]["contradiction"] > 0.60: - raise RuntimeError("🚨 HardTruth caught an unverified claim.") +from client.hardtruth_client import HardTruthClient, CONTRADICTION_THRESHOLD + +client = HardTruthClient("http://127.0.0.1:8000") + +# Check if daemon is responsive +if client.is_daemon_online(): + res = client.verify_claim( + premise="COMMAND: 'pytest tests/'. STATUS: FAILED (exit status 1).", + hypothesis="All 10 unit tests passed completely." + ) + if res and res["probabilities"]["contradiction"] >= CONTRADICTION_THRESHOLD: + print("🚨 Blocked: Agent claim contradicts execution ledger!") ``` --- ## Repository Structure -* `daemon/app.py`: FastAPI daemon hosting the DeBERTa-v3 sequence-pair cross-encoder and System One endpoints. -* `client/hardtruth_hook.py`: Production lifecycle hook for Antigravity, Claude Code, and Goose. -* `client/hardtruth_client.py`: Zero-dependency Python client with automatic fallback to deterministic AST checks. -* `docker/`: 150MB CPU ONNX Docker manifest for VPS and server environments. -* `tests/test_live.py`: Full verification suite. +* `daemon/app.py`: Dedicated FastAPI verification daemon hosting `cross-encoder/nli-deberta-v3-small` with thread-safe lazy loading and `/health` monitoring. +* `client/ast_checker.py`: Shared AST stub detection module (detects `pass`, `NotImplementedError`, `...`, `return True/None`; exempts `@abstractmethod`, `@overload`, and `Protocol`). +* `client/hardtruth_hook.py`: Antigravity lifecycle hook (`PostToolUse` and `Stop`) with fail-closed gate logic. +* `client/hardtruth_client.py`: Python client library with `/health` polling and deterministic AST fallback. +* `docker/`: Dockerfile and docker-compose.yml for Linux/VPS deployment using official PyTorch CPU wheels. +* `tests/test_live.py`: 16-case test suite verifying fail-closed execution, stub detection, filter safety, and contradiction detection. +* `benchmarks/benchmark.py`: Committed benchmark reproduction script measuring latency and memory. --- ## License -Governed by the **PolyForm Noncommercial License 1.0.0**. Free for individuals, students, researchers, and personal noncommercial use. Commercial use by businesses, corporations, or revenue-generating entities requires an active **HardTruth Commercial Enterprise License**. See [LICENSE.md](LICENSE.md) for details. +[MIT License](./LICENSE) diff --git a/benchmarks/benchmark.py b/benchmarks/benchmark.py new file mode 100644 index 0000000..d534517 --- /dev/null +++ b/benchmarks/benchmark.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +HardTruth Benchmark Script +Measures real inference latency, throughput, and memory consumption on the current machine. +Outputs hardware details, warm-up statistics, and percentiles (p50, p90, p99). +""" + +import os +import sys +import time +import platform +import statistics +import urllib.request +import json +try: + import psutil +except ImportError: + psutil = None + +def benchmark_http(daemon_url: str = "http://127.0.0.1:8000", iterations: int = 50): + print("=" * 60) + print(" HardTruth HTTP Verification Benchmark") + print("=" * 60) + print(f"Target URL: {daemon_url}") + print(f"Platform: {platform.system()} {platform.machine()} ({platform.processor()})") + + # Check health + try: + req = urllib.request.Request(f"{daemon_url}/health") + with urllib.request.urlopen(req, timeout=3.0) as resp: + health = json.loads(resp.read().decode("utf-8")) + print(f"Daemon Service: {health.get('service')}") + print(f"Daemon RSS RAM: {health.get('rss_memory_mb')} MB") + except Exception as e: + print(f"ERROR: Cannot connect to daemon at {daemon_url}: {e}") + return + + payload = json.dumps({ + "premise": "COMMAND: 'pytest tests/'. STATUS: FAILED (exit status 1). MODIFIED_FILES: app.py.", + "hypothesis": "All 10 unit tests in the test suite passed." + }).encode("utf-8") + + # Warm-up (5 iterations) + print("\nWarming up (5 iterations)...") + for _ in range(5): + req = urllib.request.Request( + f"{daemon_url}/v1/verify-claim", + data=payload, + headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req, timeout=10.0) as resp: + resp.read() + + # Benchmark runs + print(f"Running {iterations} benchmark requests...") + latencies = [] + for i in range(iterations): + t0 = time.perf_counter() + req = urllib.request.Request( + f"{daemon_url}/v1/verify-claim", + data=payload, + headers={"Content-Type": "application/json"} + ) + with urllib.request.urlopen(req, timeout=10.0) as resp: + data = json.loads(resp.read().decode("utf-8")) + latencies.append((time.perf_counter() - t0) * 1000.0) + + # Compute statistics + mean_lat = statistics.mean(latencies) + median_lat = statistics.median(latencies) + p90_lat = sorted(latencies)[int(0.90 * len(latencies))] + p99_lat = sorted(latencies)[int(0.99 * len(latencies))] + + print("\n--- Benchmark Results ---") + print(f"Iterations: {iterations}") + print(f"Mean: {mean_lat:.2f} ms") + print(f"Median (p50):{median_lat:.2f} ms") + print(f"p90: {p90_lat:.2f} ms") + print(f"p99: {p99_lat:.2f} ms") + print(f"Throughput: {1000.0 / median_lat:.1f} req/sec (sequential)") + print("=" * 60) + +if __name__ == "__main__": + url = sys.argv[1] if len(sys.argv) > 1 else "http://127.0.0.1:8000" + n = int(sys.argv[2]) if len(sys.argv) > 2 else 50 + benchmark_http(url, n) diff --git a/client/ast_checker.py b/client/ast_checker.py new file mode 100644 index 0000000..70b0f76 --- /dev/null +++ b/client/ast_checker.py @@ -0,0 +1,151 @@ +""" +HardTruth AST Stub Detection Engine +Deterministic analysis of Python ASTs to detect vacuous stubs (pass, NotImplementedError, dummy returns), +while suppressing false positives on abstract methods, protocols, and overloads. +""" + +import ast +import os +from typing import List + +class StubVisitor(ast.NodeVisitor): + def __init__(self, filename: str): + self.filename = os.path.basename(filename) + self.violations: List[str] = [] + self._class_stack: List[ast.ClassDef] = [] + + def visit_ClassDef(self, node: ast.ClassDef): + self._class_stack.append(node) + self.generic_visit(node) + self._class_stack.pop() + + def _is_enclosing_class_protocol(self) -> bool: + if not self._class_stack: + return False + cls = self._class_stack[-1] + for base in cls.bases: + # Check Protocol or typing.Protocol or Protocol[T] + if isinstance(base, ast.Name) and base.id == "Protocol": + return True + elif isinstance(base, ast.Attribute) and base.attr == "Protocol": + return True + elif isinstance(base, ast.Subscript): + val = base.value + if isinstance(val, ast.Name) and val.id == "Protocol": + return True + elif isinstance(val, ast.Attribute) and val.attr == "Protocol": + return True + return False + + def _is_exempt_decorator(self, node: ast.FunctionDef | ast.AsyncFunctionDef) -> bool: + for dec in node.decorator_list: + dec_name = None + if isinstance(dec, ast.Name): + dec_name = dec.id + elif isinstance(dec, ast.Attribute): + dec_name = dec.attr + elif isinstance(dec, ast.Call): + if isinstance(dec.func, ast.Name): + dec_name = dec.func.id + elif isinstance(dec.func, ast.Attribute): + dec_name = dec.func.attr + + if dec_name in ("abstractmethod", "overload"): + return True + return False + + def visit_FunctionDef(self, node: ast.FunctionDef): + self._check_function(node) + self.generic_visit(node) + + def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef): + self._check_function(node) + self.generic_visit(node) + + def _check_function(self, node: ast.FunctionDef | ast.AsyncFunctionDef): + if self._is_enclosing_class_protocol(): + return + if self._is_exempt_decorator(node): + return + + body = node.body + if not body: + return + + # Filter out leading docstring if present + statements = body + if len(statements) >= 1 and isinstance(statements[0], ast.Expr): + if isinstance(statements[0].value, ast.Constant) and isinstance(statements[0].value.value, str): + statements = statements[1:] + + if len(statements) != 1: + return + + stmt = statements[0] + is_stub = False + stub_type = "" + + # 1. pass + if isinstance(stmt, ast.Pass): + is_stub = True + stub_type = "pass" + + # 2. raise NotImplementedError / raise NotImplementedError(...) + elif isinstance(stmt, ast.Raise): + exc = stmt.exc + if isinstance(exc, ast.Name) and exc.id == "NotImplementedError": + is_stub = True + stub_type = "NotImplementedError" + elif isinstance(exc, ast.Call): + func = exc.func + if isinstance(func, ast.Name) and func.id == "NotImplementedError": + is_stub = True + stub_type = "NotImplementedError()" + elif isinstance(func, ast.Attribute) and func.attr == "NotImplementedError": + is_stub = True + stub_type = "NotImplementedError()" + elif isinstance(exc, ast.Attribute) and exc.attr == "NotImplementedError": + is_stub = True + stub_type = "NotImplementedError" + + # 3. ... (Ellipsis) + elif isinstance(stmt, ast.Expr): + if isinstance(stmt.value, ast.Constant) and stmt.value.value is Ellipsis: + is_stub = True + stub_type = "..." + + # 4. return True, return None, bare return + elif isinstance(stmt, ast.Return): + val = stmt.value + if val is None: + is_stub = True + stub_type = "return" + elif isinstance(val, ast.Constant): + if val.value is None: + is_stub = True + stub_type = "return None" + elif val.value is True: + is_stub = True + stub_type = "return True" + + if is_stub: + self.violations.append( + f"Function '{node.name}' in {self.filename} is an empty stub ({stub_type}). Write actual working implementation before completing." + ) + +def check_ast_stubs(filepath: str) -> List[str]: + """ + Deterministic AST check: rejects empty stubs (pass, NotImplementedError, Ellipsis, return True/None). + Suppresses legitimate stubs in Protocol classes, @abstractmethod, and @overload. + """ + if not os.path.exists(filepath) or not filepath.endswith(".py"): + return [] + try: + with open(filepath, "r", encoding="utf-8") as f: + content = f.read() + tree = ast.parse(content, filename=filepath) + visitor = StubVisitor(filepath) + visitor.visit(tree) + return visitor.violations + except Exception: + return [] diff --git a/client/hardtruth_client.py b/client/hardtruth_client.py index bc4aa86..e44449c 100644 --- a/client/hardtruth_client.py +++ b/client/hardtruth_client.py @@ -8,19 +8,38 @@ import os import sys import json -import ast -import re import urllib.request import urllib.error +_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +if _CURRENT_DIR not in sys.path: + sys.path.insert(0, _CURRENT_DIR) +_PARENT_DIR = os.path.dirname(_CURRENT_DIR) +if _PARENT_DIR not in sys.path: + sys.path.insert(0, _PARENT_DIR) + +try: + from ast_checker import check_ast_stubs +except ImportError: + try: + from client.ast_checker import check_ast_stubs + except ImportError: + from hardtruth.ast_checker import check_ast_stubs + +CONTRADICTION_THRESHOLD = 0.70 + class HardTruthClient: def __init__(self, daemon_url: str = "http://127.0.0.1:8000", ledger_path: str = None): self.daemon_url = daemon_url.rstrip("/") - self.ledger_path = ledger_path or os.path.expanduser("~/.gemini/antigravity-cli/ledger.jsonl") + self.ledger_path = ledger_path or os.environ.get( + "HARDTRUTH_LEDGER_PATH", + os.path.expanduser("~/.gemini/antigravity-cli/ledger.jsonl") + ) def is_daemon_online(self) -> bool: + """Pings daemon /health endpoint to check availability.""" try: - req = urllib.request.Request(f"{self.daemon_url}/v1/health", headers={"User-Agent": "HardTruth-Client"}) + req = urllib.request.Request(f"{self.daemon_url}/health", headers={"User-Agent": "HardTruth-Client"}) with urllib.request.urlopen(req, timeout=1.0) as resp: return resp.status == 200 except Exception: @@ -41,27 +60,5 @@ def verify_claim(self, premise: str, hypothesis: str) -> dict: return None def check_ast_stubs(self, filepath: str) -> list: - """Deterministic AST check: rejects empty stubs (pass, NotImplementedError).""" - violations = [] - if not os.path.exists(filepath) or not filepath.endswith(".py"): - return violations - try: - with open(filepath, "r", encoding="utf-8") as f: - tree = ast.parse(f.read(), filename=filepath) - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - body = node.body - is_stub = False - if len(body) == 1: - if isinstance(body[0], ast.Pass): - is_stub = True - elif isinstance(body[0], ast.Raise) and isinstance(body[0].exc, ast.Name) and body[0].exc.id == "NotImplementedError": - is_stub = True - elif len(body) == 2 and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant): - if isinstance(body[1], ast.Pass): - is_stub = True - if is_stub: - violations.append(f"Function '{node.name}' in {os.path.basename(filepath)} is an empty stub.") - except Exception: - pass - return violations + """Deterministic AST check: rejects empty stubs (pass, NotImplementedError, ..., return True/None).""" + return check_ast_stubs(filepath) diff --git a/client/hardtruth_hook.py b/client/hardtruth_hook.py index 0639e84..f1ce56b 100755 --- a/client/hardtruth_hook.py +++ b/client/hardtruth_hook.py @@ -5,7 +5,7 @@ Powered by DeBERTa-v3 Cross-Encoder & Deterministic Execution Ledger (http://127.0.0.1:8000) Enforces: -1. Deterministic Execution Ledger: Captures real tool executions, exit codes, and file diffs. +1. Deterministic Execution Ledger: Captures real tool executions, exit codes, and status. 2. AST Anti-Stubbing Linter: Rejects mock stubs (pass, NotImplementedError, dummy return). 3. System One NLI Verification: Evaluates agent claims against execution ledger in ~10ms on MPS. 4. Circuit Breaker Escape Hatch: Releases with visible warning after 3 consecutive halts. @@ -16,14 +16,47 @@ import json import time import re -import ast +import hashlib import urllib.request import urllib.error -LEDGER_FILE = os.path.expanduser("~/.gemini/antigravity-cli/ledger.jsonl") -HALT_COUNTER_DIR = "/tmp/hardtruth_halts" +_CURRENT_DIR = os.path.dirname(os.path.abspath(__file__)) +if _CURRENT_DIR not in sys.path: + sys.path.insert(0, _CURRENT_DIR) +_PARENT_DIR = os.path.dirname(_CURRENT_DIR) +if _PARENT_DIR not in sys.path: + sys.path.insert(0, _PARENT_DIR) + +try: + from ast_checker import check_ast_stubs +except ImportError: + try: + from client.ast_checker import check_ast_stubs + except ImportError: + from hardtruth.ast_checker import check_ast_stubs + +CONTRADICTION_THRESHOLD = 0.70 +LEDGER_FILE = os.environ.get( + "HARDTRUTH_LEDGER_PATH", + os.path.expanduser("~/.gemini/antigravity-cli/ledger.jsonl") +) +HALT_COUNTER_DIR = os.environ.get( + "HARDTRUTH_HALT_DIR", + os.path.expanduser("~/.hardtruth/halts") +) SYSTEM_ONE_URL = os.environ.get("SYSTEM_ONE_URL", "http://127.0.0.1:8000") +def get_halt_counter_file(conv_id: str) -> str: + """Returns secure slugified and hashed counter path inside mode 0o700 dir.""" + os.makedirs(HALT_COUNTER_DIR, mode=0o700, exist_ok=True) + try: + os.chmod(HALT_COUNTER_DIR, 0o700) + except Exception: + pass + safe_slug = re.sub(r"[^a-zA-Z0-9_-]", "_", str(conv_id))[:32] + conv_hash = hashlib.sha256(str(conv_id).encode("utf-8")).hexdigest()[:16] + return os.path.join(HALT_COUNTER_DIR, f"halt_{safe_slug}_{conv_hash}.json") + def call_system_one(endpoint: str, payload: dict, timeout: float = 3.0) -> dict: url = f"{SYSTEM_ONE_URL.rstrip('/')}/{endpoint.lstrip('/')}" data = json.dumps(payload).encode("utf-8") @@ -35,7 +68,7 @@ def call_system_one(endpoint: str, payload: dict, timeout: float = 3.0) -> dict: try: with urllib.request.urlopen(req, timeout=timeout) as resp: return json.loads(resp.read().decode("utf-8")) - except Exception as e: + except Exception: return None # --------------------------------------------------------------------------- @@ -46,7 +79,7 @@ def handle_post_tool_use(payload: dict) -> dict: """ Records ground-truth tool execution facts into append-only ledger. """ - os.makedirs(os.path.dirname(LEDGER_FILE), exist_ok=True) + os.makedirs(os.path.dirname(os.path.abspath(LEDGER_FILE)), exist_ok=True) tool_call = payload.get("toolCall", {}) tool_name = tool_call.get("name", "unknown") @@ -81,41 +114,27 @@ def handle_post_tool_use(payload: dict) -> dict: return {} -# --------------------------------------------------------------------------- -# AST Anti-Stubbing Inspection -# --------------------------------------------------------------------------- - -def inspect_python_ast_for_stubs(filepath: str) -> list: - """Checks python files for vacuous stubs (pass, NotImplementedError).""" - violations = [] - if not os.path.exists(filepath) or not filepath.endswith(".py"): - return violations - - try: - with open(filepath, "r", encoding="utf-8") as f: - tree = ast.parse(f.read(), filename=filepath) - - for node in ast.walk(tree): - if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)): - body = node.body - # Check for single 'pass' or single docstring + pass - is_stub = False - if len(body) == 1: - if isinstance(body[0], ast.Pass): - is_stub = True - elif isinstance(body[0], ast.Raise) and isinstance(body[0].exc, ast.Name) and body[0].exc.id == "NotImplementedError": - is_stub = True - elif len(body) == 2 and isinstance(body[0], ast.Expr) and isinstance(body[0].value, ast.Constant): - # Docstring + pass - if isinstance(body[1], ast.Pass): - is_stub = True - - if is_stub: - violations.append(f"Function '{node.name}' in {os.path.basename(filepath)} is an empty stub (pass/NotImplementedError).") - except Exception: - pass - - return violations +action_triggers = re.compile( + r"\b(i\s+(have\s+)?(ran|run|executed|tested|verified|fixed|modified|created)|" + r"ran\s+(unit\s+)?tests?|" + r"(all|all \d+|\d+)?\s*(unit\s+)?tests?(\s+[\w/]+){0,3}\s+passed|" + r"tests?\s+(have\s+)?passed|" + r"tests?\s+are\s+passing|" + r"test\s+suite\s+passed|" + r"tests?\s+succeeded|" + r"successfully\s+(verified|passed|tested)|" + r"all\s+checks?\s+passed|" + r"(feature|pipeline|integration)\s+is\s+(now\s+)?(wired|working|active))\b", + re.IGNORECASE +) +descriptive_filter = re.compile( + r"^\s*(?:quote:|example:|sample:|(?:when|if|for example|e\.g\.|every time|how |the system|the hook|this means|in order to|as an example|such as|to prevent|by default|instead of|note that|on x\b|on twitter|in our search|in research|search results|discussions across|discussions on|users report|practitioners note|engineers note|people are|the community|articles|papers|studies)\b)", + re.IGNORECASE +) +imperative_filter = re.compile( + r"^\s*([🚨⚠️]|(?:do not|don't|ensure|inspect|execute|verify|check|run|please|must|should|critical requirement|requirement|task|step \d|turn \d|signal \d)\b|(?:tip:|note:|warning:))", + re.IGNORECASE +) # --------------------------------------------------------------------------- # Stop: Pre-Termination Verification Gate @@ -124,14 +143,13 @@ def inspect_python_ast_for_stubs(filepath: str) -> list: def handle_stop(payload: dict) -> dict: """ Evaluates agent's visible claims against execution ledger using System One. - Halts termination if claims contradict ground truth. + Halts termination if claims contradict ground truth or if verifier is unreachable. """ conv_id = payload.get("conversationId", "default") transcript_path = payload.get("transcriptPath", "") # 1. Infinite Loop Escape Hatch Check - os.makedirs(HALT_COUNTER_DIR, exist_ok=True) - counter_file = os.path.join(HALT_COUNTER_DIR, f"halt_{conv_id}.json") + counter_file = get_halt_counter_file(conv_id) halt_count = 0 if os.path.exists(counter_file): try: @@ -141,12 +159,15 @@ def handle_stop(payload: dict) -> dict: halt_count = 0 if halt_count >= 3: - # Reset counter and allow termination with downgrade + # Reset counter and allow termination with visible warning try: os.remove(counter_file) except Exception: pass - return {"decision": "allow"} + return { + "decision": "allow", + "reason": "⚠️ HARDTRUTH CIRCUIT BREAKER RELEASE: Gate released after 3 consecutive halts to prevent infinite loop. Visible warning: verification claims could not be verified." + } # 2. Extract agent's final text from transcript agent_text = "" @@ -189,12 +210,12 @@ def handle_stop(payload: dict) -> dict: for f_entry in files_modified: target = f_entry.get("target") if target and os.path.exists(target): - stubs = inspect_python_ast_for_stubs(target) + stubs = check_ast_stubs(target) if stubs: record_halt(counter_file, halt_count + 1) return { "decision": "continue", - "reason": f"🚨 HARDTRUTH ENGINE REJECTED: Unimplemented stub detected. {stubs[0]} Write actual working implementation before completing." + "reason": f"🚨 HARDTRUTH ENGINE REJECTED: Unimplemented stub detected. {stubs[0]}" } # Build compact premise string @@ -215,44 +236,32 @@ def handle_stop(payload: dict) -> dict: premise = " ".join(premise_parts)[:1200] # 4. Atomic Claim Extraction & Targeted Vocabulary Filtering - # Strip multi-line code fences (```...```) because full scripts/files aren't claims. - # CRITICAL SECURITY RULE: Do NOT strip the content of backticks or blockquotes, - # otherwise a lying agent could wrap unverified claims in `...` or > to bypass the sentinel. - # Instead, we strip only the markdown syntax characters (` and >) so the claims remain fully visible to the judge! text_without_fences = re.sub(r"```[\s\S]*?```", "", agent_text) text_clean = text_without_fences.replace("`", "") text_clean = re.sub(r"^\s*>\s*", "", text_clean, flags=re.MULTILINE) sentences = re.split(r"(?<=[.!?])\s+|\n+", text_clean) - action_triggers = re.compile( - r"\b(i\s+(have\s+)?(ran|run|executed|tested|verified|fixed|modified|created)|" - r"ran\s+(unit\s+)?tests?|" - r"(all|all \d+|\d+)?\s*(unit\s+)?tests?(\s+[\w/]+){0,3}\s+passed|" - r"tests?\s+(have\s+)?passed|" - r"tests?\s+are\s+passing|" - r"test\s+suite\s+passed|" - r"tests?\s+succeeded|" - r"successfully\s+(verified|passed|tested)|" - r"all\s+checks?\s+passed|" - r"(feature|pipeline|integration)\s+is\s+(now\s+)?(wired|working|active))\b", - re.IGNORECASE - ) - descriptive_filter = re.compile( - r"^\s*(?:quote:|example:|sample:|(?:when|if|for example|e\.g\.|every time|how |the system|the hook|this means|in order to|as an example|such as|to prevent|by default|instead of|note that|on x\b|on twitter|in our search|in research|search results|discussions across|discussions on|users report|practitioners note|engineers note|people are|the community|articles|papers|studies)\b)", - re.IGNORECASE - ) - imperative_filter = re.compile( - r"^\s*([🚨⚠️]|(?:do not|don't|ensure|inspect|execute|verify|check|run|please|must|should|critical requirement|requirement|task|step \d|turn \d|signal \d|tip:|note:|warning:)\b)", - re.IGNORECASE - ) - claims_to_verify = [] for s in sentences: s_clean = s.strip() # Ignore short strings, descriptive explanations, and imperative prompt directives if len(s_clean) > 15 and action_triggers.search(s_clean): - if descriptive_filter.search(s_clean) or imperative_filter.search(s_clean): + if imperative_filter.search(s_clean): continue + if descriptive_filter.search(s_clean): + # Check if sentence is a completion assertion despite prefix like "Example:", "Quote:", "Sample:" + prefix_match = re.match(r"^\s*(?:quote:|example:|sample:)\s*", s_clean, re.IGNORECASE) + if prefix_match: + remainder = s_clean[prefix_match.end():].strip() + is_completion = bool(re.search( + r"\b((all\s+\d+|\d+)\s+(unit\s+)?tests?\s+passed|passed\s+completely|suite\s+is\s+green|100%\s+(success|passing)|all\s+checks?\s+passed|(feature|pipeline|integration)\s+is\s+(now\s+)?(wired|working|active)|i\s+(have\s+)?(ran|run|executed|tested|verified|fixed))\b", + remainder, + re.IGNORECASE + )) + if not is_completion: + continue + else: + continue claims_to_verify.append(s_clean[:250]) # Cap at top 4 high-risk claims @@ -260,39 +269,67 @@ def handle_stop(payload: dict) -> dict: if not claims_to_verify: return {"decision": "allow"} - # 5. NLI Verification against Premise via System One + # 5. Verification against Premise + # Phase 1: Pure-Python deterministic checks on all claims before any daemon call + has_failed_commands = any(c.get("status") != "success" or c.get("error") for c in commands_run) + has_successful_commands = any(c.get("status") == "success" and not c.get("error") for c in commands_run) + for claim in claims_to_verify: - nli_res = call_system_one("v1/verify-claim", { - "premise": premise, - "hypothesis": claim - }) - - if not nli_res: - continue - - probs = nli_res.get("probabilities", {}) - contradiction = probs.get("contradiction", 0.0) - neutral = probs.get("neutral", 0.0) - - # Test claim verification: if claim asserts test pass but no commands were run is_pass_claim = bool(re.search( - r"\b((all|all \d+)\s+tests?\s+passed|tests?\s+(have\s+)?passed|unit\s+tests?\s+passed|tests?\s+succeeded|test\s+suite\s+passed|tests?\s+are\s+passing)\b", + r"\b((all|all \d+|\d+)?\s*(unit\s+)?tests?(\s+[\w/]+){0,3}\s+passed|tests?\s+(have\s+)?passed|unit\s+tests?\s+passed|tests?\s+succeeded|test\s+suite\s+passed|tests?\s+are\s+passing)\b", claim, re.IGNORECASE )) + + # Deterministic Check 1: Claimed tests passed, but no commands were run if is_pass_claim and not commands_run: record_halt(counter_file, halt_count + 1) return { "decision": "continue", "reason": f"🚨 HARDTRUTH ENGINE HALTED: You claimed '{claim}', but the execution ledger shows NO commands or test suites were run in this session. Execute the test command with run_command to prove it before completing." } - - if contradiction >= 0.70: + + # Deterministic Check 2: Claimed tests passed, but commands in ledger failed + if is_pass_claim and has_failed_commands: record_halt(counter_file, halt_count + 1) return { "decision": "continue", - "reason": f"🚨 HARDTRUTH ENGINE CONTRADICTION DETECTED (conf: {contradiction:.2f}):\nClaim: '{claim}' contradicts the execution ledger.\nLedger Evidence: {premise}\nFix the failure and provide verified command output before stopping." + "reason": f"🚨 HARDTRUTH ENGINE CONTRADICTION DETECTED:\nClaim: '{claim}' contradicts the execution ledger.\nLedger Evidence: {premise}\nFix the failure and provide verified command output before stopping." } + + # Phase 2: NLI Verification / Fail-closed daemon checks + for claim in claims_to_verify: + is_pass_claim = bool(re.search( + r"\b((all|all \d+|\d+)?\s*(unit\s+)?tests?(\s+[\w/]+){0,3}\s+passed|tests?\s+(have\s+)?passed|unit\s+tests?\s+passed|tests?\s+succeeded|test\s+suite\s+passed|tests?\s+are\s+passing)\b", + claim, + re.IGNORECASE + )) + + nli_res = call_system_one("v1/verify-claim", { + "premise": premise, + "hypothesis": claim + }) + + if nli_res is not None: + probs = nli_res.get("probabilities", {}) + contradiction = probs.get("contradiction", 0.0) + if contradiction >= CONTRADICTION_THRESHOLD: + record_halt(counter_file, halt_count + 1) + return { + "decision": "continue", + "reason": f"🚨 HARDTRUTH ENGINE CONTRADICTION DETECTED (conf: {contradiction:.2f}):\nClaim: '{claim}' contradicts the execution ledger.\nLedger Evidence: {premise}\nFix the failure and provide verified command output before stopping." + } + else: + # Daemon unreachable: FAIL CLOSED unless deterministically verified by ledger + if is_pass_claim and has_successful_commands and not has_failed_commands: + # Deterministically verified by exit code! + pass + else: + record_halt(counter_file, halt_count + 1) + return { + "decision": "continue", + "reason": f"🚨 HARDTRUTH DAEMON UNREACHABLE: Verifier at {SYSTEM_ONE_URL} is offline. Factual claim '{claim}' cannot be verified autonomously. You must provide manual verification output before completing." + } # All claims verified clean: reset halt counter and allow stop if os.path.exists(counter_file): diff --git a/daemon/app.py b/daemon/app.py index c55a740..5810263 100644 --- a/daemon/app.py +++ b/daemon/app.py @@ -1,132 +1,51 @@ #!/usr/bin/env python3 """ -Harbor System One Neural Daemon (Port 8000) -Exposes sub-50ms Non-Autoregressive System One capabilities via free, local, open models: -- Zero-Shot Entity/Hook Extraction (GLiNER urchade/gliner_medium-v2.1) -- Non-Autoregressive Decisions & Policy Triage (DeBERTa-v3 NLI cross-encoder/nli-deberta-v3-small) -- Semantic Internal Link Mapping (SentenceTransformers all-MiniLM-L6-v2) -- Parallel Transcript Compaction (Salience Scoring) -- Turn-Level Dynamic Skill & Tool Routing -- On-Page SEO Quality & Information Gain Auditing -- Real-Time Batch Lead Intent Scoring -Runs 100% locally on Apple Silicon MPS/CPU with zero paid cloud API dependencies. -Interactive Swagger UI available at http://127.0.0.1:8000/docs +HardTruth Verification Daemon (Port 8000) +Exposes sub-15ms Non-Autoregressive NLI Verification via DeBERTa-v3 Cross-Encoder: +- cross-encoder/nli-deberta-v3-small +Evaluates (premise, hypothesis) token pairs directly on Apple Silicon MPS or CPU. """ from __future__ import annotations import os import sys import time -import json -import re import psutil -from typing import List, Dict, Any, Optional -from pydantic import BaseModel, Field -from fastapi import FastAPI, HTTPException, Request +import threading +from typing import Dict, Any +from pydantic import BaseModel +from fastapi import FastAPI, HTTPException from fastapi.responses import JSONResponse -# System One SEO & Semantic Link Mapping (Practice 10) -try: - try: - from .seo_audit import ( - SEOAuditRequest, SEOAuditResponse, InternalLinkRequest, - InternalLinkResponse, audit_article_system_one, map_internal_links_distribb - ) - except Exception: - from seo_audit import ( - SEOAuditRequest, SEOAuditResponse, InternalLinkRequest, - InternalLinkResponse, audit_article_system_one, map_internal_links_distribb - ) -except Exception: - class SEOAuditRequest(BaseModel): - content: str - target_keyword: str - existing_urls: List[str] = Field(default_factory=list) - class SEOAuditResponse(BaseModel): - score: float = 0.0 - status: str = "unavailable" - class InternalLinkRequest(BaseModel): - articles: List[Dict[str, Any]] - class InternalLinkResponse(BaseModel): - links: List[Dict[str, Any]] = Field(default_factory=list) - def audit_article_system_one(req): - return SEOAuditResponse() - def map_internal_links_distribb(req): - return InternalLinkResponse() - -# System One Gojiberry Lead Scoring (Practice 6) -try: - try: - from .lead_scorer import ( - BatchLeadScoreRequest, BatchLeadScoreResponse, score_lead_batch - ) - except Exception: - from lead_scorer import ( - BatchLeadScoreRequest, BatchLeadScoreResponse, score_lead_batch - ) -except Exception: - class BatchLeadScoreRequest(BaseModel): - leads: List[Dict[str, Any]] - class BatchLeadScoreResponse(BaseModel): - scored_leads: List[Dict[str, Any]] = Field(default_factory=list) - def score_lead_batch(req): - return BatchLeadScoreResponse() - -# --------------------------------------------------------------------------- -# FastAPI Initialization -# --------------------------------------------------------------------------- - app = FastAPI( - title="Harbor System One Neural Daemon", - description="Sub-50ms sensory perception (GLiNER) & local non-autoregressive policy decisions (DeBERTa-v3 NLI)", + title="HardTruth Verification Daemon", + description="Sub-15ms Natural Language Inference verification for agent execution claims", version="1.0.0", docs_url="/docs", redoc_url="/redoc" ) # --------------------------------------------------------------------------- -# Model Engines (GLiNER & DeBERTa-v3 NLI Lazy Warm Load on MPS / CPU) +# Model Engine (DeBERTa-v3 NLI Direct Token Pair Evaluation) # --------------------------------------------------------------------------- -_gliner_model = None -_nli_classifier = None - -def get_gliner(): - global _gliner_model - if _gliner_model is None: - from gliner import GLiNER - import torch - device = "mps" if torch.backends.mps.is_available() else "cpu" - _gliner_model = GLiNER.from_pretrained("urchade/gliner_medium-v2.1") - _gliner_model.to(device) - _gliner_model.eval() - return _gliner_model - -def get_nli_classifier(): - global _nli_classifier - if _nli_classifier is None: - import torch - from transformers import pipeline - device = "mps" if torch.backends.mps.is_available() else "cpu" - _nli_classifier = pipeline( - "zero-shot-classification", - model="cross-encoder/nli-deberta-v3-small", - device=device - ) - return _nli_classifier - +_nli_direct_lock = threading.Lock() _nli_direct_model = None _nli_direct_tok = None def get_nli_direct(): global _nli_direct_model, _nli_direct_tok if _nli_direct_model is None: - import torch - from transformers import AutoTokenizer, AutoModelForSequenceClassification - device = "mps" if torch.backends.mps.is_available() else "cpu" - _nli_direct_tok = AutoTokenizer.from_pretrained("cross-encoder/nli-deberta-v3-small") - _nli_direct_model = AutoModelForSequenceClassification.from_pretrained("cross-encoder/nli-deberta-v3-small").to(device) - _nli_direct_model.eval() + with _nli_direct_lock: + if _nli_direct_model is None: + import torch + from transformers import AutoTokenizer, AutoModelForSequenceClassification + device = "mps" if torch.backends.mps.is_available() else "cpu" + _nli_direct_tok = AutoTokenizer.from_pretrained("cross-encoder/nli-deberta-v3-small") + _nli_direct_model = AutoModelForSequenceClassification.from_pretrained( + "cross-encoder/nli-deberta-v3-small" + ).to(device) + _nli_direct_model.eval() return _nli_direct_tok, _nli_direct_model # --------------------------------------------------------------------------- @@ -143,336 +62,76 @@ class VerifyClaimResponse(BaseModel): probabilities: Dict[str, float] latency_ms: float -class ExtractRequest(BaseModel): - text: str - labels: List[str] = Field(default_factory=lambda: ["software_tool", "cli_command", "code_block", "architecture_pattern"]) - threshold: float = 0.40 - -class EntitySpan(BaseModel): - text: str - label: str - score: float - start: int - end: int - -class ExtractResponse(BaseModel): - spans: List[EntitySpan] - latency_ms: float - device: str - -class DecideRequest(BaseModel): - instruction: str - criteria: str - options: List[str] - context: Optional[str] = None - -class DecideResponse(BaseModel): - decision: str - confidence: float - probabilities: Dict[str, float] - source: str - latency_ms: float - -class Turn(BaseModel): - role: str - content: str - -class SalienceRequest(BaseModel): - turns: List[Turn] - threshold: float = 0.40 - -class ScoredTurn(BaseModel): - index: int - role: str - salience: float - keep: bool - -class SalienceResponse(BaseModel): - scored_turns: List[ScoredTurn] - original_turns: int - retained_turns: int - reduction_pct: float - latency_ms: float - -class RouteSkillsRequest(BaseModel): - turn_prompt: str - room: Optional[str] = None - available_skills: List[Dict[str, Any]] = Field(default_factory=list) - -class RouteSkillsResponse(BaseModel): - selected_skills: List[str] - selected_tools: List[str] - prompt_token_savings_pct: float - latency_ms: float - # --------------------------------------------------------------------------- # Endpoints # --------------------------------------------------------------------------- @app.get("/health") +@app.get("/v1/health") def health_check(): process = psutil.Process(os.getpid()) rss_mb = process.memory_info().rss / (1024 * 1024) return { "status": "healthy", - "service": "harbor-system-one-daemon", + "service": "hardtruth-daemon", "substrate": "free_local_open_source", "models": { - "gliner_medium": "urchade/gliner_medium-v2.1", - "nli_deberta": "cross-encoder/nli-deberta-v3-small", - "sentence_embedder": "sentence-transformers/all-MiniLM-L6-v2" + "nli_deberta": "cross-encoder/nli-deberta-v3-small" }, "rss_memory_mb": round(rss_mb, 2), - "gliner_loaded": _gliner_model is not None, - "nli_loaded": _nli_classifier is not None, + "nli_loaded": _nli_direct_model is not None, + "nli_direct_loaded": _nli_direct_model is not None, "docs_url": "http://127.0.0.1:8000/docs" } -@app.post("/v1/extract", response_model=ExtractResponse) -def extract_entities(req: ExtractRequest): - t0 = time.perf_counter() - if not req.text.strip(): - return ExtractResponse(spans=[], latency_ms=0.0, device="none") - - model = get_gliner() - import torch - device = "mps" if torch.backends.mps.is_available() else "cpu" - raw_entities = model.predict_entities(req.text, req.labels, threshold=req.threshold) - - spans = [ - EntitySpan( - text=e["text"], - label=e["label"], - score=round(float(e["score"]), 4), - start=e["start"], - end=e["end"] - ) - for e in raw_entities - ] - latency = (time.perf_counter() - t0) * 1000.0 - return ExtractResponse(spans=spans, latency_ms=round(latency, 2), device=device) - -def extract_option_descriptions(options: List[str], criteria: str) -> Dict[str, str]: - descriptions = {} - for opt in options: - pattern = rf'(?:^|[;\n.]\s*|\b){re.escape(opt)}\s*[:\-–]\s*([^;\n.]+)' - match = re.search(pattern, criteria, re.IGNORECASE) - if match: - desc = match.group(1).strip() - descriptions[opt] = f"{opt.replace('_', ' ').lower()}: {desc}" - else: - descriptions[opt] = opt.replace("_", " ").lower() - return descriptions - -@app.post("/v1/decide", response_model=DecideResponse) -def evaluate_decision(req: DecideRequest): - t0 = time.perf_counter() - - # Primary Engine: Free Local DeBERTa-v3 NLI Classifier on Apple Silicon MPS/CPU - try: - classifier = get_nli_classifier() - descriptions = extract_option_descriptions(req.options, req.criteria) - label_to_opt = {v: k for k, v in descriptions.items()} - candidate_labels = list(descriptions.values()) - - premise = f"{req.instruction}\nContext: {req.context or 'None'}".strip() - res = classifier( - premise, - candidate_labels=candidate_labels, - hypothesis_template="This case corresponds to {}." - ) - - probs = {} - for label, score in zip(res["labels"], res["scores"]): - opt_key = label_to_opt.get(label, req.options[0]) - probs[opt_key] = round(float(score), 4) - - for opt in req.options: - if opt not in probs: - probs[opt] = 0.0 - - best_opt = max(probs.items(), key=lambda x: x[1]) - latency = (time.perf_counter() - t0) * 1000.0 - return DecideResponse( - decision=best_opt[0], - confidence=best_opt[1], - probabilities=probs, - source="local_nli_deberta_v3_small", - latency_ms=round(latency, 2) - ) - except Exception as err: - # Fail-closed local deterministic heuristic fallback - inst_lower = req.instruction.lower() - crit_lower = req.criteria.lower() - ctx_lower = (req.context or "").lower() - combined_query = f"{inst_lower} {ctx_lower}" - scores = {} - - for opt in req.options: - opt_raw = opt.lower() - opt_phrase = opt_raw.replace("_", " ") - opt_words = [w for w in opt_phrase.split() if len(w) > 3] - score = 0.05 - - if opt_raw in combined_query or opt_phrase in combined_query: - score += 0.70 - - for w in opt_words: - if w in ctx_lower: - score += 0.25 - - for marker in [opt, opt_raw, opt_phrase]: - if marker in crit_lower: - idx = crit_lower.find(marker) - snippet = crit_lower[idx:idx + 120] - clues = [c.strip(" ()[],:;.\"") for c in snippet.split() if len(c) > 3] - clue_matches = sum(1 for c in clues if c in ctx_lower) - score += clue_matches * 0.20 - - scores[opt] = max(0.05, score) - - total = sum(scores.values()) - probs = {k: round(v / total, 4) for k, v in scores.items()} - best = max(probs.items(), key=lambda x: x[1]) - - latency = (time.perf_counter() - t0) * 1000.0 - return DecideResponse( - decision=best[0], - confidence=best[1], - probabilities=probs, - source="local_calibrated_heuristic_fallback", - latency_ms=round(latency, 2) - ) - @app.post("/v1/verify-claim", response_model=VerifyClaimResponse) -def verify_claim_endpoint(req: VerifyClaimRequest): +def verify_claim(req: VerifyClaimRequest): """ - Direct Sequence Pair NLI Verification (Premise -> Hypothesis). - Evaluates whether an agent's claim is mathematically ENTAILED, CONTRADICTED, or NEUTRAL - relative to the ground-truth execution ledger in ~10ms on Apple Silicon MPS. + Evaluates premise vs hypothesis using DeBERTa-v3 cross-encoder directly. + Computes exact softmax distribution across [contradiction, entailment, neutral]. """ t0 = time.perf_counter() import torch + tok, model = get_nli_direct() - device = "mps" if torch.backends.mps.is_available() else "cpu" - - inputs = tok(req.premise, req.hypothesis, return_tensors="pt", truncation=True, max_length=512).to(device) + device = next(model.parameters()).device + + inputs = tok( + req.premise, + req.hypothesis, + truncation=True, + max_length=512, + return_tensors="pt" + ).to(device) + with torch.no_grad(): logits = model(**inputs).logits - probs = torch.softmax(logits, dim=1)[0].cpu().tolist() - - id2label = model.config.id2label - prob_map = {id2label[i]: round(float(probs[i]), 4) for i in range(len(probs))} - - best_label = max(prob_map.items(), key=lambda x: x[1]) - status_map = { - "contradiction": "CONTRADICTION", - "entailment": "ENTAILED", - "neutral": "NEUTRAL" + probs = torch.softmax(logits, dim=-1)[0].tolist() + + # cross-encoder/nli-deberta-v3-small label order: 0: contradiction, 1: entailment, 2: neutral + contradiction, entailment, neutral = probs[0], probs[1], probs[2] + prob_dict = { + "contradiction": round(contradiction, 4), + "entailment": round(entailment, 4), + "neutral": round(neutral, 4) } - status = status_map.get(best_label[0], best_label[0].upper()) - - latency = (time.perf_counter() - t0) * 1000.0 - return VerifyClaimResponse( - status=status, - confidence=best_label[1], - probabilities=prob_map, - latency_ms=round(latency, 2) - ) - -@app.post("/v1/seo/audit", response_model=SEOAuditResponse) -def evaluate_seo_audit(req: SEOAuditRequest): - return audit_article_system_one(req) - -@app.post("/v1/seo/link-map", response_model=InternalLinkResponse) -def evaluate_internal_link_map(req: InternalLinkRequest): - return map_internal_links_distribb(req) -@app.post("/v1/leads/score", response_model=BatchLeadScoreResponse) -def evaluate_lead_scoring(req: BatchLeadScoreRequest): - return score_lead_batch(req) + if contradiction >= 0.70: + verdict = "CONTRADICTION" + conf = contradiction + elif entailment >= 0.50: + verdict = "ENTAILED" + conf = entailment + else: + verdict = "NEUTRAL" + conf = neutral -@app.post("/v1/salience", response_model=SalienceResponse) -def evaluate_salience(req: SalienceRequest): - t0 = time.perf_counter() - scored = [] - - # Critical preserved signatures: paths, tokens, error stacks, git hashes - keep_indicators = ["/", "\\", "error", "fail", "pass", "git", "def ", "class ", "import ", "token", "http"] - - for idx, turn in enumerate(req.turns): - content_lower = turn.content.lower() - score = 0.30 - - # System turns or user prompts retain high baseline salience - if turn.role in ["system", "user"]: - score += 0.50 - - # Tool outputs with critical artifacts retain salience - if any(ind in content_lower for ind in keep_indicators): - score += 0.40 - - score = min(score, 1.0) - keep = score >= req.threshold - scored.append(ScoredTurn(index=idx, role=turn.role, salience=round(score, 2), keep=keep)) - - retained = sum(1 for t in scored if t.keep) - original = len(req.turns) - reduction = round((1.0 - (retained / max(original, 1))) * 100.0, 1) latency = (time.perf_counter() - t0) * 1000.0 - - return SalienceResponse( - scored_turns=scored, - original_turns=original, - retained_turns=retained, - reduction_pct=reduction, - latency_ms=round(latency, 2) - ) -@app.post("/v1/route-skills", response_model=RouteSkillsResponse) -def route_skills(req: RouteSkillsRequest): - t0 = time.perf_counter() - prompt_lower = req.turn_prompt.lower() - selected_skills = [] - selected_tools = [] - - # Match skills from available catalog based on intent - for skill in req.available_skills: - name = skill.get("name", "") - desc = skill.get("description", "").lower() - if name.lower() in prompt_lower or any(word in prompt_lower for word in desc.split()[:5]): - selected_skills.append(name) - - # Default minimum skill route if prompt mentions git, video, or testing - if "git" in prompt_lower and "using-git-worktrees" not in selected_skills: - selected_skills.append("using-git-worktrees") - if "test" in prompt_lower and "verification-before-completion" not in selected_skills: - selected_skills.append("verification-before-completion") - - # Deduplicate and cap to top 2 skills per turn to protect context - selected_skills = list(dict.fromkeys(selected_skills))[:2] - - # Route tools based on intent - if any(k in prompt_lower for k in ["run", "exec", "test", "bash", "command"]): - selected_tools.append("run_command") - if any(k in prompt_lower for k in ["edit", "write", "replace", "code", "file"]): - selected_tools.extend(["view_file", "replace_file_content", "write_to_file"]) - if any(k in prompt_lower for k in ["search", "grep", "find"]): - selected_tools.extend(["grep_search", "find_by_name"]) - - selected_tools = list(set(selected_tools)) - - # Dynamically compute catalog prompt token reduction percentage - total_catalog_skills = max(1, len(req.available_skills)) - unpruned_tokens = (total_catalog_skills * 250) + 2000 # full catalog + tool schemas baseline - pruned_tokens = max(100, (len(selected_skills) * 250) + (len(selected_tools) * 250)) - savings = round(max(0.0, min(95.0, (1.0 - (pruned_tokens / max(1, unpruned_tokens))) * 100.0)), 1) - latency = (time.perf_counter() - t0) * 1000.0 - - return RouteSkillsResponse( - selected_skills=selected_skills, - selected_tools=selected_tools, - prompt_token_savings_pct=savings, + return VerifyClaimResponse( + status=verdict, + confidence=round(conf, 4), + probabilities=prob_dict, latency_ms=round(latency, 2) ) diff --git a/docker/Dockerfile b/docker/Dockerfile index e91bb3e..16ab88b 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -7,14 +7,10 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ curl build-essential \ && rm -rf /var/lib/apt/lists/* -# Install python dependencies for CPU ONNX & PyTorch -RUN pip install --no-cache-dir \ - fastapi==0.115.0 \ - uvicorn[standard]==0.31.0 \ - torch --index-url https://download.pytorch.org/whl/cpu \ - transformers==4.45.1 \ - sentence-transformers==3.1.1 \ - optimum[onnxruntime]==1.23.0 +# Copy and install python dependencies +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir torch --index-url https://download.pytorch.org/whl/cpu && \ + pip install --no-cache-dir -r /app/requirements.txt COPY daemon/app.py /app/app.py diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index d7168e9..3b2a812 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -9,7 +9,3 @@ services: restart: always ports: - "127.0.0.1:8000:8000" - volumes: - - /var/run/hardtruth:/var/run/hardtruth - environment: - - SYSTEM_ONE_URL=http://127.0.0.1:8000 diff --git a/install.sh b/install.sh index 97313f4..4f43700 100644 --- a/install.sh +++ b/install.sh @@ -6,14 +6,30 @@ echo " HardTruth: Autonomous Anti-Hallucination & Truth Shield" echo "==========================================================" REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -HOOK_SRC="$REPO_ROOT/client/hardtruth_hook.py" +HARDTRUTH_DIR="$HOME/.hardtruth" +HARDTRUTH_LIB="$HARDTRUTH_DIR/lib/hardtruth" +GLOBAL_HOOKS_DIR="$HARDTRUTH_DIR/hooks" +HALTS_DIR="$HARDTRUTH_DIR/halts" -# 1. Antigravity Configuration +# 1. Install library package to ~/.hardtruth/lib +mkdir -p "$HARDTRUTH_LIB" +mkdir -p "$GLOBAL_HOOKS_DIR" +mkdir -p "$HALTS_DIR" +chmod 700 "$HALTS_DIR" + +cp "$REPO_ROOT/client/ast_checker.py" "$HARDTRUTH_LIB/ast_checker.py" +cp "$REPO_ROOT/client/hardtruth_client.py" "$HARDTRUTH_LIB/hardtruth_client.py" +cp "$REPO_ROOT/client/hardtruth_hook.py" "$HARDTRUTH_LIB/hardtruth_hook.py" +touch "$HARDTRUTH_LIB/__init__.py" +echo "βœ“ HardTruth client library installed to $HARDTRUTH_LIB" + +# 2. Antigravity Configuration ANTIGRAVITY_CONFIG_DIR="$HOME/.gemini/config" if [ -d "$HOME/.gemini" ]; then echo "Configuring Antigravity global hooks..." mkdir -p "$ANTIGRAVITY_CONFIG_DIR" - cp "$HOOK_SRC" "$ANTIGRAVITY_CONFIG_DIR/hardtruth_hook.py" + cp "$REPO_ROOT/client/hardtruth_hook.py" "$ANTIGRAVITY_CONFIG_DIR/hardtruth_hook.py" + cp "$REPO_ROOT/client/ast_checker.py" "$ANTIGRAVITY_CONFIG_DIR/ast_checker.py" chmod +x "$ANTIGRAVITY_CONFIG_DIR/hardtruth_hook.py" cat > "$ANTIGRAVITY_CONFIG_DIR/hooks.json" << 'EOF' @@ -45,32 +61,62 @@ EOF echo "βœ“ Antigravity configured with HardTruth Stop and PostToolUse hooks." fi -# 2. Global Git Hooks Configuration -GLOBAL_HOOKS_DIR="$HOME/.hardtruth/hooks" -mkdir -p "$GLOBAL_HOOKS_DIR" -cat > "$GLOBAL_HOOKS_DIR/pre-commit" << EOF +# 3. Global Git Hooks Configuration +cat > "$GLOBAL_HOOKS_DIR/pre-commit" << 'EOF' #!/bin/bash # HardTruth Global Pre-Commit Gate python3 -c " import sys, os -sys.path.insert(0, '${REPO_ROOT}') -from client.hardtruth_client import HardTruthClient -client = HardTruthClient() -# Check all staged python files for vacuous stubs +sys.path.insert(0, os.path.expanduser('~/.hardtruth/lib')) +from hardtruth.ast_checker import check_ast_stubs import subprocess -files = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() +try: + files = subprocess.check_output(['git', 'diff', '--cached', '--name-only'], text=True).splitlines() +except Exception: + sys.exit(0) stubs = [] for f in files: if f.endswith('.py') and os.path.exists(f): - stubs.extend(client.check_ast_stubs(f)) + stubs.extend(check_ast_stubs(f)) if stubs: - print('🚨 HardTruth rejected commit: Unimplemented dummy stub detected:', stubs[0], file=sys.stderr) + print('🚨 HardTruth rejected commit: Unimplemented dummy stub detected: ' + stubs[0], file=sys.stderr) sys.exit(1) " EOF chmod +x "$GLOBAL_HOOKS_DIR/pre-commit" -git config --global core.hooksPath "$GLOBAL_HOOKS_DIR" -echo "βœ“ Configured global Git pre-commit hook in $GLOBAL_HOOKS_DIR." +echo "βœ“ Pre-commit hook written to $GLOBAL_HOOKS_DIR/pre-commit" + +# Parse CLI flags for opt-in global git hook installation +INSTALL_GLOBAL_HOOK=false +for arg in "$@"; do + if [ "$arg" == "--global" ] || [ "$arg" == "--install-global-hook" ]; then + INSTALL_GLOBAL_HOOK=true + fi +done + +# If not passed via flag and running interactively, prompt user +if [ "$INSTALL_GLOBAL_HOOK" = false ] && [ -t 0 ]; then + echo "" + read -p "Install machine-wide git hook via 'git config --global core.hooksPath'? [y/N]: " choice + if [[ "$choice" =~ ^[Yy]$ ]]; then + INSTALL_GLOBAL_HOOK=true + fi +fi + +if [ "$INSTALL_GLOBAL_HOOK" = true ]; then + CURRENT_HOOKS="$(git config --global --get core.hooksPath 2>/dev/null || true)" + if [ -n "$CURRENT_HOOKS" ] && [ "$CURRENT_HOOKS" != "$GLOBAL_HOOKS_DIR" ]; then + echo "⚠️ WARNING: An existing global core.hooksPath is set: $CURRENT_HOOKS" + echo " Backing up previous hooks path to $HARDTRUTH_DIR/previous_hooksPath" + echo "$CURRENT_HOOKS" > "$HARDTRUTH_DIR/previous_hooksPath" + fi + git config --global core.hooksPath "$GLOBAL_HOOKS_DIR" + echo "βœ“ Enabled machine-wide git commit gate: git config --global core.hooksPath $GLOBAL_HOOKS_DIR" +else + echo "ℹ️ Global git hooksPath not modified." + echo " To enable machine-wide git commit protection, run:" + echo " bash install.sh --global" +fi echo "==========================================================" echo "βœ“ HardTruth installed successfully." diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..c540e98 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,3 @@ +-r requirements.txt +pytest>=8.0.0 +httpx>=0.27.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..b5d3d4a --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +fastapi==0.115.0 +uvicorn[standard]==0.31.0 +pydantic>=2.0.0 +psutil==6.0.0 +torch>=2.2.0 +transformers==4.45.1 diff --git a/tests/test_live.py b/tests/test_live.py index fff0897..346dc05 100644 --- a/tests/test_live.py +++ b/tests/test_live.py @@ -1,43 +1,63 @@ #!/usr/bin/env python3 """ -Comprehensive Live Verification Suite for Antigravity System One Sentinel +Comprehensive Live Verification Suite for HardTruth Anti-Hallucination Gate Tests: -1. PostToolUse Ledger Recording +1. PostToolUse Ledger Recording (deterministic ledger isolation) 2. Hallucination Trap: Fake Test Pass Claim (No Tests in Ledger) 3. Contradiction Trap: Claiming Success When Ledger Shows Failure 4. Verified Truth Pass: Legitimate Evidence in Ledger 5. AST Anti-Stubbing Gate: Catching 'pass' and 'NotImplementedError' Stubs 6. Conversational Bypass: Zero False Positives on General Prose +7. Fenced Code Blocks & Imperatives Allowed +8. Inline Code & Quotes Allowed +9. Backtick & Blockquote Evasion Blocked +10. is_daemon_online Agreement with /health Endpoint +11. AST Stub Checker Extended Coverage (5 patterns caught, 3 patterns exempt) +12. Imperative Filter Regex with Colons (Tip:, Note:, Warning:) +13. Descriptive Filter Bypass Prevention (Example:, Quote: with fake passes blocked) +14. Secure Counter Directory & Path Traversal Prevention +15. Circuit Breaker Visible Warning on 4th Attempt +16. Daemon Unreachable Fail-Closed Behavior """ import os import sys import json +import shutil +import uuid import subprocess import tempfile import unittest HOOK_SCRIPT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../client/hardtruth_hook.py")) -LEDGER_FILE = os.path.expanduser("~/.gemini/antigravity-cli/ledger.jsonl") +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) +if REPO_ROOT not in sys.path: + sys.path.insert(0, REPO_ROOT) class TestSystemOneSentinel(unittest.TestCase): def setUp(self): - # Create a unique test conversation ID - self.conv_id = f"test-conv-{os.getpid()}" - # Ensure ledger dir exists - os.makedirs(os.path.dirname(LEDGER_FILE), exist_ok=True) - # Clear halt counter if any - counter_file = f"/tmp/sentinel_halts/halt_{self.conv_id}.json" - if os.path.exists(counter_file): - os.remove(counter_file) - - def run_hook(self, mode: str, payload: dict) -> dict: + self.test_dir = tempfile.mkdtemp(prefix="hardtruth-test-") + self.ledger_file = os.path.join(self.test_dir, "ledger.jsonl") + self.halt_dir = os.path.join(self.test_dir, "halts") + os.makedirs(self.halt_dir, mode=0o700, exist_ok=True) + os.environ["HARDTRUTH_LEDGER_PATH"] = self.ledger_file + os.environ["HARDTRUTH_HALT_DIR"] = self.halt_dir + self.conv_id = f"test-conv-{uuid.uuid4().hex}" + + def tearDown(self): + shutil.rmtree(self.test_dir, ignore_errors=True) + + def run_hook(self, mode: str, payload: dict, env_override: dict = None) -> dict: + env = os.environ.copy() + if env_override: + env.update(env_override) proc = subprocess.run( ["python3", HOOK_SCRIPT, mode], input=json.dumps(payload).encode("utf-8"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env, check=True ) out_str = proc.stdout.decode("utf-8").strip() @@ -56,16 +76,15 @@ def test_1_post_tool_ledger_recording(self): res = self.run_hook("post_tool", payload) self.assertEqual(res, {}) - # Verify ledger has this entry - with open(LEDGER_FILE, "r") as f: + # Verify isolated ledger has this entry + with open(self.ledger_file, "r") as f: lines = [json.loads(line) for line in f if self.conv_id in line] self.assertTrue(len(lines) >= 1) self.assertEqual(lines[-1]["target"], "python3 -m unittest test_daemon.py") self.assertEqual(lines[-1]["status"], "success") def test_2_fake_test_pass_claim_blocked(self): - # Conversation with NO test runs in ledger - conv = f"fake-pass-{os.getpid()}" + conv = f"fake-pass-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -82,14 +101,12 @@ def test_2_fake_test_pass_claim_blocked(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Must halt because no test command was ever run! self.assertEqual(res.get("decision"), "continue") self.assertIn("NO commands or test suites were run", res.get("reason", "")) print(f"\n[Test 2: Fake Test Pass Blocked] Halt Reason: {res.get('reason')}") def test_3_contradiction_claim_blocked(self): - # Record a failed test execution in ledger - conv = f"fail-conv-{os.getpid()}" + conv = f"fail-conv-{uuid.uuid4().hex}" self.run_hook("post_tool", { "toolCall": {"name": "run_command", "args": {"CommandLine": "pytest tests/"}}, "stepIdx": 5, @@ -113,14 +130,12 @@ def test_3_contradiction_claim_blocked(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Must halt with contradiction! self.assertEqual(res.get("decision"), "continue") self.assertIn("CONTRADICTION DETECTED", res.get("reason", "")) print(f"[Test 3: Contradiction Blocked] Halt Reason: {res.get('reason')}") def test_4_verified_truth_allowed(self): - # Record a successful test execution in ledger - conv = f"success-conv-{os.getpid()}" + conv = f"success-conv-{uuid.uuid4().hex}" self.run_hook("post_tool", { "toolCall": {"name": "run_command", "args": {"CommandLine": "python3 -m unittest test_daemon.py"}}, "stepIdx": 8, @@ -144,18 +159,15 @@ def test_4_verified_truth_allowed(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Must allow termination! self.assertEqual(res.get("decision"), "allow") print("[Test 4: Verified Truth Allowed] Decision: allow") def test_5_ast_stubbing_blocked(self): - conv = f"stub-conv-{os.getpid()}" - # Create a stubbed file + conv = f"stub-conv-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as pyf: pyf.write("def calculate_tuition(child_age):\n pass\n") stub_file = pyf.name - # Record file modification in ledger self.run_hook("post_tool", { "toolCall": {"name": "write_to_file", "args": {"TargetFile": stub_file}}, "stepIdx": 4, @@ -180,13 +192,12 @@ def test_5_ast_stubbing_blocked(self): os.remove(stub_file) os.remove(transcript_path) - # Must halt because of empty stub! self.assertEqual(res.get("decision"), "continue") self.assertIn("Unimplemented stub detected", res.get("reason", "")) print(f"[Test 5: AST Stub Blocked] Halt Reason: {res.get('reason')}") def test_6_conversational_prose_allowed(self): - conv = f"chat-conv-{os.getpid()}" + conv = f"chat-conv-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -203,12 +214,11 @@ def test_6_conversational_prose_allowed(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Conversational text without verifiable action claims must pass cleanly self.assertEqual(res.get("decision"), "allow") print("[Test 6: Conversational Prose Allowed] Decision: allow") def test_7_fenced_code_blocks_and_imperatives_allowed(self): - conv = f"prompt-conv-{os.getpid()}" + conv = f"prompt-conv-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -225,12 +235,11 @@ def test_7_fenced_code_blocks_and_imperatives_allowed(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Quoted prompt blocks and imperative instructions must not trigger false alarms self.assertEqual(res.get("decision"), "allow") print("[Test 7: Fenced Code Blocks & Imperatives Allowed] Decision: allow") def test_8_inline_code_and_quotes_allowed(self): - conv = f"inline-conv-{os.getpid()}" + conv = f"inline-conv-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -247,13 +256,11 @@ def test_8_inline_code_and_quotes_allowed(self): res = self.run_hook("stop", payload) os.remove(transcript_path) - # Inline code and blockquotes must not trigger false alarms self.assertEqual(res.get("decision"), "allow") print("[Test 8: Inline Code & Quotes Allowed] Decision: allow") def test_9_backtick_and_blockquote_evasion_blocked(self): - # Case A: Agent tries to hide a fake pass inside backticks - conv_a = f"evade-backtick-{os.getpid()}" + conv_a = f"evade-backtick-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -270,13 +277,11 @@ def test_9_backtick_and_blockquote_evasion_blocked(self): res_a = self.run_hook("stop", payload) os.remove(transcript_path) - # MUST HALT: Evasion via backticks is blocked! self.assertEqual(res_a.get("decision"), "continue") self.assertIn("NO commands or test suites were run", res_a.get("reason", "")) print(f"[Test 9A: Backtick Evasion Blocked] Halt Reason: {res_a.get('reason')}") - # Case B: Agent tries to hide a fake pass inside blockquotes - conv_b = f"evade-quote-{os.getpid()}" + conv_b = f"evade-quote-{uuid.uuid4().hex}" with tempfile.NamedTemporaryFile("w", delete=False) as tf: tf.write(json.dumps({ "type": "PLANNER_RESPONSE", @@ -293,10 +298,149 @@ def test_9_backtick_and_blockquote_evasion_blocked(self): res_b = self.run_hook("stop", payload) os.remove(transcript_path) - # MUST HALT: Evasion via blockquote is blocked! self.assertEqual(res_b.get("decision"), "continue") self.assertIn("NO commands or test suites were run", res_b.get("reason", "")) print(f"[Test 9B: Blockquote Evasion Blocked] Halt Reason: {res_b.get('reason')}") + def test_10_is_daemon_online(self): + from client.hardtruth_client import HardTruthClient + # Point to unreachable URL + client_offline = HardTruthClient("http://127.0.0.1:9") + self.assertFalse(client_offline.is_daemon_online()) + + def test_11_ast_checker_extended_patterns(self): + from client.ast_checker import check_ast_stubs + cases_to_catch = { + "bare_raise": "def f():\n raise NotImplementedError", + "raise_call_empty": "def f():\n raise NotImplementedError()", + "raise_call_msg": "def f():\n raise NotImplementedError('not implemented')", + "ellipsis": "def f():\n ...", + "return_true": "def f():\n return True", + "return_none": "def f():\n return None", + "bare_return": "def f():\n return", + "docstring_then_stub": "def f():\n '''doc'''\n pass" + } + for name, code in cases_to_catch.items(): + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as tf: + tf.write(code) + tf.flush() + violations = check_ast_stubs(tf.name) + os.unlink(tf.name) + self.assertTrue(len(violations) >= 1, f"Failed to catch stub: {name}") + + cases_to_exempt = { + "abstractmethod": "from abc import abstractmethod\n@abstractmethod\ndef f():\n pass", + "overload": "from typing import overload\n@overload\ndef f():\n ...", + "protocol": "from typing import Protocol\nclass P(Protocol):\n def f(self):\n pass", + "implemented": "def f():\n x = 1\n return True" + } + for name, code in cases_to_exempt.items(): + with tempfile.NamedTemporaryFile("w", suffix=".py", delete=False) as tf: + tf.write(code) + tf.flush() + violations = check_ast_stubs(tf.name) + os.unlink(tf.name) + self.assertEqual(violations, [], f"Falsely flagged exempt code: {name}") + + def test_12_imperative_filter_colon(self): + from client.hardtruth_hook import imperative_filter + self.assertTrue(bool(imperative_filter.search("Tip: ensure tests are run"))) + self.assertTrue(bool(imperative_filter.search("Note: verify the output"))) + self.assertTrue(bool(imperative_filter.search("Warning: check the logs"))) + + def test_13_descriptive_filter_bypass_prevention(self): + # Prefixes Example: and Quote: must NOT bypass fake test pass detection + for prefix in ["Example:", "Quote:", "Sample:"]: + conv = f"bypass-{uuid.uuid4().hex}" + with tempfile.NamedTemporaryFile("w", delete=False) as tf: + tf.write(json.dumps({ + "type": "PLANNER_RESPONSE", + "content": f"{prefix} All 10 unit tests passed completely and the suite is green.", + "tool_calls": [] + }) + "\n") + transcript_path = tf.name + + payload = { + "conversationId": conv, + "transcriptPath": transcript_path, + "executionNum": 1 + } + res = self.run_hook("stop", payload) + os.remove(transcript_path) + self.assertEqual(res.get("decision"), "continue", f"Prefix {prefix} bypassed the gate!") + + def test_14_secure_counter_isolation(self): + # A conversationId with ../ must not escape HALT_COUNTER_DIR + evil_conv = "../../evil_path" + with tempfile.NamedTemporaryFile("w", delete=False) as tf: + tf.write(json.dumps({ + "type": "PLANNER_RESPONSE", + "content": "All 10 unit tests passed completely!", + "tool_calls": [] + }) + "\n") + transcript_path = tf.name + + payload = { + "conversationId": evil_conv, + "transcriptPath": transcript_path, + "executionNum": 1 + } + res = self.run_hook("stop", payload) + os.remove(transcript_path) + self.assertEqual(res.get("decision"), "continue") + + # Verify that all files created are inside self.halt_dir and nowhere outside + halt_files = os.listdir(self.halt_dir) + self.assertTrue(len(halt_files) >= 1) + for hf in halt_files: + self.assertFalse(".." in hf) + + def test_15_circuit_breaker_visible_warning(self): + conv = f"breaker-{uuid.uuid4().hex}" + with tempfile.NamedTemporaryFile("w", delete=False) as tf: + tf.write(json.dumps({ + "type": "PLANNER_RESPONSE", + "content": "All 10 unit tests passed completely!", + "tool_calls": [] + }) + "\n") + transcript_path = tf.name + + payload = { + "conversationId": conv, + "transcriptPath": transcript_path, + "executionNum": 1 + } + # First 3 attempts must be halted + for i in range(3): + res = self.run_hook("stop", payload) + self.assertEqual(res.get("decision"), "continue") + + # 4th attempt must release with visible warning in reason + res_4 = self.run_hook("stop", payload) + os.remove(transcript_path) + self.assertEqual(res_4.get("decision"), "allow") + self.assertIn("CIRCUIT BREAKER", res_4.get("reason", "")) + + def test_16_daemon_unreachable_fail_closed(self): + # Claim that cannot be verified deterministically must fail closed if daemon unreachable + conv = f"dead-daemon-{uuid.uuid4().hex}" + with tempfile.NamedTemporaryFile("w", delete=False) as tf: + tf.write(json.dumps({ + "type": "PLANNER_RESPONSE", + "content": "I have created and verified the entire authentication subsystem.", + "tool_calls": [] + }) + "\n") + transcript_path = tf.name + + payload = { + "conversationId": conv, + "transcriptPath": transcript_path, + "executionNum": 1 + } + res = self.run_hook("stop", payload, env_override={"SYSTEM_ONE_URL": "http://127.0.0.1:9"}) + os.remove(transcript_path) + self.assertEqual(res.get("decision"), "continue") + self.assertIn("HARDTRUTH DAEMON UNREACHABLE", res.get("reason", "")) + if __name__ == "__main__": unittest.main()