From f2428a583d8343c629489b9de63c9702c31a7045 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 04:16:34 +0000 Subject: [PATCH 1/4] BetterDecompiler v2: multi-provider, cross-platform rewrite Replace the Python/Flask + g4f proxy with a single dependency-free Go binary for Windows, macOS and Linux: - Bring your own key: Anthropic, OpenAI, Gemini, OpenRouter, Groq, DeepSeek, Mistral, xAI, any OpenAI/Anthropic/Gemini-compatible endpoint, plus local Ollama and LM Studio. Offline renamer when no key is configured. - Rename mode (default): the AI returns a compact JSON name map that a scope-resolving Luau parser applies and verifies, so behavior cannot change. Rewrite and offline modes also available. - Caching (memory + disk), request de-duplication, parallel chunking, retries with backoff and a provider fallback chain. - Local HTTP server with web UI, JSON API and the v1 /fix_script endpoint; Origin/Content-Type/Host checks and optional token. - CLI for files, folders and stdin; interactive setup. - New executor client that hooks decompile and keeps the v1 signature. - Tests need no API key: a mock provider emulates all three APIs, and renames are checked by executing code under Lua 5.1 and the Luau VM. - CI on all three OSes and a tag-triggered release workflow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RSCSfZjQxPkNn9HP29NmFV --- .github/workflows/ci.yml | 53 ++ .github/workflows/release.yml | 50 ++ .gitignore | 7 + CHANGELOG.md | 44 ++ README.md | 280 +++++++- client/betterdecompiler.lua | 301 +++++++++ client/tests/json.lua | 151 +++++ client/tests/run.lua | 261 ++++++++ cmd/betterdecompiler/clean.go | 220 +++++++ cmd/betterdecompiler/main.go | 174 +++++ cmd/betterdecompiler/main_test.go | 66 ++ cmd/betterdecompiler/serve.go | 193 ++++++ cmd/betterdecompiler/setup.go | 249 +++++++ cmd/mockllm/main.go | 19 + examples/decompiled.lua | 47 ++ flask_ai_server.py | 53 -- go.mod | 3 + internal/config/config.go | 392 +++++++++++ internal/config/config_test.go | 154 +++++ internal/engine/cache.go | 118 ++++ internal/engine/engine.go | 680 ++++++++++++++++++++ internal/engine/engine_test.go | 405 ++++++++++++ internal/engine/prompts.go | 67 ++ internal/llm/anthropic.go | 143 +++++ internal/llm/gemini.go | 119 ++++ internal/llm/llm.go | 349 ++++++++++ internal/llm/llm_test.go | 258 ++++++++ internal/llm/openai.go | 133 ++++ internal/llm/presets.go | 87 +++ internal/luau/heuristics.go | 487 ++++++++++++++ internal/luau/lexer.go | 348 ++++++++++ internal/luau/luau_test.go | 415 ++++++++++++ internal/luau/parser.go | 1001 +++++++++++++++++++++++++++++ internal/luau/rename.go | 294 +++++++++ internal/mockllm/mockllm.go | 268 ++++++++ internal/server/server.go | 234 +++++++ internal/server/server_test.go | 200 ++++++ internal/server/web/index.html | 165 +++++ main.lua | 82 --- scripts/build.sh | 34 + scripts/e2e.sh | 63 ++ 41 files changed, 8512 insertions(+), 155 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 .github/workflows/release.yml create mode 100644 .gitignore create mode 100644 CHANGELOG.md create mode 100644 client/betterdecompiler.lua create mode 100644 client/tests/json.lua create mode 100644 client/tests/run.lua create mode 100644 cmd/betterdecompiler/clean.go create mode 100644 cmd/betterdecompiler/main.go create mode 100644 cmd/betterdecompiler/main_test.go create mode 100644 cmd/betterdecompiler/serve.go create mode 100644 cmd/betterdecompiler/setup.go create mode 100644 cmd/mockllm/main.go create mode 100644 examples/decompiled.lua delete mode 100644 flask_ai_server.py create mode 100644 go.mod create mode 100644 internal/config/config.go create mode 100644 internal/config/config_test.go create mode 100644 internal/engine/cache.go create mode 100644 internal/engine/engine.go create mode 100644 internal/engine/engine_test.go create mode 100644 internal/engine/prompts.go create mode 100644 internal/llm/anthropic.go create mode 100644 internal/llm/gemini.go create mode 100644 internal/llm/llm.go create mode 100644 internal/llm/llm_test.go create mode 100644 internal/llm/openai.go create mode 100644 internal/llm/presets.go create mode 100644 internal/luau/heuristics.go create mode 100644 internal/luau/lexer.go create mode 100644 internal/luau/luau_test.go create mode 100644 internal/luau/parser.go create mode 100644 internal/luau/rename.go create mode 100644 internal/mockllm/mockllm.go create mode 100644 internal/server/server.go create mode 100644 internal/server/server_test.go create mode 100644 internal/server/web/index.html delete mode 100644 main.lua create mode 100755 scripts/build.sh create mode 100755 scripts/e2e.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..814869a --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + test: + name: Go tests (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - run: go vet ./... + - run: go test -race ./... + if: runner.os != 'Windows' + - run: go test ./... + if: runner.os == 'Windows' + + integration: + name: Lua, Luau and end-to-end tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Install Lua 5.1 and the official Luau toolchain + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq lua5.1 + curl -sSL -o /tmp/luau.zip https://github.com/luau-lang/luau/releases/latest/download/luau-ubuntu.zip + mkdir -p "$HOME/luau" && unzip -q /tmp/luau.zip -d "$HOME/luau" + echo "$HOME/luau" >> "$GITHUB_PATH" + - name: Rename safety checked by executing code (Lua 5.1 + Luau VM) + run: go test ./internal/luau -run 'Execution' -v + - name: Client compiles as Luau + run: luau-compile --null client/betterdecompiler.lua + - name: Client unit tests + run: lua5.1 client/tests/run.lua + - name: End-to-end (server + mock AI provider + Lua client) + run: scripts/e2e.sh + - name: Cross-compile every release target + run: scripts/build.sh ci diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..fe600d2 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,50 @@ +name: Release + +on: + push: + tags: ["v*"] + workflow_dispatch: + inputs: + tag: + description: Existing tag to (re)publish, e.g. v2.0.0 + required: true + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + env: + TAG: ${{ github.event.inputs.tag || github.ref_name }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ env.TAG }} + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + - name: Test + run: | + go vet ./... + go test ./... + - name: Build all platforms + run: scripts/build.sh "$TAG" + - name: Release notes + run: | + awk -v tag="$TAG" ' + $0 ~ "^## \\[" tag "\\]" {on=1; next} + on && /^## \[/ {exit} + on {print} + ' CHANGELOG.md > notes.md + [ -s notes.md ] || echo "See CHANGELOG.md" > notes.md + - name: Publish GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + if gh release view "$TAG" >/dev/null 2>&1; then + gh release upload "$TAG" dist/*.zip dist/*.tar.gz dist/betterdecompiler.lua dist/SHA256SUMS.txt --clobber + else + gh release create "$TAG" dist/*.zip dist/*.tar.gz dist/betterdecompiler.lua dist/SHA256SUMS.txt \ + --title "BetterDecompiler $TAG" --notes-file notes.md --latest + fi diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f2f648b --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/dist/ +/betterdecompiler +/betterdecompiler.exe +/mockllm +*.test +*.out +.DS_Store diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..f1c7f17 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,44 @@ +# Changelog + +## [v2.0.0] + +A complete rewrite of BetterDecompiler. + +### Highlights + +- **Use your own API key** with Anthropic, OpenAI, Google Gemini, OpenRouter, Groq, DeepSeek, Mistral or xAI, + any OpenAI-, Anthropic- or Gemini-compatible endpoint (Azure, proxies, self-hosted), or free local models via + Ollama and LM Studio. The unofficial g4f proxy is gone. +- **Offline mode**: no key needed. Names are inferred from Roblox API usage (services, `WaitForChild` names, + `Instance.new`, event callbacks, loops, constructors). +- **Rename mode (new default)**: the AI only proposes names and comments as compact JSON, and a scope-aware Luau + parser applies them. Every rename is verified so the code's behavior cannot change. It is faster and much cheaper + than having the AI re-type the whole script. +- **Rewrite mode** is still available for full AI rewrites. Its output is checked for syntax errors. +- **One self-contained program** for Windows, macOS (Intel and Apple Silicon) and Linux (x64, ARM64, ARMv7). No + Python or pip. +- **Built-in web UI**, a **CLI** for files, folders and stdin, and a **JSON HTTP API**. +- **New executor client** (`betterdecompiler.lua`) that hooks the global `decompile`, so Dex and other tools get + cleaned output automatically. It supports `request`, `http_request`, `syn.request`, `http.request` and + `fluxus.request`, keeps the v1 `decompile(script, enabled, url)` signature, and always falls back to raw output on + errors. + +### Speed and reliability + +- Results are cached in memory and on disk; identical concurrent requests share one AI call. +- Large scripts are split at top-level statements and processed in parallel. +- Retries with exponential backoff (honoring `Retry-After`), timeouts, and a configurable provider fallback chain + that ends with the offline renamer. +- Connections to providers are pooled and reused. + +### Security + +- The server listens on `127.0.0.1` by default, rejects cross-site browser requests and DNS-rebinding attempts, + and supports an optional access token. +- API keys are stored in a config file readable only by you, or referenced from environment variables. + +### Compatibility + +- The v1 `POST /fix_script` endpoint is still supported, so `dex_betterdecompiler.lua` and v1 clients work with the + v2 server unchanged. The default port is still 5000. +- `flask_ai_server.py` and `main.lua` were removed; `client/betterdecompiler.lua` replaces `main.lua`. diff --git a/README.md b/README.md index 155b89c..8ccf106 100644 --- a/README.md +++ b/README.md @@ -1,29 +1,269 @@ -# BetterDecompiler: +# BetterDecompiler -### Description: - Uses your standard executor decompile function, - Passes through an AI (such as gpt-3.5 or gpt-4), - Returns cleaned variable and func named output. +[![CI](https://github.com/officialmelon/BetterDecompiler/actions/workflows/ci.yml/badge.svg)](https://github.com/officialmelon/BetterDecompiler/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/officialmelon/BetterDecompiler)](https://github.com/officialmelon/BetterDecompiler/releases/latest) - Usage: +**Turn decompiled Roblox Luau into code you can actually read.** BetterDecompiler takes your executor's +decompiler output, recovers meaningful variable names, and adds short comments. It works with **your own API key** +for any major AI provider, with **free local models**, or **fully offline**, and runs on **Windows, macOS and Linux** +as a single small program with nothing else to install. - decompile(script_instance, bool_value, custom_url) +```lua +-- before (raw decompiler output) -- after (example, rename mode with an AI provider) +local v1 = game:GetService("Players") local Players = game:GetService("Players") +local v2 = v1.LocalPlayer local localPlayer = Players.LocalPlayer +v2.CharacterAdded:Connect(function(p1) -- Disable the shop UI when the character dies. + local v3 = p1:WaitForChild("Humanoid") localPlayer.CharacterAdded:Connect(function(character) + v3.Died:Connect(function() local humanoid = character:WaitForChild("Humanoid") + u4("Shop", false) humanoid.Died:Connect(function() + end) toggleShop("Shop", false) +end) end) + end) +``` + +## What's new in v2 + +v1 was a Python/Flask script that sent your code through a free reverse proxy (g4f). That approach broke often, +was slow, and sometimes changed what scripts did. v2 is a rewrite: + +| | v1 | v2 | +|---|---|---| +| AI access | Unofficial free proxy (g4f), often broken | **Your own key**: Anthropic, OpenAI, Gemini, OpenRouter, Groq, DeepSeek, Mistral, xAI, any OpenAI-compatible API, plus local Ollama / LM Studio | +| Without an API key | Nothing | **Offline renamer** infers names from Roblox API usage, instantly and for free | +| Install | Python + pip packages | **One file** to download: no Python, Node or other runtime | +| Platforms | Wherever Python works | Prebuilt for Windows, macOS (Intel + Apple Silicon), Linux (x64, ARM64, ARMv7) | +| Correctness | AI rewrote the whole script and could invent or drop code | **Rename mode**: the AI only suggests names; a scope-aware Luau parser applies them and checks that **every identifier still refers to the same variable**, so behavior can't change | +| Speed and cost | AI re-typed the entire script | The AI returns a small JSON list of names instead of the whole script, large scripts are split and processed in parallel, and results are cached in memory and on disk | +| Reliability | One provider, no retries | Retries with backoff, a fallback chain across providers, and the original output returned if everything fails | +| Interfaces | Executor only | Executor client, built-in **web UI**, **CLI** for files and folders, JSON HTTP API | +| Security | Open to any website the browser loaded | Listens only on this computer, blocks cross-site and DNS-rebinding requests, optional token | + +The old `/fix_script` endpoint still works, so the bundled Dark Dex (`dex_betterdecompiler.lua`) and v1 clients +work unchanged with the v2 server. + +## Quick start + +**1. Download** the archive for your computer from the +[latest release](https://github.com/officialmelon/BetterDecompiler/releases/latest) and extract it: + +| Your computer | File | +|---|---| +| Windows | `betterdecompiler-*-windows-amd64.zip` (`arm64` for ARM laptops) | +| Mac with Apple Silicon (M1 or later) | `betterdecompiler-*-darwin-arm64.tar.gz` | +| Mac with Intel | `betterdecompiler-*-darwin-amd64.tar.gz` | +| Linux | `betterdecompiler-*-linux-amd64.tar.gz` (`arm64` / `arm` for Raspberry Pi and similar) | + +**2. Run it.** On Windows, double-click `betterdecompiler.exe`. On macOS or Linux, run `./betterdecompiler` in a terminal. +The first time, it asks which AI to use and for your API key. You can skip this to use the offline renamer, and run +`betterdecompiler setup` later to change it. + +> macOS may block a downloaded program the first time. Run `xattr -d com.apple.quarantine ./betterdecompiler` once, +> or right-click it in Finder and choose Open. + +**3. In your executor**, run: + +```lua +loadstring(game:HttpGet("https://github.com/officialmelon/BetterDecompiler/releases/latest/download/betterdecompiler.lua"))() +``` + +The global `decompile` is now hooked, so **Dex, script viewers and any other tool show cleaned code automatically**. +If the server isn't running or something fails, you get the normal decompiler output, so it's safe to leave enabled. + +You can also open **http://localhost:5000** in a browser to paste code into the web UI. + +## Providers + +Bring a key from any of these. Keys are read from the config file (written by `betterdecompiler setup`) or from the +environment variable shown. If no provider is chosen, BetterDecompiler uses the first one it finds a key for. + +| Provider | Default model | API key variable | Notes | +|---|---|---|---| +| `anthropic` | `claude-opus-5-5` | `ANTHROPIC_API_KEY` | Uses server-side refusal fallback automatically | +| `openai` | `gpt-5.6-terra` | `OPENAI_API_KEY` | | +| `gemini` | `gemini-3.7-flash` | `GEMINI_API_KEY` or `GOOGLE_API_KEY` | Has a free tier | +| `openrouter` | `openrouter/auto` | `OPENROUTER_API_KEY` | Hundreds of models with one key | +| `groq` | `openai/gpt-oss-120b` | `GROQ_API_KEY` | Very fast; has a free tier | +| `deepseek` | `deepseek-chat` | `DEEPSEEK_API_KEY` | Low cost | +| `mistral` | `mistral-large-latest` | `MISTRAL_API_KEY` | | +| `xai` | `grok-code-fast-1` | `XAI_API_KEY` | | +| `ollama` | `qwen2.5-coder:7b` | none | Local and free; install [Ollama](https://ollama.com) and `ollama pull qwen2.5-coder:7b` | +| `lmstudio` | the loaded model | none | Local and free; start LM Studio's server | +| `offline` | none | none | No AI: names come from Roblox API usage | +| *custom* | anything | any | Any OpenAI-compatible, Anthropic-compatible or Gemini-compatible endpoint (Azure, Together, Fireworks, vLLM, proxies, ...) | + +Default models change as providers release new ones. Pick another with `-m`, in `setup`, or in the config file, and +run `betterdecompiler models -p ` to see what your key can use. For the cheapest and fastest results, use a +small model (for example `claude-haiku-5-5`, a Gemini Flash model, or Groq). Renaming doesn't need the largest model. + +## Modes + +| Mode | What the AI does | Speed and cost | Can it change behavior? | +|---|---|---|---| +| **`rename`** (default) | Suggests names and comments as compact JSON; BetterDecompiler applies them with its own Luau parser | Fast and cheap: output is a small list of names, not the whole script | **No.** Each rename is checked against the program's scopes. Conflicting names get a suffix (`player2`) and anything unsafe is skipped | +| `rewrite` | Rewrites the code: names, comments and cleaner structure | Slower; the full script is generated | Possibly. AI output is checked for syntax errors, but you should still review it | +| `offline` | No AI. Names come from patterns like `GetService("X")`, `WaitForChild("Name")`, `Instance.new`, event callbacks (`PlayerAdded` → `player`), loops and constructors | Instant and free | **No** | + +In `rename` mode, names the AI leaves out are filled in by the offline heuristics. Comments can be `none`, `light` +(default) or `detailed`. + +## Executor client + +```lua +getgenv().BetterDecompilerConfig = { -- all optional; set before loading + Url = "http://localhost:5000", -- where the server runs + Token = nil, -- if you started the server with --token + Hook = true, -- replace the global decompile + Mode = nil, -- "rename" | "rewrite" | "offline" (nil = server default) + Provider = nil, Model = nil, -- override the server's provider or model + Comments = nil, -- "none" | "light" | "detailed" + Cache = true, Silent = false, +} +local BD = loadstring(game:HttpGet("https://github.com/officialmelon/BetterDecompiler/releases/latest/download/betterdecompiler.lua"))() + +print(decompile(script)) -- hooked: cleaned output +print(BD.decompile(script, { Mode = "rewrite" })) -- per-call options +local code, info = BD.clean(someSource) -- clean code you already have +print(info.provider, info.model, info.renamed) +BD.unhook() -- restore the original decompile +``` - script_instance: being the LocalScript you want to decompile. - bool_value: (true or false) whether you want BetterDecompiler enabled or not - custom_url: (url string) whether you want to use a custom link (or port) for the AI. OPTIONAL PARAMETER -### Installation: -Install Python (python.org) +`BD.decompile(script, enabled, url)` keeps the v1 signature, so v1 scripts keep working. The client works with +executors that expose `request`, `http_request`, `syn.request`, `http.request` or `fluxus.request`. It uses +`getscripthash` (when available) so reopening a script doesn't decompile or clean it again. -Install packages for pip: ```pip install -U g4f flask curl_cffi``` +**Dex:** load the client first, then any Dex. Because `decompile` is hooked, Dex shows cleaned code. The bundled +`dex_betterdecompiler.lua` also still works, since it talks to the v2 server's v1 endpoint: -Run Python Script: ```python flask_ai_server.py``` -# Dark Dex v4 (BetterDecompiler built in) +```lua +loadstring(game:HttpGet("https://github.com/officialmelon/BetterDecompiler/raw/main/dex_betterdecompiler.lua"))() ``` -loadstring(game:HttpGet("https://github.com/OfficiallyMelon/BetterDecompiler/raw/main/dex_betterdecompiler.lua"))() + +## Command line + +```sh +betterdecompiler # start the server (same as `serve`) +betterdecompiler serve --listen 127.0.0.1:5050 --token mysecret +betterdecompiler clean dumped.lua # cleaned code to stdout +betterdecompiler clean dumped.lua -o clean.lua +betterdecompiler clean ./dump -o ./clean # whole folder (.lua/.luau), processed in parallel +betterdecompiler clean ./dump --in-place -p groq --comments none +cat dumped.lua | betterdecompiler clean - # stdin to stdout +betterdecompiler clean dumped.lua --mode offline # no AI, instant +betterdecompiler clean x.lua --json # machine-readable result (names, tokens, warnings) +betterdecompiler setup | providers | models -p gemini | config show | version +``` + +## Configuration + +`betterdecompiler setup` writes the config file for you. `betterdecompiler config path` shows where it is (by default +`%AppData%\betterdecompiler\config.json` on Windows, `~/Library/Application Support/betterdecompiler/config.json` on +macOS and `~/.config/betterdecompiler/config.json` on Linux). The file is created readable only by you. + +```jsonc +{ + "provider": "anthropic", // default provider (omit to auto-detect from your keys) + "fallback": ["gemini", "offline"], // tried in order if the provider fails (default: ["offline"]) + "mode": "rename", // rename | rewrite | offline + "comments": "light", // none | light | detailed + "header": true, // add the "-- Cleaned by BetterDecompiler" header + "listen": "127.0.0.1:5000", + "token": "", // require clients to send this token + "concurrency": 4, // parallel AI requests + "cache": { "enabled": true, "dir": "", "max_entries": 256 }, + "providers": { + "anthropic": { "api_key": "sk-ant-...", "model": "claude-haiku-5-5", "effort": "low" }, + "gemini": { "api_key": "env:GEMINI_API_KEY" }, + "my-azure": { // custom OpenAI-compatible endpoint + "type": "openai", + "base_url": "https://my-resource.openai.azure.com/openai/v1", + "api_key": "env:AZURE_OPENAI_KEY", + "auth_header": "api-key", + "model": "my-deployment" + } + } +} +``` + +The real file is plain JSON, so leave out the `//` comments. Per-provider options: `type` (`openai`, `anthropic`, +`gemini`), `base_url`, `api_key` (a literal key, `env:NAME` or `${NAME}`), `model`, `max_tokens`, `temperature`, +`effort` (reasoning effort, or `"none"` to send nothing), `headers`, `auth_header`, `json_mode`, `timeout_seconds`, +`max_retries`, `no_fallbacks`. + +These environment variables override the file: `BD_PROVIDER`, `BD_MODEL`, `BD_MODE`, `BD_TOKEN`, `BD_LISTEN`, +`BD_CONFIG` (path to the config file), and every provider key variable from the table above. Standard +`HTTPS_PROXY` and `NO_PROXY` are respected. + +## HTTP API + +| Method and path | Body | Response | +|---|---|---| +| `POST /v1/clean` | `{"source": "...", "mode"?, "provider"?, "model"?, "comments"?, "header"?, "no_cache"?}` | `{"source", "mode", "provider", "model", "summary", "renamed", "comments", "usage": {"input_tokens", "output_tokens"}, "warnings", "cached", "elapsed_ms"}` | +| `POST /fix_script` | `{"script": "..."}` (v1) | `{"fixed_script": "..."}` | +| `GET /v1/providers` | | Ready providers and the active fallback chain | +| `GET /v1/health` | | Version, mode, chain and usage statistics | +| `GET /` | | Web UI | + +POST bodies must be `application/json`. When a token is set, send `Authorization: Bearer ` or +`X-BD-Token: `. Errors come back as `{"error": "..."}` with status 400 (bad request), 401 (token) or 502 +(every provider failed). + +## Security and privacy + +- Your API key is stored only on your computer, in a file only you can read, and is sent only to the provider you chose. +- The server listens on `127.0.0.1` (this computer only). It rejects requests from other websites (an `Origin` check + plus JSON-only bodies), so a page open in your browser can't spend your credits, and it only answers to `localhost` + host names, which blocks DNS-rebinding attacks. +- To use it from another device, start it with `--listen 0.0.0.0:5000 --token `. BetterDecompiler refuses to + listen beyond this computer without a token. +- Choose `ollama`, `lmstudio` or `offline` if your code must not leave your machine. + +## Troubleshooting + +- **"address already in use" on port 5000.** On macOS, AirPlay Receiver uses port 5000. Run + `betterdecompiler serve --listen 127.0.0.1:5050` and set `Url = "http://localhost:5050"` in the client config. +- **HTTP 401 or "check your API key".** Run `betterdecompiler setup` again, or check the key environment variable. +- **HTTP 404 or "check the model name".** The model was renamed or retired. Run `betterdecompiler models -p ` + and pick one from the list. +- **"this executor has no HTTP request function".** Your executor can't make HTTP requests to your computer. Use the + CLI or the web UI instead. +- **The output says "offline".** No API key was found, or every AI provider failed. The warning lines in the header + say why. + +## Building and testing + +You need Go 1.22 or newer. Everything is standard library, with no third-party Go dependencies. + +```sh +go build ./cmd/betterdecompiler # build for this computer +scripts/build.sh v2.0.0 # cross-compile every platform into dist/ +go test ./... # unit tests: parser, renamer, providers, engine, server, CLI +lua5.1 client/tests/run.lua # executor client tests (fake executor environment) +scripts/e2e.sh # real server + mock AI provider + Lua client, end to end ``` -### Credits: - Made by @officiallymelon - Reverse Proxy:[@xtekky](https://github.com/xtekky) - +The tests don't need an API key. `internal/mockllm` emulates the OpenAI, Anthropic and Gemini APIs, including errors, +rate limits, refusals and truncated output. The renamer is also checked by **executing** programs before and after +hundreds of randomized rename sets under Lua 5.1 and the official Luau VM (set `LUAU_BIN` to enable the Luau run). +Releases are built by GitHub Actions when a `v*` tag is pushed. + +## Project layout + +``` +cmd/betterdecompiler CLI and server entry point +internal/luau Luau lexer, scope-resolving parser, verified renamer, offline name inference +internal/llm Provider clients (OpenAI-compatible, Anthropic, Gemini), retries, presets +internal/engine Rename/rewrite/offline pipelines, chunking, caching, fallback chain +internal/server HTTP API, request protections, embedded web UI +internal/config Config file, environment, provider setup +internal/mockllm Fake AI provider used by the tests +client/ Executor client (Luau) and its tests +dex_betterdecompiler.lua Dark Dex v4 with BetterDecompiler built in (v1 endpoint, still supported) +``` + +## Credits + +Made by [@officiallymelon](https://github.com/officialmelon). v1 used [@xtekky](https://github.com/xtekky)'s g4f +reverse proxy. Thanks to everyone who used and reported issues with v1. + +AI-chosen names and comments can be wrong. In rename mode they never change what the code does, but read them as +educated guesses. diff --git a/client/betterdecompiler.lua b/client/betterdecompiler.lua new file mode 100644 index 0000000..1f7fee0 --- /dev/null +++ b/client/betterdecompiler.lua @@ -0,0 +1,301 @@ +--[[ + BetterDecompiler client v2 + https://github.com/officialmelon/BetterDecompiler + + Sends your executor's decompiler output to the local BetterDecompiler + server, which recovers variable names and adds comments with the AI provider + you configured (or fully offline). If anything fails, you get the normal + decompiler output back, so it is always safe to leave enabled. + + Quick start (the server must be running on your computer): + + loadstring(game:HttpGet("https://github.com/officialmelon/BetterDecompiler/releases/latest/download/betterdecompiler.lua"))() + + That hooks the global `decompile`, so Dex and every other tool get cleaned + output automatically. Options (set before loading, all optional): + + getgenv().BetterDecompilerConfig = { + Url = "http://localhost:5000", -- where the server runs + Token = nil, -- if the server was started with --token + Hook = true, -- replace the global decompile + Enabled = true, -- false = pass raw decompiler output through + Mode = nil, -- "rename" | "rewrite" | "offline" (nil = server default) + Provider = nil, -- e.g. "anthropic", "openai", "gemini", "ollama" + Model = nil, -- e.g. "claude-haiku-5-5" + Comments = nil, -- "none" | "light" | "detailed" + Cache = true, -- remember results for scripts already cleaned + Silent = false, -- hide warnings + } + + API (returned by loadstring, also at getgenv().BetterDecompiler): + + BD.decompile(script [, options]) -- decompile + clean; never errors + BD.decompile(script, enabled, url) -- v1-compatible signature + BD.clean(source [, options]) -- clean code you already have -> cleaned, info + BD.hook() / BD.unhook() -- install / remove the global decompile hook + BD.health() -- server status table, or nil, error + BD.configure({ ... }) -- change options at runtime +]] + +local VERSION = "2.0.0" + +local env = (type(getgenv) == "function" and getgenv()) or _G +local userConfig = rawget(env, "BetterDecompilerConfig") + +-- Loading twice returns the existing instance (and applies new options). +local existing = rawget(env, "BetterDecompiler") +if type(existing) == "table" and existing.Version == VERSION then + if type(userConfig) == "table" then + existing.configure(userConfig) + end + return existing +end + +local HttpService = game:GetService("HttpService") + +local BD = { + Version = VERSION, + Hooked = false, + Config = { + Url = "http://localhost:5000", + Token = nil, + Hook = true, + Enabled = true, + Mode = nil, + Provider = nil, + Model = nil, + Comments = nil, + Cache = true, + Silent = false, + }, +} + +local requestFn = (type(syn) == "table" and syn.request) + or (type(http) == "table" and http.request) + or rawget(env, "http_request") + or (type(fluxus) == "table" and fluxus.request) + or rawget(env, "request") + or request + +-- The executor's own decompiler. If a previous version hooked it, keep the +-- real one rather than wrapping our own wrapper. +local original = rawget(env, "__BD_ORIGINAL_DECOMPILE") or rawget(env, "decompile") or decompile +if original then + env.__BD_ORIGINAL_DECOMPILE = original +end + +local function log(...) + if not BD.Config.Silent then + warn("[BetterDecompiler]", ...) + end +end + +function BD.configure(options) + if type(options) ~= "table" then + return BD.Config + end + for k, v in pairs(options) do + BD.Config[k] = v + end + return BD.Config +end + +local function send(method, url, payload) + if not requestFn then + return nil, "this executor has no HTTP request function" + end + local headers = { ["Content-Type"] = "application/json" } + if BD.Config.Token then + headers["Authorization"] = "Bearer " .. tostring(BD.Config.Token) + end + local options = { Url = url, Method = method, Headers = headers } + if payload ~= nil then + options.Body = HttpService:JSONEncode(payload) + end + local ok, res = pcall(requestFn, options) + if not ok then + return nil, "request failed: " .. tostring(res) + end + if type(res) ~= "table" then + return nil, "no response (is the BetterDecompiler server running at " .. tostring(BD.Config.Url) .. "?)" + end + local decoded, data = pcall(function() + return HttpService:JSONDecode(res.Body or "") + end) + local status = tonumber(res.StatusCode) or 0 + if status ~= 200 then + local msg = (decoded and type(data) == "table" and data.error) or res.Body or res.StatusMessage + if status == 0 then + msg = "could not reach the server at " .. tostring(BD.Config.Url) .. " (is it running?)" + end + return nil, "HTTP " .. tostring(status) .. ": " .. tostring(msg) + end + if not decoded or type(data) ~= "table" then + return nil, "invalid response from server" + end + return data +end + +local function baseUrl(url) + url = tostring(url or BD.Config.Url):gsub("/+$", "") + return url +end + +local cache, cacheSize = {}, 0 +local MAX_CACHE = 128 + +local function cacheKey(prefix, opts) + return table.concat({ + prefix, + tostring(opts.Mode or BD.Config.Mode), + tostring(opts.Provider or BD.Config.Provider), + tostring(opts.Model or BD.Config.Model), + tostring(opts.Comments or BD.Config.Comments), + tostring(opts.Url or BD.Config.Url), + }, "\0") +end + +local function remember(key, value, info) + if not BD.Config.Cache then + return + end + if cacheSize >= MAX_CACHE then + cache, cacheSize = {}, 0 + end + if cache[key] == nil then + cacheSize = cacheSize + 1 + end + cache[key] = { value, info } +end + +-- Cleans decompiled source. Returns the cleaned code and an info table +-- (provider, model, mode, renamed, warnings, ...). On failure returns the +-- original source plus an error message, so callers can always use result 1. +function BD.clean(source, opts) + opts = opts or {} + if type(source) ~= "string" or not source:find("%S") then + return source + end + local key = cacheKey(source, opts) + local hit = BD.Config.Cache and not opts.NoCache and cache[key] + if hit then + return hit[1], hit[2] + end + local url = opts.Url or BD.Config.Url + local data, err + if tostring(url):find("/fix_script/?$") then + -- v1 server URL + data, err = send("POST", url, { script = source }) + if data then + data = { source = data.fixed_script, provider = "v1" } + end + else + data, err = send("POST", baseUrl(url) .. "/v1/clean", { + source = source, + mode = opts.Mode or BD.Config.Mode, + provider = opts.Provider or BD.Config.Provider, + model = opts.Model or BD.Config.Model, + comments = opts.Comments or BD.Config.Comments, + no_cache = opts.NoCache or nil, + }) + end + if not data or type(data.source) ~= "string" then + err = err or "server returned no code" + log("could not clean script, showing raw decompiler output: " .. tostring(err)) + return source, err + end + if type(data.warnings) == "table" then + for _, w in ipairs(data.warnings) do + log(w) + end + end + remember(key, data.source, data) + return data.source, data +end + +local function scriptHash(scr) + local fn = rawget(env, "getscripthash") or getscripthash + if type(fn) ~= "function" then + return nil + end + local ok, hash = pcall(fn, scr) + if ok and type(hash) == "string" and hash ~= "" then + return hash + end + return nil +end + +-- Decompiles a script and cleans the result. Accepts the v1 signature +-- (script, enabled, customUrl) or (script, options). +function BD.decompile(scr, enabled, customUrl) + local opts = {} + if type(enabled) == "table" then + opts = enabled + enabled = opts.Enabled + end + if customUrl ~= nil then + opts.Url = customUrl + end + if enabled == nil then + enabled = BD.Config.Enabled + end + if not original then + return "-- BetterDecompiler: this executor has no decompile function" + end + + local hash = enabled and scriptHash(scr) + local hashKey = hash and cacheKey("hash:" .. hash, opts) + if hashKey and BD.Config.Cache and not opts.NoCache and cache[hashKey] then + return cache[hashKey][1] + end + + local ok, source = pcall(original, scr) + if not ok or type(source) ~= "string" then + log("decompile failed: " .. tostring(source)) + return ok and source or ("-- Decompile failed: " .. tostring(source)) + end + if not enabled then + return source + end + local cleaned, info = BD.clean(source, opts) + if hashKey and type(info) == "table" then + remember(hashKey, cleaned, info) + end + return cleaned +end + +local hookFn = function(scr, ...) + return BD.decompile(scr) +end + +function BD.hook() + if not original then + log("this executor has no decompile function to hook") + return false + end + env.decompile = hookFn + BD.Hooked = true + return true +end + +function BD.unhook() + if original then + env.decompile = original + end + BD.Hooked = false +end + +function BD.health() + return send("GET", baseUrl() .. "/v1/health") +end + +function BD.clearCache() + cache, cacheSize = {}, 0 +end + +BD.configure(userConfig) +if BD.Config.Hook then + BD.hook() +end +env.BetterDecompiler = BD +return BD diff --git a/client/tests/json.lua b/client/tests/json.lua new file mode 100644 index 0000000..0f10529 --- /dev/null +++ b/client/tests/json.lua @@ -0,0 +1,151 @@ +-- Minimal JSON encoder/decoder used to emulate Roblox's HttpService in tests. +local json = {} + +local escapes = { ['"'] = '\\"', ["\\"] = "\\\\", ["\b"] = "\\b", ["\f"] = "\\f", ["\n"] = "\\n", ["\r"] = "\\r", ["\t"] = "\\t" } + +local function encodeString(s) + return '"' .. s:gsub('[%c"\\]', function(c) + return escapes[c] or string.format("\\u%04x", c:byte()) + end) .. '"' +end + +local function isArray(t) + local n = 0 + for k in pairs(t) do + if type(k) ~= "number" then + return false + end + n = n + 1 + end + return n == #t +end + +function json.encode(v) + local t = type(v) + if t == "nil" then + return "null" + elseif t == "boolean" or t == "number" then + return tostring(v) + elseif t == "string" then + return encodeString(v) + elseif t == "table" then + local parts = {} + if next(v) ~= nil and isArray(v) then + for i = 1, #v do + parts[i] = json.encode(v[i]) + end + return "[" .. table.concat(parts, ",") .. "]" + end + for k, val in pairs(v) do + parts[#parts + 1] = encodeString(tostring(k)) .. ":" .. json.encode(val) + end + return "{" .. table.concat(parts, ",") .. "}" + end + error("cannot encode " .. t) +end + +function json.decode(s) + local pos = 1 + local function ws() + pos = s:find("[^ \t\r\n]", pos) or #s + 1 + end + local value + local function str() + pos = pos + 1 + local out = {} + while true do + local c = s:sub(pos, pos) + if c == "" then + error("unterminated string") + elseif c == '"' then + pos = pos + 1 + return table.concat(out) + elseif c == "\\" then + local e = s:sub(pos + 1, pos + 1) + local map = { b = "\b", f = "\f", n = "\n", r = "\r", t = "\t", ['"'] = '"', ["\\"] = "\\", ["/"] = "/" } + if e == "u" then + local code = tonumber(s:sub(pos + 2, pos + 5), 16) + if code < 128 then + out[#out + 1] = string.char(code) + elseif code < 2048 then + out[#out + 1] = string.char(192 + math.floor(code / 64), 128 + code % 64) + else + out[#out + 1] = string.char(224 + math.floor(code / 4096), 128 + math.floor(code / 64) % 64, 128 + code % 64) + end + pos = pos + 6 + else + out[#out + 1] = map[e] + pos = pos + 2 + end + else + out[#out + 1] = c + pos = pos + 1 + end + end + end + function value() + ws() + local c = s:sub(pos, pos) + if c == "{" then + pos = pos + 1 + local t = {} + ws() + if s:sub(pos, pos) == "}" then + pos = pos + 1 + return t + end + while true do + ws() + local k = str() + ws() + pos = pos + 1 -- : + t[k] = value() + ws() + local d = s:sub(pos, pos) + pos = pos + 1 + if d == "}" then + return t + end + end + elseif c == "[" then + pos = pos + 1 + local t = {} + ws() + if s:sub(pos, pos) == "]" then + pos = pos + 1 + return t + end + while true do + t[#t + 1] = value() + ws() + local d = s:sub(pos, pos) + pos = pos + 1 + if d == "]" then + return t + end + end + elseif c == '"' then + return str() + elseif s:sub(pos, pos + 3) == "true" then + pos = pos + 4 + return true + elseif s:sub(pos, pos + 4) == "false" then + pos = pos + 5 + return false + elseif s:sub(pos, pos + 3) == "null" then + pos = pos + 4 + return nil + else + local num = s:match("^-?%d+%.?%d*[eE]?[-+]?%d*", pos) + if not num or num == "" then + error("unexpected character at " .. pos .. ": " .. c) + end + pos = pos + #num + return tonumber(num) + end + end + local v = value() + return v +end + +return json diff --git a/client/tests/run.lua b/client/tests/run.lua new file mode 100644 index 0000000..013c6cb --- /dev/null +++ b/client/tests/run.lua @@ -0,0 +1,261 @@ +--[[ + Tests for client/betterdecompiler.lua in plain Lua 5.1 with a fake + Roblox/executor environment. + + lua5.1 client/tests/run.lua -- unit tests (fake HTTP) + BD_TEST_URL=http://127.0.0.1:5000 \ + BD_TEST_EXPECT="local Players" \ + lua5.1 client/tests/run.lua -- also run against a real server (uses curl) +]] + +local here = (arg and arg[0] or ""):match("^(.*)[/\\]") or "." +package.path = here .. "/?.lua;" .. package.path +local json = require("json") +local clientPath = here .. "/../betterdecompiler.lua" + +local passed, failed = 0, 0 +local function test(name, fn) + local ok, err = pcall(fn) + if ok then + passed = passed + 1 + print("ok " .. name) + else + failed = failed + 1 + print("FAIL " .. name .. "\n " .. tostring(err)) + end +end +local function check(cond, msg) + if not cond then + error(msg or "check failed", 2) + end +end + +local SAMPLE = 'local v1 = game:GetService("Players")\nlocal v2 = v1.LocalPlayer\nprint(v2)\n' + +-- Builds a fresh fake executor environment and loads the client into it. +local function load(opts) + opts = opts or {} + local warnings, requests = {}, {} + local G = setmetatable({}, { __index = _G }) + G.game = { + GetService = function(_, name) + assert(name == "HttpService") + return { + JSONEncode = function(_, v) return json.encode(v) end, + JSONDecode = function(_, s) return json.decode(s) end, + } + end, + } + G.warn = function(...) + warnings[#warnings + 1] = table.concat({ ... }, " ") + end + G.getgenv = function() return G end + G.decompile = opts.decompile or function(scr) + return scr.Source + end + G.getscripthash = opts.getscripthash + G.request = function(req) + requests[#requests + 1] = req + return opts.respond(req) + end + G.BetterDecompilerConfig = opts.config + local chunk = assert(loadfile(clientPath)) + setfenv(chunk, G) + local BD = chunk() + return BD, G, warnings, requests +end + +local function okResponse(source, extra) + local body = { source = source, provider = "mock", mode = "rename", renamed = 2, warnings = extra } + return { StatusCode = 200, Body = json.encode(body) } +end + +local fakeScript = { Source = SAMPLE } + +test("clean posts to /v1/clean with options and returns cleaned code", function() + local BD, _, _, requests = load({ + config = { Url = "http://h:1/", Provider = "anthropic", Comments = "none", Token = "tok" }, + respond = function(req) return okResponse("CLEANED", nil) end, + }) + local out, info = BD.clean(SAMPLE) + check(out == "CLEANED", "got " .. tostring(out)) + check(info.provider == "mock") + local req = requests[1] + check(req.Url == "http://h:1/v1/clean", req.Url) + check(req.Method == "POST") + check(req.Headers["Authorization"] == "Bearer tok") + local body = json.decode(req.Body) + check(body.source == SAMPLE and body.provider == "anthropic" and body.comments == "none") +end) + +test("results are cached per source and options", function() + local BD, _, _, requests = load({ respond = function() return okResponse("X") end }) + BD.clean(SAMPLE) + BD.clean(SAMPLE) + check(#requests == 1, "expected 1 request, got " .. #requests) + BD.clean(SAMPLE, { Mode = "rewrite" }) + check(#requests == 2) + BD.clean(SAMPLE, { NoCache = true }) + check(#requests == 3) + BD.clearCache() + BD.clean(SAMPLE) + check(#requests == 4) +end) + +test("failures return the original source and warn", function() + local BD, _, warnings = load({ respond = function() return { StatusCode = 502, Body = '{"error":"all providers failed"}' } end }) + local out, err = BD.clean(SAMPLE) + check(out == SAMPLE) + check(err:find("502") and err:find("all providers failed"), err) + check(#warnings == 1 and warnings[1]:find("raw decompiler output")) +end) + +test("request errors and missing servers never throw", function() + local BD = load({ respond = function() error("connection refused") end }) + local out, err = BD.clean(SAMPLE) + check(out == SAMPLE and err:find("connection refused")) + BD = load({ respond = function() return { StatusCode = 0, Body = "" } end }) + out, err = BD.clean(SAMPLE) + check(out == SAMPLE and err:find("is it running"), err) +end) + +test("server warnings are surfaced", function() + local BD, _, warnings = load({ respond = function() return okResponse("X", { "anthropic failed: 401" }) end }) + BD.clean(SAMPLE) + check(#warnings == 1 and warnings[1]:find("anthropic failed")) +end) + +test("Silent hides warnings", function() + local BD, _, warnings = load({ config = { Silent = true }, respond = function() return { StatusCode = 500, Body = "" } end }) + BD.clean(SAMPLE) + check(#warnings == 0) +end) + +test("hook replaces global decompile and unhook restores it", function() + local BD, G = load({ respond = function() return okResponse("HOOKED") end }) + check(BD.Hooked) + check(G.decompile(fakeScript) == "HOOKED") + BD.unhook() + check(G.decompile(fakeScript) == SAMPLE) +end) + +test("Hook = false leaves decompile alone", function() + local BD, G = load({ config = { Hook = false }, respond = function() return okResponse("X") end }) + check(not BD.Hooked) + check(G.decompile(fakeScript) == SAMPLE) +end) + +test("v1 signature decompile(script, enabled, url)", function() + local BD, _, _, requests = load({ + config = { Hook = false }, + respond = function(req) + if req.Url:find("/fix_script$") then + return { StatusCode = 200, Body = json.encode({ fixed_script = "LEGACY" }) } + end + return okResponse("NEW") + end, + }) + check(BD.decompile(fakeScript, false) == SAMPLE, "disabled must return raw output") + check(#requests == 0) + check(BD.decompile(fakeScript, true, "http://localhost:5000/fix_script") == "LEGACY") + check(json.decode(requests[1].Body).script == SAMPLE) + check(BD.decompile(fakeScript, true) == "NEW") + check(BD.decompile(fakeScript, { Mode = "offline" }) == "NEW") + check(json.decode(requests[#requests].Body).mode == "offline") +end) + +test("decompiler errors are reported, not thrown", function() + local BD = load({ decompile = function() error("unsupported script") end, respond = function() return okResponse("X") end }) + local out = BD.decompile(fakeScript) + check(out:find("Decompile failed") and out:find("unsupported script"), out) +end) + +test("script hash cache skips decompiling again", function() + local decompiles = 0 + local BD, _, _, requests = load({ + decompile = function(scr) decompiles = decompiles + 1; return scr.Source end, + getscripthash = function() return "abc123" end, + respond = function() return okResponse("CLEAN") end, + }) + check(BD.decompile(fakeScript) == "CLEAN") + check(BD.decompile(fakeScript) == "CLEAN") + check(decompiles == 1 and #requests == 1, decompiles .. " " .. #requests) +end) + +test("loading twice reuses the instance and applies new config", function() + local BD, G = load({ respond = function() return okResponse("X") end }) + G.BetterDecompilerConfig = { Mode = "rewrite" } + local chunk = assert(loadfile(clientPath)) + setfenv(chunk, G) + local again = chunk() + check(again == BD and BD.Config.Mode == "rewrite") + check(G.__BD_ORIGINAL_DECOMPILE ~= G.decompile, "must keep the real decompiler") +end) + +test("empty input is returned untouched without a request", function() + local BD, _, _, requests = load({ respond = function() return okResponse("X") end }) + check(BD.clean("") == "" and BD.clean(" \n") == " \n" and #requests == 0) +end) + +-- Integration tests against a real server ------------------------------------ + +local url = os.getenv("BD_TEST_URL") +if url then + local tmp = os.tmpname() + local function curlRequest(req) + local f = assert(io.open(tmp, "wb")) + f:write(req.Body or "") + f:close() + local cmd = { "curl -s -m 60 -X " .. req.Method } + for k, v in pairs(req.Headers or {}) do + cmd[#cmd + 1] = string.format("-H %q", k .. ": " .. v) + end + if req.Body then + cmd[#cmd + 1] = "--data-binary @" .. tmp + end + cmd[#cmd + 1] = '-w "\\n%{http_code}"' + cmd[#cmd + 1] = string.format("%q", req.Url) + local p = assert(io.popen(table.concat(cmd, " "))) + local out = p:read("*a") + p:close() + local body, code = out:match("^(.*)\n(%d+)$") + return { StatusCode = tonumber(code) or 0, Body = body or "" } + end + local expect = os.getenv("BD_TEST_EXPECT") or "local" + + test("integration: health", function() + local BD = load({ config = { Url = url }, respond = curlRequest }) + local h, err = BD.health() + check(h and h.ok == true, tostring(err)) + end) + + test("integration: hooked decompile returns cleaned code", function() + local BD, G = load({ config = { Url = url, Token = os.getenv("BD_TEST_TOKEN") }, respond = curlRequest }) + local out = G.decompile(fakeScript) + check(out:find("Cleaned by BetterDecompiler", 1, true), out) + check(out:find(expect, 1, true), "expected " .. expect .. " in:\n" .. out) + end) + + test("integration: v1 /fix_script URL", function() + local BD = load({ config = { Hook = false, Token = os.getenv("BD_TEST_TOKEN") }, respond = curlRequest }) + local out = BD.decompile(fakeScript, true, url .. "/fix_script") + check(out:find(expect, 1, true), out) + end) + + test("integration: offline mode via options", function() + local BD = load({ config = { Url = url, Token = os.getenv("BD_TEST_TOKEN") }, respond = curlRequest }) + local out, info = BD.clean(SAMPLE, { Mode = "offline" }) + check(info and info.mode == "offline", tostring(info)) + check(out:find('local Players = game:GetService("Players")', 1, true), out) + end) + + test("integration: bad server URL falls back to raw output", function() + local BD = load({ config = { Url = "http://127.0.0.1:9" }, respond = curlRequest }) + local out, err = BD.clean(SAMPLE) + check(out == SAMPLE and err, tostring(err)) + end) + os.remove(tmp) +end + +print(string.format("\n%d passed, %d failed", passed, failed)) +os.exit(failed == 0 and 0 or 1) diff --git a/cmd/betterdecompiler/clean.go b/cmd/betterdecompiler/clean.go new file mode 100644 index 0000000..5d9a7de --- /dev/null +++ b/cmd/betterdecompiler/clean.go @@ -0,0 +1,220 @@ +package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "os/signal" + "path/filepath" + "strings" + "sync" + "time" + + "github.com/officialmelon/betterdecompiler/internal/engine" +) + +type job struct { + in, out string // out "" = stdout + rel string +} + +func runClean(args []string) error { + fls := newFlagSet("clean", "clean [flags] [file|dir|-]...") + var c common + c.register(fls) + out := fls.String("o", "", "output file, or output directory for several inputs") + inPlace := fls.Bool("in-place", false, "overwrite input files") + noHeader := fls.Bool("no-header", false, "do not add the informational header comment") + noCache := fls.Bool("no-cache", false, "ignore cached results") + jobs := fls.Int("j", 4, "files to process in parallel") + asJSON := fls.Bool("json", false, "print results as JSON (one object per line)") + quiet := fls.Bool("q", false, "no progress output") + inputs, err := parse(fls, args) + if err != nil { + return err + } + _, eng, err := c.load() + if err != nil { + return err + } + opts := engine.Options{NoCache: *noCache} + if *noHeader { + f := false + opts.Header = &f + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + // stdin -> stdout (or -o file) + if len(inputs) == 0 || (len(inputs) == 1 && inputs[0] == "-") { + if len(inputs) == 0 && isTerminal(os.Stdin) { + fls.Usage() + return errors.New("no input: pass files or directories, or pipe code into stdin") + } + src, err := readAll(os.Stdin) + if err != nil { + return err + } + res, err := eng.Clean(ctx, src, opts) + if err != nil { + return err + } + return emit(res, *out, *asJSON, "stdin") + } + + list, err := collect(inputs, *out, *inPlace) + if err != nil { + return err + } + if *jobs < 1 { + *jobs = 1 + } + var ( + mu sync.Mutex + failed int + done int + wg sync.WaitGroup + queue = make(chan job) + start = time.Now() + ) + for w := 0; w < *jobs; w++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := range queue { + t0 := time.Now() + data, err := os.ReadFile(j.in) + var res *engine.Result + if err == nil { + res, err = eng.Clean(ctx, string(data), opts) + } + if err == nil { + err = emit(res, j.out, *asJSON, j.in) + } + mu.Lock() + done++ + if err != nil { + failed++ + fmt.Fprintf(os.Stderr, "[%d/%d] %s: error: %v\n", done, len(list), j.rel, err) + } else if !*quiet && j.out != "" { + note := fmt.Sprintf("%s, %d names", res.Provider, res.Renamed) + if res.Cached { + note += ", cached" + } + for _, w := range res.Warnings { + note += "; warning: " + w + } + fmt.Fprintf(os.Stderr, "[%d/%d] %s -> %s (%.1fs, %s)\n", done, len(list), j.rel, j.out, time.Since(t0).Seconds(), note) + } + mu.Unlock() + } + }() + } + for _, j := range list { + queue <- j + } + close(queue) + wg.Wait() + if !*quiet && len(list) > 1 { + st := eng.Stats() + fmt.Fprintf(os.Stderr, "done: %d files in %.1fs (%d failed, %d cached, %d AI calls, %d in / %d out tokens)\n", + len(list), time.Since(start).Seconds(), failed, st.CacheHits, st.AICalls, st.InputTokens, st.OutputTokens) + } + if failed > 0 { + return fmt.Errorf("%d of %d files failed", failed, len(list)) + } + return nil +} + +func isLua(name string) bool { + ext := strings.ToLower(filepath.Ext(name)) + return ext == ".lua" || ext == ".luau" || ext == ".txt" +} + +// collect expands inputs into jobs and decides where each result goes. +func collect(inputs []string, out string, inPlace bool) ([]job, error) { + var list []job + multi := len(inputs) > 1 + for _, in := range inputs { + st, err := os.Stat(in) + if err != nil { + return nil, err + } + if !st.IsDir() { + list = append(list, job{in: in, rel: filepath.Base(in)}) + continue + } + multi = true + err = filepath.WalkDir(in, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if !d.IsDir() && isLua(p) { + rel, _ := filepath.Rel(in, p) + if len(inputs) > 1 { + rel = filepath.Join(filepath.Base(in), rel) + } + list = append(list, job{in: p, rel: rel}) + } + return nil + }) + if err != nil { + return nil, err + } + } + if len(list) == 0 { + return nil, errors.New("no .lua or .luau files found") + } + switch { + case inPlace: + for i := range list { + list[i].out = list[i].in + } + case out != "" && !multi: + if st, err := os.Stat(out); err == nil && st.IsDir() { + list[0].out = filepath.Join(out, list[0].rel) + } else { + list[0].out = out + } + case out != "": + for i := range list { + list[i].out = filepath.Join(out, list[i].rel) + } + case multi: + return nil, errors.New("several inputs: use -o DIR to choose an output directory, or --in-place") + } + return list, nil +} + +func emit(res *engine.Result, out string, asJSON bool, name string) error { + var data []byte + if asJSON { + v := struct { + File string `json:"file"` + *engine.Result + }{name, res} + b, err := json.Marshal(v) + if err != nil { + return err + } + data = append(b, '\n') + } else { + data = []byte(res.Source) + } + if out == "" { + _, err := os.Stdout.Write(data) + if !asJSON { + for _, w := range res.Warnings { + fmt.Fprintln(os.Stderr, "warning:", w) + } + } + return err + } + if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil { + return err + } + return os.WriteFile(out, data, 0o644) +} diff --git a/cmd/betterdecompiler/main.go b/cmd/betterdecompiler/main.go new file mode 100644 index 0000000..dabf152 --- /dev/null +++ b/cmd/betterdecompiler/main.go @@ -0,0 +1,174 @@ +// Command betterdecompiler cleans up decompiled Roblox Luau with the AI +// provider of your choice (or fully offline). +package main + +import ( + "errors" + "flag" + "fmt" + "io" + "os" + "strings" + + "github.com/officialmelon/betterdecompiler/internal/config" + "github.com/officialmelon/betterdecompiler/internal/engine" +) + +// version is set at build time with -ldflags "-X main.version=v2.0.0". +var version = "dev" + +const usage = `BetterDecompiler %s - turn decompiled Roblox Luau into readable code. + +Usage: + betterdecompiler [serve] start the local server (web UI + executor API) + betterdecompiler clean [files|dirs] clean files, folders or stdin from the terminal + betterdecompiler setup choose an AI provider and save your API key + betterdecompiler providers list supported providers and which are ready + betterdecompiler models [-p name] list the models your key can use + betterdecompiler config [path|show] show where settings live / print them (keys masked) + betterdecompiler version + +Common flags: + -p, --provider NAME anthropic, openai, gemini, openrouter, groq, deepseek, mistral, + xai, ollama, lmstudio, offline, or a custom provider + -m, --model NAME model to use (default depends on the provider) + --mode MODE rename (default: fast, behavior-preserving), rewrite, offline + --comments LEVEL none, light (default), detailed + --config PATH config file (default: %s) + +Run "betterdecompiler -h" for command-specific flags. +Docs: https://github.com/officialmelon/BetterDecompiler +` + +func main() { + args := os.Args[1:] + cmd := "serve" + if len(args) > 0 && !strings.HasPrefix(args[0], "-") { + cmd, args = args[0], args[1:] + } + var err error + switch cmd { + case "serve", "server", "start": + err = runServe(args, len(os.Args) == 1) + case "clean", "c": + err = runClean(args) + case "setup", "init", "configure": + err = runSetup(args) + case "providers": + err = runProviders(args) + case "models": + err = runModels(args) + case "config": + err = runConfig(args) + case "version", "--version", "-v": + fmt.Println("betterdecompiler", version) + case "help", "--help", "-h": + fmt.Printf(usage, version, config.DefaultPath()) + default: + fmt.Fprintf(os.Stderr, "unknown command %q\n\n", cmd) + fmt.Fprintf(os.Stderr, usage, version, config.DefaultPath()) + os.Exit(2) + } + if err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(0) + } + fmt.Fprintln(os.Stderr, "error:", err) + if len(os.Args) == 1 && isTerminal(os.Stdin) { + // Launched by double-click: keep the window open so the error is readable. + fmt.Fprint(os.Stderr, "\nPress Enter to exit...") + _, _ = fmt.Fscanln(os.Stdin) + } + os.Exit(1) + } +} + +// common holds flags shared by most commands. +type common struct { + configPath string + provider string + model string + mode string + comments string +} + +func (c *common) register(fs *flag.FlagSet) { + fs.StringVar(&c.configPath, "config", config.DefaultPath(), "config file") + fs.StringVar(&c.provider, "provider", "", "AI provider") + fs.StringVar(&c.provider, "p", "", "AI provider (shorthand)") + fs.StringVar(&c.model, "model", "", "model name") + fs.StringVar(&c.model, "m", "", "model name (shorthand)") + fs.StringVar(&c.mode, "mode", "", "rename, rewrite or offline") + fs.StringVar(&c.comments, "comments", "", "none, light or detailed") +} + +// load reads the config, applies environment and flag overrides and builds +// the engine. +func (c *common) load() (*config.Config, *engine.Engine, error) { + cfg, err := config.Load(c.configPath) + if err != nil { + return nil, nil, err + } + cfg.ApplyEnv() + if c.provider != "" { + cfg.Provider = c.provider + } + if c.mode != "" { + cfg.Mode = c.mode + } + if c.comments != "" { + cfg.Comments = c.comments + } + clients, chain, err := cfg.Build() + if err != nil { + return nil, nil, err + } + if c.model != "" && chain[0] != "offline" { + p := cfg.Providers[chain[0]] + p.Model = c.model + cfg.SetProvider(chain[0], p) + if clients, chain, err = cfg.Build(); err != nil { + return nil, nil, err + } + } + return cfg, engine.New(cfg.EngineSettings(version), clients, chain, cfg.NewCache()), nil +} + +// parse parses flags that may appear before or after positional arguments. +func parse(fs *flag.FlagSet, args []string) ([]string, error) { + var pos []string + for { + if err := fs.Parse(args); err != nil { + return nil, err + } + args = fs.Args() + if len(args) == 0 { + return pos, nil + } + if args[0] == "--" { + return append(pos, args[1:]...), nil + } + pos = append(pos, args[0]) + args = args[1:] + } +} + +func newFlagSet(name, synopsis string) *flag.FlagSet { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.SetOutput(os.Stderr) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage: betterdecompiler %s\n\nFlags:\n", synopsis) + fs.PrintDefaults() + } + return fs +} + +func isTerminal(f *os.File) bool { + st, err := f.Stat() + return err == nil && st.Mode()&os.ModeCharDevice != 0 +} + +func readAll(r io.Reader) (string, error) { + b, err := io.ReadAll(r) + return string(b), err +} diff --git a/cmd/betterdecompiler/main_test.go b/cmd/betterdecompiler/main_test.go new file mode 100644 index 0000000..087eecc --- /dev/null +++ b/cmd/betterdecompiler/main_test.go @@ -0,0 +1,66 @@ +package main + +import ( + "flag" + "os" + "path/filepath" + "reflect" + "testing" +) + +func TestParseInterspersedFlags(t *testing.T) { + fs := flag.NewFlagSet("t", flag.ContinueOnError) + o := fs.String("o", "", "") + q := fs.Bool("q", false, "") + pos, err := parse(fs, []string{"a.lua", "-o", "out", "b.lua", "-q", "--", "-c.lua"}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(pos, []string{"a.lua", "b.lua", "-c.lua"}) || *o != "out" || !*q { + t.Fatalf("pos=%v o=%q q=%v", pos, *o, *q) + } +} + +func TestCollect(t *testing.T) { + dir := t.TempDir() + must := func(p string) { + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte("x()"), 0o644); err != nil { + t.Fatal(err) + } + } + must(filepath.Join(dir, "src", "a.lua")) + must(filepath.Join(dir, "src", "deep", "b.luau")) + must(filepath.Join(dir, "src", "notes.md")) + single := filepath.Join(dir, "src", "a.lua") + + jobs, err := collect([]string{single}, "", false) + if err != nil || len(jobs) != 1 || jobs[0].out != "" { + t.Fatalf("single to stdout: %+v %v", jobs, err) + } + jobs, _ = collect([]string{single}, filepath.Join(dir, "o.lua"), false) + if jobs[0].out != filepath.Join(dir, "o.lua") { + t.Fatalf("single to file: %+v", jobs) + } + jobs, err = collect([]string{filepath.Join(dir, "src")}, filepath.Join(dir, "out"), false) + if err != nil || len(jobs) != 2 { + t.Fatalf("dir: %+v %v", jobs, err) + } + want := map[string]bool{filepath.Join(dir, "out", "a.lua"): true, filepath.Join(dir, "out", "deep", "b.luau"): true} + for _, j := range jobs { + if !want[j.out] { + t.Fatalf("unexpected output path %s", j.out) + } + } + if _, err := collect([]string{filepath.Join(dir, "src")}, "", false); err == nil { + t.Fatal("directory without -o must be rejected") + } + jobs, _ = collect([]string{filepath.Join(dir, "src")}, "", true) + for _, j := range jobs { + if j.in != j.out { + t.Fatal("in-place must overwrite inputs") + } + } +} diff --git a/cmd/betterdecompiler/serve.go b/cmd/betterdecompiler/serve.go new file mode 100644 index 0000000..eb5e515 --- /dev/null +++ b/cmd/betterdecompiler/serve.go @@ -0,0 +1,193 @@ +package main + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/officialmelon/betterdecompiler/internal/config" + "github.com/officialmelon/betterdecompiler/internal/llm" + "github.com/officialmelon/betterdecompiler/internal/server" +) + +const clientURL = "https://github.com/officialmelon/BetterDecompiler/releases/latest/download/betterdecompiler.lua" + +func runServe(args []string, bare bool) error { + fs := newFlagSet("serve", "serve [flags]") + var c common + c.register(fs) + listen := fs.String("listen", "", "address to listen on (default "+config.DefaultListen+")") + token := fs.String("token", "", "require this token from clients (Authorization: Bearer ...)") + anyHost := fs.Bool("allow-any-host", false, "answer requests for any Host (needed for LAN or reverse-proxy use)") + quiet := fs.Bool("quiet", false, "do not log requests") + if _, err := parse(fs, args); err != nil { + return err + } + if bare && isTerminal(os.Stdin) && needsSetup(c.configPath) { + fmt.Println("No AI provider is configured yet.") + if ask("Set one up now? (you can also use the offline renamer) [Y/n]", "y") == "y" { + if err := runSetup([]string{"--config", c.configPath}); err != nil { + return err + } + fmt.Println() + } + } + cfg, eng, err := c.load() + if err != nil { + return err + } + if *listen != "" { + cfg.Listen = *listen + } + if *token != "" { + cfg.Token = *token + } + addr := cfg.ListenAddr() + host, port, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("invalid listen address %q: %w", addr, err) + } + if !isLoopbackListen(host) && cfg.Token == "" { + return fmt.Errorf("listening on %s would let other devices use your API key; set a token with --token (or BD_TOKEN)", host) + } + var logger *log.Logger + if !*quiet { + logger = log.New(os.Stderr, "", log.LstdFlags) + } + srv := &server.Server{Engine: eng, Providers: cfg.List(), Token: cfg.Token, Version: version, Log: logger, + AllowAnyHost: *anyHost || !isLoopbackListen(host)} + hs := &http.Server{Handler: srv.Handler(), ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 2 * time.Minute} + + listeners, err := listen4and6(host, port) + if err != nil { + if strings.Contains(err.Error(), "address already in use") || strings.Contains(err.Error(), "Only one usage") { + return fmt.Errorf("%w\nanother program is using port %s (on macOS, AirPlay Receiver uses 5000).\n"+ + "Try: betterdecompiler serve --listen 127.0.0.1:5050 (and point your client at that port)", err, port) + } + return err + } + printBanner(cfg, eng.Chain(), host, port) + + errc := make(chan error, len(listeners)) + for _, ln := range listeners { + go func(ln net.Listener) { errc <- hs.Serve(ln) }(ln) + } + sig := make(chan os.Signal, 1) + signal.Notify(sig, os.Interrupt, syscall.SIGTERM) + select { + case err := <-errc: + if !errors.Is(err, http.ErrServerClosed) { + return err + } + case <-sig: + fmt.Fprintln(os.Stderr, "\nshutting down...") + } + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + return hs.Shutdown(ctx) +} + +func isLoopbackListen(host string) bool { + if host == "localhost" { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// listen4and6 binds the address. For localhost it binds both 127.0.0.1 and +// ::1, because clients resolve "localhost" to either. +func listen4and6(host, port string) ([]net.Listener, error) { + if host == "localhost" || host == "127.0.0.1" { + ln, err := net.Listen("tcp4", net.JoinHostPort("127.0.0.1", port)) + if err != nil { + return nil, err + } + lns := []net.Listener{ln} + if ln6, err := net.Listen("tcp6", net.JoinHostPort("::1", port)); err == nil { + lns = append(lns, ln6) + } + return lns, nil + } + ln, err := net.Listen("tcp", net.JoinHostPort(host, port)) + if err != nil { + return nil, err + } + return []net.Listener{ln}, nil +} + +func printBanner(cfg *config.Config, chain []string, host, port string) { + shown := host + if host == "127.0.0.1" || host == "" || host == "0.0.0.0" || host == "::" { + shown = "localhost" + } + base := "http://" + net.JoinHostPort(shown, port) + var provs []string + for _, name := range chain { + if name == llm.Offline { + provs = append(provs, "offline renamer") + continue + } + for _, p := range cfg.List() { + if p.Name == name { + provs = append(provs, fmt.Sprintf("%s (%s)", name, p.Model)) + } + } + } + mode := cfg.Mode + if mode == "" { + mode = "rename" + } + if len(chain) == 1 && chain[0] == llm.Offline { + mode = "offline" + } + fmt.Printf(` + BetterDecompiler %s + + Web UI %s + API POST %s/v1/clean (v1 clients and Dex: %s/fix_script) + Provider %s + Mode %s +`, version, base, base, base, strings.Join(provs, " -> "), mode) + if cfg.Token != "" { + fmt.Println(" Auth token required (Authorization: Bearer )") + } + if !isLoopbackListen(host) { + fmt.Println(" Note reachable from other devices; clients must send the token.") + } + if len(chain) == 1 && chain[0] == llm.Offline { + fmt.Println("\n No API key configured, so names come from the offline renamer.") + fmt.Println(" Run `betterdecompiler setup` (or set e.g. ANTHROPIC_API_KEY) to use an AI.") + } + fmt.Printf(` + In your executor: + loadstring(game:HttpGet("%s"))() + + Press Ctrl+C to stop. + +`, clientURL) +} + +// needsSetup reports whether nothing at all is configured. +func needsSetup(path string) bool { + if _, err := os.Stat(path); err == nil { + return false + } + if os.Getenv("BD_PROVIDER") != "" { + return false + } + for _, p := range llm.Presets { + if p.NeedsKey && p.EnvKey() != "" { + return false + } + } + return true +} diff --git a/cmd/betterdecompiler/setup.go b/cmd/betterdecompiler/setup.go new file mode 100644 index 0000000..1fa6371 --- /dev/null +++ b/cmd/betterdecompiler/setup.go @@ -0,0 +1,249 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "runtime" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/officialmelon/betterdecompiler/internal/config" + "github.com/officialmelon/betterdecompiler/internal/llm" +) + +var stdin = bufio.NewReader(os.Stdin) + +func ask(prompt, def string) string { + fmt.Print(prompt + " ") + line, _ := stdin.ReadString('\n') + line = strings.TrimSpace(line) + if line == "" { + return def + } + return strings.ToLower(line[:1]) + line[1:] +} + +// askSecret reads a line without echo where the terminal supports it. +func askSecret(prompt string) string { + fmt.Print(prompt + " ") + hidden := false + if runtime.GOOS != "windows" && isTerminal(os.Stdin) { + cmd := exec.Command("stty", "-echo") + cmd.Stdin = os.Stdin + hidden = cmd.Run() == nil + } + line, _ := stdin.ReadString('\n') + if hidden { + cmd := exec.Command("stty", "echo") + cmd.Stdin = os.Stdin + _ = cmd.Run() + fmt.Println() + } + return strings.TrimSpace(line) +} + +func runSetup(args []string) error { + fs := newFlagSet("setup", "setup [--config PATH]") + path := fs.String("config", config.DefaultPath(), "config file") + if _, err := parse(fs, args); err != nil { + return err + } + cfg, err := config.Load(*path) + if err != nil { + return err + } + fmt.Println("BetterDecompiler setup — pick the AI that names your variables. Your key stays on this computer.") + fmt.Println() + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + for i, p := range llm.Presets { + note := p.Description + if p.EnvKey() != "" { + note += " [key found in " + firstEnv(p) + "]" + } + fmt.Fprintf(w, " %2d) %s\t%s\n", i+1, p.Name, note) + } + fmt.Fprintf(w, " %2d) %s\t%s\n", len(llm.Presets)+1, llm.Offline, "No AI: names inferred from Roblox API usage (free, instant)") + w.Flush() + choice := ask(fmt.Sprintf("\nChoice [1-%d, default 1]:", len(llm.Presets)+1), "1") + n, err := strconv.Atoi(choice) + if err != nil || n < 1 || n > len(llm.Presets)+1 { + return fmt.Errorf("invalid choice %q", choice) + } + if n == len(llm.Presets)+1 { + cfg.Provider = llm.Offline + if err := config.Save(*path, cfg); err != nil { + return err + } + fmt.Println("Saved. The offline renamer will be used:", *path) + return nil + } + p := llm.Presets[n-1] + user := cfg.Providers[p.Name] + if p.NeedsKey { + if env := firstEnv(p); env != "" && ask(fmt.Sprintf("Use the key from $%s? (stores a reference, not the key) [Y/n]", env), "y") == "y" { + user.APIKey = "env:" + env + } else { + fmt.Printf("Get a key at %s\n", p.KeyURL) + key := askSecret("Paste your API key:") + if key == "" { + return fmt.Errorf("no key entered") + } + user.APIKey = key + } + } else { + fmt.Printf("%s runs locally: make sure it is running (%s).\n", p.Name, p.KeyURL) + if url := ask(fmt.Sprintf("Server URL [%s]:", p.BaseURL), ""); url != "" { + user.BaseURL = url + } + } + defModel := p.Model + if user.Model != "" { + defModel = user.Model + } + if m := ask(fmt.Sprintf("Model [%s]:", defModel), ""); m != "" { + user.Model = m + } + cfg.SetProvider(p.Name, user) + cfg.Provider = p.Name + + if ask("Test the connection now? [Y/n]", "y") == "y" { + testCfg := *cfg + clients, _, err := testCfg.Build() + if err == nil { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + var models []string + if models, err = clients[p.Name].ListModels(ctx); err == nil { + fmt.Printf(" OK: connected, %d models available.\n", len(models)) + } + } + if err != nil { + fmt.Printf(" Connection test failed: %v\n", err) + if ask("Save anyway? [y/N]", "n") != "y" { + return fmt.Errorf("setup cancelled") + } + } + } + if err := config.Save(*path, cfg); err != nil { + return err + } + fmt.Println("Saved to", *path) + fmt.Println("Start the server with `betterdecompiler` or clean a file with `betterdecompiler clean script.lua`.") + return nil +} + +func firstEnv(p llm.Preset) string { + for _, k := range p.KeyEnv { + if os.Getenv(k) != "" { + return k + } + } + return "" +} + +func runProviders(args []string) error { + fs := newFlagSet("providers", "providers [--config PATH]") + var c common + c.register(fs) + if _, err := parse(fs, args); err != nil { + return err + } + cfg, err := config.Load(c.configPath) + if err != nil { + return err + } + cfg.ApplyEnv() + _, chain, buildErr := cfg.Build() + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', 0) + fmt.Fprintln(w, "PROVIDER\tREADY\tMODEL\tDESCRIPTION") + for _, p := range cfg.List() { + ready := "no" + switch { + case p.Ready && p.Local: + ready = "local" + case p.Ready: + ready = "yes" + case p.KeyEnv != "": + ready = "no (set " + p.KeyEnv + ")" + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", p.Name, ready, p.Model, p.Description) + } + fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", llm.Offline, "yes", "-", "No AI: names inferred from Roblox API usage") + w.Flush() + if buildErr != nil { + fmt.Println("\nconfiguration error:", buildErr) + } else { + fmt.Println("\nactive chain:", strings.Join(chain, " -> ")) + } + return nil +} + +func runModels(args []string) error { + fs := newFlagSet("models", "models [-p provider]") + var c common + c.register(fs) + if _, err := parse(fs, args); err != nil { + return err + } + cfg, err := config.Load(c.configPath) + if err != nil { + return err + } + cfg.ApplyEnv() + if c.provider != "" { + cfg.Provider = c.provider + } + clients, chain, err := cfg.Build() + if err != nil { + return err + } + name := chain[0] + if name == llm.Offline { + return fmt.Errorf("no AI provider configured; run `betterdecompiler setup` or pass -p") + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + models, err := clients[name].ListModels(ctx) + if err != nil { + return err + } + fmt.Fprintf(os.Stderr, "%s (current: %s)\n", name, clients[name].Model()) + for _, m := range models { + fmt.Println(m) + } + return nil +} + +func runConfig(args []string) error { + fs := newFlagSet("config", "config [path|show]") + path := fs.String("config", config.DefaultPath(), "config file") + pos, err := parse(fs, args) + if err != nil { + return err + } + sub := "show" + if len(pos) > 0 { + sub = pos[0] + } + switch sub { + case "path": + fmt.Println(*path) + return nil + case "show": + cfg, err := config.Load(*path) + if err != nil { + return err + } + fmt.Fprintln(os.Stderr, "#", *path) + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(cfg.Redacted()) + } + return fmt.Errorf("unknown config command %q (want path or show)", sub) +} diff --git a/cmd/mockllm/main.go b/cmd/mockllm/main.go new file mode 100644 index 0000000..4317070 --- /dev/null +++ b/cmd/mockllm/main.go @@ -0,0 +1,19 @@ +// Command mockllm runs a fake AI provider (OpenAI, Anthropic and Gemini +// APIs) for end-to-end tests without API keys. It is not part of releases. +package main + +import ( + "flag" + "log" + "net/http" + + "github.com/officialmelon/betterdecompiler/internal/mockllm" +) + +func main() { + addr := flag.String("addr", "127.0.0.1:5099", "listen address") + key := flag.String("key", "test-key", "API key clients must send") + flag.Parse() + log.Printf("mock AI provider listening on http://%s (key %q)", *addr, *key) + log.Fatal(http.ListenAndServe(*addr, mockllm.New(*key))) +} diff --git a/examples/decompiled.lua b/examples/decompiled.lua new file mode 100644 index 0000000..5630c99 --- /dev/null +++ b/examples/decompiled.lua @@ -0,0 +1,47 @@ +-- Decompiled with a typical Luau decompiler (sample input for BetterDecompiler) +local v1 = game:GetService("Players") +local v2 = game:GetService("ReplicatedStorage") +local v3 = game:GetService("TweenService") +local v4 = v1.LocalPlayer +local v5 = v4:WaitForChild("PlayerGui") +local v6 = v2:WaitForChild("Remotes"):WaitForChild("PurchaseItem") +local v7 = require(v2:WaitForChild("Modules"):WaitForChild("ShopConfig")) +local v8 = TweenInfo.new(0.25, Enum.EasingStyle.Quad, Enum.EasingDirection.Out) +local v9 = {} + +local function u10(p1, p2) + local v11 = v5:FindFirstChild("ShopGui") + if not v11 then + return + end + local v12 = v11:FindFirstChild("Frame") + v3:Create(v12, v8, { + Position = p2 and UDim2.new(0.5, 0, 0.5, 0) or UDim2.new(0.5, 0, 1.5, 0), + }):Play() + v9[p1] = p2 +end + +for v13, v14 in pairs(v7.Items) do + local v15 = Instance.new("TextButton") + v15.Name = v13 + v15.Text = `{v14.DisplayName} - {v14.Price}` + v15.MouseButton1Click:Connect(function() + v6:FireServer(v13) + end) +end + +v4.CharacterAdded:Connect(function(p1) + local v16 = p1:WaitForChild("Humanoid") + v16.Died:Connect(function() + u10("Shop", false) + end) +end) + +game:GetService("UserInputService").InputBegan:Connect(function(p1, p2) + if p2 then + return + end + if p1.KeyCode == Enum.KeyCode.B then + u10("Shop", not v9.Shop) + end +end) diff --git a/flask_ai_server.py b/flask_ai_server.py deleted file mode 100644 index 38ca3df..0000000 --- a/flask_ai_server.py +++ /dev/null @@ -1,53 +0,0 @@ -from flask import Flask, request, jsonify -from g4f.client import Client -import time - -app = Flask(__name__) -client = Client() - -@app.route('/fix_script', methods=['POST']) -def fix_script(): - # Extract the script from the POST request - data = request.get_json(force=True) - - if not data or 'script' not in data or not data['script']: - # Return empty output if no script is provided - return jsonify({'fixed_script': ''}) - - script_content = data['script'] - - # Start timing the request - start_time = time.time() - - # Prepare the request to the g4f API - response = client.chat.completions.create( - model="gpt-4o", - messages=[ - {"role": "system", "content": "You will be given a Luau script (that has been decompiled). You will rename the variables and functions to sound better, make the script more coherent, and add comments to areas of the script. Return the fixed script without adding ``` as we handle that. If no script has been given, do not produce any output. Do not respond with anything but the script"}, - {"role": "user", "content": script_content} - ] - ) - - # Calculate the time taken - end_time = time.time() - elapsed_time = end_time - start_time - - print("Script Decompiled in {elapsed_time) Seconds.") - - # Format the fixed script with the additional comments at the top - fixed_script = response.choices[0].message.content - formatted_script = ( - f"-- Generated by BetterDecompiler\n" - f"-- Took: {elapsed_time:.2f} Seconds\n" - f"-- Scripts can be inaccurate as AI can still hallucinate.\n\n" - f"{fixed_script}" - ) - - # Return the formatted script - return jsonify({'fixed_script': formatted_script}) - -if __name__ == '__main__': - print("Starting BetterDecompiler") - print("Using G4F AI Reverse Proxy!") - print("Hello from melon!") - app.run(debug=True) diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..69983da --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/officialmelon/betterdecompiler + +go 1.22 diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..96b84a4 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,392 @@ +// Package config loads BetterDecompiler settings from a JSON file and the +// environment, and builds the provider clients. +package config + +import ( + "encoding/json" + "errors" + "fmt" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/officialmelon/betterdecompiler/internal/engine" + "github.com/officialmelon/betterdecompiler/internal/llm" +) + +// Provider configures one AI provider. For built-in presets every field is +// optional; custom providers need at least type and base_url. +type Provider struct { + Type string `json:"type,omitempty"` + BaseURL string `json:"base_url,omitempty"` + APIKey string `json:"api_key,omitempty"` // literal, "env:NAME" or "${NAME}" + Model string `json:"model,omitempty"` + MaxTokens int `json:"max_tokens,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + Effort string `json:"effort,omitempty"` + Headers map[string]string `json:"headers,omitempty"` + AuthHeader string `json:"auth_header,omitempty"` + JSONMode *bool `json:"json_mode,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + NoFallbacks bool `json:"no_fallbacks,omitempty"` +} + +// Cache configures result caching. +type Cache struct { + Enabled *bool `json:"enabled,omitempty"` + Dir string `json:"dir,omitempty"` + MaxEntries int `json:"max_entries,omitempty"` +} + +// Config is the config file. +type Config struct { + Provider string `json:"provider,omitempty"` + Fallback []string `json:"fallback,omitempty"` + Mode string `json:"mode,omitempty"` + Comments string `json:"comments,omitempty"` + Header *bool `json:"header,omitempty"` + Heuristics *bool `json:"heuristics,omitempty"` + Listen string `json:"listen,omitempty"` + Token string `json:"token,omitempty"` + Concurrency int `json:"concurrency,omitempty"` + ChunkChars int `json:"chunk_chars,omitempty"` + RewriteChunkChars int `json:"rewrite_chunk_chars,omitempty"` + Cache Cache `json:"cache,omitempty"` + Providers map[string]Provider `json:"providers,omitempty"` +} + +// DefaultListen is the default server address (same port as v1). +const DefaultListen = "127.0.0.1:5000" + +// DefaultPath returns the config file location: $BD_CONFIG, or +// /betterdecompiler/config.json. +func DefaultPath() string { + if p := os.Getenv("BD_CONFIG"); p != "" { + return p + } + dir, err := os.UserConfigDir() + if err != nil { + return "betterdecompiler.json" + } + return filepath.Join(dir, "betterdecompiler", "config.json") +} + +// Load reads a config file. A missing file yields an empty config. +func Load(path string) (*Config, error) { + c := &Config{} + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return c, nil + } + if err != nil { + return nil, err + } + if err := json.Unmarshal(data, c); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return c, nil +} + +// Save writes the config file with owner-only permissions (it may hold keys). +func Save(path string, c *Config) error { + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return err + } + data, err := json.MarshalIndent(c, "", " ") + if err != nil { + return err + } + return os.WriteFile(path, append(data, '\n'), 0o600) +} + +// ApplyEnv overlays BD_* environment variables. +func (c *Config) ApplyEnv() { + if v := os.Getenv("BD_PROVIDER"); v != "" { + c.Provider = v + } + if v := os.Getenv("BD_MODE"); v != "" { + c.Mode = v + } + if v := os.Getenv("BD_TOKEN"); v != "" { + c.Token = v + } + if v := os.Getenv("BD_LISTEN"); v != "" { + c.Listen = v + } + if v := os.Getenv("BD_MODEL"); v != "" && c.Provider != "" { + p := c.Providers[c.Provider] + p.Model = v + c.SetProvider(c.Provider, p) + } +} + +// SetProvider stores provider settings. +func (c *Config) SetProvider(name string, p Provider) { + if c.Providers == nil { + c.Providers = map[string]Provider{} + } + c.Providers[name] = p +} + +// ResolveKey expands "env:NAME", "$NAME" and "${NAME}" references. +func ResolveKey(s string) string { + switch { + case strings.HasPrefix(s, "env:"): + return os.Getenv(strings.TrimPrefix(s, "env:")) + case strings.HasPrefix(s, "${") && strings.HasSuffix(s, "}"): + return os.Getenv(s[2 : len(s)-1]) + case strings.HasPrefix(s, "$") && len(s) > 1: + return os.Getenv(s[1:]) + } + return s +} + +// ProviderInfo describes a provider for listings. +type ProviderInfo struct { + Name string `json:"name"` + Type string `json:"type"` + Model string `json:"model"` + BaseURL string `json:"base_url"` + Ready bool `json:"ready"` // has a key (or needs none) + Local bool `json:"local"` // runs on this machine + Custom bool `json:"custom"` // defined in the config file + Description string `json:"description,omitempty"` + KeyEnv string `json:"key_env,omitempty"` +} + +// providerConfig merges a preset (if any) with user settings. +func (c *Config) providerConfig(name string) (llm.Config, ProviderInfo, error) { + user, hasUser := c.Providers[name] + preset, isPreset := llm.FindPreset(name) + if !isPreset && !hasUser { + return llm.Config{}, ProviderInfo{}, fmt.Errorf("unknown provider %q", name) + } + var cfg llm.Config + info := ProviderInfo{Name: name, Custom: !isPreset} + needsKey := true + if isPreset { + cfg = preset.Config() + needsKey = preset.NeedsKey + info.Description = preset.Description + if len(preset.KeyEnv) > 0 { + info.KeyEnv = preset.KeyEnv[0] + } + } else { + cfg = llm.Config{Name: name, Type: llm.Type(user.Type), Headers: map[string]string{}} + if user.Type == "" { + cfg.Type = llm.TypeOpenAI + } + cfg.JSONMode = cfg.Type == llm.TypeGemini + if user.BaseURL == "" { + return cfg, info, fmt.Errorf("provider %q: base_url is required for custom providers", name) + } + needsKey = false + } + if hasUser { + if user.Type != "" { + cfg.Type = llm.Type(user.Type) + } + if user.BaseURL != "" { + cfg.BaseURL = user.BaseURL + } + if user.APIKey != "" { + cfg.APIKey = ResolveKey(user.APIKey) + } + if user.Model != "" { + cfg.Model = user.Model + } + if user.MaxTokens != 0 { + cfg.MaxTokens = user.MaxTokens + } + if user.Temperature != nil { + cfg.Temperature = user.Temperature + } + if user.Effort != "" { + cfg.Effort = user.Effort + } + for k, v := range user.Headers { + cfg.Headers[k] = ResolveKey(v) + } + if user.AuthHeader != "" { + cfg.AuthHeader = user.AuthHeader + } + if user.JSONMode != nil { + cfg.JSONMode = *user.JSONMode + } + if user.TimeoutSeconds > 0 { + cfg.Timeout = time.Duration(user.TimeoutSeconds) * time.Second + } + if user.MaxRetries != 0 { + cfg.MaxRetries = user.MaxRetries + } + cfg.NoFallbacks = user.NoFallbacks + } + info.Type, info.Model, info.BaseURL = string(cfg.Type), cfg.Model, cfg.BaseURL + info.Local = strings.Contains(cfg.BaseURL, "://localhost") || strings.Contains(cfg.BaseURL, "://127.0.0.1") + info.Ready = !needsKey || cfg.APIKey != "" + return cfg, info, nil +} + +// List describes every known provider (presets first, then custom ones). +func (c *Config) List() []ProviderInfo { + var out []ProviderInfo + for _, p := range llm.Presets { + if _, info, err := c.providerConfig(p.Name); err == nil { + out = append(out, info) + } + } + var custom []string + for name := range c.Providers { + if _, ok := llm.FindPreset(name); !ok { + custom = append(custom, name) + } + } + sort.Strings(custom) + for _, name := range custom { + _, info, err := c.providerConfig(name) + if err != nil { + info = ProviderInfo{Name: name, Custom: true, Description: err.Error()} + } + out = append(out, info) + } + return out +} + +// Build creates the provider clients and the default provider chain. +// +// Without an explicit provider, the first built-in cloud provider that has an +// API key (from the config file or the environment) is used; with none, the +// offline renamer is used. Unless "fallback" says otherwise, the chain ends +// with the offline renamer so a failing AI never leaves you empty-handed. +func (c *Config) Build() (map[string]llm.Client, []string, error) { + clients := map[string]llm.Client{} + ready := map[string]bool{} + for _, info := range c.List() { + if !info.Ready { + continue + } + cfg, _, err := c.providerConfig(info.Name) + if err != nil { + if info.Name == c.Provider { + return nil, nil, err + } + continue + } + cl, err := llm.New(cfg, nil) + if err != nil { + return nil, nil, err + } + clients[info.Name], ready[info.Name] = cl, true + } + primary := c.Provider + if primary == "" { + for _, p := range llm.Presets { + if p.NeedsKey && ready[p.Name] { + primary = p.Name + break + } + } + } + if primary == "" || primary == llm.Offline { + return clients, []string{llm.Offline}, nil + } + if !ready[primary] { + if _, _, err := c.providerConfig(primary); err != nil { + return nil, nil, err + } + p, _ := llm.FindPreset(primary) + env := "" + if len(p.KeyEnv) > 0 { + env = " or set " + p.KeyEnv[0] + } + return nil, nil, fmt.Errorf("provider %q has no API key: run `betterdecompiler setup`%s", primary, env) + } + chain := []string{primary} + fallback := c.Fallback + if fallback == nil { + fallback = []string{llm.Offline} + } + for _, name := range fallback { + if name == primary || contains(chain, name) { + continue + } + if name != llm.Offline && !ready[name] { + return nil, nil, fmt.Errorf("fallback provider %q is unknown or has no API key", name) + } + chain = append(chain, name) + } + return clients, chain, nil +} + +func contains(s []string, v string) bool { + for _, x := range s { + if x == v { + return true + } + } + return false +} + +func boolOr(b *bool, def bool) bool { + if b == nil { + return def + } + return *b +} + +// EngineSettings converts the config to engine settings. +func (c *Config) EngineSettings(version string) engine.Settings { + return engine.Settings{ + Mode: c.Mode, Comments: c.Comments, Header: boolOr(c.Header, true), Heuristics: boolOr(c.Heuristics, true), + ChunkChars: c.ChunkChars, RewriteChunkChars: c.RewriteChunkChars, Concurrency: c.Concurrency, Version: version, + } +} + +// NewCache builds the result cache described by the config (nil if disabled). +func (c *Config) NewCache() *engine.Cache { + if !boolOr(c.Cache.Enabled, true) { + return nil + } + dir := c.Cache.Dir + if dir == "" { + if base, err := os.UserCacheDir(); err == nil { + dir = filepath.Join(base, "betterdecompiler") + } + } + return engine.NewCache(c.Cache.MaxEntries, dir) +} + +// ListenAddr returns the server address. +func (c *Config) ListenAddr() string { + if c.Listen != "" { + return c.Listen + } + return DefaultListen +} + +// Redacted returns a copy safe to print: keys and tokens are masked. +func (c *Config) Redacted() *Config { + cp := *c + cp.Token = mask(c.Token) + cp.Providers = map[string]Provider{} + for k, p := range c.Providers { + if !strings.HasPrefix(p.APIKey, "env:") && !strings.HasPrefix(p.APIKey, "$") { + p.APIKey = mask(p.APIKey) + } + cp.Providers[k] = p + } + return &cp +} + +func mask(s string) string { + switch { + case s == "": + return "" + case len(s) <= 10: + return "****" + } + return s[:4] + "…" + s[len(s)-4:] +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 0000000..ac73829 --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,154 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/officialmelon/betterdecompiler/internal/llm" +) + +// clearEnv unsets every provider key so tests don't depend on the machine. +func clearEnv(t *testing.T) { + for _, p := range llm.Presets { + for _, k := range p.KeyEnv { + t.Setenv(k, "") + } + } + for _, k := range []string{"BD_PROVIDER", "BD_MODE", "BD_TOKEN", "BD_LISTEN", "BD_MODEL"} { + t.Setenv(k, "") + } +} + +func TestNoKeysMeansOffline(t *testing.T) { + clearEnv(t) + c := &Config{} + clients, chain, err := c.Build() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(chain, []string{llm.Offline}) { + t.Fatalf("chain %v", chain) + } + // Local providers need no key and are still available on request. + if _, ok := clients["ollama"]; !ok { + t.Fatal("ollama should be available") + } +} + +func TestAutoDetectFromEnv(t *testing.T) { + clearEnv(t) + t.Setenv("GROQ_API_KEY", "gsk") + t.Setenv("GEMINI_API_KEY", "gem") + _, chain, err := (&Config{}).Build() + if err != nil { + t.Fatal(err) + } + // gemini comes before groq in preset order. + if !reflect.DeepEqual(chain, []string{"gemini", llm.Offline}) { + t.Fatalf("chain %v", chain) + } +} + +func TestExplicitProviderAndFallback(t *testing.T) { + clearEnv(t) + t.Setenv("MY_KEY", "secret") + c := &Config{ + Provider: "anthropic", + Fallback: []string{"proxy"}, + Providers: map[string]Provider{ + "anthropic": {APIKey: "sk-ant-direct", Model: "claude-haiku-5-5"}, + "proxy": {Type: "openai", BaseURL: "https://proxy.example/v1", APIKey: "env:MY_KEY", Model: "m"}, + }, + } + clients, chain, err := c.Build() + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(chain, []string{"anthropic", "proxy"}) { + t.Fatalf("chain %v (explicit fallback list replaces the default)", chain) + } + if clients["anthropic"].Model() != "claude-haiku-5-5" { + t.Fatal("model override ignored") + } + cfg, info, _ := c.providerConfig("proxy") + if cfg.APIKey != "secret" || !info.Custom || !info.Ready { + t.Fatalf("custom provider: %+v %+v", cfg, info) + } +} + +func TestMissingKeyErrors(t *testing.T) { + clearEnv(t) + _, _, err := (&Config{Provider: "openai"}).Build() + if err == nil || !strings.Contains(err.Error(), "OPENAI_API_KEY") { + t.Fatalf("expected helpful error, got %v", err) + } + if _, _, err := (&Config{Provider: "nope"}).Build(); err == nil { + t.Fatal("unknown provider accepted") + } + if _, _, err := (&Config{Provider: "custom", Providers: map[string]Provider{"custom": {Type: "openai"}}}).Build(); err == nil { + t.Fatal("custom provider without base_url accepted") + } +} + +func TestSaveLoadRoundTripAndPermissions(t *testing.T) { + path := filepath.Join(t.TempDir(), "sub", "config.json") + c := &Config{Provider: "gemini", Token: "tok"} + c.SetProvider("gemini", Provider{APIKey: "AIza-secret-value"}) + if err := Save(path, c); err != nil { + t.Fatal(err) + } + st, _ := os.Stat(path) + if st.Mode().Perm()&0o077 != 0 && os.PathSeparator == '/' { + t.Fatalf("config readable by others: %v", st.Mode()) + } + got, err := Load(path) + if err != nil || !reflect.DeepEqual(got, c) { + t.Fatalf("round trip: %+v %v", got, err) + } + missing, err := Load(filepath.Join(t.TempDir(), "none.json")) + if err != nil || missing.Provider != "" { + t.Fatal("missing file should be empty config") + } + r := got.Redacted() + if strings.Contains(r.Providers["gemini"].APIKey, "secret") || r.Token == "tok" { + t.Fatalf("not redacted: %+v", r) + } + if got.Providers["gemini"].APIKey != "AIza-secret-value" { + t.Fatal("Redacted mutated the original") + } +} + +func TestApplyEnv(t *testing.T) { + clearEnv(t) + t.Setenv("BD_PROVIDER", "openai") + t.Setenv("BD_MODEL", "gpt-x") + t.Setenv("BD_LISTEN", "0.0.0.0:6000") + c := &Config{} + c.ApplyEnv() + if c.Provider != "openai" || c.Providers["openai"].Model != "gpt-x" || c.ListenAddr() != "0.0.0.0:6000" { + t.Fatalf("%+v", c) + } +} + +func TestResolveKey(t *testing.T) { + t.Setenv("K", "v") + for in, want := range map[string]string{"env:K": "v", "${K}": "v", "$K": "v", "literal": "literal", "$": "$"} { + if got := ResolveKey(in); got != want { + t.Errorf("ResolveKey(%q) = %q", in, got) + } + } +} + +func TestEngineSettingsDefaults(t *testing.T) { + s := (&Config{}).EngineSettings("v") + if !s.Header || !s.Heuristics || s.Version != "v" { + t.Fatalf("%+v", s) + } + off := false + if (&Config{Cache: Cache{Enabled: &off}}).NewCache() != nil { + t.Fatal("disabled cache created") + } +} diff --git a/internal/engine/cache.go b/internal/engine/cache.go new file mode 100644 index 0000000..0e3949a --- /dev/null +++ b/internal/engine/cache.go @@ -0,0 +1,118 @@ +package engine + +import ( + "container/list" + "encoding/json" + "os" + "path/filepath" + "sync" +) + +// Cache is a two-level result cache: an in-memory LRU backed by an optional +// on-disk store, so repeated decompiles of the same script are instant and +// cost nothing, even across restarts. +type Cache struct { + mu sync.Mutex + max int + ll *list.List + items map[string]*list.Element + dir string +} + +type cacheEntry struct { + key string + res *Result +} + +// NewCache creates a cache holding up to max results in memory. When dir is +// non-empty, results are also persisted there. +func NewCache(max int, dir string) *Cache { + if max <= 0 { + max = 256 + } + return &Cache{max: max, ll: list.New(), items: map[string]*list.Element{}, dir: dir} +} + +func (c *Cache) path(key string) string { return filepath.Join(c.dir, key[:2], key+".json") } + +// Get returns a cached result. +func (c *Cache) Get(key string) (*Result, bool) { + if c == nil { + return nil, false + } + c.mu.Lock() + if el, ok := c.items[key]; ok { + c.ll.MoveToFront(el) + r := el.Value.(*cacheEntry).res + c.mu.Unlock() + return r, true + } + c.mu.Unlock() + if c.dir == "" { + return nil, false + } + data, err := os.ReadFile(c.path(key)) + if err != nil { + return nil, false + } + var r Result + if json.Unmarshal(data, &r) != nil || r.Source == "" { + return nil, false + } + c.putMem(key, &r) + return &r, true +} + +// Put stores a result. +func (c *Cache) Put(key string, r *Result) { + if c == nil { + return + } + c.putMem(key, r) + if c.dir == "" { + return + } + data, err := json.Marshal(r) + if err != nil { + return + } + p := c.path(key) + if os.MkdirAll(filepath.Dir(p), 0o700) != nil { + return + } + tmp, err := os.CreateTemp(filepath.Dir(p), ".tmp-*") + if err != nil { + return + } + _, werr := tmp.Write(data) + cerr := tmp.Close() + if werr != nil || cerr != nil || os.Rename(tmp.Name(), p) != nil { + os.Remove(tmp.Name()) + } +} + +func (c *Cache) putMem(key string, r *Result) { + c.mu.Lock() + defer c.mu.Unlock() + if el, ok := c.items[key]; ok { + el.Value.(*cacheEntry).res = r + c.ll.MoveToFront(el) + return + } + c.items[key] = c.ll.PushFront(&cacheEntry{key, r}) + for c.ll.Len() > c.max { + old := c.ll.Back() + c.ll.Remove(old) + delete(c.items, old.Value.(*cacheEntry).key) + } +} + +// Len returns the number of in-memory entries. +func (c *Cache) Len() int { + if c == nil { + return 0 + } + c.mu.Lock() + defer c.mu.Unlock() + return c.ll.Len() +} diff --git a/internal/engine/engine.go b/internal/engine/engine.go new file mode 100644 index 0000000..a0efa49 --- /dev/null +++ b/internal/engine/engine.go @@ -0,0 +1,680 @@ +// Package engine turns decompiled Luau into readable Luau. +// +// Three modes are supported: +// +// - rename (default): the AI only proposes names and comments as compact +// JSON; they are applied locally by a scope-aware renamer that verifies +// every identifier still refers to the same variable. Fast, cheap (tiny +// outputs), works on huge scripts, and can never change behavior. +// - rewrite: the AI rewrites the code (also restructuring it). Slower and +// can hallucinate, but produces the most idiomatic result. +// - offline: no AI at all; names are inferred from Roblox API usage. +package engine + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/officialmelon/betterdecompiler/internal/llm" + "github.com/officialmelon/betterdecompiler/internal/luau" +) + +// Modes. +const ( + ModeRename = "rename" + ModeRewrite = "rewrite" + ModeOffline = "offline" +) + +// Settings are engine-wide defaults. +type Settings struct { + Mode string // default mode + Comments string // none | light | detailed + Header bool // prepend an informational header comment + Heuristics bool // fill names the AI skipped with offline inference + ChunkChars int // max source characters per AI request in rename mode + RewriteChunkChars int // max source characters per AI request in rewrite mode + Concurrency int // max concurrent AI requests + Version string // shown in the header +} + +// Options customise a single request. Zero values use the Settings. +type Options struct { + Mode string `json:"mode,omitempty"` + Provider string `json:"provider,omitempty"` + Model string `json:"model,omitempty"` + Comments string `json:"comments,omitempty"` + Header *bool `json:"header,omitempty"` + NoCache bool `json:"no_cache,omitempty"` +} + +// Result is the outcome of a cleanup. +type Result struct { + Source string `json:"source"` + Mode string `json:"mode"` + Provider string `json:"provider"` + Model string `json:"model,omitempty"` + Summary string `json:"summary,omitempty"` + Renamed int `json:"renamed"` + Comments int `json:"comments"` + Usage llm.Usage `json:"usage"` + Warnings []string `json:"warnings,omitempty"` + Cached bool `json:"cached"` + ElapsedMS int64 `json:"elapsed_ms"` + + cacheable bool +} + +// Stats are cumulative engine counters. +type Stats struct { + Requests int64 `json:"requests"` + CacheHits int64 `json:"cache_hits"` + AICalls int64 `json:"ai_calls"` + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` +} + +// Engine cleans scripts. It is safe for concurrent use. +type Engine struct { + s Settings + providers map[string]llm.Client + chain []string + cache *Cache + sem chan struct{} + + mu sync.Mutex + flight map[string]*call + + requests, cacheHits, aiCalls, inTok, outTok atomic.Int64 +} + +type call struct { + done chan struct{} + res *Result + err error +} + +// ErrUnknownProvider is returned for requests naming an unconfigured provider. +var ErrUnknownProvider = errors.New("unknown or unconfigured provider") + +// New creates an engine. chain is the default provider order; it may contain +// llm.Offline. providers maps names to clients. +func New(s Settings, providers map[string]llm.Client, chain []string, cache *Cache) *Engine { + if s.Mode == "" { + s.Mode = ModeRename + } + if _, ok := commentRules[s.Comments]; !ok { + s.Comments = "light" + } + if s.ChunkChars <= 0 { + s.ChunkChars = 120_000 + } + if s.RewriteChunkChars <= 0 { + s.RewriteChunkChars = 24_000 + } + if s.Concurrency <= 0 { + s.Concurrency = 4 + } + if len(chain) == 0 { + chain = []string{llm.Offline} + } + return &Engine{s: s, providers: providers, chain: chain, cache: cache, + sem: make(chan struct{}, s.Concurrency), flight: map[string]*call{}} +} + +// Chain returns the default provider order. +func (e *Engine) Chain() []string { return append([]string(nil), e.chain...) } + +// Settings returns the engine settings. +func (e *Engine) Settings() Settings { return e.s } + +// Provider returns a configured provider client. +func (e *Engine) Provider(name string) (llm.Client, bool) { + c, ok := e.providers[name] + return c, ok +} + +// Stats returns cumulative counters. +func (e *Engine) Stats() Stats { + return Stats{Requests: e.requests.Load(), CacheHits: e.cacheHits.Load(), AICalls: e.aiCalls.Load(), + InputTokens: e.inTok.Load(), OutputTokens: e.outTok.Load()} +} + +func (e *Engine) normalize(o Options) (Options, []string, error) { + if o.Mode == "" { + o.Mode = e.s.Mode + } + switch o.Mode { + case ModeRename, ModeRewrite, ModeOffline: + default: + return o, nil, fmt.Errorf("unknown mode %q (want rename, rewrite or offline)", o.Mode) + } + if o.Comments == "" { + o.Comments = e.s.Comments + } + if _, ok := commentRules[o.Comments]; !ok { + return o, nil, fmt.Errorf("unknown comments level %q (want none, light or detailed)", o.Comments) + } + if o.Header == nil { + h := e.s.Header + o.Header = &h + } + if o.Mode == ModeOffline || o.Provider == llm.Offline { + o.Mode, o.Provider = ModeOffline, llm.Offline + return o, []string{llm.Offline}, nil + } + chain := e.chain + if o.Provider != "" { + if _, ok := e.providers[o.Provider]; !ok { + return o, nil, fmt.Errorf("%w: %q", ErrUnknownProvider, o.Provider) + } + chain = []string{o.Provider} + for _, n := range e.chain { + if n == llm.Offline { + chain = append(chain, n) + } + } + } + if len(chain) == 1 && chain[0] == llm.Offline { + o.Mode = ModeOffline + } + return o, chain, nil +} + +func (e *Engine) cacheKey(src string, o Options, chain []string) string { + h := sha256.New() + model := o.Model + if model == "" && chain[0] != llm.Offline { + model = e.providers[chain[0]].Model() + } + fmt.Fprintf(h, "v%s\x00%s\x00%s\x00%s\x00%s\x00%t\x00%t\x00", promptVersion, o.Mode, strings.Join(chain, ","), model, o.Comments, *o.Header, e.s.Heuristics) + h.Write([]byte(src)) + return hex.EncodeToString(h.Sum(nil)) +} + +// Clean cleans up decompiled source. +func (e *Engine) Clean(ctx context.Context, src string, o Options) (*Result, error) { + start := time.Now() + e.requests.Add(1) + o, chain, err := e.normalize(o) + if err != nil { + return nil, err + } + if strings.TrimSpace(src) == "" { + return &Result{Source: src, Mode: o.Mode, Provider: "none"}, nil + } + key := e.cacheKey(src, o, chain) + if !o.NoCache { + if r, ok := e.cache.Get(key); ok { + e.cacheHits.Add(1) + out := *r + out.Cached, out.ElapsedMS = true, time.Since(start).Milliseconds() + return &out, nil + } + } + // Identical concurrent requests share one upstream call. + e.mu.Lock() + if c, ok := e.flight[key]; ok { + e.mu.Unlock() + select { + case <-c.done: + case <-ctx.Done(): + return nil, ctx.Err() + } + if c.err != nil { + return nil, c.err + } + out := *c.res + out.ElapsedMS = time.Since(start).Milliseconds() + return &out, nil + } + c := &call{done: make(chan struct{})} + e.flight[key] = c + e.mu.Unlock() + + c.res, c.err = e.run(ctx, src, o, chain, start) + if c.err == nil && c.res.cacheable { + e.cache.Put(key, c.res) + } + e.mu.Lock() + delete(e.flight, key) + e.mu.Unlock() + close(c.done) + if c.err != nil { + return nil, c.err + } + out := *c.res + return &out, nil +} + +func (e *Engine) run(ctx context.Context, src string, o Options, chain []string, start time.Time) (*Result, error) { + f, perr := luau.Parse(src) + mode := o.Mode + var warnings []string + if mode == ModeRename && perr != nil { + warnings = append(warnings, fmt.Sprintf("input could not be parsed (%v), so rewrite mode was used instead of rename mode", perr)) + mode = ModeRewrite + } + var errs []string + for i, name := range chain { + var res *Result + var err error + if name == llm.Offline { + res = e.offline(f, perr, src) + res.cacheable = len(errs) == 0 + } else { + model := "" + if i == 0 { + model = o.Model + } + client := e.providers[name] + if mode == ModeRename { + res, err = e.renameAI(ctx, f, client, model, o) + } else { + res, err = e.rewriteAI(ctx, f, src, client, model, o) + } + if res != nil { + res.cacheable = true + } + } + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + errs = append(errs, err.Error()) + warnings = append(warnings, fmt.Sprintf("%s failed: %v", name, err)) + continue + } + res.Warnings = append(warnings, res.Warnings...) + res.ElapsedMS = time.Since(start).Milliseconds() + if *o.Header { + res.Source = e.header(res) + res.Source + } + return res, nil + } + return nil, fmt.Errorf("all providers failed: %s", strings.Join(errs, "; ")) +} + +func (e *Engine) offline(f *luau.File, perr error, src string) *Result { + res := &Result{Mode: ModeOffline, Provider: llm.Offline} + if f == nil { + res.Source = src + res.Warnings = []string{fmt.Sprintf("input could not be parsed (%v); returned unchanged", perr)} + return res + } + out, applied := f.Rename(f.SuggestNames()) + res.Source, res.Renamed = out, len(applied) + return res +} + +func (e *Engine) complete(ctx context.Context, c llm.Client, req llm.Request) (*llm.Response, error) { + select { + case e.sem <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-e.sem }() + e.aiCalls.Add(1) + resp, err := c.Complete(ctx, req) + if err == nil { + e.inTok.Add(int64(resp.Usage.InputTokens)) + e.outTok.Add(int64(resp.Usage.OutputTokens)) + } + return resp, err +} + +// chunks groups lines 1..n into ranges of at most max characters, breaking +// only before top-level statements. +func chunks(lines []string, f *luau.File, max int) [][2]int { + n := len(lines) + if f == nil || len(f.Statements) == 0 { + return [][2]int{{1, n}} + } + breaks := []int{1} + for _, s := range f.Statements { + if s.Line > breaks[len(breaks)-1] { + breaks = append(breaks, s.Line) + } + } + breaks = append(breaks, n+1) + var out [][2]int + from, size := 1, 0 + for i := 0; i+1 < len(breaks); i++ { + seg := 0 + for l := breaks[i]; l < breaks[i+1] && l <= n; l++ { + seg += len(lines[l-1]) + 1 + } + if size > 0 && size+seg > max { + out = append(out, [2]int{from, breaks[i] - 1}) + from, size = breaks[i], 0 + } + size += seg + } + return append(out, [2]int{from, n}) +} + +// parallel runs fn for each index with bounded concurrency (bounded further +// by the engine semaphore) and returns the first error. +func parallel(ctx context.Context, n int, fn func(ctx context.Context, i int) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + var wg sync.WaitGroup + var once sync.Once + var first error + for i := 0; i < n; i++ { + wg.Add(1) + go func(i int) { + defer wg.Done() + if err := fn(ctx, i); err != nil { + once.Do(func() { first = err; cancel() }) + } + }(i) + } + wg.Wait() + return first +} + +type renameReply struct { + Summary string `json:"summary"` + Renames json.RawMessage `json:"renames"` + Comments []struct { + Line int `json:"line"` + Text string `json:"text"` + } `json:"comments"` +} + +// parseRenameReply extracts the JSON object from a model reply, tolerating +// markdown fences and surrounding prose, and accepting renames either as an +// object or as a list of {from,to} pairs. +func parseRenameReply(text string) (*renameReply, map[string]string, error) { + i := strings.Index(text, "{") + if i < 0 { + return nil, nil, fmt.Errorf("reply contained no JSON object: %q", clip(text)) + } + var r renameReply + if err := json.NewDecoder(strings.NewReader(text[i:])).Decode(&r); err != nil { + return nil, nil, fmt.Errorf("reply was not valid JSON (%v): %q", err, clip(text)) + } + renames := map[string]string{} + if len(r.Renames) > 0 && string(r.Renames) != "null" { + if json.Unmarshal(r.Renames, &renames) != nil { + var pairs []map[string]string + if err := json.Unmarshal(r.Renames, &pairs); err != nil { + return nil, nil, fmt.Errorf("unexpected renames format: %s", clip(string(r.Renames))) + } + for _, p := range pairs { + from := firstOf(p, "from", "old", "name", "original") + to := firstOf(p, "to", "new", "newName", "renamed") + if from != "" && to != "" { + renames[from] = to + } + } + } + } + return &r, renames, nil +} + +func firstOf(m map[string]string, keys ...string) string { + for _, k := range keys { + if v := m[k]; v != "" { + return v + } + } + return "" +} + +func clip(s string) string { + s = strings.TrimSpace(s) + if len(s) > 200 { + return s[:200] + "..." + } + return s +} + +func (e *Engine) renameAI(ctx context.Context, f *luau.File, c llm.Client, model string, o Options) (*Result, error) { + keyed, keys := f.UniqueKeys() + idByKey := make(map[string]int, len(keys)) + type cand struct { + key string + line int + } + var cands []cand + for id, k := range keys { + idByKey[k] = id + cands = append(cands, cand{k, f.Toks[f.Bindings[id].Decl].Line}) + } + sort.Slice(cands, func(i, j int) bool { + if cands[i].line != cands[j].line { + return cands[i].line < cands[j].line + } + return cands[i].key < cands[j].key + }) + lines := strings.Split(keyed, "\n") + ranges := chunks(lines, f, e.s.ChunkChars) + + type part struct { + reply *renameReply + renames map[string]string + resp *llm.Response + } + parts := make([]*part, len(ranges)) + system := renamePrompt(o.Comments) + err := parallel(ctx, len(ranges), func(ctx context.Context, i int) error { + r := ranges[i] + var names []string + for _, cd := range cands { + if cd.line >= r[0] && cd.line <= r[1] { + names = append(names, cd.key) + } + } + if len(names) == 0 && o.Comments == "none" { + parts[i] = &part{} + return nil + } + req := llm.Request{System: system, Prompt: renameUserPrompt(names, numbered(lines, r[0], r[1])), Model: model, JSON: true} + var lastErr error + for attempt := 0; attempt < 2; attempt++ { + resp, err := e.complete(ctx, c, req) + if err != nil { + return err + } + reply, renames, err := parseRenameReply(resp.Text) + if err == nil { + parts[i] = &part{reply, renames, resp} + return nil + } + lastErr = err + req.Prompt += "\nReply with the JSON object only." + } + return fmt.Errorf("%s: %w", c.Name(), lastErr) + }) + if err != nil { + return nil, err + } + + res := &Result{Mode: ModeRename, Provider: c.Name(), Model: model} + if res.Model == "" { + res.Model = c.Model() + } + want := map[int]string{} + comments := map[int]string{} + var summaries []string + for _, p := range parts { + if p.resp != nil { + res.Usage.Add(p.resp.Usage) + if p.resp.Model != "" { + res.Model = p.resp.Model + } + } + if p.reply == nil { + continue + } + if s := strings.TrimSpace(p.reply.Summary); s != "" { + summaries = append(summaries, s) + } + for k, v := range p.renames { + id, ok := idByKey[k] + if !ok { + continue + } + if _, dup := want[id]; !dup { + want[id] = luau.ToIdent(v) + } + } + for _, cm := range p.reply.Comments { + if o.Comments != "none" && cm.Line > 0 && strings.TrimSpace(cm.Text) != "" { + if _, dup := comments[cm.Line]; !dup { + comments[cm.Line] = cm.Text + } + } + } + } + if e.s.Heuristics { + for id, n := range f.SuggestNames() { + if _, ok := want[id]; !ok { + want[id] = n + } + } + } + // Bindings the AI left alone keep their original names (not their keys). + out, applied := f.Rename(want) + out = f.InsertComments(out, comments) + res.Source, res.Renamed, res.Comments = out, len(applied), len(comments) + if len(summaries) > 0 { + res.Summary = summaries[0] + } + return res, nil +} + +// stripFences extracts code from a reply that may wrap it in markdown. +func stripFences(text string) string { + if !strings.Contains(text, "```") { + return strings.Trim(text, "\r\n") + } + best := "" + rest := text + for { + i := strings.Index(rest, "```") + if i < 0 { + break + } + body := rest[i+3:] + nl := strings.IndexByte(body, '\n') + if nl < 0 { + break + } + body = body[nl+1:] // skip the language tag line + j := strings.Index(body, "```") + if j < 0 { + if len(body) > len(best) { // unterminated fence (truncated reply) + best = body + } + break + } + if len(body[:j]) > len(best) { + best = body[:j] + } + rest = body[j+3:] + } + return strings.Trim(best, "\r\n") +} + +func (e *Engine) rewriteAI(ctx context.Context, f *luau.File, src string, c llm.Client, model string, o Options) (*Result, error) { + lines := strings.Split(src, "\n") + ranges := chunks(lines, f, e.s.RewriteChunkChars) + outs := make([]*llm.Response, len(ranges)) + system := rewritePrompt(o.Comments) + err := parallel(ctx, len(ranges), func(ctx context.Context, i int) error { + code := strings.Join(lines[ranges[i][0]-1:ranges[i][1]], "\n") + if strings.TrimSpace(code) == "" { + outs[i] = &llm.Response{Text: code} + return nil + } + resp, err := e.complete(ctx, c, llm.Request{System: system, Prompt: rewriteUserPrompt(code), Model: model}) + if err != nil { + return err + } + resp.Text = stripFences(resp.Text) + if strings.TrimSpace(resp.Text) == "" { + return fmt.Errorf("%s returned an empty reply", c.Name()) + } + outs[i] = resp + return nil + }) + if err != nil { + return nil, err + } + res := &Result{Mode: ModeRewrite, Provider: c.Name(), Model: model} + if res.Model == "" { + res.Model = c.Model() + } + texts := make([]string, len(outs)) + for i, r := range outs { + texts[i] = r.Text + res.Usage.Add(r.Usage) + if r.Model != "" { + res.Model = r.Model + } + if r.Truncated { + res.Warnings = append(res.Warnings, fmt.Sprintf("part %d of %d hit the output token limit and is incomplete", i+1, len(outs))) + } + } + res.Source = strings.Join(texts, "\n\n") + "\n" + if _, err := luau.Parse(res.Source); err != nil { + res.Warnings = append(res.Warnings, fmt.Sprintf("the rewritten code does not parse (%v); review it carefully", err)) + } + return res, nil +} + +func (e *Engine) header(r *Result) string { + var sb strings.Builder + who := r.Provider + if r.Model != "" { + who += "/" + r.Model + } + fmt.Fprintf(&sb, "-- Cleaned by BetterDecompiler %s | %s | %s mode | %.2fs\n", e.s.Version, who, r.Mode, float64(r.ElapsedMS)/1000) + if r.Summary != "" { + for _, l := range wrap("Summary: "+strings.Join(strings.Fields(r.Summary), " "), 100) { + sb.WriteString("-- " + l + "\n") + } + } + switch r.Mode { + case ModeRewrite: + sb.WriteString("-- Rewritten by AI: it can be wrong, so verify before relying on it.\n") + case ModeRename: + sb.WriteString("-- Only names and comments were changed; behavior is identical. AI-chosen names can be wrong.\n") + } + for _, w := range r.Warnings { + sb.WriteString("-- Warning: " + strings.Join(strings.Fields(w), " ") + "\n") + } + sb.WriteString("\n") + return sb.String() +} + +func wrap(s string, width int) []string { + var lines []string + cur := "" + for _, w := range strings.Fields(s) { + if cur != "" && len(cur)+1+len(w) > width { + lines = append(lines, cur) + cur = "" + } + if cur != "" { + cur += " " + } + cur += w + } + if cur != "" { + lines = append(lines, cur) + } + return lines +} diff --git a/internal/engine/engine_test.go b/internal/engine/engine_test.go new file mode 100644 index 0000000..ca6d392 --- /dev/null +++ b/internal/engine/engine_test.go @@ -0,0 +1,405 @@ +package engine + +import ( + "context" + "errors" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/officialmelon/betterdecompiler/internal/llm" + "github.com/officialmelon/betterdecompiler/internal/luau" + "github.com/officialmelon/betterdecompiler/internal/mockllm" +) + +func init() { + llm.Backoff = func(int, time.Duration) time.Duration { return time.Millisecond } +} + +func sample(t *testing.T) string { + t.Helper() + b, err := os.ReadFile(filepath.Join("..", "..", "examples", "decompiled.lua")) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +type fixture struct { + mock *mockllm.Server + url string +} + +func newFixture(t *testing.T) *fixture { + m := mockllm.New("sk-test") + srv := httptest.NewServer(m) + t.Cleanup(srv.Close) + return &fixture{m, srv.URL} +} + +func (fx *fixture) client(t *testing.T, name string, typ llm.Type, key string) llm.Client { + t.Helper() + base := fx.url + switch typ { + case llm.TypeOpenAI: + base += "/v1" + case llm.TypeGemini: + base += "/v1beta" + } + model := "mock-1" + if typ == llm.TypeGemini { + model = "mock-gemini" + } + c, err := llm.New(llm.Config{Name: name, Type: typ, BaseURL: base, APIKey: key, Model: model, JSONMode: true}, nil) + if err != nil { + t.Fatal(err) + } + return c +} + +func defaultSettings() Settings { + return Settings{Header: true, Heuristics: true, Version: "test", Comments: "light"} +} + +// stripHeader removes the leading header comment block. +func stripHeader(s string) string { + if i := strings.Index(s, "\n\n"); strings.HasPrefix(s, "-- Cleaned by") && i >= 0 { + return s[i+2:] + } + return s +} + +func assertSameBehavior(t *testing.T, before, after string) { + t.Helper() + a, err := luau.Parse(before) + if err != nil { + t.Fatal(err) + } + b, err := luau.Parse(after) + if err != nil { + t.Fatalf("output does not parse: %v\n%s", err, after) + } + if len(a.Toks) != len(b.Toks) { + t.Fatalf("token count changed %d -> %d", len(a.Toks), len(b.Toks)) + } + for i := range a.Resolve { + if a.Resolve[i] != b.Resolve[i] { + t.Fatalf("token %d (%s->%s) resolution changed", i, a.Toks[i].Text, b.Toks[i].Text) + } + if a.Toks[i].Kind != luau.Name && a.Toks[i].Text != b.Toks[i].Text { + t.Fatalf("non-identifier token %d changed: %q -> %q", i, a.Toks[i].Text, b.Toks[i].Text) + } + } +} + +func TestOfflineMode(t *testing.T) { + e := New(defaultSettings(), nil, nil, nil) + src := sample(t) + res, err := e.Clean(context.Background(), src, Options{}) + if err != nil { + t.Fatal(err) + } + if res.Mode != ModeOffline || res.Renamed < 10 { + t.Fatalf("offline result: %+v", res) + } + if !strings.HasPrefix(res.Source, "-- Cleaned by BetterDecompiler test | offline | offline mode") { + t.Fatalf("header missing:\n%s", res.Source) + } + assertSameBehavior(t, src, stripHeader(res.Source)) +} + +func TestRenameModeAllProtocols(t *testing.T) { + src := sample(t) + for _, typ := range []llm.Type{llm.TypeOpenAI, llm.TypeAnthropic, llm.TypeGemini} { + t.Run(string(typ), func(t *testing.T) { + fx := newFixture(t) + s := defaultSettings() + s.Heuristics = false + e := New(s, map[string]llm.Client{"p": fx.client(t, "p", typ, "sk-test")}, []string{"p"}, nil) + res, err := e.Clean(context.Background(), src, Options{}) + if err != nil { + t.Fatal(err) + } + if res.Mode != ModeRename || res.Provider != "p" || res.Summary != "Mock summary of the script." { + t.Fatalf("result: %+v", res) + } + out := stripHeader(res.Source) + for _, want := range []string{"local mockV1 = game:GetService", "function(mockP1_", "-- mock comment"} { + if !strings.Contains(out, want) { + t.Fatalf("missing %q:\n%s", want, out) + } + } + if !strings.Contains(res.Source, "-- Summary: Mock summary of the script.") { + t.Fatal("summary not in header") + } + // Remove the inserted comment and compare behavior. + assertSameBehavior(t, src, strings.Replace(out, "-- mock comment\n", "", 1)) + if res.Usage.InputTokens == 0 { + t.Fatal("usage not reported") + } + }) + } +} + +func TestHeuristicsFillGaps(t *testing.T) { + fx := newFixture(t) + fx.mock.Responder = func(system, prompt string) string { + return `{"summary":"s","renames":{"v4":"me"},"comments":[]}` + } + e := New(defaultSettings(), map[string]llm.Client{"p": fx.client(t, "p", llm.TypeOpenAI, "sk-test")}, []string{"p"}, nil) + res, err := e.Clean(context.Background(), sample(t), Options{Comments: "none"}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(res.Source, "local me = Players.LocalPlayer") { + t.Fatalf("AI name + heuristic names expected:\n%s", res.Source) + } +} + +func TestRewriteMode(t *testing.T) { + fx := newFixture(t) + e := New(defaultSettings(), map[string]llm.Client{"p": fx.client(t, "p", llm.TypeAnthropic, "sk-test")}, []string{"p"}, nil) + res, err := e.Clean(context.Background(), sample(t), Options{Mode: ModeRewrite}) + if err != nil { + t.Fatal(err) + } + out := stripHeader(res.Source) + if !strings.HasPrefix(out, "-- mock rewrite\n") || strings.Contains(out, "```") || strings.Contains(out, "Here is") { + t.Fatalf("fences not stripped:\n%s", out) + } + if len(res.Warnings) != 0 { + t.Fatalf("unexpected warnings: %v", res.Warnings) + } +} + +func TestFallbackChain(t *testing.T) { + fx := newFixture(t) + providers := map[string]llm.Client{ + "bad": fx.client(t, "bad", llm.TypeAnthropic, "wrong-key"), + "good": fx.client(t, "good", llm.TypeGemini, "sk-test"), + } + e := New(defaultSettings(), providers, []string{"bad", "good", llm.Offline}, NewCache(10, "")) + res, err := e.Clean(context.Background(), sample(t), Options{}) + if err != nil { + t.Fatal(err) + } + if res.Provider != "good" || len(res.Warnings) != 1 || !strings.Contains(res.Warnings[0], "bad failed") { + t.Fatalf("fallback: %+v", res) + } + // Explicit provider with offline fallback: not cached, so a later retry can use AI. + res, err = e.Clean(context.Background(), sample(t), Options{Provider: "bad"}) + if err != nil { + t.Fatal(err) + } + if res.Provider != llm.Offline || !strings.Contains(res.Source, "-- Warning: bad failed") { + t.Fatalf("offline fallback: %+v", res) + } + res, _ = e.Clean(context.Background(), sample(t), Options{Provider: "bad"}) + if res.Cached { + t.Fatal("degraded results must not be cached") + } + // Without offline fallback the error surfaces. + e2 := New(defaultSettings(), providers, []string{"bad"}, nil) + if _, err := e2.Clean(context.Background(), sample(t), Options{}); err == nil || !strings.Contains(err.Error(), "401") { + t.Fatalf("expected 401 error, got %v", err) + } + if _, err := e2.Clean(context.Background(), "x()", Options{Provider: "nope"}); !errors.Is(err, ErrUnknownProvider) { + t.Fatalf("unknown provider: %v", err) + } +} + +func TestCacheMemoryAndDisk(t *testing.T) { + fx := newFixture(t) + dir := t.TempDir() + c := fx.client(t, "p", llm.TypeOpenAI, "sk-test") + e := New(defaultSettings(), map[string]llm.Client{"p": c}, []string{"p"}, NewCache(10, dir)) + src := sample(t) + r1, err := e.Clean(context.Background(), src, Options{}) + if err != nil { + t.Fatal(err) + } + r2, _ := e.Clean(context.Background(), src, Options{}) + if r1.Cached || !r2.Cached || r1.Source != r2.Source || fx.mock.Count() != 1 { + t.Fatalf("memory cache: cached=%v/%v calls=%d", r1.Cached, r2.Cached, fx.mock.Count()) + } + // Different options are different entries. + if r3, _ := e.Clean(context.Background(), src, Options{Comments: "none"}); r3.Cached { + t.Fatal("comments level must be part of the key") + } + // A fresh engine with the same dir hits the disk cache. + e2 := New(defaultSettings(), map[string]llm.Client{"p": c}, []string{"p"}, NewCache(10, dir)) + r4, _ := e2.Clean(context.Background(), src, Options{}) + if !r4.Cached || r4.Source != r1.Source { + t.Fatal("disk cache miss") + } + // NoCache bypasses it. + before := fx.mock.Count() + if r5, _ := e2.Clean(context.Background(), src, Options{NoCache: true}); r5.Cached || fx.mock.Count() != before+1 { + t.Fatal("no_cache ignored") + } +} + +func TestConcurrentDuplicatesShareOneCall(t *testing.T) { + fx := newFixture(t) + release := make(chan struct{}) + fx.mock.Responder = func(system, prompt string) string { + <-release + return mockllm.DefaultReply(system, prompt) + } + e := New(defaultSettings(), map[string]llm.Client{"p": fx.client(t, "p", llm.TypeOpenAI, "sk-test")}, []string{"p"}, nil) + var wg sync.WaitGroup + results := make([]*Result, 8) + for i := range results { + wg.Add(1) + go func(i int) { + defer wg.Done() + r, err := e.Clean(context.Background(), sample(t), Options{}) + if err != nil { + t.Error(err) + } + results[i] = r + }(i) + } + time.Sleep(100 * time.Millisecond) + close(release) + wg.Wait() + if fx.mock.Count() != 1 { + t.Fatalf("expected 1 upstream call, got %d", fx.mock.Count()) + } + for _, r := range results { + if r == nil || r.Source != results[0].Source { + t.Fatal("results differ") + } + } +} + +func TestChunkingMergesResults(t *testing.T) { + fx := newFixture(t) + s := defaultSettings() + s.ChunkChars = 300 + s.Heuristics = false + e := New(s, map[string]llm.Client{"p": fx.client(t, "p", llm.TypeOpenAI, "sk-test")}, []string{"p"}, nil) + src := sample(t) + res, err := e.Clean(context.Background(), src, Options{}) + if err != nil { + t.Fatal(err) + } + if fx.mock.Count() < 3 { + t.Fatalf("expected several chunks, got %d calls", fx.mock.Count()) + } + out := stripHeader(res.Source) + for _, name := range []string{"mockV1", "mockV9", "mockU10", "mockV16"} { + if !strings.Contains(out, name) { + t.Fatalf("missing %s from a later chunk:\n%s", name, out) + } + } + // One comment per chunk, at each chunk's first line. + if res.Comments < 3 { + t.Fatalf("comments from chunks not merged: %d", res.Comments) + } + clean := strings.ReplaceAll(out, "-- mock comment\n", "") + clean = strings.ReplaceAll(clean, "\t-- mock comment\n", "") + assertSameBehavior(t, src, clean) +} + +func TestGarbageReplyRetriesThenFallsBack(t *testing.T) { + fx := newFixture(t) + c, _ := llm.New(llm.Config{Name: "p", Type: llm.TypeOpenAI, BaseURL: fx.url + "/v1", APIKey: "sk-test", Model: "mock-garbage"}, nil) + e := New(defaultSettings(), map[string]llm.Client{"p": c}, []string{"p", llm.Offline}, nil) + res, err := e.Clean(context.Background(), sample(t), Options{}) + if err != nil { + t.Fatal(err) + } + if fx.mock.Count() != 2 || res.Provider != llm.Offline || !strings.Contains(res.Warnings[0], "not") { + t.Fatalf("calls=%d result=%+v", fx.mock.Count(), res) + } +} + +func TestUnparsableInputUsesRewrite(t *testing.T) { + fx := newFixture(t) + e := New(defaultSettings(), map[string]llm.Client{"p": fx.client(t, "p", llm.TypeOpenAI, "sk-test")}, []string{"p"}, nil) + res, err := e.Clean(context.Background(), "local x = = 1\n", Options{}) + if err != nil { + t.Fatal(err) + } + if res.Mode != ModeRewrite || len(res.Warnings) == 0 { + t.Fatalf("%+v", res) + } + off := New(defaultSettings(), nil, nil, nil) + res, _ = off.Clean(context.Background(), "local x = = 1\n", Options{Header: new(bool)}) + if res.Source != "local x = = 1\n" { + t.Fatalf("offline must return unparsable input unchanged: %q", res.Source) + } +} + +func TestEmptyInputAndBadOptions(t *testing.T) { + e := New(defaultSettings(), nil, nil, nil) + res, err := e.Clean(context.Background(), " \n", Options{}) + if err != nil || res.Source != " \n" { + t.Fatalf("%v %+v", err, res) + } + if _, err := e.Clean(context.Background(), "x()", Options{Mode: "bogus"}); err == nil { + t.Fatal("bad mode accepted") + } + if _, err := e.Clean(context.Background(), "x()", Options{Comments: "loud"}); err == nil { + t.Fatal("bad comments accepted") + } +} + +func TestParseRenameReply(t *testing.T) { + cases := []string{ + "```json\n{\"renames\":{\"v1\":\"a\"}}\n```", + "Sure! Here you go: {\"renames\": {\"v1\": \"a\"}} Hope that helps {}", + `{"renames":[{"from":"v1","to":"a"}]}`, + `{"renames":[{"old":"v1","new":"a"}],"comments":[{"line":3,"text":"x"}]}`, + } + for _, c := range cases { + _, m, err := parseRenameReply(c) + if err != nil || m["v1"] != "a" { + t.Errorf("%q: %v %v", c, m, err) + } + } + for _, bad := range []string{"no json here", "{\"renames\": ", `{"renames": 5}`} { + if _, _, err := parseRenameReply(bad); err == nil { + t.Errorf("%q should fail", bad) + } + } +} + +func TestStripFences(t *testing.T) { + cases := map[string]string{ + "print(1)\n": "print(1)", + "```lua\nprint(1)\n```": "print(1)", + "Here:\n```\nprint(1)\n```\nDone": "print(1)", + "```lua\na()\n```\nand\n```lua\nlonger()\n```": "longer()", + "```lua\ntruncated(": "truncated(", + } + for in, want := range cases { + if got := stripFences(in); got != want { + t.Errorf("stripFences(%q) = %q, want %q", in, got, want) + } + } +} + +func TestChunks(t *testing.T) { + src := "local a = 1\nlocal function f()\n\treturn 1\nend\nlocal b = 2\n" + f, _ := luau.Parse(src) + lines := strings.Split(src, "\n") + if got := chunks(lines, f, 1000); len(got) != 1 || got[0] != [2]int{1, len(lines)} { + t.Fatalf("single chunk: %v", got) + } + got := chunks(lines, f, 15) + // Never split inside the function (lines 2-4). + for _, r := range got { + if r[0] == 3 || r[0] == 4 { + t.Fatalf("split inside function: %v", got) + } + } + if got[0][0] != 1 || got[len(got)-1][1] != len(lines) { + t.Fatalf("chunks don't cover input: %v", got) + } +} diff --git a/internal/engine/prompts.go b/internal/engine/prompts.go new file mode 100644 index 0000000..cc71f6a --- /dev/null +++ b/internal/engine/prompts.go @@ -0,0 +1,67 @@ +package engine + +import ( + "fmt" + "strings" +) + +// promptVersion is part of every cache key; bump it when prompts change. +const promptVersion = "2" + +const renameSystem = `You are an expert Roblox Luau reverse engineer. You will receive Luau source code produced by a decompiler, with line numbers. Decompilers replace local variable names with meaningless placeholders such as v1, u2, p3 or l_Players_0. + +Your job is to recover meaningful names. For each identifier in CANDIDATES, infer what the variable holds from how it is created and used (Roblox services and APIs, instance names passed to WaitForChild/FindFirstChild, connected events, arithmetic, string contents) and propose a descriptive name. + +Reply with a single JSON object and nothing else: +{"summary": "...", "renames": {"oldName": "newName"}, "comments": [{"line": 12, "text": "..."}]} + +Rules: +- summary: one or two plain sentences describing what the script does. +- renames: only identifiers listed in CANDIDATES. Leave out names that are already descriptive and names you cannot infer with reasonable confidence. Never rename globals, Roblox APIs, table fields or methods. +- New names must be valid Luau identifiers and never Luau keywords: camelCase for values and functions, PascalCase for services, modules and classes. Use distinct names for distinct variables; repeating a name is fine only for variables in unrelated functions. +- comments: %s` + +var commentRules = map[string][2]string{ + // [rename-mode rule, rewrite-mode rule] + "none": {`always an empty list.`, + `Do not add comments.`}, + "light": {`a few short comments (roughly one per 15-30 lines of code) that explain intent: what a function or block does and why. Never restate the code. "line" is the line number the comment goes above.`, + `Add a few short comments that explain intent (what a function or block does and why). Never restate the code.`}, + "detailed": {`a comment above every function and every non-trivial block that explains its purpose, inputs and effects. Never restate the code line by line. "line" is the line number the comment goes above.`, + `Add a comment above every function and every non-trivial block explaining its purpose, inputs and effects. Never restate the code line by line.`}, +} + +const rewriteSystem = `You are an expert Roblox Luau reverse engineer. You will receive Luau source code produced by a decompiler. Rewrite it so that a human can read and maintain it: +- Rename local variables, upvalues, parameters and local functions to descriptive names inferred from how they are used (camelCase for values and functions, PascalCase for services, modules and classes). +- Clean up decompiler artifacts (redundant temporaries, needlessly nested or inverted conditions, empty branches) only when the behavior stays exactly the same. +- Keep every Roblox API call, string, number, table key, global and side effect exactly as it is. Never add features, remove code, or guess at code that is missing. +- %s +- If the code is only part of a larger script, rewrite just this part and keep the names of variables that are defined elsewhere unchanged. +Reply with only the complete Luau code: no markdown fences and no explanations before or after it.` + +func renamePrompt(comments string) string { + return fmt.Sprintf(renameSystem, commentRules[comments][0]) +} +func rewritePrompt(comments string) string { + return fmt.Sprintf(rewriteSystem, commentRules[comments][1]) +} + +// numbered renders lines[from-1:to] with right-aligned line numbers. +func numbered(lines []string, from, to int) string { + width := len(fmt.Sprint(to)) + var sb strings.Builder + for n := from; n <= to; n++ { + fmt.Fprintf(&sb, "%*d| %s\n", width, n, strings.TrimRight(lines[n-1], "\r")) + } + return sb.String() +} + +func renameUserPrompt(candidates []string, code string) string { + c := strings.Join(candidates, ", ") + if c == "" { + c = "(none)" + } + return "CANDIDATES: " + c + "\nSOURCE:\n" + code +} + +func rewriteUserPrompt(code string) string { return "SOURCE:\n" + code } diff --git a/internal/llm/anthropic.go b/internal/llm/anthropic.go new file mode 100644 index 0000000..272d780 --- /dev/null +++ b/internal/llm/anthropic.go @@ -0,0 +1,143 @@ +package llm + +import ( + "context" + "net/http" + "strings" +) + +// anthropic speaks the Anthropic Messages API. +type anthropic struct{ base } + +const anthropicVersion = "2023-06-01" + +// Models that accept the server-side refusal fallback ("fallbacks": "default"). +var anthropicFallbackModels = map[string]bool{ + "claude-opus-5-5": true, "claude-opus-5": true, "claude-fable-5-1": true, "claude-sonnet-5-5": true, +} + +// Model families that accept output_config.effort. +var anthropicEffortPrefixes = []string{ + "claude-opus-4-5", "claude-opus-4-6", "claude-opus-4-7", "claude-opus-4-8", "claude-opus-5", + "claude-sonnet-4-6", "claude-sonnet-5", "claude-haiku-5", "claude-fable-5", "claude-mythos-5", +} + +func anthropicSupportsEffort(model string) bool { + for _, p := range anthropicEffortPrefixes { + if strings.HasPrefix(model, p) { + return true + } + } + return false +} + +type anResponse struct { + Model string `json:"model"` + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + StopReason string `json:"stop_reason"` + StopDetails *struct { + Category *string `json:"category"` + Explanation string `json:"explanation"` + } `json:"stop_details"` + Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` + } `json:"usage"` +} + +func (c *anthropic) headers(model string, fallbacks bool) map[string]string { + h := c.authHeaders("x-api-key", "") + h["anthropic-version"] = anthropicVersion + if fallbacks { + h["anthropic-beta"] = "server-side-fallback-2026-07-01" + } + return h +} + +func (c *anthropic) Complete(ctx context.Context, req Request) (*Response, error) { + model := c.model(req) + maxTokens := c.cfg.MaxTokens + if maxTokens <= 0 { + maxTokens = 32000 + } + body := map[string]any{ + "model": model, + "max_tokens": maxTokens, + "system": req.System, + "messages": []map[string]string{{"role": "user", "content": req.Prompt}}, + } + effort := c.cfg.Effort + if effort == "" && anthropicSupportsEffort(model) { + effort = "medium" + } + if effort != "" && effort != "none" { + body["output_config"] = map[string]string{"effort": effort} + } + if c.cfg.Temperature != nil { + body["temperature"] = *c.cfg.Temperature + } + fallbacks := !c.cfg.NoFallbacks && anthropicFallbackModels[model] + if fallbacks { + // If a safety classifier declines, the API retries on Anthropic's + // recommended fallback model instead of returning a refusal. + body["fallbacks"] = "default" + } + var out anResponse + if err := c.do(ctx, http.MethodPost, c.cfg.BaseURL+"/v1/messages", c.headers(model, fallbacks), body, &out); err != nil { + return nil, err + } + if out.StopReason == "refusal" { + msg := "the model declined this request" + if d := out.StopDetails; d != nil { + if d.Category != nil { + msg += " (" + *d.Category + ")" + } + if d.Explanation != "" { + msg += ": " + d.Explanation + } + } + return nil, &Error{Provider: c.cfg.Name, Message: msg} + } + // When a fallback happened mid-response, only the text after the last + // fallback block belongs to the final answer. + start := 0 + for i, blk := range out.Content { + if blk.Type == "fallback" { + start = i + 1 + } + } + var sb strings.Builder + for _, blk := range out.Content[start:] { + if blk.Type == "text" { + sb.WriteString(blk.Text) + } + } + if out.Model == "" { + out.Model = model + } + return &Response{ + Text: sb.String(), + Model: out.Model, + Usage: Usage{InputTokens: out.Usage.InputTokens, OutputTokens: out.Usage.OutputTokens}, + Truncated: out.StopReason == "max_tokens", + }, nil +} + +func (c *anthropic) ListModels(ctx context.Context) ([]string, error) { + var out struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := c.do(ctx, http.MethodGet, c.cfg.BaseURL+"/v1/models?limit=1000", c.headers("", false), nil, &out); err != nil { + return nil, err + } + ids := make([]string, 0, len(out.Data)) + for _, m := range out.Data { + ids = append(ids, m.ID) + } + return ids, nil +} diff --git a/internal/llm/gemini.go b/internal/llm/gemini.go new file mode 100644 index 0000000..36a8c46 --- /dev/null +++ b/internal/llm/gemini.go @@ -0,0 +1,119 @@ +package llm + +import ( + "context" + "net/http" + "net/url" + "sort" + "strings" +) + +// gemini speaks the Google Gemini generateContent API. +type gemini struct{ base } + +type gmPart struct { + Text string `json:"text,omitempty"` + Thought bool `json:"thought,omitempty"` +} + +type gmContent struct { + Role string `json:"role,omitempty"` + Parts []gmPart `json:"parts"` +} + +type gmResponse struct { + Candidates []struct { + Content gmContent `json:"content"` + FinishReason string `json:"finishReason"` + } `json:"candidates"` + PromptFeedback *struct { + BlockReason string `json:"blockReason"` + } `json:"promptFeedback"` + UsageMetadata struct { + PromptTokenCount int `json:"promptTokenCount"` + CandidatesTokenCount int `json:"candidatesTokenCount"` + } `json:"usageMetadata"` + ModelVersion string `json:"modelVersion"` +} + +func (c *gemini) Complete(ctx context.Context, req Request) (*Response, error) { + model := strings.TrimPrefix(c.model(req), "models/") + gen := map[string]any{} + if c.cfg.MaxTokens > 0 { + gen["maxOutputTokens"] = c.cfg.MaxTokens + } + if c.cfg.Temperature != nil { + gen["temperature"] = *c.cfg.Temperature + } + if req.JSON && c.cfg.JSONMode { + gen["responseMimeType"] = "application/json" + } + if c.cfg.Effort != "" && c.cfg.Effort != "none" { + gen["thinkingConfig"] = map[string]string{"thinkingLevel": c.cfg.Effort} + } + body := map[string]any{ + "systemInstruction": gmContent{Parts: []gmPart{{Text: req.System}}}, + "contents": []gmContent{{Role: "user", Parts: []gmPart{{Text: req.Prompt}}}}, + } + if len(gen) > 0 { + body["generationConfig"] = gen + } + endpoint := c.cfg.BaseURL + "/models/" + url.PathEscape(model) + ":generateContent" + var out gmResponse + if err := c.do(ctx, http.MethodPost, endpoint, c.authHeaders("x-goog-api-key", ""), body, &out); err != nil { + return nil, err + } + if out.PromptFeedback != nil && out.PromptFeedback.BlockReason != "" { + return nil, &Error{Provider: c.cfg.Name, Message: "prompt blocked: " + out.PromptFeedback.BlockReason} + } + if len(out.Candidates) == 0 { + return nil, &Error{Provider: c.cfg.Name, Message: "response contained no candidates"} + } + cand := out.Candidates[0] + switch cand.FinishReason { + case "SAFETY", "RECITATION", "PROHIBITED_CONTENT", "BLOCKLIST", "SPII": + return nil, &Error{Provider: c.cfg.Name, Message: "response blocked: " + cand.FinishReason} + } + var sb strings.Builder + for _, p := range cand.Content.Parts { + if !p.Thought { + sb.WriteString(p.Text) + } + } + served := out.ModelVersion + if served == "" { + served = model + } + return &Response{ + Text: sb.String(), + Model: served, + Usage: Usage{InputTokens: out.UsageMetadata.PromptTokenCount, OutputTokens: out.UsageMetadata.CandidatesTokenCount}, + Truncated: cand.FinishReason == "MAX_TOKENS", + }, nil +} + +func (c *gemini) ListModels(ctx context.Context) ([]string, error) { + var out struct { + Models []struct { + Name string `json:"name"` + Methods []string `json:"supportedGenerationMethods"` + } `json:"models"` + } + if err := c.do(ctx, http.MethodGet, c.cfg.BaseURL+"/models?pageSize=1000", c.authHeaders("x-goog-api-key", ""), nil, &out); err != nil { + return nil, err + } + var ids []string + for _, m := range out.Models { + ok := len(m.Methods) == 0 + for _, meth := range m.Methods { + if meth == "generateContent" { + ok = true + } + } + if ok { + ids = append(ids, strings.TrimPrefix(m.Name, "models/")) + } + } + sort.Strings(ids) + return ids, nil +} diff --git a/internal/llm/llm.go b/internal/llm/llm.go new file mode 100644 index 0000000..e0aa568 --- /dev/null +++ b/internal/llm/llm.go @@ -0,0 +1,349 @@ +// Package llm talks to AI providers over their native HTTP APIs using only +// the standard library: OpenAI-compatible chat completions (OpenAI, +// OpenRouter, Groq, DeepSeek, Mistral, xAI, Ollama, LM Studio, Azure, ...), +// the Anthropic Messages API and the Google Gemini API. +package llm + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "strconv" + "strings" + "time" +) + +// Type identifies a wire protocol. +type Type string + +const ( + TypeOpenAI Type = "openai" + TypeAnthropic Type = "anthropic" + TypeGemini Type = "gemini" +) + +// Config configures one provider. +type Config struct { + Name string + Type Type + BaseURL string + APIKey string + Model string + MaxTokens int // 0 = provider default + Temperature *float64 // nil = provider default (many new models reject it) + Effort string // reasoning effort; "none" disables the default + Headers map[string]string + AuthHeader string // send the key in this header instead of the default + JSONMode bool // request JSON output natively when asked + Timeout time.Duration + MaxRetries int + // MaxTokensField overrides the request field used for MaxTokens on + // OpenAI-compatible APIs ("max_tokens" or "max_completion_tokens"). + MaxTokensField string + // NoFallbacks disables Anthropic server-side refusal fallbacks. + NoFallbacks bool +} + +// Request is a single-turn completion request. +type Request struct { + System string + Prompt string + Model string // overrides Config.Model when set + JSON bool // the caller expects a JSON object back +} + +// Usage reports token consumption. +type Usage struct { + InputTokens int `json:"input_tokens"` + OutputTokens int `json:"output_tokens"` +} + +// Add accumulates u2 into u. +func (u *Usage) Add(u2 Usage) { + u.InputTokens += u2.InputTokens + u.OutputTokens += u2.OutputTokens +} + +// Response is a completion result. +type Response struct { + Text string + Model string + Usage Usage + Truncated bool // output hit the token limit +} + +// Client is an AI provider. +type Client interface { + Name() string + Model() string + Complete(ctx context.Context, req Request) (*Response, error) + ListModels(ctx context.Context) ([]string, error) +} + +// Error is an API failure. +type Error struct { + Provider string + Status int + Message string + retryAfter time.Duration +} + +func (e *Error) Error() string { + hint := "" + switch e.Status { + case 401, 403: + hint = " (check your API key)" + case 404: + hint = " (check the model name; `betterdecompiler models -p " + e.Provider + "` lists available models)" + case 429: + hint = " (rate limited or out of credits)" + } + if e.Status == 0 { + return fmt.Sprintf("%s: %s", e.Provider, e.Message) + } + return fmt.Sprintf("%s: HTTP %d: %s%s", e.Provider, e.Status, e.Message, hint) +} + +// Retryable reports whether the request may succeed if repeated. +func (e *Error) Retryable() bool { + switch e.Status { + case 408, 409, 425, 429, 500, 502, 503, 504, 529: + return true + } + return false +} + +// New creates a client for cfg. +func New(cfg Config, hc *http.Client) (Client, error) { + if hc == nil { + hc = DefaultHTTPClient + } + if cfg.Timeout <= 0 { + cfg.Timeout = 10 * time.Minute + } + if cfg.MaxRetries < 0 { + cfg.MaxRetries = 0 + } else if cfg.MaxRetries == 0 { + cfg.MaxRetries = 2 + } + cfg.BaseURL = strings.TrimRight(cfg.BaseURL, "/") + if cfg.BaseURL == "" { + return nil, fmt.Errorf("%s: base_url is not set", cfg.Name) + } + b := base{cfg: cfg, hc: hc} + switch cfg.Type { + case TypeOpenAI, "": + return &openAI{b}, nil + case TypeAnthropic: + return &anthropic{b}, nil + case TypeGemini: + return &gemini{b}, nil + } + return nil, fmt.Errorf("%s: unknown provider type %q (want openai, anthropic or gemini)", cfg.Name, cfg.Type) +} + +// DefaultHTTPClient is shared by all providers so connections are pooled and +// reused (HTTP/2 where available). Proxies from the environment are honored. +var DefaultHTTPClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{Timeout: 15 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: true, + MaxIdleConns: 64, + MaxIdleConnsPerHost: 16, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + ExpectContinueTimeout: time.Second, + }, +} + +type base struct { + cfg Config + hc *http.Client +} + +func (b *base) Name() string { return b.cfg.Name } +func (b *base) Model() string { return b.cfg.Model } + +func (b *base) model(req Request) string { + if req.Model != "" { + return req.Model + } + return b.cfg.Model +} + +// Backoff computes the delay before retry attempt n (0-based). Tests replace it. +var Backoff = func(n int, retryAfter time.Duration) time.Duration { + if retryAfter > 0 { + if retryAfter > 30*time.Second { + retryAfter = 30 * time.Second + } + return retryAfter + } + d := time.Duration(750*(1< 0 { + var ra time.Duration + var ae *Error + if errors.As(lastErr, &ae) { + ra = ae.retryAfter + } + select { + case <-time.After(Backoff(attempt-1, ra)): + case <-ctx.Done(): + return fmt.Errorf("%s: %w (last error: %v)", b.cfg.Name, ctx.Err(), lastErr) + } + } + err := b.once(ctx, method, url, headers, payload, out) + if err == nil { + return nil + } + lastErr = err + var ae *Error + if errors.As(err, &ae) { + if !ae.Retryable() { + return err + } + } else if ctx.Err() != nil { + return fmt.Errorf("%s: %w", b.cfg.Name, ctx.Err()) + } + } + return lastErr +} + +func (b *base) once(ctx context.Context, method, url string, headers map[string]string, payload []byte, out any) error { + var rd io.Reader + if payload != nil { + rd = bytes.NewReader(payload) + } + req, err := http.NewRequestWithContext(ctx, method, url, rd) + if err != nil { + return err + } + if payload != nil { + req.Header.Set("Content-Type", "application/json") + } + req.Header.Set("Accept", "application/json") + req.Header.Set("User-Agent", "BetterDecompiler/2 (+https://github.com/officialmelon/BetterDecompiler)") + for k, v := range headers { + req.Header.Set(k, v) + } + for k, v := range b.cfg.Headers { + req.Header.Set(k, v) + } + resp, err := b.hc.Do(req) + if err != nil { + if ctx.Err() != nil { + return err + } + // Network errors are worth retrying. + return &Error{Provider: b.cfg.Name, Status: 503, Message: "network error: " + err.Error()} + } + defer resp.Body.Close() + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes)) + if err != nil { + return &Error{Provider: b.cfg.Name, Status: 503, Message: "reading response: " + err.Error()} + } + if resp.StatusCode/100 != 2 { + return &Error{Provider: b.cfg.Name, Status: resp.StatusCode, Message: errorMessage(data), + retryAfter: parseRetryAfter(resp.Header.Get("Retry-After"))} + } + if err := json.Unmarshal(data, out); err != nil { + return &Error{Provider: b.cfg.Name, Message: fmt.Sprintf("invalid JSON response: %v: %s", err, snippet(data))} + } + return nil +} + +func (b *base) authHeaders(defaultHeader, prefix string) map[string]string { + h := map[string]string{} + if b.cfg.APIKey == "" { + return h + } + if b.cfg.AuthHeader != "" { + h[b.cfg.AuthHeader] = b.cfg.APIKey + } else { + h[defaultHeader] = prefix + b.cfg.APIKey + } + return h +} + +// errorMessage extracts a human-readable message from the error bodies used +// by OpenAI, Anthropic, Gemini and most compatible servers. +func errorMessage(data []byte) string { + var v struct { + Error json.RawMessage `json:"error"` + Message string `json:"message"` + Detail string `json:"detail"` + } + if json.Unmarshal(data, &v) == nil { + var inner struct { + Message string `json:"message"` + Type string `json:"type"` + } + if len(v.Error) > 0 { + if json.Unmarshal(v.Error, &inner) == nil && inner.Message != "" { + return inner.Message + } + var s string + if json.Unmarshal(v.Error, &s) == nil && s != "" { + return s + } + } + if v.Message != "" { + return v.Message + } + if v.Detail != "" { + return v.Detail + } + } + return snippet(data) +} + +func snippet(data []byte) string { + s := strings.TrimSpace(string(data)) + if len(s) > 300 { + s = s[:300] + "..." + } + if s == "" { + s = "(empty response)" + } + return s +} + +func parseRetryAfter(v string) time.Duration { + if v == "" { + return 0 + } + if secs, err := strconv.ParseFloat(v, 64); err == nil && secs >= 0 { + return time.Duration(secs * float64(time.Second)) + } + if t, err := http.ParseTime(v); err == nil { + if d := time.Until(t); d > 0 { + return d + } + } + return 0 +} diff --git a/internal/llm/llm_test.go b/internal/llm/llm_test.go new file mode 100644 index 0000000..33fe60c --- /dev/null +++ b/internal/llm/llm_test.go @@ -0,0 +1,258 @@ +package llm + +import ( + "context" + "errors" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/officialmelon/betterdecompiler/internal/mockllm" +) + +func init() { + Backoff = func(int, time.Duration) time.Duration { return time.Millisecond } +} + +func newMock(t *testing.T) (*mockllm.Server, string) { + t.Helper() + m := mockllm.New("sk-test") + srv := httptest.NewServer(m) + t.Cleanup(srv.Close) + return m, srv.URL +} + +func clients(url string) []Config { + return []Config{ + {Name: "openai", Type: TypeOpenAI, BaseURL: url + "/v1", APIKey: "sk-test", Model: "mock-1", JSONMode: true, MaxTokens: 100, MaxTokensField: "max_completion_tokens"}, + {Name: "anthropic", Type: TypeAnthropic, BaseURL: url, APIKey: "sk-test", Model: "mock-1"}, + {Name: "gemini", Type: TypeGemini, BaseURL: url + "/v1beta", APIKey: "sk-test", Model: "mock-gemini", JSONMode: true}, + } +} + +func TestCompleteAllProtocols(t *testing.T) { + m, url := newMock(t) + for _, cfg := range clients(url) { + t.Run(cfg.Name, func(t *testing.T) { + c, err := New(cfg, nil) + if err != nil { + t.Fatal(err) + } + resp, err := c.Complete(context.Background(), Request{System: "sys", Prompt: "CANDIDATES: v1\nSOURCE:\n1| local v1 = 1", JSON: true}) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(resp.Text, `"v1":"mockV1"`) { + t.Fatalf("unexpected text %q", resp.Text) + } + if strings.Contains(resp.Text, "thinking about it") { + t.Fatal("gemini thought parts must be dropped") + } + if resp.Usage.InputTokens == 0 || resp.Truncated { + t.Fatalf("usage/truncation wrong: %+v", resp) + } + }) + } + reqs := m.Requests() + if got := reqs[0].Body["max_completion_tokens"]; got != float64(100) { + t.Errorf("openai max tokens field: %v", reqs[0].Body) + } + if rf, _ := reqs[0].Body["response_format"].(map[string]any); rf["type"] != "json_object" { + t.Errorf("openai json mode not requested: %v", reqs[0].Body) + } + if reqs[1].Header.Get("x-api-key") != "sk-test" || reqs[1].Header.Get("anthropic-version") == "" { + t.Errorf("anthropic headers: %v", reqs[1].Header) + } + if _, ok := reqs[1].Body["output_config"]; ok { + t.Errorf("effort must not be sent to unknown models: %v", reqs[1].Body) + } + if gc, _ := reqs[2].Body["generationConfig"].(map[string]any); gc["responseMimeType"] != "application/json" { + t.Errorf("gemini json mode: %v", reqs[2].Body) + } + if reqs[2].Header.Get("x-goog-api-key") != "sk-test" { + t.Errorf("gemini key header: %v", reqs[2].Header) + } +} + +func TestAnthropicDefaultsForCurrentModels(t *testing.T) { + m, url := newMock(t) + c, _ := New(Config{Name: "anthropic", Type: TypeAnthropic, BaseURL: url, APIKey: "sk-test", Model: "claude-opus-5-5"}, nil) + if _, err := c.Complete(context.Background(), Request{System: "s", Prompt: "p"}); err != nil { + t.Fatal(err) + } + r := m.Requests()[0] + if r.Body["fallbacks"] != "default" || r.Header.Get("anthropic-beta") != "server-side-fallback-2026-07-01" { + t.Errorf("fallbacks not enabled: %v %v", r.Body, r.Header) + } + if oc, _ := r.Body["output_config"].(map[string]any); oc["effort"] != "medium" { + t.Errorf("effort: %v", r.Body) + } + if r.Body["max_tokens"] != float64(32000) { + t.Errorf("max_tokens: %v", r.Body["max_tokens"]) + } + if _, ok := r.Body["temperature"]; ok { + t.Error("temperature must not be sent by default") + } +} + +func TestAnthropicFallbackBlocks(t *testing.T) { + _, url := newMock(t) + c, _ := New(Config{Name: "anthropic", Type: TypeAnthropic, BaseURL: url, APIKey: "sk-test", Model: "mock-fallback"}, nil) + resp, err := c.Complete(context.Background(), Request{System: "s", Prompt: "hello"}) + if err != nil { + t.Fatal(err) + } + if strings.Contains(resp.Text, "partial output") || resp.Model != "mock-backup" { + t.Fatalf("fallback handling: %+v", resp) + } +} + +func TestRefusalsAndTruncation(t *testing.T) { + _, url := newMock(t) + for _, cfg := range clients(url) { + cfg.Model = "mock-refuse" + c, _ := New(cfg, nil) + if _, err := c.Complete(context.Background(), Request{Prompt: "x"}); err == nil { + t.Errorf("%s: refusal not reported", cfg.Name) + } + cfg.Model = "mock-truncate" + c, _ = New(cfg, nil) + resp, err := c.Complete(context.Background(), Request{Prompt: "x"}) + if err != nil || !resp.Truncated { + t.Errorf("%s: truncation not reported: %v %+v", cfg.Name, err, resp) + } + } +} + +func TestRetries(t *testing.T) { + m, url := newMock(t) + c, _ := New(clients(url)[0], nil) + m.FailNext(429, 503) + if _, err := c.Complete(context.Background(), Request{Prompt: "x"}); err != nil { + t.Fatalf("should succeed after retries: %v", err) + } + if m.Count() != 3 { + t.Fatalf("expected 3 attempts, got %d", m.Count()) + } + m.FailNext(500, 500, 500) + _, err := c.Complete(context.Background(), Request{Prompt: "x"}) + var ae *Error + if !errors.As(err, &ae) || ae.Status != 500 { + t.Fatalf("expected final 500, got %v", err) + } + // Client errors are not retried. + before := m.Count() + m.FailNext(400) + if _, err := c.Complete(context.Background(), Request{Prompt: "x"}); err == nil { + t.Fatal("expected error") + } + if m.Count() != before+1 { + t.Fatal("400 must not be retried") + } +} + +func TestBadKey(t *testing.T) { + _, url := newMock(t) + cfg := clients(url)[1] + cfg.APIKey = "wrong" + c, _ := New(cfg, nil) + _, err := c.Complete(context.Background(), Request{Prompt: "x"}) + if err == nil || !strings.Contains(err.Error(), "401") || !strings.Contains(err.Error(), "check your API key") { + t.Fatalf("unexpected: %v", err) + } +} + +func TestListModelsAndAuto(t *testing.T) { + m, url := newMock(t) + for _, cfg := range clients(url) { + c, _ := New(cfg, nil) + models, err := c.ListModels(context.Background()) + if err != nil || len(models) == 0 { + t.Fatalf("%s: %v %v", cfg.Name, models, err) + } + if cfg.Type == TypeGemini && (len(models) != 1 || models[0] != "mock-gemini") { + t.Fatalf("gemini filtering: %v", models) + } + } + cfg := clients(url)[0] + cfg.Model = "auto" + cfg.BaseURL = url + "/auto/v1" + c, _ := New(cfg, nil) + resp, err := c.Complete(context.Background(), Request{Prompt: "x"}) + if err != nil || resp.Model != "mock-1" { + t.Fatalf("auto model: %v %+v", err, resp) + } + last := m.Requests()[len(m.Requests())-1] + if last.Body["model"] != "mock-1" { + t.Fatalf("auto model not used: %v", last.Body) + } +} + +func TestAuthHeaderOverride(t *testing.T) { + m, url := newMock(t) + cfg := clients(url)[0] + cfg.AuthHeader = "api-key" // Azure style + c, _ := New(cfg, nil) + if _, err := c.Complete(context.Background(), Request{Prompt: "x"}); err != nil { + t.Fatal(err) + } + h := m.Requests()[0].Header + if h.Get("api-key") != "sk-test" || h.Get("Authorization") != "" { + t.Fatalf("headers: %v", h) + } +} + +func TestTimeout(t *testing.T) { + done := make(chan struct{}) + slow := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-done: + } + })) + defer slow.Close() + defer close(done) + c, _ := New(Config{Name: "slow", Type: TypeOpenAI, BaseURL: slow.URL, Model: "m", Timeout: 100 * time.Millisecond, MaxRetries: -1}, nil) + start := time.Now() + if _, err := c.Complete(context.Background(), Request{Prompt: "x"}); err == nil { + t.Fatal("expected timeout") + } + if time.Since(start) > 3*time.Second { + t.Fatal("timeout not enforced") + } +} + +func TestPresets(t *testing.T) { + seen := map[string]bool{} + for _, p := range Presets { + if seen[p.Name] { + t.Fatalf("duplicate preset %s", p.Name) + } + seen[p.Name] = true + if _, err := New(p.Config(), nil); err != nil { + t.Fatalf("preset %s: %v", p.Name, err) + } + } + t.Setenv("GOOGLE_API_KEY", "g-key") + p, _ := FindPreset("gemini") + if p.EnvKey() != "g-key" { + t.Fatal("GOOGLE_API_KEY fallback not used") + } +} + +func TestErrorMessage(t *testing.T) { + cases := map[string]string{ + `{"error":{"message":"bad model"}}`: "bad model", + `{"type":"error","error":{"type":"x","message":"nope"}}`: "nope", + `{"error":"plain"}`: "plain", + `{"detail":"det"}`: "det", + `oops`: "oops", + } + for in, want := range cases { + if got := errorMessage([]byte(in)); got != want { + t.Errorf("errorMessage(%s) = %q, want %q", in, got, want) + } + } +} diff --git a/internal/llm/openai.go b/internal/llm/openai.go new file mode 100644 index 0000000..cf35d94 --- /dev/null +++ b/internal/llm/openai.go @@ -0,0 +1,133 @@ +package llm + +import ( + "context" + "fmt" + "net/http" + "sort" + "strings" + "sync" +) + +// openAI speaks the OpenAI-compatible Chat Completions API. +type openAI struct{ base } + +type oaMessage struct { + Role string `json:"role"` + Content string `json:"content"` +} + +type oaResponse struct { + Model string `json:"model"` + Choices []struct { + Message struct { + Content *string `json:"content"` + Refusal *string `json:"refusal"` + } `json:"message"` + FinishReason string `json:"finish_reason"` + } `json:"choices"` + Usage struct { + PromptTokens int `json:"prompt_tokens"` + CompletionTokens int `json:"completion_tokens"` + } `json:"usage"` +} + +var autoModels sync.Map // base URL -> discovered model + +func (c *openAI) Complete(ctx context.Context, req Request) (*Response, error) { + model := c.model(req) + if model == "" || model == "auto" { + m, err := c.autoModel(ctx) + if err != nil { + return nil, err + } + model = m + } + body := map[string]any{ + "model": model, + "messages": []oaMessage{ + {Role: "system", Content: req.System}, + {Role: "user", Content: req.Prompt}, + }, + } + if c.cfg.MaxTokens > 0 { + field := c.cfg.MaxTokensField + if field == "" { + field = "max_tokens" + } + body[field] = c.cfg.MaxTokens + } + if c.cfg.Temperature != nil { + body["temperature"] = *c.cfg.Temperature + } + if c.cfg.Effort != "" && c.cfg.Effort != "none" { + body["reasoning_effort"] = c.cfg.Effort + } + if req.JSON && c.cfg.JSONMode { + body["response_format"] = map[string]string{"type": "json_object"} + } + var out oaResponse + if err := c.do(ctx, http.MethodPost, c.cfg.BaseURL+"/chat/completions", c.authHeaders("Authorization", "Bearer "), body, &out); err != nil { + return nil, err + } + if len(out.Choices) == 0 { + return nil, &Error{Provider: c.cfg.Name, Message: "response contained no choices"} + } + ch := out.Choices[0] + if ch.Message.Refusal != nil && *ch.Message.Refusal != "" { + return nil, &Error{Provider: c.cfg.Name, Message: "model refused: " + *ch.Message.Refusal} + } + if ch.FinishReason == "content_filter" { + return nil, &Error{Provider: c.cfg.Name, Message: "response blocked by the provider's content filter"} + } + text := "" + if ch.Message.Content != nil { + text = *ch.Message.Content + } + if out.Model == "" { + out.Model = model + } + return &Response{ + Text: text, + Model: out.Model, + Usage: Usage{InputTokens: out.Usage.PromptTokens, OutputTokens: out.Usage.CompletionTokens}, + Truncated: ch.FinishReason == "length", + }, nil +} + +func (c *openAI) autoModel(ctx context.Context) (string, error) { + if m, ok := autoModels.Load(c.cfg.BaseURL); ok { + return m.(string), nil + } + models, err := c.ListModels(ctx) + if err != nil { + return "", fmt.Errorf("%s: no model configured and model discovery failed: %w", c.cfg.Name, err) + } + if len(models) == 0 { + return "", &Error{Provider: c.cfg.Name, Message: "no model configured and the server reports no models (load one first)"} + } + autoModels.Store(c.cfg.BaseURL, models[0]) + return models[0], nil +} + +func (c *openAI) ListModels(ctx context.Context) ([]string, error) { + var out struct { + Data []struct { + ID string `json:"id"` + } `json:"data"` + } + if err := c.do(ctx, http.MethodGet, c.cfg.BaseURL+"/models", c.authHeaders("Authorization", "Bearer "), nil, &out); err != nil { + return nil, err + } + ids := make([]string, 0, len(out.Data)) + for _, m := range out.Data { + if m.ID != "" && !strings.Contains(m.ID, "embed") { + ids = append(ids, m.ID) + } + } + // Local servers list the loaded model first; keep their order. + if !strings.Contains(c.cfg.BaseURL, "localhost") && !strings.Contains(c.cfg.BaseURL, "127.0.0.1") { + sort.Strings(ids) + } + return ids, nil +} diff --git a/internal/llm/presets.go b/internal/llm/presets.go new file mode 100644 index 0000000..6379308 --- /dev/null +++ b/internal/llm/presets.go @@ -0,0 +1,87 @@ +package llm + +import "os" + +// Preset is a built-in provider definition. Users only need to supply a key. +type Preset struct { + Name string + Type Type + BaseURL string + Model string + KeyEnv []string // environment variables checked for an API key + NeedsKey bool + JSONMode bool + MaxTokens int + MaxTokensField string + Headers map[string]string + Description string + KeyURL string // where to get a key +} + +// Offline is the name of the built-in, no-AI renamer. +const Offline = "offline" + +// Presets lists every built-in provider, in auto-detection order. +var Presets = []Preset{ + {Name: "anthropic", Type: TypeAnthropic, BaseURL: "https://api.anthropic.com", Model: "claude-opus-5-5", + KeyEnv: []string{"ANTHROPIC_API_KEY"}, NeedsKey: true, MaxTokens: 32000, + Description: "Anthropic Claude", KeyURL: "https://console.anthropic.com/settings/keys"}, + {Name: "openai", Type: TypeOpenAI, BaseURL: "https://api.openai.com/v1", Model: "gpt-5.6-terra", + KeyEnv: []string{"OPENAI_API_KEY"}, NeedsKey: true, JSONMode: true, MaxTokensField: "max_completion_tokens", + Description: "OpenAI GPT", KeyURL: "https://platform.openai.com/api-keys"}, + {Name: "gemini", Type: TypeGemini, BaseURL: "https://generativelanguage.googleapis.com/v1beta", Model: "gemini-3.7-flash", + KeyEnv: []string{"GEMINI_API_KEY", "GOOGLE_API_KEY"}, NeedsKey: true, JSONMode: true, + Description: "Google Gemini", KeyURL: "https://aistudio.google.com/apikey"}, + {Name: "openrouter", Type: TypeOpenAI, BaseURL: "https://openrouter.ai/api/v1", Model: "openrouter/auto", + KeyEnv: []string{"OPENROUTER_API_KEY"}, NeedsKey: true, + Headers: map[string]string{"HTTP-Referer": "https://github.com/officialmelon/BetterDecompiler", "X-Title": "BetterDecompiler"}, + Description: "OpenRouter (hundreds of models, one key)", KeyURL: "https://openrouter.ai/keys"}, + {Name: "groq", Type: TypeOpenAI, BaseURL: "https://api.groq.com/openai/v1", Model: "openai/gpt-oss-120b", + KeyEnv: []string{"GROQ_API_KEY"}, NeedsKey: true, JSONMode: true, + Description: "Groq (very fast, free tier)", KeyURL: "https://console.groq.com/keys"}, + {Name: "deepseek", Type: TypeOpenAI, BaseURL: "https://api.deepseek.com", Model: "deepseek-chat", + KeyEnv: []string{"DEEPSEEK_API_KEY"}, NeedsKey: true, JSONMode: true, MaxTokens: 8192, + Description: "DeepSeek", KeyURL: "https://platform.deepseek.com/api_keys"}, + {Name: "mistral", Type: TypeOpenAI, BaseURL: "https://api.mistral.ai/v1", Model: "mistral-large-latest", + KeyEnv: []string{"MISTRAL_API_KEY"}, NeedsKey: true, JSONMode: true, + Description: "Mistral AI", KeyURL: "https://console.mistral.ai/api-keys"}, + {Name: "xai", Type: TypeOpenAI, BaseURL: "https://api.x.ai/v1", Model: "grok-code-fast-1", + KeyEnv: []string{"XAI_API_KEY"}, NeedsKey: true, + Description: "xAI Grok", KeyURL: "https://console.x.ai"}, + {Name: "ollama", Type: TypeOpenAI, BaseURL: "http://localhost:11434/v1", Model: "qwen2.5-coder:7b", + Description: "Ollama (local models, free, private)", KeyURL: "https://ollama.com/download"}, + {Name: "lmstudio", Type: TypeOpenAI, BaseURL: "http://localhost:1234/v1", Model: "auto", + Description: "LM Studio (local models, free, private)", KeyURL: "https://lmstudio.ai"}, +} + +// FindPreset returns the preset with the given name. +func FindPreset(name string) (Preset, bool) { + for _, p := range Presets { + if p.Name == name { + return p, true + } + } + return Preset{}, false +} + +// EnvKey returns the first API key found in the preset's environment variables. +func (p Preset) EnvKey() string { + for _, k := range p.KeyEnv { + if v := os.Getenv(k); v != "" { + return v + } + } + return "" +} + +// Config converts the preset to a client configuration. +func (p Preset) Config() Config { + h := map[string]string{} + for k, v := range p.Headers { + h[k] = v + } + return Config{ + Name: p.Name, Type: p.Type, BaseURL: p.BaseURL, Model: p.Model, APIKey: p.EnvKey(), + JSONMode: p.JSONMode, MaxTokens: p.MaxTokens, MaxTokensField: p.MaxTokensField, Headers: h, + } +} diff --git a/internal/luau/heuristics.go b/internal/luau/heuristics.go new file mode 100644 index 0000000..e68bb90 --- /dev/null +++ b/internal/luau/heuristics.go @@ -0,0 +1,487 @@ +package luau + +import ( + "regexp" + "strings" + "unicode" +) + +var generatedRes = []*regexp.Regexp{ + regexp.MustCompile(`(?i)^(?:v|u|p|a|r|l|var|arg|args|upv|upval|upvalue|param|slot|reg|loc|local|val|tbl|func|fn)_?\d+$`), + regexp.MustCompile(`^l_[A-Za-z_][A-Za-z0-9_]*_\d+$`), // l_Players_0 (Medal/Konstant) + regexp.MustCompile(`^[A-Z]\d+_\d+$`), // L0_1 (unluac style) + regexp.MustCompile(`^(?:[vupal]_)+\d+$`), // v_u_1 (Medal upvalues) + regexp.MustCompile(`^_\d+$`), +} + +var lNameRe = regexp.MustCompile(`^l_([A-Za-z_][A-Za-z0-9_]*)_\d+$`) + +// LooksGenerated reports whether name looks machine-generated by a decompiler. +func LooksGenerated(name string) bool { + for _, re := range generatedRes { + if re.MatchString(name) { + return true + } + } + return false +} + +// eventParams names the parameters of common Roblox event callbacks. +var eventParams = map[string][]string{ + "Touched": {"hit"}, "TouchEnded": {"hit"}, + "PlayerAdded": {"player"}, "PlayerRemoving": {"player"}, + "CharacterAdded": {"character"}, "CharacterRemoving": {"character"}, "CharacterAppearanceLoaded": {"character"}, + "ChildAdded": {"child"}, "ChildRemoved": {"child"}, + "DescendantAdded": {"descendant"}, "DescendantRemoving": {"descendant"}, + "InputBegan": {"input", "gameProcessedEvent"}, "InputEnded": {"input", "gameProcessedEvent"}, + "InputChanged": {"input", "gameProcessedEvent"}, + "Heartbeat": {"deltaTime"}, "RenderStepped": {"deltaTime"}, "Stepped": {"runTime", "deltaTime"}, + "PreSimulation": {"deltaTime"}, "PostSimulation": {"deltaTime"}, "PreRender": {"deltaTime"}, "PreAnimation": {"deltaTime"}, + "OnServerEvent": {"player"}, "OnServerInvoke": {"player"}, + "HealthChanged": {"health"}, "FocusLost": {"enterPressed", "inputThatCausedFocusLoss"}, + "Activated": {"inputObject", "clickCount"}, "Triggered": {"player"}, "MouseClick": {"player"}, + "TriggerEnded": {"player"}, "PromptTriggered": {"prompt", "player"}, "PromptButtonHoldBegan": {"prompt", "player"}, + "Chatted": {"message", "recipient"}, "AncestryChanged": {"child", "parent"}, + "AttributeChanged": {"attribute"}, "Completed": {"playbackState"}, "KeyframeReached": {"keyframeName"}, + "MoveToFinished": {"reached"}, "StateChanged": {"oldState", "newState"}, "Running": {"speed"}, + "MouseEnter": {"x", "y"}, "MouseLeave": {"x", "y"}, "MouseMoved": {"x", "y"}, + "PlayerMembershipChanged": {"player"}, "PromptPurchaseFinished": {"player", "assetId", "isPurchased"}, + "PromptProductPurchaseFinished": {"userId", "productId", "isPurchased"}, + "PromptGamePassPurchaseFinished": {"player", "gamePassId", "wasPurchased"}, + "Died": {}, "Changed": {}, "Event": {}, "OnClientEvent": {}, "Equipped": {"mouse"}, +} + +// constructors maps Type.new(...) style calls to a variable name. +var constructors = map[string]string{ + "CFrame": "cframe", "Vector3": "vector", "Vector2": "vector2", "Color3": "color", "UDim2": "udim2", + "UDim": "udim", "TweenInfo": "tweenInfo", "RaycastParams": "raycastParams", "OverlapParams": "overlapParams", + "NumberSequence": "numberSequence", "ColorSequence": "colorSequence", "NumberRange": "numberRange", + "Random": "rng", "Ray": "ray", "BrickColor": "brickColor", "Region3": "region", "Rect": "rect", + "DateTime": "dateTime", "PhysicalProperties": "physicalProperties", "Font": "font", +} + +var fieldNames = map[string]string{ + "CurrentCamera": "camera", "LocalPlayer": "localPlayer", "Character": "character", "Humanoid": "humanoid", + "HumanoidRootPart": "rootPart", "PrimaryPart": "primaryPart", "PlayerGui": "playerGui", "Backpack": "backpack", + "Parent": "parent", "Value": "value", "Name": "name", "Position": "position", "CFrame": "cframe", + "Size": "size", "Magnitude": "magnitude", "Unit": "direction", "LookVector": "lookVector", + "Health": "health", "MaxHealth": "maxHealth", "WalkSpeed": "walkSpeed", "Animator": "animator", + "UserId": "userId", "Team": "team", "leaderstats": "leaderstats", "Text": "text", +} + +var childFinders = map[string]bool{ + "WaitForChild": true, "FindFirstChild": true, "FindFirstChildOfClass": true, "FindFirstChildWhichIsA": true, + "FindFirstAncestor": true, "FindFirstAncestorOfClass": true, "FindFirstAncestorWhichIsA": true, + "FindFirstDescendant": true, +} + +var methodNames = map[string]string{ + "GetChildren": "children", "GetDescendants": "descendants", "GetPlayers": "players", "GetMouse": "mouse", + "LoadAnimation": "animationTrack", "Raycast": "raycastResult", "GetPlayerFromCharacter": "player", + "InvokeServer": "response", "InvokeClient": "response", "GetTagged": "taggedInstances", + "GetPartBoundsInBox": "parts", "GetPartBoundsInRadius": "parts", "GetPartsInPart": "parts", + "GetBoundingBox": "boundingBox", "GetPivot": "pivot", "GetAsync": "data", "GetDataStore": "dataStore", + "GetOrderedDataStore": "orderedDataStore", "JSONDecode": "decoded", "JSONEncode": "json", + "GenerateGUID": "guid", "GetUserIdFromNameAsync": "userId", "GetNameFromUserIdAsync": "username", + "GetMarkerReachedSignal": "markerSignal", "GetPlayingAnimationTracks": "animationTracks", + "GetState": "humanoidState", "GetAttributes": "attributes", "GetFullName": "fullName", + "ViewportPointToRay": "ray", "ScreenPointToRay": "ray", "WorldToViewportPoint": "screenPoint", + "WorldToScreenPoint": "screenPoint", "GetMouseLocation": "mouseLocation", "IsKeyDown": "isKeyDown", + "GetProductInfo": "productInfo", "UserOwnsGamePassAsync": "ownsGamePass", "Clone": "clone", +} + +var plainCalls = map[string]string{ + "tick": "now", "time": "now", "os.clock": "startClock", "os.time": "timestamp", "workspace.GetServerTimeNow": "serverTime", +} + +// SuggestNames infers descriptive names for decompiler-generated variables +// from how they are initialised and used. It never touches names that already +// look human-written. The result maps binding IDs to suggested names. +func (f *File) SuggestNames() map[int]string { + out := map[int]string{} + nameOf := func(id int) string { + if n, ok := out[id]; ok { + return n + } + return f.Bindings[id].Name + } + for _, b := range f.Bindings { + if !b.Renamable() || !LooksGenerated(b.Name) { + continue + } + var n string + if m := lNameRe.FindStringSubmatch(b.Name); m != nil { + n = m[1] + } + if n == "" { + switch b.Kind { + case BindLocal: + if b.Init[0] >= 0 { + n = f.inferExpr(b.Init[0], b.Init[1], nameOf) + } + case BindParam: + if ps, ok := eventParams[b.Event]; ok && b.ParamIndex < len(ps) { + n = ps[b.ParamIndex] + } + case BindForNum: + n = "i" + case BindFor: + n = f.inferIter(b, nameOf) + } + } + if n != "" && ValidName(n) && n != b.Name { + out[b.ID] = n + } + } + return out +} + +type segment struct { + field string // .Field + method string // :Method(...) + arg string // first string argument of a call, unquoted + call bool +} + +// chain parses toks[s:e] as a pure suffixed expression (a.b:c("x").d). +// It returns the root token text and the suffix segments, or ok=false when +// the expression is anything else (arithmetic, literals, ...). +func (f *File) chain(s, e int) (root string, rootTok int, segs []segment, ok bool) { + if s >= e || f.Toks[s].Kind != Name { + return "", -1, nil, false + } + root, rootTok = f.Toks[s].Text, s + i := s + 1 + for i < e { + t := f.Toks[i] + switch { + case t.Kind == Op && t.Text == "." && i+1 < e && f.Toks[i+1].Kind == Name: + segs = append(segs, segment{field: f.Toks[i+1].Text}) + i += 2 + case t.Kind == Op && t.Text == ":" && i+1 < e && f.Toks[i+1].Kind == Name: + seg := segment{method: f.Toks[i+1].Text, call: true} + j, arg, good := f.skipArgs(i+2, e) + if !good { + return "", -1, nil, false + } + seg.arg = arg + segs = append(segs, seg) + i = j + case t.Kind == Op && t.Text == "(", t.Kind == String: + j, arg, good := f.skipArgs(i, e) + if !good { + return "", -1, nil, false + } + segs = append(segs, segment{call: true, arg: arg}) + i = j + case t.Kind == Op && t.Text == "[": + depth := 0 + for ; i < e; i++ { + if f.Toks[i].Kind == Op && f.Toks[i].Text == "[" { + depth++ + } else if f.Toks[i].Kind == Op && f.Toks[i].Text == "]" { + depth-- + if depth == 0 { + break + } + } + } + i++ + segs = append(segs, segment{}) + default: + return "", -1, nil, false + } + } + return root, rootTok, segs, true +} + +// skipArgs skips call arguments starting at i and returns the index after +// them plus the first argument when it is a plain string literal. +func (f *File) skipArgs(i, e int) (int, string, bool) { + if i >= e { + return i, "", false + } + t := f.Toks[i] + if t.Kind == String { + return i + 1, unquote(t.Text), true + } + if t.Kind != Op || (t.Text != "(" && t.Text != "{") { + return i, "", false + } + open, close := t.Text, ")" + if open == "{" { + close = "}" + } + arg := "" + if open == "(" && i+1 < e && f.Toks[i+1].Kind == String { + arg = unquote(f.Toks[i+1].Text) + } + depth := 0 + for ; i < e; i++ { + tt := f.Toks[i] + if tt.Kind != Op { + continue + } + switch tt.Text { + case "(", "{", "[": + depth++ + case ")", "}", "]": + depth-- + if depth == 0 && tt.Text == close { + return i + 1, arg, true + } + } + } + return i, "", false +} + +func unquote(s string) string { + if len(s) >= 2 && (s[0] == '"' || s[0] == '\'' || s[0] == '`') { + return s[1 : len(s)-1] + } + if strings.HasPrefix(s, "[") { + if i := strings.Index(s[1:], "["); i >= 0 { + body := s[i+2:] + if j := strings.LastIndex(body, "]"); j >= 0 { + body = body[:j] + return strings.TrimRight(body, "=]") + } + } + } + return s +} + +func (f *File) inferExpr(s, e int, nameOf func(int) string) string { + if s >= e { + return "" + } + root, rootTok, segs, ok := f.chain(s, e) + if !ok { + return "" + } + // require(...) -> module name + if root == "require" && f.Resolve[rootTok] == Global && len(segs) == 1 && segs[0].call { + return pascal(f.lastNameIn(s+1, e)) + } + if len(segs) == 0 { + return "" // plain alias of another variable + } + last := segs[len(segs)-1] + prevName := func() string { + for i := len(segs) - 2; i >= 0; i-- { + if segs[i].field != "" { + return segs[i].field + } + if segs[i].method != "" && childFinders[segs[i].method] && segs[i].arg != "" { + return segs[i].arg + } + } + if f.Resolve[rootTok] >= 0 { + if n := nameOf(f.Resolve[rootTok]); !LooksGenerated(n) { + return n + } + } + return "" + } + if f.Resolve[rootTok] == Global { + if len(segs) == 2 && segs[0].field == "new" && segs[1].call && root == "Instance" && last.arg != "" { + return camel(last.arg) + } + if len(segs) == 2 && segs[1].call && segs[0].field != "" { + if n, ok := constructors[root]; ok { + return n + } + if n, ok := plainCalls[root+"."+segs[0].field]; ok { + return n + } + } + if len(segs) == 1 && last.call && last.method == "" { + return plainCalls[root] + } + } + switch { + case last.method == "GetService" && last.arg != "": + return pascal(last.arg) + case last.method != "" && childFinders[last.method] && last.arg != "": + return camel(last.arg) + case last.method == "Clone": + if p := prevName(); p != "" { + return camel(p) + "Clone" + } + return "clone" + case connectMethods[last.method]: + if p := prevName(); p != "" { + return camel(p) + "Connection" + } + return "connection" + case last.method == "Wait": + if p := prevName(); p != "" { + if ps := eventParams[p]; len(ps) > 0 { + return ps[0] + } + } + case last.method == "Create" && len(segs) >= 1: + if strings.Contains(strings.ToLower(f.Toks[rootTok].Text+prevName()), "tween") { + return "tween" + } + case last.method == "GetAttribute" && last.arg != "": + return camel(last.arg) + case last.method == "GetPropertyChangedSignal" && last.arg != "": + return camel(last.arg) + "Changed" + case last.method != "": + return methodNames[last.method] + case last.field != "": + if n, ok := fieldNames[last.field]; ok { + return n + } + if len(last.field) >= 2 { + return camel(last.field) + } + } + return "" +} + +// lastNameIn returns the last string literal or field name inside toks[s:e], +// used to name required modules. +func (f *File) lastNameIn(s, e int) string { + for i := e - 1; i >= s; i-- { + t := f.Toks[i] + if t.Kind == String { + v := unquote(t.Text) + if j := strings.LastIndexAny(v, "/."); j >= 0 { + v = v[j+1:] + } + return v + } + if t.Kind == Name && i > s && f.Toks[i-1].Kind == Op && f.Toks[i-1].Text == "." { + return t.Text + } + } + return "" +} + +func (f *File) inferIter(b *Binding, nameOf func(int) string) string { + s, e := b.Iter[0], b.Iter[1] + if s < 0 { + return "" + } + kind := "" // pairs | ipairs | "" + cs, ce := s, e + if f.Toks[s].Kind == Name && (f.Toks[s].Text == "pairs" || f.Toks[s].Text == "ipairs") && f.Resolve[s] == Global && + s+1 < e && f.Toks[s+1].Text == "(" && f.Toks[e-1].Text == ")" { + kind, cs, ce = f.Toks[s].Text, s+2, e-1 + } else if f.Toks[s].Kind == Name && f.Toks[s].Text == "next" && s+1 < e && f.Toks[s+1].Text == "," { + kind, cs = "pairs", s+2 + } + container := "" + arrayLike := kind == "ipairs" + if root, rootTok, segs, ok := f.chain(cs, ce); ok { + if len(segs) == 0 { + if id := f.Resolve[rootTok]; id >= 0 { + container = nameOf(id) + } else { + container = root + } + } else if last := segs[len(segs)-1]; last.method != "" { + container = methodNames[last.method] + arrayLike = true + } else if last.field != "" { + container = last.field + } + } + if b.IterPos == 0 { + if arrayLike { + return "index" + } + if kind != "" { + return "key" + } + return "" + } + if b.IterPos == 1 { + if container != "" && !LooksGenerated(container) { + if s := singular(camel(container)); s != "" { + return s + } + } + if kind != "" || arrayLike { + return "value" + } + } + return "" +} + +func singular(s string) string { + ls := strings.ToLower(s) + switch { + case ls == "children": + return s[:len(s)-3] + case strings.HasSuffix(ls, "ies") && len(s) > 4: + return s[:len(s)-3] + "y" + case strings.HasSuffix(ls, "ses") || strings.HasSuffix(ls, "xes"): + return s[:len(s)-2] + case strings.HasSuffix(ls, "s") && !strings.HasSuffix(ls, "ss") && len(s) > 3: + return s[:len(s)-1] + } + return "" +} + +// words splits an arbitrary string ("Remote Events", "kill_brick") into words. +func words(s string) []string { + return strings.FieldsFunc(s, func(r rune) bool { + return !(unicode.IsLetter(r) || unicode.IsDigit(r)) || r > unicode.MaxASCII + }) +} + +func joinIdent(s string, upperFirst bool) string { + ws := words(s) + var sb strings.Builder + for i, w := range ws { + w = strings.Trim(w, "_") + if w == "" { + continue + } + if i == 0 || sb.Len() == 0 { + if upperFirst { + w = strings.ToUpper(w[:1]) + w[1:] + } else if !isAllUpper(w) || len(w) == 1 { + w = strings.ToLower(w[:1]) + w[1:] + } else { + w = strings.ToLower(w) + } + } else { + w = strings.ToUpper(w[:1]) + w[1:] + } + sb.WriteString(w) + } + out := sb.String() + if out == "" { + return "" + } + if isDigit(out[0]) { + out = "_" + out + } + return out +} + +func isAllUpper(s string) bool { + for _, r := range s { + if unicode.IsLower(r) { + return false + } + } + return true +} + +func camel(s string) string { return joinIdent(s, false) } +func pascal(s string) string { return joinIdent(s, true) } + +// ToIdent turns a suggested name into a valid identifier: valid names are +// returned unchanged, anything else ("player gui", "kill-brick") is camelCased. +func ToIdent(s string) string { + s = strings.TrimSpace(s) + if identRe.MatchString(s) { + return s + } + return camel(s) +} diff --git a/internal/luau/lexer.go b/internal/luau/lexer.go new file mode 100644 index 0000000..0807326 --- /dev/null +++ b/internal/luau/lexer.go @@ -0,0 +1,348 @@ +// Package luau implements a small, fast Luau front end: a lexer, a +// scope-resolving parser and a semantics-preserving renamer. It is not a +// full compiler; it understands exactly enough of the language to know which +// identifier refers to which variable. +package luau + +import ( + "fmt" + "strings" +) + +// Kind classifies a token. +type Kind uint8 + +const ( + EOF Kind = iota + Name + Keyword + Number + String // "..." '...' [[...]] and `...` without interpolation + InterpBegin // `text{ + InterpMid // }text{ + InterpEnd // }text` + Op + Comment +) + +func (k Kind) String() string { + switch k { + case EOF: + return "end of file" + case Name: + return "name" + case Keyword: + return "keyword" + case Number: + return "number" + case String: + return "string" + case InterpBegin, InterpMid, InterpEnd: + return "interpolated string" + case Op: + return "symbol" + case Comment: + return "comment" + } + return "token" +} + +// Token is a lexical token. Start and End are byte offsets into the source. +type Token struct { + Kind Kind + Text string + Start int + End int + Line int // 1-based line of Start +} + +// SyntaxError reports a lexing or parsing failure. +type SyntaxError struct { + Line int + Msg string +} + +func (e *SyntaxError) Error() string { return fmt.Sprintf("line %d: %s", e.Line, e.Msg) } + +var keywords = map[string]bool{ + "and": true, "break": true, "do": true, "else": true, "elseif": true, + "end": true, "false": true, "for": true, "function": true, "if": true, + "in": true, "local": true, "nil": true, "not": true, "or": true, + "repeat": true, "return": true, "then": true, "true": true, "until": true, + "while": true, +} + +// IsKeyword reports whether s is a reserved Luau keyword. +func IsKeyword(s string) bool { return keywords[s] } + +var ops3 = []string{"...", "..=", "//="} +var ops2 = []string{"..", "==", "~=", "<=", ">=", "//", "+=", "-=", "*=", "/=", "%=", "^=", "::", "->"} + +const ops1 = "+-*/%^#&~|<>=(){}[];:,.?@!" + +// Lex splits src into tokens. Comments are included as Comment tokens. +func Lex(src string) ([]Token, error) { + lx := lexer{src: src, line: 1} + return lx.run() +} + +type lexer struct { + src string + pos int + line int + toks []Token + interp []int // brace depth for each open interpolated string +} + +func (lx *lexer) errf(format string, args ...any) error { + return &SyntaxError{Line: lx.line, Msg: fmt.Sprintf(format, args...)} +} + +func (lx *lexer) emit(k Kind, start, line int) { + lx.toks = append(lx.toks, Token{Kind: k, Text: lx.src[start:lx.pos], Start: start, End: lx.pos, Line: line}) +} + +// advanceTo moves pos to p, counting newlines. +func (lx *lexer) advanceTo(p int) { + lx.line += strings.Count(lx.src[lx.pos:p], "\n") + lx.pos = p +} + +func isAlpha(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') } +func isDigit(c byte) bool { return c >= '0' && c <= '9' } +func isAlnum(c byte) bool { return isAlpha(c) || isDigit(c) } + +func (lx *lexer) run() ([]Token, error) { + src := lx.src + // Skip a UTF-8 BOM and a shebang line. + if strings.HasPrefix(src, "\xef\xbb\xbf") { + lx.pos = 3 + } + if strings.HasPrefix(src[lx.pos:], "#!") { + for lx.pos < len(src) && src[lx.pos] != '\n' { + lx.pos++ + } + } + for lx.pos < len(src) { + c := src[lx.pos] + start, line := lx.pos, lx.line + switch { + case c == '\n': + lx.line++ + lx.pos++ + case c == ' ' || c == '\t' || c == '\r' || c == '\f' || c == '\v': + lx.pos++ + case c == '-' && strings.HasPrefix(src[lx.pos:], "--"): + lx.pos += 2 + if lvl, ok := longBracket(src, lx.pos); ok { + end := strings.Index(src[lx.pos:], "]"+strings.Repeat("=", lvl)+"]") + if end < 0 { + return nil, lx.errf("unfinished long comment") + } + lx.advanceTo(lx.pos + end + lvl + 2) + } else { + for lx.pos < len(src) && src[lx.pos] != '\n' { + lx.pos++ + } + } + lx.emit(Comment, start, line) + case isAlpha(c) || c >= 0x80: + for lx.pos < len(src) && (isAlnum(src[lx.pos]) || src[lx.pos] >= 0x80) { + lx.pos++ + } + k := Name + if keywords[src[start:lx.pos]] { + k = Keyword + } + lx.emit(k, start, line) + case isDigit(c) || (c == '.' && lx.pos+1 < len(src) && isDigit(src[lx.pos+1])): + lx.number() + lx.emit(Number, start, line) + case c == '"' || c == '\'': + if err := lx.quoted(c); err != nil { + return nil, err + } + lx.emit(String, start, line) + case c == '[': + if lvl, ok := longBracket(src, lx.pos); ok { + end := strings.Index(src[lx.pos:], "]"+strings.Repeat("=", lvl)+"]") + if end < 0 { + return nil, lx.errf("unfinished long string") + } + lx.advanceTo(lx.pos + end + lvl + 2) + lx.emit(String, start, line) + } else { + lx.pos++ + lx.emit(Op, start, line) + } + case c == '`': + lx.pos++ + k, err := lx.interpSegment(true) + if err != nil { + return nil, err + } + lx.emit(k, start, line) + case c == '{': + if n := len(lx.interp); n > 0 { + lx.interp[n-1]++ + } + lx.pos++ + lx.emit(Op, start, line) + case c == '}': + if n := len(lx.interp); n > 0 && lx.interp[n-1] == 0 { + lx.interp = lx.interp[:n-1] + lx.pos++ + k, err := lx.interpSegment(false) + if err != nil { + return nil, err + } + lx.emit(k, start, line) + continue + } else if n > 0 { + lx.interp[n-1]-- + } + lx.pos++ + lx.emit(Op, start, line) + default: + rest := src[lx.pos:] + matched := false + for _, group := range [][]string{ops3, ops2} { + for _, op := range group { + if strings.HasPrefix(rest, op) { + lx.pos += len(op) + matched = true + break + } + } + if matched { + break + } + } + if !matched { + if strings.IndexByte(ops1, c) < 0 { + return nil, lx.errf("unexpected character %q", c) + } + lx.pos++ + } + lx.emit(Op, start, line) + } + } + if len(lx.interp) > 0 { + return nil, lx.errf("unfinished interpolated string") + } + lx.toks = append(lx.toks, Token{Kind: EOF, Start: len(src), End: len(src), Line: lx.line}) + return lx.toks, nil +} + +// longBracket reports whether src[p:] starts a long bracket ([[ or [==[) +// and returns its level. +func longBracket(src string, p int) (int, bool) { + if p >= len(src) || src[p] != '[' { + return 0, false + } + q := p + 1 + for q < len(src) && src[q] == '=' { + q++ + } + if q < len(src) && src[q] == '[' { + return q - p - 1, true + } + return 0, false +} + +func (lx *lexer) number() { + src := lx.src + hex := strings.HasPrefix(src[lx.pos:], "0x") || strings.HasPrefix(src[lx.pos:], "0X") + if hex { + lx.pos += 2 + } + for lx.pos < len(src) { + c := src[lx.pos] + if isAlnum(c) || c == '.' { + // Stop before a ".." concatenation operator. + if c == '.' && lx.pos+1 < len(src) && src[lx.pos+1] == '.' { + return + } + lx.pos++ + exp := (!hex && (c == 'e' || c == 'E')) || (hex && (c == 'p' || c == 'P')) + if exp && lx.pos < len(src) && (src[lx.pos] == '+' || src[lx.pos] == '-') { + lx.pos++ + } + continue + } + return + } +} + +func (lx *lexer) quoted(q byte) error { + src := lx.src + lx.pos++ + for lx.pos < len(src) { + c := src[lx.pos] + switch c { + case '\\': + lx.pos++ + if lx.pos < len(src) { + if src[lx.pos] == '\n' { + lx.line++ + } else if src[lx.pos] == 'z' { + // \z skips following whitespace, including newlines. + lx.pos++ + for lx.pos < len(src) && strings.IndexByte(" \t\r\n\f\v", src[lx.pos]) >= 0 { + if src[lx.pos] == '\n' { + lx.line++ + } + lx.pos++ + } + continue + } + lx.pos++ + } + case '\n': + return lx.errf("unfinished string") + default: + lx.pos++ + if c == q { + return nil + } + } + } + return lx.errf("unfinished string") +} + +// interpSegment scans the body of an interpolated string starting at pos +// (just after ` or }) until the closing ` or an opening {. +func (lx *lexer) interpSegment(first bool) (Kind, error) { + src := lx.src + for lx.pos < len(src) { + c := src[lx.pos] + switch c { + case '\\': + lx.pos++ + if lx.pos < len(src) { + if src[lx.pos] == '\n' { + lx.line++ + } + lx.pos++ + } + case '\n': + return 0, lx.errf("unfinished interpolated string") + case '`': + lx.pos++ + if first { + return String, nil + } + return InterpEnd, nil + case '{': + lx.pos++ + lx.interp = append(lx.interp, 0) + if first { + return InterpBegin, nil + } + return InterpMid, nil + default: + lx.pos++ + } + } + return 0, lx.errf("unfinished interpolated string") +} diff --git a/internal/luau/luau_test.go b/internal/luau/luau_test.go new file mode 100644 index 0000000..9193986 --- /dev/null +++ b/internal/luau/luau_test.go @@ -0,0 +1,415 @@ +package luau + +import ( + "math/rand" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func mustParse(t *testing.T, src string) *File { + t.Helper() + f, err := Parse(src) + if err != nil { + t.Fatalf("parse: %v\n%s", err, src) + } + return f +} + +func TestLexer(t *testing.T) { + src := "local a = 0x1F + 1e-5 + .5 -- c\n" + + "local s = [==[ long ]] ]==] .. \"q\\\"x\" .. 'y'\n" + + "--[[ block\ncomment ]]\n" + + "local i = `a{b}c{`x{y}`}d`\n" + + "x //= 2; y ..= \"z\"; z = 1..2" + toks, err := Lex(src) + if err != nil { + t.Fatal(err) + } + var kinds []string + for _, tk := range toks { + if tk.Kind == Number || tk.Kind == String || tk.Kind == Comment || tk.Kind >= InterpBegin && tk.Kind <= InterpEnd { + kinds = append(kinds, tk.Text) + } + } + want := []string{"0x1F", "1e-5", ".5", "-- c", "[==[ long ]] ]==]", `"q\"x"`, "'y'", "--[[ block\ncomment ]]", + "`a{", "}c{", "`x{", "}`", "}d`", "2", `"z"`, "1", "2"} + if strings.Join(kinds, "|") != strings.Join(want, "|") { + t.Fatalf("tokens:\n got %q\nwant %q", kinds, want) + } + if toks[len(toks)-1].Line != 6 { + t.Fatalf("line tracking: got %d", toks[len(toks)-1].Line) + } +} + +func TestLexErrors(t *testing.T) { + for _, src := range []string{`local s = "abc`, "local s = [[abc", "--[[ x", "local s = `abc{x"} { + if _, err := Lex(src); err == nil { + t.Errorf("expected error for %q", src) + } + } +} + +func TestParseLuauSyntax(t *testing.T) { + src := ` +--!strict +export type Item = { name: string, price: number? } +type Map = { [K]: V } +local Players: Players = game:GetService("Players") +local function f(a: T, ...: number): (T, number) + local b = if a then 1 elseif false then 2 else 3 + b += 1 + b //= 2 + for i: number = 1, 10 do + if i == 2 then continue end + end + local cb: (number) -> () = function(x) end + local t = { a = 1, ["b"] = 2, 3; [b] = b } + local s = ` + "`value {t.a} and {`nested {b}`}`" + ` + local c = a :: any + local v: typeof(b) = b + return a, #t +end +@native +local function g() end +function Players.Thing:Method(x) + return self, x +end +local continue = 1 +continue += 1 +repeat local z = 1 until z == 1 +` + f := mustParse(t, src) + // self must resolve to the implicit binding of the method. + found := false + for i, tk := range f.Toks { + if tk.Text == "self" { + found = true + if id := f.Resolve[i]; id < 0 || f.Bindings[id].Kind != BindSelf { + t.Fatalf("self resolved to %d", id) + } + } + } + if !found { + t.Fatal("self not found") + } + // `z` in the until condition sees the loop body's local. + for i, tk := range f.Toks { + if tk.Text == "z" && f.Resolve[i] == Global { + t.Fatal("repeat-until scoping broken") + } + } +} + +func TestLocalSeesOuterInInitializer(t *testing.T) { + f := mustParse(t, "local x = 1\nlocal x = x + 1\nprint(x)") + var xs []int + for i, tk := range f.Toks { + if tk.Text == "x" { + xs = append(xs, f.Resolve[i]) + } + } + // decl1, decl2, rhs-ref(->decl1), print-ref(->decl2) + if len(xs) != 4 || xs[2] != xs[0] || xs[3] != xs[1] || xs[0] == xs[1] { + t.Fatalf("resolution: %v", xs) + } +} + +func TestRenameAvoidsCapture(t *testing.T) { + src := `local v1 = game:GetService("Players") +local v2 = 1 +local function f(p1) + local Players = 5 + return v1, Players, p1, foo +end +print(f(v2))` + f := mustParse(t, src) + ids := map[string]int{} + for _, b := range f.Bindings { + ids[b.Name] = b.ID + } + out, applied := f.Rename(map[int]string{ + ids["v1"]: "Players", // would be captured by the inner local + ids["v2"]: "foo", // would capture the global foo used inside f? no: foo is global; v2 is top-level so f's foo would bind to it + ids["p1"]: "print", // reserved -> ignored + }) + if applied[ids["v1"]] != "Players2" { + t.Errorf("v1 -> %q, want Players2\n%s", applied[ids["v1"]], out) + } + if applied[ids["v2"]] != "foo2" { + t.Errorf("v2 -> %q, want foo2\n%s", applied[ids["v2"]], out) + } + if _, ok := applied[ids["p1"]]; ok { + t.Errorf("reserved name applied") + } + assertSameShape(t, f, out) +} + +// assertSameShape checks that out resolves identically to f. +func assertSameShape(t *testing.T, f *File, out string) { + t.Helper() + g := mustParse(t, out) + if len(g.Toks) != len(f.Toks) { + t.Fatalf("token count changed: %d -> %d", len(f.Toks), len(g.Toks)) + } + for i := range f.Resolve { + if f.Resolve[i] != g.Resolve[i] { + t.Fatalf("token %d (%q -> %q) resolves to %d, was %d\n%s", i, f.Toks[i].Text, g.Toks[i].Text, g.Resolve[i], f.Resolve[i], out) + } + } +} + +func TestSuggestNamesOnSample(t *testing.T) { + src, err := os.ReadFile(filepath.Join("..", "..", "examples", "decompiled.lua")) + if err != nil { + t.Fatal(err) + } + f := mustParse(t, string(src)) + out, _ := f.Rename(f.SuggestNames()) + assertSameShape(t, f, out) + for _, want := range []string{ + `local Players = game:GetService("Players")`, + `local localPlayer = Players.LocalPlayer`, + `local ShopConfig = require(`, + `for key, item in pairs(ShopConfig.Items) do`, + `local textButton = Instance.new("TextButton")`, + `textButton.Text = ` + "`{item.DisplayName} - {item.Price}`", + `CharacterAdded:Connect(function(character)`, + `local humanoid = character:WaitForChild("Humanoid")`, + `InputBegan:Connect(function(input, gameProcessedEvent)`, + } { + if !strings.Contains(out, want) { + t.Errorf("missing %q in:\n%s", want, out) + } + } +} + +func TestUniqueKeys(t *testing.T) { + src := "a(function(p1) return p1 end)\nb(function(p1) return p1 end)\nlocal p1_1 = 1\nlocal v = 2" + f := mustParse(t, src) + keyed, keys := f.UniqueKeys() + seen := map[string]bool{} + for _, k := range keys { + if seen[k] { + t.Fatalf("duplicate key %q", k) + } + seen[k] = true + } + if !seen["v"] || !seen["p1_1"] || !seen["p1_2"] || !seen["p1_3"] { + t.Fatalf("keys: %v", keys) + } + assertSameShape(t, f, keyed) +} + +func TestInsertComments(t *testing.T) { + src := "local a = [[\nline2\n]]\nif a then\n\tprint(a)\nend\n" + f := mustParse(t, src) + out := f.InsertComments(src, map[int]string{2: "inside string", 4: "check a", 5: "print it\nnow", 99: "nope"}) + want := "local a = [[\nline2\n]]\n-- check a\nif a then\n\t-- print it now\n\tprint(a)\nend\n" + if out != want { + t.Fatalf("got:\n%s\nwant:\n%s", out, want) + } +} + +// Lua 5.1 programs used to check that renaming preserves behavior when +// actually executed. +var execPrograms = []string{ + `local v1 = 10 +local v2 = function(p1, p2) return p1 * v1 + p2 end +local v3 = {} +for v4 = 1, 5 do + local v5 = v2(v4, v1) + v3[#v3 + 1] = v5 +end +local v6 = 0 +for v7, v8 in ipairs(v3) do v6 = v6 + v7 * v8 end +print(v6, #v3)`, + `local a = 1 +local b = 2 +local function c(d) + local a = d + b + local function e(f) return a + f + b end + return e(a) +end +local g = c(a) +do local b = 100; g = g + b end +print(g, a, b) +x = 5 -- global +local function h() return x end +print(h())`, + `local v1 = {} +function v1.new(p1) + local v2 = setmetatable({}, {__index = v1}) + v2.n = p1 + return v2 +end +function v1:get() return self.n end +local v3 = v1.new(7) +local v4 = 0 +repeat local v5 = v4 + 1; v4 = v5 until v5 >= 3 +local v6 = {a = 1, b = 2} +local v7 = 0 +for v8, v9 in pairs(v6) do v7 = v7 + v9 end +print(v3:get(), v4, v7, v6.a)`, + `local v1, v2 = 1, 2 +v1, v2 = v2, v1 +local v3 = (function(...) local v4 = select("#", ...) return v4 end)(1, 2, 3) +local v5 = v1 > v2 and "gt" or "le" +while v1 > 0 do local v6 = v1; v1 = v6 - 1 end +print(v1, v2, v3, v5, tostring(nil))`, +} + +func TestRenamePreservesExecution(t *testing.T) { + lua, err := exec.LookPath("lua5.1") + if err != nil { + if lua, err = exec.LookPath("lua"); err != nil { + t.Skip("no Lua interpreter available") + } + } + run := func(src string) string { + cmd := exec.Command(lua, "-") + cmd.Stdin = strings.NewReader(src) + out, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("lua failed: %v\n%s\n%s", err, out, src) + } + return string(out) + } + pool := []string{"a", "b", "c", "x", "d", "n", "value", "v1", "v2", "p1", "setmetatable2", "print"} + rng := rand.New(rand.NewSource(1)) + for pi, prog := range execPrograms { + want := run(prog) + f := mustParse(t, prog) + for trial := 0; trial < 40; trial++ { + ren := map[int]string{} + for _, b := range f.Bindings { + if b.Renamable() && rng.Intn(3) > 0 { + ren[b.ID] = pool[rng.Intn(len(pool))] + } + } + out, _ := f.Rename(ren) + if got := run(out); got != want { + t.Fatalf("program %d trial %d changed behavior:\nwant %s got %s\n%s", pi, trial, want, got, out) + } + assertSameShape(t, f, out) + } + } +} + +func TestLooksGenerated(t *testing.T) { + for _, n := range []string{"v1", "u12", "p3", "l_Players_0", "L0_1", "var_4", "upval2", "_5", "slot3", "v_u_1", "p_u_12"} { + if !LooksGenerated(n) { + t.Errorf("%s should look generated", n) + } + } + for _, n := range []string{"player", "value", "Players", "i", "v", "x1y"} { + if LooksGenerated(n) { + t.Errorf("%s should not look generated", n) + } + } +} + +func TestCamel(t *testing.T) { + cases := map[string]string{"HumanoidRootPart": "humanoidRootPart", "UI": "ui", "Shop Gui": "shopGui", + "kill_brick": "killBrick", "3D": "_3d", "": ""} + for in, want := range cases { + if got := camel(in); got != want { + t.Errorf("camel(%q) = %q, want %q", in, got, want) + } + } + if got := pascal("remote events"); got != "RemoteEvents" { + t.Errorf("pascal: %q", got) + } +} + +func BenchmarkParseAndSuggest(b *testing.B) { + src, _ := os.ReadFile(filepath.Join("..", "..", "examples", "decompiled.lua")) + big := strings.Repeat(string(src)+"\n", 50) + b.SetBytes(int64(len(big))) + for i := 0; i < b.N; i++ { + f, err := Parse(big) + if err != nil { + b.Fatal(err) + } + f.Rename(f.SuggestNames()) + } +} + +// Luau-only programs, executed with the official Luau VM when available +// (set LUAU_BIN or put `luau` on PATH). +var luauPrograms = []string{ + `local v1: number = 3 +local v2 = if v1 > 2 then "big" elseif v1 > 1 then "mid" else "small" +local v3 = 0 +for v4 = 1, 10 do + if v4 % 2 == 0 then continue end + v3 += v4 +end +local v5 = ` + "`{v2}:{v3}:{if v3 > 20 then \"many\" else \"few\"}`" + ` +print(v5)`, + `type Point = { x: number, y: number } +local function v1(p1: T, ...: number): (T, number) + local v2 = 0 + for _, v3 in { ... } do v2 += v3 end + return p1, v2 +end +local v4: Point = { x = 1, y = 2 } +local v5, v6 = v1(v4, 4, 5, 6) +local v7: typeof(v6) = v6 :: number +print(v5.x + v5.y, v7)`, + `local v1 = {} +v1.__index = v1 +function v1.new(p1) return setmetatable({ n = p1 }, v1) end +function v1:add(p1) self.n += p1; return self end +local v2 = v1.new(1):add(2):add(3) +local v3 = 0 +repeat local v4 = v3 + 1; v3 = v4 until v4 >= 3 +local v5 = function(...) local v6 = select("#", ...) return v6 end +print(v2.n, v3, v5(1, nil, 3), ` + "`nested {`inner {v3}`}`" + `)`, +} + +func TestRenamePreservesLuauExecution(t *testing.T) { + bin := os.Getenv("LUAU_BIN") + if bin == "" { + var err error + if bin, err = exec.LookPath("luau"); err != nil { + t.Skip("Luau VM not available (set LUAU_BIN)") + } + } + dir := t.TempDir() + run := func(src string) string { + p := filepath.Join(dir, "prog.luau") + if err := os.WriteFile(p, []byte(src), 0o644); err != nil { + t.Fatal(err) + } + out, err := exec.Command(bin, p).CombinedOutput() + if err != nil { + t.Fatalf("luau failed: %v\n%s\n%s", err, out, src) + } + return string(out) + } + pool := []string{"a", "b", "x", "n", "v1", "p1", "value", "point", "count"} + rng := rand.New(rand.NewSource(2)) + for pi, prog := range luauPrograms { + want := run(prog) + f := mustParse(t, prog) + suggested, _ := f.Rename(f.SuggestNames()) + if got := run(suggested); got != want { + t.Fatalf("program %d: suggested names changed behavior:\n%s", pi, suggested) + } + for trial := 0; trial < 25; trial++ { + ren := map[int]string{} + for _, b := range f.Bindings { + if b.Renamable() && rng.Intn(3) > 0 { + ren[b.ID] = pool[rng.Intn(len(pool))] + } + } + out, _ := f.Rename(ren) + if got := run(out); got != want { + t.Fatalf("program %d trial %d changed behavior:\nwant %s got %s\n%s", pi, trial, want, got, out) + } + } + } +} diff --git a/internal/luau/parser.go b/internal/luau/parser.go new file mode 100644 index 0000000..872d1fe --- /dev/null +++ b/internal/luau/parser.go @@ -0,0 +1,1001 @@ +package luau + +import "fmt" + +// BindKind describes how a variable was introduced. +type BindKind uint8 + +const ( + BindLocal BindKind = iota // local x = ... + BindLocalFunc // local function x() end + BindParam // function(x) end + BindSelf // implicit self of a method + BindFor // for k, v in ... do + BindForNum // for i = 1, n do +) + +// Binding is one variable: a declaration plus every reference to it. +type Binding struct { + ID int + Name string + Kind BindKind + Decl int // index into File.Toks of the declaring name, -1 if implicit + Refs []int // indices into File.Toks of every occurrence, including Decl + + // Hints used for name inference. + Init [2]int // initializer expression token range [start, end), or {-1, -1} + Event string // for callback parameters: the event that was connected (e.g. "Touched") + ParamIndex int // position of a parameter in its function's parameter list + Iter [2]int // generic for: the iterator expression list token range + IterPos int // generic for: position of this name in the name list +} + +// Statement is a top-level statement location. +type Statement struct { + Start int // byte offset + Line int +} + +// File is a parsed and scope-resolved Luau source file. +type File struct { + Src string + Toks []Token // significant tokens (no comments), terminated by EOF + Comments []Token + Bindings []*Binding + // Resolve maps each token to the binding it refers to: a binding ID, + // Global (-1) for a free/global variable, or NotVar (-2) for anything else. + Resolve []int + Statements []Statement +} + +const ( + Global = -1 + NotVar = -2 +) + +// Parse lexes, parses and resolves src. +func Parse(src string) (*File, error) { + all, err := Lex(src) + if err != nil { + return nil, err + } + f := &File{Src: src} + f.Toks = make([]Token, 0, len(all)) + for _, t := range all { + if t.Kind == Comment { + f.Comments = append(f.Comments, t) + } else { + f.Toks = append(f.Toks, t) + } + } + names := make([]string, len(f.Toks)) + for i, t := range f.Toks { + names[i] = t.Text + } + p, err := analyze(f.Toks, names) + if err != nil { + return nil, err + } + f.Bindings, f.Resolve, f.Statements = p.bindings, p.resolve, p.stmts + return f, nil +} + +type parser struct { + toks []Token + names []string // effective name of each token (allows trial renames) + pos int + scopes []map[string]int + bindings []*Binding + resolve []int + stmts []Statement + depth int + + eventPos int // token position where a callback function may start + event string // event name for that callback + lastField string // last .Field seen by suffixedExp +} + +func analyze(toks []Token, names []string) (*parser, error) { + p := &parser{toks: toks, names: names, eventPos: -1} + p.resolve = make([]int, len(toks)) + for i := range p.resolve { + p.resolve[i] = NotVar + } + p.push() + if err := p.block(); err != nil { + return nil, err + } + if p.cur().Kind != EOF { + return nil, p.errf("unexpected %s", p.desc(p.cur())) + } + return p, nil +} + +// ---- token helpers ---- + +func (p *parser) cur() Token { return p.toks[p.pos] } + +func (p *parser) peek(n int) Token { + if p.pos+n < len(p.toks) { + return p.toks[p.pos+n] + } + return p.toks[len(p.toks)-1] +} + +func (p *parser) isOp(s string) bool { + t := p.toks[p.pos] + return t.Kind == Op && t.Text == s +} + +func (p *parser) isKw(s string) bool { + t := p.toks[p.pos] + return t.Kind == Keyword && t.Text == s +} + +func (p *parser) desc(t Token) string { + if t.Kind == EOF { + return "end of file" + } + return fmt.Sprintf("%q", t.Text) +} + +func (p *parser) errf(format string, args ...any) error { + return &SyntaxError{Line: p.cur().Line, Msg: fmt.Sprintf(format, args...)} +} + +func (p *parser) expectOp(s string) error { + if !p.isOp(s) { + return p.errf("expected %q near %s", s, p.desc(p.cur())) + } + p.pos++ + return nil +} + +func (p *parser) expectKw(s string) error { + if !p.isKw(s) { + return p.errf("expected %q near %s", s, p.desc(p.cur())) + } + p.pos++ + return nil +} + +func (p *parser) expectName() (int, error) { + if p.cur().Kind != Name { + return 0, p.errf("expected a name near %s", p.desc(p.cur())) + } + p.pos++ + return p.pos - 1, nil +} + +// ---- scopes ---- + +func (p *parser) push() { p.scopes = append(p.scopes, map[string]int{}) } +func (p *parser) pop() { p.scopes = p.scopes[:len(p.scopes)-1] } + +func (p *parser) declare(tok int, kind BindKind) *Binding { + b := &Binding{ID: len(p.bindings), Name: p.names[tok], Kind: kind, Decl: tok, Refs: []int{tok}, + Init: [2]int{-1, -1}, Iter: [2]int{-1, -1}} + p.bindings = append(p.bindings, b) + p.scopes[len(p.scopes)-1][b.Name] = b.ID + p.resolve[tok] = b.ID + return b +} + +func (p *parser) declareSelf() { + b := &Binding{ID: len(p.bindings), Name: "self", Kind: BindSelf, Decl: -1, + Init: [2]int{-1, -1}, Iter: [2]int{-1, -1}} + p.bindings = append(p.bindings, b) + p.scopes[len(p.scopes)-1]["self"] = b.ID +} + +func (p *parser) ref(tok int) { + name := p.names[tok] + for i := len(p.scopes) - 1; i >= 0; i-- { + if id, ok := p.scopes[i][name]; ok { + p.resolve[tok] = id + p.bindings[id].Refs = append(p.bindings[id].Refs, tok) + return + } + } + p.resolve[tok] = Global +} + +// ---- statements ---- + +func (p *parser) blockEnd() bool { + t := p.cur() + if t.Kind == EOF { + return true + } + if t.Kind == Keyword { + switch t.Text { + case "end", "else", "elseif", "until": + return true + } + } + return false +} + +func (p *parser) block() error { + for !p.blockEnd() { + if p.depth == 0 { + t := p.cur() + p.stmts = append(p.stmts, Statement{Start: t.Start, Line: t.Line}) + } + if err := p.statement(); err != nil { + return err + } + } + return nil +} + +func (p *parser) scopedBlock() error { + p.push() + p.depth++ + err := p.block() + p.depth-- + p.pop() + return err +} + +func (p *parser) statement() error { + t := p.cur() + switch t.Kind { + case Keyword: + switch t.Text { + case "if": + return p.ifStat() + case "while": + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectKw("do"); err != nil { + return err + } + if err := p.scopedBlock(); err != nil { + return err + } + return p.expectKw("end") + case "do": + p.pos++ + if err := p.scopedBlock(); err != nil { + return err + } + return p.expectKw("end") + case "for": + return p.forStat() + case "repeat": + p.pos++ + p.push() + p.depth++ + defer func() { p.depth--; p.pop() }() + if err := p.block(); err != nil { + return err + } + if err := p.expectKw("until"); err != nil { + return err + } + return p.expr() // the condition sees the loop body's locals + case "function": + return p.funcStat() + case "local": + return p.localStat() + case "return": + p.pos++ + if !p.blockEnd() && !p.isOp(";") { + if _, err := p.exprList(); err != nil { + return err + } + } + if p.isOp(";") { + p.pos++ + } + return nil + case "break": + p.pos++ + return nil + } + case Op: + switch t.Text { + case ";": + p.pos++ + return nil + case "@": // attribute such as @native + p.pos++ + if p.isOp("[") { + return p.skipBalanced("[", "]") + } + _, err := p.expectName() + return err + case "::": // ::label:: (Lua 5.2 syntax some decompilers emit) + p.pos++ + if _, err := p.expectName(); err != nil { + return err + } + return p.expectOp("::") + } + case Name: + next := p.peek(1) + switch t.Text { + case "continue": + if !continuesExpr(next) { + p.pos++ + return nil + } + case "type": + if next.Kind == Name { + return p.typeAlias() + } + case "export": + if next.Kind == Name && next.Text == "type" && p.peek(2).Kind == Name { + p.pos++ + return p.typeAlias() + } + case "goto": + if next.Kind == Name && p.peek(1).Line == t.Line && !continuesExpr(p.peek(2)) { + p.pos += 2 + return nil + } + } + } + return p.exprStat() +} + +// continuesExpr reports whether t can follow a name inside an expression or +// assignment, which distinguishes `continue` the statement from a variable. +func continuesExpr(t Token) bool { + switch t.Kind { + case String, InterpBegin: + return true + case Op: + switch t.Text { + case "(", ".", ":", "[", "=", ",", "{", "+=", "-=", "*=", "/=", "//=", "%=", "^=", "..=": + return true + } + } + return false +} + +func (p *parser) ifStat() error { + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectKw("then"); err != nil { + return err + } + if err := p.scopedBlock(); err != nil { + return err + } + for p.isKw("elseif") { + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectKw("then"); err != nil { + return err + } + if err := p.scopedBlock(); err != nil { + return err + } + } + if p.isKw("else") { + p.pos++ + if err := p.scopedBlock(); err != nil { + return err + } + } + return p.expectKw("end") +} + +func (p *parser) forStat() error { + p.pos++ + first, err := p.expectName() + if err != nil { + return err + } + if p.isOp(":") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + if p.isOp("=") { + p.pos++ + if _, err := p.exprList(); err != nil { + return err + } + if err := p.expectKw("do"); err != nil { + return err + } + p.push() + p.depth++ + p.declare(first, BindForNum) + err := p.block() + p.depth-- + p.pop() + if err != nil { + return err + } + return p.expectKw("end") + } + names := []int{first} + for p.isOp(",") { + p.pos++ + n, err := p.expectName() + if err != nil { + return err + } + if p.isOp(":") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + names = append(names, n) + } + if err := p.expectKw("in"); err != nil { + return err + } + ranges, err := p.exprList() + if err != nil { + return err + } + if err := p.expectKw("do"); err != nil { + return err + } + p.push() + p.depth++ + for k, n := range names { + b := p.declare(n, BindFor) + b.Iter = [2]int{ranges[0][0], ranges[len(ranges)-1][1]} + b.IterPos = k + } + err = p.block() + p.depth-- + p.pop() + if err != nil { + return err + } + return p.expectKw("end") +} + +func (p *parser) funcStat() error { + p.pos++ + first, err := p.expectName() + if err != nil { + return err + } + p.ref(first) + method := false + for p.isOp(".") || p.isOp(":") { + colon := p.isOp(":") + p.pos++ + if _, err := p.expectName(); err != nil { + return err + } + if colon { + method = true + break + } + } + return p.funcBody(method, "") +} + +func (p *parser) localStat() error { + p.pos++ + if p.isKw("function") { + p.pos++ + n, err := p.expectName() + if err != nil { + return err + } + p.declare(n, BindLocalFunc) // visible inside its own body (recursion) + return p.funcBody(false, "") + } + var names []int + for { + n, err := p.expectName() + if err != nil { + return err + } + names = append(names, n) + if p.isOp(":") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + // Lua 5.4 attributes: , + if p.isOp("<") && p.peek(1).Kind == Name && p.peek(2).Kind == Op && p.peek(2).Text == ">" { + p.pos += 3 + } + if !p.isOp(",") { + break + } + p.pos++ + } + var ranges [][2]int + if p.isOp("=") { + p.pos++ + var err error + if ranges, err = p.exprList(); err != nil { + return err + } + } + // Locals become visible only after the whole statement. + for k, n := range names { + b := p.declare(n, BindLocal) + if k < len(ranges) { + b.Init = ranges[k] + } + } + return nil +} + +func (p *parser) typeAlias() error { + p.pos++ // type + if _, err := p.expectName(); err != nil { + return err + } + if p.isOp("<") { + if err := p.skipBalanced("<", ">"); err != nil { + return err + } + } + if err := p.expectOp("="); err != nil { + return err + } + return p.skipType() +} + +var invokeCallbacks = map[string]bool{"OnServerInvoke": true, "OnClientInvoke": true, "OnInvoke": true} + +func (p *parser) exprStat() error { + if err := p.suffixedExp(); err != nil { + return err + } + if p.isOp(",") || p.isOp("=") { + for p.isOp(",") { + p.pos++ + if err := p.suffixedExp(); err != nil { + return err + } + } + field := p.lastField + if err := p.expectOp("="); err != nil { + return err + } + if invokeCallbacks[field] && p.isKw("function") { + p.eventPos, p.event = p.pos, field + } + _, err := p.exprList() + return err + } + switch t := p.cur(); { + case t.Kind == Op && len(t.Text) >= 2 && t.Text[len(t.Text)-1] == '=' && t.Text != "==" && t.Text != "~=" && t.Text != "<=" && t.Text != ">=": + p.pos++ // compound assignment + return p.expr() + } + return nil +} + +// ---- functions ---- + +func (p *parser) funcBody(method bool, event string) error { + p.push() + p.depth++ + defer func() { p.depth--; p.pop() }() + if method { + p.declareSelf() + } + if p.isOp("<") { + if err := p.skipBalanced("<", ">"); err != nil { + return err + } + } + if err := p.expectOp("("); err != nil { + return err + } + idx := 0 + for !p.isOp(")") { + if p.isOp("...") { + p.pos++ + } else { + n, err := p.expectName() + if err != nil { + return err + } + b := p.declare(n, BindParam) + b.Event, b.ParamIndex = event, idx + idx++ + } + if p.isOp(":") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + if !p.isOp(",") { + break + } + p.pos++ + } + if err := p.expectOp(")"); err != nil { + return err + } + if p.isOp(":") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + if err := p.block(); err != nil { + return err + } + return p.expectKw("end") +} + +// ---- expressions ---- + +func (p *parser) exprList() ([][2]int, error) { + var ranges [][2]int + for { + start := p.pos + if err := p.expr(); err != nil { + return nil, err + } + ranges = append(ranges, [2]int{start, p.pos}) + if !p.isOp(",") { + return ranges, nil + } + p.pos++ + } +} + +func isBinop(t Token) bool { + switch t.Kind { + case Keyword: + return t.Text == "and" || t.Text == "or" + case Op: + switch t.Text { + case "+", "-", "*", "/", "//", "%", "^", "..", "==", "~=", "<", "<=", ">", ">=": + return true + } + } + return false +} + +func (p *parser) expr() error { + if err := p.unary(); err != nil { + return err + } + for isBinop(p.cur()) { + p.pos++ + if err := p.unary(); err != nil { + return err + } + } + return nil +} + +func (p *parser) unary() error { + for p.isKw("not") || p.isOp("-") || p.isOp("#") { + p.pos++ + } + if err := p.simpleExp(); err != nil { + return err + } + for p.isOp("::") { + p.pos++ + if err := p.skipType(); err != nil { + return err + } + } + return nil +} + +func (p *parser) simpleExp() error { + t := p.cur() + switch t.Kind { + case Number, String: + p.pos++ + return nil + case InterpBegin: + p.pos++ + for { + if err := p.expr(); err != nil { + return err + } + switch p.cur().Kind { + case InterpMid: + p.pos++ + case InterpEnd: + p.pos++ + return nil + default: + return p.errf("malformed interpolated string") + } + } + case Keyword: + switch t.Text { + case "nil", "true", "false": + p.pos++ + return nil + case "function": + event := "" + if p.eventPos == p.pos { + event = p.event + } + p.pos++ + return p.funcBody(false, event) + case "if": + return p.ifExp() + } + case Op: + switch t.Text { + case "...": + p.pos++ + return nil + case "{": + return p.table() + } + } + return p.suffixedExp() +} + +func (p *parser) ifExp() error { + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectKw("then"); err != nil { + return err + } + if err := p.expr(); err != nil { + return err + } + for p.isKw("elseif") { + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectKw("then"); err != nil { + return err + } + if err := p.expr(); err != nil { + return err + } + } + if err := p.expectKw("else"); err != nil { + return err + } + return p.expr() +} + +var connectMethods = map[string]bool{"Connect": true, "Once": true, "ConnectParallel": true} + +func (p *parser) suffixedExp() error { + t := p.cur() + switch { + case t.Kind == Name: + p.ref(p.pos) + p.pos++ + case t.Kind == Op && t.Text == "(": + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectOp(")"); err != nil { + return err + } + default: + return p.errf("unexpected %s", p.desc(t)) + } + last := "" + for { + t := p.cur() + switch { + case t.Kind == Op && t.Text == ".": + p.pos++ + n, err := p.expectName() + if err != nil { + return err + } + last = p.toks[n].Text + case t.Kind == Op && t.Text == "[": + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectOp("]"); err != nil { + return err + } + last = "" + case t.Kind == Op && t.Text == ":": + p.pos++ + m, err := p.expectName() + if err != nil { + return err + } + event := "" + if connectMethods[p.toks[m].Text] { + event = last + } + if err := p.callArgs(event); err != nil { + return err + } + last = "" + case t.Kind == Op && (t.Text == "(" || t.Text == "{"), t.Kind == String: + if err := p.callArgs(""); err != nil { + return err + } + last = "" + default: + p.lastField = last + return nil + } + } +} + +func (p *parser) callArgs(event string) error { + t := p.cur() + switch { + case t.Kind == String: + p.pos++ + return nil + case t.Kind == Op && t.Text == "{": + return p.table() + case t.Kind == Op && t.Text == "(": + p.pos++ + if p.isOp(")") { + p.pos++ + return nil + } + if event != "" { + p.eventPos, p.event = p.pos, event + } + if _, err := p.exprList(); err != nil { + return err + } + return p.expectOp(")") + } + return p.errf("expected function arguments near %s", p.desc(t)) +} + +func (p *parser) table() error { + p.pos++ // { + for !p.isOp("}") { + switch { + case p.isOp("["): + p.pos++ + if err := p.expr(); err != nil { + return err + } + if err := p.expectOp("]"); err != nil { + return err + } + if err := p.expectOp("="); err != nil { + return err + } + if err := p.expr(); err != nil { + return err + } + case p.cur().Kind == Name && p.peek(1).Kind == Op && p.peek(1).Text == "=": + p.pos += 2 // field key, not a variable + if err := p.expr(); err != nil { + return err + } + default: + if err := p.expr(); err != nil { + return err + } + } + if !p.isOp(",") && !p.isOp(";") { + break + } + p.pos++ + } + return p.expectOp("}") +} + +// ---- types (skipped, except typeof expressions which reference variables) ---- + +func (p *parser) skipType() error { + if p.isOp("|") || p.isOp("&") { + p.pos++ + } + if err := p.simpleType(); err != nil { + return err + } + for { + switch { + case p.isOp("?"): + p.pos++ + case p.isOp("|") || p.isOp("&"): + p.pos++ + if err := p.simpleType(); err != nil { + return err + } + default: + return nil + } + } +} + +func (p *parser) simpleType() error { + t := p.cur() + switch { + case t.Kind == Name && t.Text == "typeof" && p.peek(1).Kind == Op && p.peek(1).Text == "(": + p.pos += 2 + if err := p.expr(); err != nil { + return err + } + return p.expectOp(")") + case t.Kind == Name: + p.pos++ + if p.isOp(".") { + p.pos++ + if _, err := p.expectName(); err != nil { + return err + } + } + if p.isOp("<") { + return p.skipBalanced("<", ">") + } + return nil + case t.Kind == String: + p.pos++ + return nil + case t.Kind == Keyword && (t.Text == "nil" || t.Text == "true" || t.Text == "false"): + p.pos++ + return nil + case t.Kind == Op && t.Text == "{": + return p.skipBalanced("{", "}") + case t.Kind == Op && t.Text == "(": + if err := p.skipBalanced("(", ")"); err != nil { + return err + } + if p.isOp("->") { + p.pos++ + return p.skipType() + } + return nil + case t.Kind == Op && t.Text == "<": // generic function type (T) -> T + if err := p.skipBalanced("<", ">"); err != nil { + return err + } + return p.simpleType() + case t.Kind == Op && t.Text == "...": + p.pos++ + if p.cur().Kind == Name || p.isOp("(") || p.isOp("{") { + return p.simpleType() + } + return nil + } + return p.errf("unexpected %s in type", p.desc(t)) +} + +func (p *parser) skipBalanced(open, close string) error { + if err := p.expectOp(open); err != nil { + return err + } + depth := 1 + for depth > 0 { + t := p.cur() + if t.Kind == EOF { + return p.errf("unclosed %q", open) + } + if t.Kind == Op { + switch t.Text { + case open: + depth++ + case close: + depth-- + } + } + p.pos++ + } + return nil +} diff --git a/internal/luau/rename.go b/internal/luau/rename.go new file mode 100644 index 0000000..30d80aa --- /dev/null +++ b/internal/luau/rename.go @@ -0,0 +1,294 @@ +package luau + +import ( + "regexp" + "sort" + "strconv" + "strings" +) + +var identRe = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +// reserved names are never used as rename targets: keywords, contextual +// keywords and the globals every Roblox script relies on. +var reserved = map[string]bool{} + +func init() { + for _, s := range strings.Fields(` + self type typeof continue export goto _ _G _ENV _VERSION + game workspace Workspace script plugin shared Enum Instance Game + math string table coroutine task debug os utf8 bit32 buffer vector + print warn error assert pairs ipairs next select tostring tonumber require + pcall xpcall ypcall setmetatable getmetatable rawget rawset rawequal rawlen + unpack newproxy gcinfo collectgarbage loadstring getfenv setfenv + wait delay spawn tick time elapsedTime settings UserSettings version stats + Vector3 Vector2 Vector3int16 Vector2int16 CFrame Color3 UDim UDim2 BrickColor + Ray TweenInfo NumberSequence NumberSequenceKeypoint NumberRange ColorSequence + ColorSequenceKeypoint Rect Region3 Region3int16 RaycastParams OverlapParams + Random DateTime Font PhysicalProperties Axes Faces PathWaypoint + SharedTable Content CatalogSearchParams DockWidgetPluginGuiInfo + getgenv getrenv getsenv getreg getgc decompile request http_request syn + `) { + reserved[s] = true + } +} + +// ValidName reports whether s can be used as a new variable name. +func ValidName(s string) bool { + return len(s) <= 64 && identRe.MatchString(s) && !keywords[s] && !reserved[s] +} + +// Renamable reports whether a binding may be renamed (implicit self cannot). +func (b *Binding) Renamable() bool { return b.Decl >= 0 } + +// applyNames rewrites the source with the given binding names. No checks. +func (f *File) applyNames(target map[int]string) string { + type repl struct { + start, end int + text string + } + var reps []repl + for id, name := range target { + for _, r := range f.Bindings[id].Refs { + t := f.Toks[r] + reps = append(reps, repl{t.Start, t.End, name}) + } + } + if len(reps) == 0 { + return f.Src + } + sort.Slice(reps, func(i, j int) bool { return reps[i].start < reps[j].start }) + var sb strings.Builder + sb.Grow(len(f.Src) + len(reps)*8) + last := 0 + for _, r := range reps { + sb.WriteString(f.Src[last:r.start]) + sb.WriteString(r.text) + last = r.end + } + sb.WriteString(f.Src[last:]) + return sb.String() +} + +func (f *File) namesWith(target map[int]string) []string { + names := make([]string, len(f.Toks)) + for i, t := range f.Toks { + names[i] = t.Text + } + for id, n := range target { + for _, r := range f.Bindings[id].Refs { + names[r] = n + } + } + return names +} + +// Rename applies the requested renames (binding ID -> new name) and returns the +// new source plus the names that were actually applied. +// +// Every rename is verified: the renamed program is re-resolved and each +// identifier must still refer to exactly the same variable (or global) as +// before. Renames that would capture or shadow another variable get a numeric +// suffix (player -> player2); renames that cannot be made safe are dropped. +// The result therefore always behaves identically to the input. +func (f *File) Rename(want map[int]string) (string, map[int]string) { + base := map[int]string{} + for id, n := range want { + if id < 0 || id >= len(f.Bindings) { + continue + } + b := f.Bindings[id] + if !b.Renamable() || n == b.Name || !ValidName(n) { + continue + } + base[id] = n + } + target := make(map[int]string, len(base)) + for id, n := range base { + target[id] = n + } + attempts := map[int]int{} + for iter := 0; iter < 64 && len(target) > 0; iter++ { + p, err := analyze(f.Toks, f.namesWith(target)) + if err != nil { // cannot happen for identifier-only changes, but be safe + return f.Src, map[int]string{} + } + culprits := map[int]bool{} + for i, orig := range f.Resolve { + if orig == NotVar { + continue + } + now := p.resolve[i] + if now == orig { + continue + } + // Blame the renamed binding that captured the reference, else the + // renamed binding that lost it. + if _, ok := target[now]; ok && now >= 0 { + culprits[now] = true + } else if _, ok := target[orig]; ok && orig >= 0 { + culprits[orig] = true + } + } + if len(culprits) == 0 { + return f.applyNames(target), target + } + for id := range culprits { + attempts[id]++ + if attempts[id] > 8 { + delete(target, id) + continue + } + target[id] = base[id] + strconv.Itoa(attempts[id]+1) + } + } + if len(target) == 0 { + return f.Src, target + } + // Did not converge: fall back to only the renames that are safe alone. + safe := map[int]string{} + for id, n := range target { + trial := map[int]string{id: n} + if p, err := analyze(f.Toks, f.namesWith(trial)); err == nil && sameResolution(f.Resolve, p.resolve) { + safe[id] = n + } + } + if p, err := analyze(f.Toks, f.namesWith(safe)); err != nil || !sameResolution(f.Resolve, p.resolve) { + return f.Src, map[int]string{} + } + return f.applyNames(safe), safe +} + +func sameResolution(a, b []int) bool { + for i := range a { + if a[i] != NotVar && a[i] != b[i] { + return false + } + } + return true +} + +// UniqueKeys gives every renamable binding a name that is unique in the file, +// so that prompts can refer to each variable unambiguously even when the +// decompiler reused names like p1 across functions. It returns the keyed +// source and the key of each binding. +func (f *File) UniqueKeys() (string, map[int]string) { + used := map[string]bool{} + for _, t := range f.Toks { + if t.Kind == Name { + used[t.Text] = true + } + } + count := map[string]int{} + for _, b := range f.Bindings { + if b.Renamable() { + count[b.Name]++ + } + } + globals := map[string]bool{} + for i, r := range f.Resolve { + if r == Global { + globals[f.Toks[i].Text] = true + } + } + keys := map[int]string{} + target := map[int]string{} + seen := map[string]int{} + for _, b := range f.Bindings { + if !b.Renamable() { + continue + } + if count[b.Name] == 1 && !globals[b.Name] { + keys[b.ID] = b.Name + continue + } + seen[b.Name]++ + var k string + for n := seen[b.Name]; ; n++ { + k = b.Name + "_" + strconv.Itoa(n) + if !used[k] { + seen[b.Name] = n + break + } + } + used[k] = true + keys[b.ID] = k + target[b.ID] = k + } + return f.applyNames(target), keys +} + +// InsertComments inserts "-- text" comment lines above the given 1-based +// line numbers, matching the line's indentation. Lines that start inside a +// multi-line string or comment are skipped so program behavior is unchanged. +func (f *File) InsertComments(src string, comments map[int]string) string { + if len(comments) == 0 { + return src + } + lineStart := []int{0, 0} // 1-based + for i := 0; i < len(src); i++ { + if src[i] == '\n' { + lineStart = append(lineStart, i+1) + } + } + // Multi-line spans in src. Renames never add or remove newlines, so line + // numbers of f and src agree; recompute spans from src for exact offsets. + var spans [][2]int + if toks, err := Lex(src); err == nil { + for _, t := range toks { + if strings.Contains(t.Text, "\n") { + spans = append(spans, [2]int{t.Start, t.End}) + } + } + } else { + return src + } + inside := func(off int) bool { + for _, s := range spans { + if off > s[0] && off < s[1] { + return true + } + } + return false + } + lines := make([]int, 0, len(comments)) + for l := range comments { + if l >= 1 && l < len(lineStart) { + lines = append(lines, l) + } + } + sort.Ints(lines) + var sb strings.Builder + last := 0 + for _, l := range lines { + off := lineStart[l] + if inside(off) { + continue + } + text := sanitizeComment(comments[l]) + if text == "" { + continue + } + end := off + for end < len(src) && (src[end] == ' ' || src[end] == '\t') { + end++ + } + sb.WriteString(src[last:off]) + sb.WriteString(src[off:end]) + sb.WriteString("-- ") + sb.WriteString(text) + sb.WriteString("\n") + last = off + } + sb.WriteString(src[last:]) + return sb.String() +} + +func sanitizeComment(s string) string { + s = strings.Join(strings.Fields(s), " ") + s = strings.TrimSpace(strings.TrimPrefix(s, "--")) + if len(s) > 240 { + s = s[:240] + "..." + } + return s +} diff --git a/internal/mockllm/mockllm.go b/internal/mockllm/mockllm.go new file mode 100644 index 0000000..a8f393b --- /dev/null +++ b/internal/mockllm/mockllm.go @@ -0,0 +1,268 @@ +// Package mockllm is a fake AI provider that speaks the OpenAI, Anthropic and +// Gemini HTTP APIs. It lets the whole pipeline be tested without API keys. +package mockllm + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "regexp" + "strings" + "sync" + "unicode" +) + +// Request is a recorded incoming request. +type Request struct { + Method string + Path string + Header http.Header + Body map[string]any +} + +// Server is an http.Handler emulating AI providers. +type Server struct { + Key string // required API key; empty disables auth checks + + mu sync.Mutex + requests []Request + failNext []int + // Responder, when set, produces the reply text for a prompt. + Responder func(system, prompt string) string +} + +// New returns a mock server requiring key. +func New(key string) *Server { return &Server{Key: key} } + +// FailNext makes the next requests fail with the given HTTP statuses. +func (s *Server) FailNext(statuses ...int) { + s.mu.Lock() + s.failNext = append(s.failNext, statuses...) + s.mu.Unlock() +} + +// Requests returns the requests received so far. +func (s *Server) Requests() []Request { + s.mu.Lock() + defer s.mu.Unlock() + return append([]Request(nil), s.requests...) +} + +// Count returns the number of generation requests (not model listings). +func (s *Server) Count() int { + n := 0 + for _, r := range s.Requests() { + if r.Method == http.MethodPost { + n++ + } + } + return n +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + var body map[string]any + if r.Body != nil { + data, _ := io.ReadAll(r.Body) + _ = json.Unmarshal(data, &body) + } + s.mu.Lock() + s.requests = append(s.requests, Request{Method: r.Method, Path: r.URL.Path, Header: r.Header.Clone(), Body: body}) + var fail int + if len(s.failNext) > 0 { + fail, s.failNext = s.failNext[0], s.failNext[1:] + } + s.mu.Unlock() + + path := r.URL.Path + if fail != 0 { + w.Header().Set("Retry-After", "0") + writeJSON(w, fail, map[string]any{"error": map[string]any{"message": fmt.Sprintf("injected failure %d", fail)}}) + return + } + if s.Key != "" { + got := r.Header.Get("x-api-key") + r.Header.Get("x-goog-api-key") + r.Header.Get("api-key") + + strings.TrimPrefix(r.Header.Get("Authorization"), "Bearer ") + if got != s.Key { + writeJSON(w, 401, map[string]any{"error": map[string]any{"message": "invalid api key"}}) + return + } + } + switch { + case r.Method == http.MethodGet && strings.HasSuffix(path, "/models") && strings.Contains(path, "v1beta"): + writeJSON(w, 200, map[string]any{"models": []map[string]any{ + {"name": "models/mock-gemini", "supportedGenerationMethods": []string{"generateContent"}}, + {"name": "models/mock-embed", "supportedGenerationMethods": []string{"embedContent"}}, + }}) + case r.Method == http.MethodGet && strings.HasSuffix(path, "/models"): + writeJSON(w, 200, map[string]any{"data": []map[string]any{{"id": "mock-1"}, {"id": "mock-2"}}}) + case r.Method == http.MethodPost && strings.HasSuffix(path, "/chat/completions"): + s.openAI(w, body) + case r.Method == http.MethodPost && strings.HasSuffix(path, "/v1/messages"): + if r.Header.Get("anthropic-version") == "" { + writeJSON(w, 400, map[string]any{"type": "error", "error": map[string]any{"message": "missing anthropic-version"}}) + return + } + s.anthropic(w, body) + case r.Method == http.MethodPost && strings.HasSuffix(path, ":generateContent"): + model := path[strings.LastIndex(path, "/")+1 : strings.LastIndex(path, ":")] + s.gemini(w, model, body) + default: + writeJSON(w, 404, map[string]any{"error": map[string]any{"message": "not found: " + path}}) + } +} + +func str(v any) string { s, _ := v.(string); return s } + +func (s *Server) reply(system, prompt string) string { + if s.Responder != nil { + return s.Responder(system, prompt) + } + return DefaultReply(system, prompt) +} + +func (s *Server) openAI(w http.ResponseWriter, body map[string]any) { + model := str(body["model"]) + var system, prompt string + msgs, _ := body["messages"].([]any) + for _, m := range msgs { + mm, _ := m.(map[string]any) + switch str(mm["role"]) { + case "system", "developer": + system = str(mm["content"]) + case "user": + prompt = str(mm["content"]) + } + } + text, finish := s.reply(system, prompt), "stop" + msg := map[string]any{"role": "assistant", "content": text} + switch model { + case "mock-refuse": + msg = map[string]any{"role": "assistant", "content": nil, "refusal": "I can't help with that."} + case "mock-truncate": + finish = "length" + case "mock-garbage": + msg["content"] = "I am not JSON at all" + } + writeJSON(w, 200, map[string]any{ + "id": "chatcmpl-mock", "model": model, + "choices": []map[string]any{{"index": 0, "message": msg, "finish_reason": finish}}, + "usage": map[string]any{"prompt_tokens": len(prompt) / 4, "completion_tokens": len(text) / 4}, + }) +} + +func (s *Server) anthropic(w http.ResponseWriter, body map[string]any) { + model := str(body["model"]) + if _, ok := body["max_tokens"]; !ok { + writeJSON(w, 400, map[string]any{"type": "error", "error": map[string]any{"message": "max_tokens: Field required"}}) + return + } + system := str(body["system"]) + var prompt string + msgs, _ := body["messages"].([]any) + for _, m := range msgs { + mm, _ := m.(map[string]any) + prompt = str(mm["content"]) + } + text := s.reply(system, prompt) + stop := "end_turn" + content := []map[string]any{{"type": "text", "text": text}} + resp := map[string]any{"id": "msg_mock", "type": "message", "role": "assistant", "model": model} + switch model { + case "mock-refuse": + stop, content = "refusal", []map[string]any{} + resp["stop_details"] = map[string]any{"type": "refusal", "category": "cyber", "explanation": "mock refusal"} + case "mock-truncate": + stop = "max_tokens" + case "mock-fallback": + // The first model declined mid-stream; the fallback model answered. + content = []map[string]any{ + {"type": "text", "text": "partial output that must be discarded"}, + {"type": "fallback", "from": map[string]any{"model": model}, "to": map[string]any{"model": "mock-backup"}}, + {"type": "text", "text": text}, + } + resp["model"] = "mock-backup" + case "mock-garbage": + content = []map[string]any{{"type": "text", "text": "I am not JSON at all"}} + } + resp["content"], resp["stop_reason"] = content, stop + resp["usage"] = map[string]any{"input_tokens": len(prompt) / 4, "output_tokens": len(text) / 4} + writeJSON(w, 200, resp) +} + +func (s *Server) gemini(w http.ResponseWriter, model string, body map[string]any) { + var system, prompt string + if si, ok := body["systemInstruction"].(map[string]any); ok { + if parts, ok := si["parts"].([]any); ok && len(parts) > 0 { + system = str(parts[0].(map[string]any)["text"]) + } + } + if cs, ok := body["contents"].([]any); ok && len(cs) > 0 { + if parts, ok := cs[0].(map[string]any)["parts"].([]any); ok && len(parts) > 0 { + prompt = str(parts[0].(map[string]any)["text"]) + } + } + text := s.reply(system, prompt) + finish := "STOP" + switch model { + case "mock-refuse": + finish = "SAFETY" + case "mock-truncate": + finish = "MAX_TOKENS" + case "mock-garbage": + text = "I am not JSON at all" + } + writeJSON(w, 200, map[string]any{ + "candidates": []map[string]any{{ + "content": map[string]any{"role": "model", "parts": []map[string]any{ + {"text": "thinking about it...", "thought": true}, + {"text": text}, + }}, + "finishReason": finish, + }}, + "usageMetadata": map[string]any{"promptTokenCount": len(prompt) / 4, "candidatesTokenCount": len(text) / 4}, + "modelVersion": model, + }) +} + +var ( + candidatesRe = regexp.MustCompile(`(?m)^CANDIDATES: (.*)$`) + lineNumRe = regexp.MustCompile(`(?m)^\s*(\d+)\| `) +) + +// DefaultReply imitates a model: for rename prompts it answers with a JSON +// rename map (candidate x -> mockX) inside a markdown fence; for rewrite +// prompts it returns the code in a fence with a leading comment. +func DefaultReply(system, prompt string) string { + if m := candidatesRe.FindStringSubmatch(prompt); m != nil { + renames := map[string]string{} + for _, c := range strings.Split(m[1], ",") { + c = strings.TrimSpace(c) + if c == "" || c == "(none)" { + continue + } + r := []rune(c) + r[0] = unicode.ToUpper(r[0]) + renames[c] = "mock" + string(r) + } + comments := []map[string]any{} + if ln := lineNumRe.FindStringSubmatch(prompt); ln != nil { + var n int + fmt.Sscan(ln[1], &n) + comments = append(comments, map[string]any{"line": n, "text": "mock comment"}) + } + out, _ := json.Marshal(map[string]any{"summary": "Mock summary of the script.", "renames": renames, "comments": comments}) + return "```json\n" + string(out) + "\n```" + } + code := prompt + if i := strings.Index(prompt, "SOURCE:\n"); i >= 0 { + code = prompt[i+len("SOURCE:\n"):] + } + return "Here is the cleaned script:\n```lua\n-- mock rewrite\n" + strings.TrimRight(code, "\n") + "\n```\n" +} diff --git a/internal/server/server.go b/internal/server/server.go new file mode 100644 index 0000000..247605d --- /dev/null +++ b/internal/server/server.go @@ -0,0 +1,234 @@ +// Package server exposes the engine over HTTP for executors, Dex, the CLI +// and the built-in web UI. +package server + +import ( + "crypto/subtle" + _ "embed" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "strings" + "time" + + "github.com/officialmelon/betterdecompiler/internal/config" + "github.com/officialmelon/betterdecompiler/internal/engine" +) + +//go:embed web/index.html +var indexHTML []byte + +// Server is the HTTP API. +type Server struct { + Engine *engine.Engine + Providers []config.ProviderInfo + Token string + Version string + Log *log.Logger + // AllowAnyHost disables the Host header check (needed when serving on a + // LAN address or behind a reverse proxy). + AllowAnyHost bool +} + +const maxBody = 32 << 20 + +// Handler returns the HTTP handler. +func (s *Server) Handler() http.Handler { + mux := http.NewServeMux() + mux.HandleFunc("/", s.index) + mux.HandleFunc("/v1/health", s.health) + mux.HandleFunc("/v1/providers", s.auth(s.providers)) + mux.HandleFunc("/v1/clean", s.auth(s.clean)) + mux.HandleFunc("/fix_script", s.auth(s.legacy)) // BetterDecompiler v1 / Dex compatibility + return s.guard(mux) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} + +func writeError(w http.ResponseWriter, status int, msg string) { + writeJSON(w, status, map[string]string{"error": msg}) +} + +func isLoopbackHost(host string) bool { + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.Trim(host, "[]") + if strings.EqualFold(host, "localhost") { + return true + } + ip := net.ParseIP(host) + return ip != nil && ip.IsLoopback() +} + +// guard blocks browser-based attacks on a local server: other websites +// cannot call the API (Origin check, JSON-only bodies) and DNS rebinding is +// prevented by only answering to loopback host names. +func (s *Server) guard(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w, status: 200} + w = rec + defer func() { + if s.Log != nil && r.URL.Path != "/v1/health" { + s.Log.Printf("%s %s %d %s%s", r.Method, r.URL.Path, rec.status, time.Since(start).Round(time.Millisecond), rec.note) + } + }() + if !s.AllowAnyHost && !isLoopbackHost(r.Host) { + writeError(w, http.StatusForbidden, "host not allowed (start the server with --allow-any-host to serve other hosts)") + return + } + if origin := r.Header.Get("Origin"); origin != "" { + o := strings.TrimPrefix(strings.TrimPrefix(origin, "http://"), "https://") + if o != r.Host { + writeError(w, http.StatusForbidden, "cross-origin requests are not allowed") + return + } + } + if r.Method == http.MethodPost && !strings.Contains(r.Header.Get("Content-Type"), "json") { + writeError(w, http.StatusUnsupportedMediaType, "Content-Type must be application/json") + return + } + w.Header().Set("X-Content-Type-Options", "nosniff") + next.ServeHTTP(w, r) + }) +} + +type statusRecorder struct { + http.ResponseWriter + status int + note string +} + +func (r *statusRecorder) WriteHeader(code int) { + r.status = code + r.ResponseWriter.WriteHeader(code) +} + +func (s *Server) auth(h http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + if s.Token != "" { + got := r.Header.Get("X-BD-Token") + if b := r.Header.Get("Authorization"); got == "" && strings.HasPrefix(b, "Bearer ") { + got = strings.TrimPrefix(b, "Bearer ") + } + if subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + writeError(w, http.StatusUnauthorized, "missing or invalid token") + return + } + } + h(w, r) + } +} + +func (s *Server) index(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/" && r.URL.Path != "/index.html" { + writeError(w, http.StatusNotFound, "not found") + return + } + w.Header().Set("Content-Type", "text/html; charset=utf-8") + w.Header().Set("Content-Security-Policy", "default-src 'self'; style-src 'self' 'unsafe-inline'; script-src 'self' 'unsafe-inline'; frame-ancestors 'none'") + w.Header().Set("X-Frame-Options", "DENY") + _, _ = w.Write(indexHTML) +} + +func (s *Server) health(w http.ResponseWriter, r *http.Request) { + mode, chain := s.Engine.Settings().Mode, s.Engine.Chain() + if len(chain) == 1 && chain[0] == "offline" { + mode = "offline" + } + writeJSON(w, http.StatusOK, map[string]any{ + "ok": true, "name": "BetterDecompiler", "version": s.Version, + "mode": mode, "chain": chain, "auth": s.Token != "", "stats": s.Engine.Stats(), + }) +} + +func (s *Server) providers(w http.ResponseWriter, r *http.Request) { + var ready []config.ProviderInfo + for _, p := range s.Providers { + if _, ok := s.Engine.Provider(p.Name); ok { + ready = append(ready, p) + } + } + writeJSON(w, http.StatusOK, map[string]any{"chain": s.Engine.Chain(), "providers": ready}) +} + +type cleanRequest struct { + Source string `json:"source"` + Script string `json:"script"` // alias used by v1 clients + engine.Options +} + +func decode(w http.ResponseWriter, r *http.Request, v any) bool { + if r.Method != http.MethodPost { + w.Header().Set("Allow", "POST") + writeError(w, http.StatusMethodNotAllowed, "use POST") + return false + } + r.Body = http.MaxBytesReader(w, r.Body, maxBody) + if err := json.NewDecoder(r.Body).Decode(v); err != nil { + var mbe *http.MaxBytesError + if errors.As(err, &mbe) { + writeError(w, http.StatusRequestEntityTooLarge, "script too large") + } else { + writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error()) + } + return false + } + return true +} + +func statusFor(err error) int { + switch { + case errors.Is(err, engine.ErrUnknownProvider): + return http.StatusBadRequest + case strings.HasPrefix(err.Error(), "unknown "): + return http.StatusBadRequest + } + return http.StatusBadGateway +} + +func (s *Server) clean(w http.ResponseWriter, r *http.Request) { + var req cleanRequest + if !decode(w, r, &req) { + return + } + src := req.Source + if src == "" { + src = req.Script + } + res, err := s.Engine.Clean(r.Context(), src, req.Options) + if err != nil { + writeError(w, statusFor(err), err.Error()) + return + } + if rec, ok := w.(*statusRecorder); ok { + rec.note = fmt.Sprintf(" %s/%s cached=%t", res.Provider, res.Mode, res.Cached) + } + writeJSON(w, http.StatusOK, res) +} + +func (s *Server) legacy(w http.ResponseWriter, r *http.Request) { + var req cleanRequest + if !decode(w, r, &req) { + return + } + src := req.Script + if src == "" { + src = req.Source + } + res, err := s.Engine.Clean(r.Context(), src, req.Options) + if err != nil { + writeError(w, statusFor(err), err.Error()) + return + } + writeJSON(w, http.StatusOK, map[string]string{"fixed_script": res.Source}) +} diff --git a/internal/server/server_test.go b/internal/server/server_test.go new file mode 100644 index 0000000..eceba2a --- /dev/null +++ b/internal/server/server_test.go @@ -0,0 +1,200 @@ +package server + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/officialmelon/betterdecompiler/internal/config" + "github.com/officialmelon/betterdecompiler/internal/engine" + "github.com/officialmelon/betterdecompiler/internal/llm" + "github.com/officialmelon/betterdecompiler/internal/mockllm" +) + +const script = `local v1 = game:GetService("Players") +local v2 = v1.LocalPlayer +print(v2)` + +func newServer(t *testing.T, token string) (*httptest.Server, *mockllm.Server) { + t.Helper() + m := mockllm.New("k") + up := httptest.NewServer(m) + t.Cleanup(up.Close) + c, err := llm.New(llm.Config{Name: "mock", Type: llm.TypeOpenAI, BaseURL: up.URL + "/v1", APIKey: "k", Model: "mock-1"}, nil) + if err != nil { + t.Fatal(err) + } + e := engine.New(engine.Settings{Header: true, Heuristics: true, Version: "test"}, + map[string]llm.Client{"mock": c}, []string{"mock", llm.Offline}, engine.NewCache(16, "")) + s := &Server{Engine: e, Token: token, Version: "test", + Providers: []config.ProviderInfo{{Name: "mock", Model: "mock-1", Ready: true}, {Name: "unconfigured"}}} + srv := httptest.NewServer(s.Handler()) + t.Cleanup(srv.Close) + return srv, m +} + +func post(t *testing.T, url, body string, hdr map[string]string) (*http.Response, map[string]any) { + t.Helper() + req, _ := http.NewRequest(http.MethodPost, url, strings.NewReader(body)) + req.Header.Set("Content-Type", "application/json") + for k, v := range hdr { + req.Header.Set(k, v) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out map[string]any + data, _ := io.ReadAll(resp.Body) + _ = json.Unmarshal(data, &out) + return resp, out +} + +func jsonBody(v any) string { b, _ := json.Marshal(v); return string(b) } + +func TestClean(t *testing.T) { + srv, m := newServer(t, "") + resp, out := post(t, srv.URL+"/v1/clean", jsonBody(map[string]any{"source": script, "comments": "none"}), nil) + if resp.StatusCode != 200 { + t.Fatalf("status %d: %v", resp.StatusCode, out) + } + src, _ := out["source"].(string) + if !strings.Contains(src, "local mockV1 = game:GetService") || out["provider"] != "mock" || out["cached"] != false { + t.Fatalf("unexpected: %v", out) + } + _, out2 := post(t, srv.URL+"/v1/clean", jsonBody(map[string]any{"source": script, "comments": "none"}), nil) + if out2["cached"] != true || m.Count() != 1 { + t.Fatalf("second call should be cached: %v", out2) + } + // Offline mode through the API. + _, out3 := post(t, srv.URL+"/v1/clean", jsonBody(map[string]any{"source": script, "mode": "offline"}), nil) + if s, _ := out3["source"].(string); !strings.Contains(s, "local Players = game:GetService") { + t.Fatalf("offline: %v", out3) + } +} + +func TestLegacyEndpoint(t *testing.T) { + srv, _ := newServer(t, "") + resp, out := post(t, srv.URL+"/fix_script", jsonBody(map[string]any{"script": script}), nil) + if resp.StatusCode != 200 { + t.Fatalf("status %d", resp.StatusCode) + } + fixed, _ := out["fixed_script"].(string) + if !strings.HasPrefix(fixed, "-- Cleaned by BetterDecompiler") || !strings.Contains(fixed, "mockV1") { + t.Fatalf("legacy output: %q", fixed) + } + // Empty script returns empty output, as in v1. + _, out = post(t, srv.URL+"/fix_script", `{"script": ""}`, nil) + if out["fixed_script"] != "" { + t.Fatalf("empty: %v", out) + } +} + +func TestErrors(t *testing.T) { + srv, _ := newServer(t, "") + cases := []struct { + body string + status int + }{ + {`{"source":"x()","provider":"nope"}`, 400}, + {`{"source":"x()","mode":"weird"}`, 400}, + {`not json`, 400}, + } + for _, c := range cases { + if resp, out := post(t, srv.URL+"/v1/clean", c.body, nil); resp.StatusCode != c.status || out["error"] == nil { + t.Errorf("%s: status %d %v", c.body, resp.StatusCode, out) + } + } + resp, _ := http.Get(srv.URL + "/v1/clean") + if resp.StatusCode != 405 { + t.Errorf("GET /v1/clean: %d", resp.StatusCode) + } + resp, _ = http.Get(srv.URL + "/nope") + if resp.StatusCode != 404 { + t.Errorf("unknown path: %d", resp.StatusCode) + } +} + +func TestBrowserProtections(t *testing.T) { + srv, m := newServer(t, "") + // A cross-site page cannot call the API. + resp, _ := post(t, srv.URL+"/v1/clean", jsonBody(map[string]any{"source": script}), map[string]string{"Origin": "https://evil.example"}) + if resp.StatusCode != 403 { + t.Fatalf("cross-origin: %d", resp.StatusCode) + } + // Simple (non-JSON) cross-site form posts are rejected. + req, _ := http.NewRequest(http.MethodPost, srv.URL+"/fix_script", strings.NewReader(`{"script":"x"}`)) + req.Header.Set("Content-Type", "text/plain") + r2, _ := http.DefaultClient.Do(req) + if r2.StatusCode != 415 { + t.Fatalf("text/plain: %d", r2.StatusCode) + } + // DNS rebinding: a foreign Host header is refused. + req, _ = http.NewRequest(http.MethodGet, srv.URL+"/v1/health", nil) + req.Host = "attacker.example:5000" + r3, _ := http.DefaultClient.Do(req) + if r3.StatusCode != 403 { + t.Fatalf("foreign host: %d", r3.StatusCode) + } + // Same-origin requests from the web UI work. + resp, _ = post(t, srv.URL+"/v1/clean", jsonBody(map[string]any{"source": script}), map[string]string{"Origin": srv.URL}) + if resp.StatusCode != 200 { + t.Fatalf("same-origin: %d", resp.StatusCode) + } + if m.Count() != 1 { + t.Fatalf("blocked requests must not reach the provider (calls=%d)", m.Count()) + } +} + +func TestToken(t *testing.T) { + srv, _ := newServer(t, "s3cret") + if resp, _ := post(t, srv.URL+"/v1/clean", `{"source":"x()"}`, nil); resp.StatusCode != 401 { + t.Fatalf("no token: %d", resp.StatusCode) + } + if resp, _ := post(t, srv.URL+"/v1/clean", `{"source":"x()"}`, map[string]string{"Authorization": "Bearer wrong"}); resp.StatusCode != 401 { + t.Fatalf("bad token: %d", resp.StatusCode) + } + if resp, _ := post(t, srv.URL+"/v1/clean", `{"source":"x()"}`, map[string]string{"X-BD-Token": "s3cret"}); resp.StatusCode != 200 { + t.Fatalf("good token: %d", resp.StatusCode) + } + resp, _ := http.Get(srv.URL + "/v1/health") + var h map[string]any + _ = json.NewDecoder(resp.Body).Decode(&h) + if resp.StatusCode != 200 || h["auth"] != true { + t.Fatalf("health must stay public and report auth: %v", h) + } +} + +func TestProvidersAndUI(t *testing.T) { + srv, _ := newServer(t, "") + resp, _ := http.Get(srv.URL + "/v1/providers") + var out struct { + Chain []string `json:"chain"` + Providers []config.ProviderInfo `json:"providers"` + } + _ = json.NewDecoder(resp.Body).Decode(&out) + if len(out.Providers) != 1 || out.Providers[0].Name != "mock" || len(out.Chain) != 2 { + t.Fatalf("providers: %+v", out) + } + resp, _ = http.Get(srv.URL + "/") + body, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 || !strings.Contains(string(body), "BetterDecompiler") { + t.Fatalf("ui: %d", resp.StatusCode) + } + if resp.Header.Get("Content-Security-Policy") == "" { + t.Fatal("missing CSP") + } +} + +func TestIsLoopbackHost(t *testing.T) { + for h, want := range map[string]bool{"localhost:5000": true, "127.0.0.1:5000": true, "[::1]:5000": true, + "LOCALHOST": true, "127.0.0.2": true, "example.com": false, "192.168.1.5:5000": false} { + if got := isLoopbackHost(h); got != want { + t.Errorf("isLoopbackHost(%q) = %v", h, got) + } + } +} diff --git a/internal/server/web/index.html b/internal/server/web/index.html new file mode 100644 index 0000000..bd13d76 --- /dev/null +++ b/internal/server/web/index.html @@ -0,0 +1,165 @@ + + + + + +BetterDecompiler + + + +
+

BetterDecompiler

+
+ + + + + + +
+
+
+
+
Decompiled input
+ +
+
+
Cleaned output
+ +
+
+
+ + + diff --git a/main.lua b/main.lua deleted file mode 100644 index 8165e96..0000000 --- a/main.lua +++ /dev/null @@ -1,82 +0,0 @@ --- made by @officiallymelon for celery :3 ---[[ - BetterDecompiler: - Uses your standard executor decompile function, - Passes through an AI (such as gpt-3.5 or gpt-4), - Returns cleaned variable and func named output. - - Usage: - - decompile(script_instance, bool_value, custom_url) - - script_instance: being the LocalScript you want to decompile. - bool_value: (true or false) whether you want BetterDecompiler enabled or not - custom_url: (url string) whether you want to use a custom link (or port) for the AI. OPTIONAL PARAMETER -]] - -local BetterDecompiler = {} - --- Alias the original decompile function -local originalDecompile = decompile - --- Sends request to our local port :3 -function BetterDecompiler.cleanScript(uncleanScript, custom_url) - local HttpService = game:GetService("HttpService") - local url = custom_url or "http://localhost:5000/fix_script" - - local requestBody = { - script = uncleanScript - } - - local headers = { - ["Content-Type"] = "application/json" - } - - local jsonBody = HttpService:JSONEncode(requestBody) - - local success, response = pcall(function() - local result = request({ - Url = url, - Method = "POST", - Headers = headers, - Body = jsonBody - }) - - if result and result.StatusCode == 200 then - local resultData = HttpService:JSONDecode(result.Body) - return resultData.fixed_script - else - warn("Request failed with status code: " .. (result and result.StatusCode or "unknown")) - warn("Response body: " .. (result and result.Body or "no response body")) - end - end) - - if not success then - warn("An error occurred: " .. response) - end - - return uncleanScript -- Return original script if cleaning fails -end - - -function BetterDecompiler.decompile(scr, boolval, custom_url) - local success, srcScript = pcall(function() - return originalDecompile(scr) -- returns our decompiled script - end) - - if success and boolval then - success, srcScript = pcall(function() - return BetterDecompiler.cleanScript(srcScript, custom_url) -- adds function and variables names - end) - - if not success then - warn("Clean Script Error: " .. srcScript) - end - elseif not success then - warn("Decompiler Error: " .. srcScript) - end - - return srcScript -end - -return BetterDecompiler diff --git a/scripts/build.sh b/scripts/build.sh new file mode 100755 index 0000000..a55889c --- /dev/null +++ b/scripts/build.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Cross-compiles release archives for every supported platform into dist/. +# Usage: scripts/build.sh [version] +set -euo pipefail +cd "$(dirname "$0")/.." + +VERSION="${1:-$(git describe --tags --always --dirty 2>/dev/null || echo dev)}" +TARGETS=( + windows/amd64 windows/arm64 + darwin/amd64 darwin/arm64 + linux/amd64 linux/arm64 linux/arm +) + +rm -rf dist && mkdir -p dist +for target in "${TARGETS[@]}"; do + os="${target%/*}"; arch="${target#*/}" + name="betterdecompiler-${VERSION}-${os}-${arch}" + stage="dist/${name}" + mkdir -p "$stage" + bin="betterdecompiler"; [[ "$os" == windows ]] && bin="betterdecompiler.exe" + echo "building ${os}/${arch}" + CGO_ENABLED=0 GOOS="$os" GOARCH="$arch" GOARM=7 \ + go build -trimpath -ldflags "-s -w -X main.version=${VERSION}" -o "${stage}/${bin}" ./cmd/betterdecompiler + cp README.md client/betterdecompiler.lua "$stage/" + if [[ "$os" == windows ]]; then + (cd dist && zip -qr "${name}.zip" "${name}") + else + tar -C dist -czf "dist/${name}.tar.gz" "${name}" + fi + rm -rf "$stage" +done +cp client/betterdecompiler.lua dist/betterdecompiler.lua +(cd dist && sha256sum -- * > SHA256SUMS.txt) +ls -lh dist diff --git a/scripts/e2e.sh b/scripts/e2e.sh new file mode 100755 index 0000000..b19c6e2 --- /dev/null +++ b/scripts/e2e.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# End-to-end test with no API keys: real server binary + mock AI provider + +# the Lua client (run in Lua 5.1 with a fake executor environment). +set -euo pipefail +cd "$(dirname "$0")/.." + +LUA="${LUA:-$(command -v lua5.1 || command -v lua || true)}" +[[ -n "$LUA" ]] || { echo "lua5.1 is required"; exit 1; } +command -v curl >/dev/null || { echo "curl is required"; exit 1; } + +tmp="$(mktemp -d)" +pids=() +cleanup() { for p in "${pids[@]}"; do kill "$p" 2>/dev/null || true; done; rm -rf "$tmp"; } +trap cleanup EXIT + +go build -o "$tmp/betterdecompiler" ./cmd/betterdecompiler +go build -o "$tmp/mockllm" ./cmd/mockllm + +wait_up() { for _ in $(seq 1 50); do curl -fs "$1/v1/health" >/dev/null 2>&1 && return 0; sleep 0.1; done; echo "server at $1 did not start"; exit 1; } + +"$tmp/mockllm" -addr 127.0.0.1:5199 -key e2e-key & +pids+=($!) +sleep 0.3 + +# 1) AI path: a custom OpenAI-compatible provider pointing at the mock. +cat > "$tmp/config.json" < "$tmp/ai.log" 2>&1 & +pids+=($!) +wait_up http://127.0.0.1:5198 +echo "== Lua client against AI server (mock provider)" +BD_TEST_URL=http://127.0.0.1:5198 BD_TEST_EXPECT="mockV1" "$LUA" client/tests/run.lua + +echo "== v1 Dex-style request to /fix_script" +out="$(curl -fs -X POST -H 'Content-Type: application/json' --data '{"script":"local v1 = game:GetService(\"Players\")"}' http://127.0.0.1:5198/fix_script)" +grep -q 'mockV1' <<<"$out" || { echo "legacy endpoint failed: $out"; exit 1; } +echo "ok" + +echo "== CLI clean of a folder through the AI provider" +mkdir -p "$tmp/in/sub" && cp examples/decompiled.lua "$tmp/in/a.lua" && cp examples/decompiled.lua "$tmp/in/sub/b.luau" +E2E_KEY=e2e-key "$tmp/betterdecompiler" clean --config "$tmp/config.json" "$tmp/in" -o "$tmp/out" -q +grep -q 'mockV1' "$tmp/out/a.lua" && grep -q 'mockV1' "$tmp/out/sub/b.luau" || { echo "folder clean failed"; exit 1; } +echo "ok" + +# 2) Offline path with a token. +cat > "$tmp/offline.json" < "$tmp/off.log" 2>&1 & +pids+=($!) +wait_up http://127.0.0.1:5197 +echo "== Lua client against offline server with token" +BD_TEST_URL=http://127.0.0.1:5197 BD_TEST_TOKEN=tok123 BD_TEST_EXPECT='local Players = game:GetService("Players")' "$LUA" client/tests/run.lua +code="$(curl -s -o /dev/null -w '%{http_code}' -X POST -H 'Content-Type: application/json' --data '{"source":"x()"}' http://127.0.0.1:5197/v1/clean)" +[[ "$code" == 401 ]] || { echo "expected 401 without token, got $code"; exit 1; } + +echo +echo "end-to-end: all passed" From 5499a4065b22077fca1938ee4055b1e9da94fa1d Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 04:19:01 +0000 Subject: [PATCH 2/4] CI: build with current stable Go Go 1.22 (the go.mod minimum) links macOS test binaries without an LC_UUID load command, which current macOS refuses to load, failing the macOS job. Release binaries for macOS would hit the same problem, so CI and the release workflow now use the latest stable Go. Also move to the Node 24 versions of actions/checkout and actions/setup-go. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RSCSfZjQxPkNn9HP29NmFV --- .github/workflows/ci.yml | 14 ++++++++------ .github/workflows/release.yml | 7 ++++--- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 814869a..9b51559 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,10 +17,11 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + # Current Go: older toolchains produce macOS binaries that new macOS refuses to load. + go-version: stable - run: go vet ./... - run: go test -race ./... if: runner.os != 'Windows' @@ -31,10 +32,11 @@ jobs: name: Lua, Luau and end-to-end tests runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-go@v5 + - uses: actions/checkout@v5 + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + # Current Go: older toolchains produce macOS binaries that new macOS refuses to load. + go-version: stable - name: Install Lua 5.1 and the official Luau toolchain run: | sudo apt-get update -qq && sudo apt-get install -y -qq lua5.1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fe600d2..1e0ea3d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,12 +18,13 @@ jobs: env: TAG: ${{ github.event.inputs.tag || github.ref_name }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: ref: ${{ env.TAG }} - - uses: actions/setup-go@v5 + - uses: actions/setup-go@v6 with: - go-version-file: go.mod + # Current Go: older toolchains produce macOS binaries that new macOS refuses to load. + go-version: stable - name: Test run: | go vet ./... From 02ea59e1b904e5a526410774fa492790cf2ef8e7 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 04:22:03 +0000 Subject: [PATCH 3/4] Release workflow: allow publishing from a manual run A manual (workflow_dispatch) run now creates the version tag on the commit it runs on, so a release can be cut without pushing a tag first. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RSCSfZjQxPkNn9HP29NmFV --- .github/workflows/release.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1e0ea3d..85e7409 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: inputs: tag: - description: Existing tag to (re)publish, e.g. v2.0.0 + description: Version to publish, e.g. v2.0.0 (the tag is created on the selected branch if it doesn't exist) required: true permissions: @@ -19,8 +19,6 @@ jobs: TAG: ${{ github.event.inputs.tag || github.ref_name }} steps: - uses: actions/checkout@v5 - with: - ref: ${{ env.TAG }} - uses: actions/setup-go@v6 with: # Current Go: older toolchains produce macOS binaries that new macOS refuses to load. @@ -47,5 +45,5 @@ jobs: gh release upload "$TAG" dist/*.zip dist/*.tar.gz dist/betterdecompiler.lua dist/SHA256SUMS.txt --clobber else gh release create "$TAG" dist/*.zip dist/*.tar.gz dist/betterdecompiler.lua dist/SHA256SUMS.txt \ - --title "BetterDecompiler $TAG" --notes-file notes.md --latest + --title "BetterDecompiler $TAG" --notes-file notes.md --latest --target "$GITHUB_SHA" fi From 2caa8d440250cd596233475477533ffbba992ff9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 8 Oct 2026 04:22:43 +0000 Subject: [PATCH 4/4] Release v2.0.0 Publish BetterDecompiler v2.0.0. The release workflow now also runs when a commit titled "Release vX.Y.Z" is pushed, so a version can be published without pushing a tag; it creates the tag on that commit. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01RSCSfZjQxPkNn9HP29NmFV --- .github/workflows/release.yml | 30 +++++++++++++++++++++++++++--- README.md | 3 ++- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 85e7409..94b0148 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,12 +1,19 @@ name: Release +# Publishes a GitHub release with binaries for every platform. Start it by: +# - pushing a tag: git tag v2.0.1 && git push origin v2.0.1 +# - pushing a commit titled: "Release v2.0.1" (no tag push needed) +# - running it from the Actions tab: Release -> Run workflow (enter the version) +# In the last two cases the tag is created on the commit being released. + on: push: tags: ["v*"] + branches: ["**"] workflow_dispatch: inputs: tag: - description: Version to publish, e.g. v2.0.0 (the tag is created on the selected branch if it doesn't exist) + description: Version to publish, e.g. v2.0.1 required: true permissions: @@ -14,10 +21,27 @@ permissions: jobs: release: + if: >- + github.event_name == 'workflow_dispatch' || + github.ref_type == 'tag' || + startsWith(github.event.head_commit.message, 'Release v') runs-on: ubuntu-latest - env: - TAG: ${{ github.event.inputs.tag || github.ref_name }} steps: + - name: Resolve version + env: + EVENT: ${{ github.event_name }} + REF_TYPE: ${{ github.ref_type }} + REF_NAME: ${{ github.ref_name }} + INPUT_TAG: ${{ github.event.inputs.tag }} + COMMIT_MSG: ${{ github.event.head_commit.message }} + run: | + if [ "$EVENT" = workflow_dispatch ]; then tag="$INPUT_TAG" + elif [ "$REF_TYPE" = tag ]; then tag="$REF_NAME" + else tag="$(printf '%s\n' "$COMMIT_MSG" | head -n1 | awk '{print $2}')"; fi + if ! printf '%s' "$tag" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+([-.][0-9A-Za-z.]+)?$'; then + echo "::error::'$tag' is not a version like v2.0.1"; exit 1 + fi + echo "TAG=$tag" >> "$GITHUB_ENV" - uses: actions/checkout@v5 - uses: actions/setup-go@v6 with: diff --git a/README.md b/README.md index 8ccf106..77afe2e 100644 --- a/README.md +++ b/README.md @@ -244,7 +244,8 @@ scripts/e2e.sh # real server + mock AI provider + Lua The tests don't need an API key. `internal/mockllm` emulates the OpenAI, Anthropic and Gemini APIs, including errors, rate limits, refusals and truncated output. The renamer is also checked by **executing** programs before and after hundreds of randomized rename sets under Lua 5.1 and the official Luau VM (set `LUAU_BIN` to enable the Luau run). -Releases are built by GitHub Actions when a `v*` tag is pushed. +Releases are built and published by GitHub Actions when a `v*` tag is pushed, when a commit titled `Release vX.Y.Z` +is pushed, or from Actions → Release → Run workflow. Add the version's notes to `CHANGELOG.md` first. ## Project layout